2022-04-28T19:27:57.592 INFO:root:teuthology version: 1.1.1.dev328+g004701de 2022-04-28T19:27:57.593 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade:octopus-x/parallel/{0-distro/ubuntu_18.04 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api}} --verbose --archive /home/teuthworker/archive/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495 --name dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi --owner scheduled_dis@teuthology -- /home/teuthworker/archive/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495/orig.config.yaml 2022-04-28T19:27:57.643 DEBUG:teuthology.report:Pushing job info to http://paddles.front.sepia.ceph.com/ 2022-04-28T19:27:57.720 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495 branch: pacific description: upgrade:octopus-x/parallel/{0-distro/ubuntu_18.04 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api}} email: idryomov@gmail.com first_in_suite: false job_id: '6811495' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' name: dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '18.04' overrides: admin_socket: branch: pacific ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-whitelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d workunit: branch: pacific sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d owner: scheduled_dis@teuthology priority: 51 repo: https://github.com/ceph/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d sleep_before_teardown: 100000 suite: upgrade:octopus-x suite_branch: pacific suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d targets: smithi059.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJqdwrmBwa1gKkrobrp/Nf7z3Y7RvO8idltDM0wxe+PTA6NnheMPsRSiqY2q2GWBFyxrIjI8LrJFPLFpGAJvNbZ1prusCDgFzJH7sELEAsHcubqVySF6MXGuDQgOB62fvQkBpjmAzEAJSTXXbroibWPa3EyW4YyE9dCk+5BWRSmyrus5oC7pUkQNk54rfKacq2sI6k5Hsph88qmx9f39NEH3jF02UvOcv5LnT2sSDUsYvbmSgF1wj1Xouw/AGQhANc+38ruFav+emBGCS2OR8zzTkyl3/rjlG55CBS/yjsEW9NKfHiVY+gkckeoFC3TicXaFLspoiXgF8JfEaPRnqh smithi139.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDMoTnbe/SX8UgiXZmb58uxXVdvd/FYxjDTghbgZlBLq/vmkD4kAyRy/EtRemoznusHSSrHOp2T4s+L2N7ScrFIxGFeWT/da3PUHhEmohXrJScZTMGPffW6ea8PKXWFfkmJQsAPeYHcKCLT3z9OXfeMlw4i9lXlqjxPIYg4+MEXXSmEGl8uOSYdm5r0A2ViFOY0BytPjDuIgBzQs/ig8UWv4FOyZ6IWphzVziPi+h5wwA/j2hmEv+Y2GF+jYb4tXINe9ybMlwVn7R2OgfwmXqg7vOPZUEXgpxWOy5KmXOY+C450f91bDemHWTvM+EsiMBuWEfxOMeJ6ixnGcqyk/5Gt tasks: - install: branch: octopus - print: '**** done install task...' - print: '**** done start installing octopus cephadm ...' - cephadm: add_mons_via_daemon_add: true avoid_pacific_features: true cephadm_branch: octopus cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:octopus - print: '**** done end installing octopus cephadm ...' - cephadm.shell: mon.a: - ceph fs volume create foo - ceph.healthy: null - print: '**** done creating new fs' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' teuthology_branch: master teuthology_sha1: 004701def07b67dd914d7b1775457fec6788a495 timestamp: 2022-04-28_19:20:58 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: dis verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.5191 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: octopus clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: octopus clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: octopus clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: octopus clients: client.0: - rbd/test_librbd.sh - print: '**** done end test_rbd_api.yaml' 2022-04-28T19:27:57.721 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa; will attempt to use it 2022-04-28T19:27:57.722 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa/tasks 2022-04-28T19:27:57.722 INFO:teuthology.run_tasks:Running task internal.check_packages... 2022-04-28T19:27:57.723 INFO:teuthology.task.internal:Checking packages... 2022-04-28T19:27:57.744 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '4fa079ba14503defa8dc257d7c2d506ebefcfe6d' 2022-04-28T19:27:57.744 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-04-28T19:27:57.744 INFO:teuthology.packaging:ref: None 2022-04-28T19:27:57.744 INFO:teuthology.packaging:tag: None 2022-04-28T19:27:57.744 INFO:teuthology.packaging:branch: pacific 2022-04-28T19:27:57.744 INFO:teuthology.packaging:sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:27:57.744 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=pacific 2022-04-28T19:27:57.969 INFO:teuthology.task.internal:Found packages for ceph version 16.2.7-969-g29754ed4-1bionic 2022-04-28T19:27:57.970 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2022-04-28T19:27:57.980 INFO:teuthology.task.internal:no buildpackages task found 2022-04-28T19:27:57.981 INFO:teuthology.run_tasks:Running task internal.save_config... 2022-04-28T19:27:58.031 INFO:teuthology.task.internal:Saving configuration 2022-04-28T19:27:58.051 INFO:teuthology.run_tasks:Running task internal.check_lock... 2022-04-28T19:27:58.116 INFO:teuthology.task.internal.check_lock:Checking locks... 2022-04-28T19:27:58.139 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi059.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '18.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-04-28 19:22:48.663266', 'locked_by': 'scheduled_dis@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILW+W23A7Y4XbIuUpqDnFpQ942zFWzQYVmnAzEkI6OFa'} 2022-04-28T19:27:58.162 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi139.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '18.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2022-04-28 19:22:48.664738', 'locked_by': 'scheduled_dis@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHIaaWgto9qHxeVxQtkLmSY6wLpowzD62/gxACtUPbwn'} 2022-04-28T19:27:58.163 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2022-04-28T19:27:58.198 INFO:teuthology.task.internal:roles: ubuntu@smithi059.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2022-04-28T19:27:58.198 INFO:teuthology.task.internal:roles: ubuntu@smithi139.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2022-04-28T19:27:58.199 INFO:teuthology.run_tasks:Running task console_log... 2022-04-28T19:27:58.313 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f2eeaf5e158>, signals=[15]) 2022-04-28T19:27:58.313 INFO:teuthology.run_tasks:Running task internal.connect... 2022-04-28T19:27:58.325 INFO:teuthology.task.internal:Opening connections... 2022-04-28T19:27:58.326 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi059.front.sepia.ceph.com 2022-04-28T19:27:58.328 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-04-28T19:27:58.418 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi139.front.sepia.ceph.com 2022-04-28T19:27:58.419 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-04-28T19:27:58.545 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2022-04-28T19:27:58.553 DEBUG:teuthology.orchestra.run.smithi059:> uname -m 2022-04-28T19:27:58.559 INFO:teuthology.orchestra.run.smithi059.stdout:x86_64 2022-04-28T19:27:58.560 DEBUG:teuthology.orchestra.run.smithi059:> cat /etc/os-release 2022-04-28T19:27:58.606 INFO:teuthology.orchestra.run.smithi059.stdout:NAME="Ubuntu" 2022-04-28T19:27:58.606 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION="18.04.6 LTS (Bionic Beaver)" 2022-04-28T19:27:58.606 INFO:teuthology.orchestra.run.smithi059.stdout:ID=ubuntu 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:ID_LIKE=debian 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:PRETTY_NAME="Ubuntu 18.04.6 LTS" 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION_ID="18.04" 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:HOME_URL="https://www.ubuntu.com/" 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:VERSION_CODENAME=bionic 2022-04-28T19:27:58.607 INFO:teuthology.orchestra.run.smithi059.stdout:UBUNTU_CODENAME=bionic 2022-04-28T19:27:58.608 INFO:teuthology.lock.ops:Updating smithi059.front.sepia.ceph.com on lock server 2022-04-28T19:27:58.628 DEBUG:teuthology.orchestra.run.smithi139:> uname -m 2022-04-28T19:27:58.673 INFO:teuthology.orchestra.run.smithi139.stdout:x86_64 2022-04-28T19:27:58.673 DEBUG:teuthology.orchestra.run.smithi139:> cat /etc/os-release 2022-04-28T19:27:58.723 INFO:teuthology.orchestra.run.smithi139.stdout:NAME="Ubuntu" 2022-04-28T19:27:58.723 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION="18.04.6 LTS (Bionic Beaver)" 2022-04-28T19:27:58.723 INFO:teuthology.orchestra.run.smithi139.stdout:ID=ubuntu 2022-04-28T19:27:58.723 INFO:teuthology.orchestra.run.smithi139.stdout:ID_LIKE=debian 2022-04-28T19:27:58.723 INFO:teuthology.orchestra.run.smithi139.stdout:PRETTY_NAME="Ubuntu 18.04.6 LTS" 2022-04-28T19:27:58.724 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION_ID="18.04" 2022-04-28T19:27:58.724 INFO:teuthology.orchestra.run.smithi139.stdout:HOME_URL="https://www.ubuntu.com/" 2022-04-28T19:27:58.724 INFO:teuthology.orchestra.run.smithi139.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2022-04-28T19:27:58.724 INFO:teuthology.orchestra.run.smithi139.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2022-04-28T19:27:58.724 INFO:teuthology.orchestra.run.smithi139.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2022-04-28T19:27:58.724 INFO:teuthology.orchestra.run.smithi139.stdout:VERSION_CODENAME=bionic 2022-04-28T19:27:58.725 INFO:teuthology.orchestra.run.smithi139.stdout:UBUNTU_CODENAME=bionic 2022-04-28T19:27:58.725 INFO:teuthology.lock.ops:Updating smithi139.front.sepia.ceph.com on lock server 2022-04-28T19:27:58.746 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2022-04-28T19:27:58.761 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2022-04-28T19:27:58.785 INFO:teuthology.task.internal:Checking for old test directory... 2022-04-28T19:27:58.786 DEBUG:teuthology.orchestra.run.smithi059:> test '!' -e /home/ubuntu/cephtest 2022-04-28T19:27:58.788 DEBUG:teuthology.orchestra.run.smithi139:> test '!' -e /home/ubuntu/cephtest 2022-04-28T19:27:58.794 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2022-04-28T19:27:58.817 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2022-04-28T19:27:58.817 DEBUG:teuthology.orchestra.run.smithi059:> test -z $(ls -A /var/lib/ceph) 2022-04-28T19:27:58.838 DEBUG:teuthology.orchestra.run.smithi139:> test -z $(ls -A /var/lib/ceph) 2022-04-28T19:27:58.861 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2022-04-28T19:27:58.917 INFO:teuthology.run_tasks:Running task kernel... 2022-04-28T19:27:58.953 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2022-04-28T19:27:58.953 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2022-04-28T19:27:58.954 DEBUG:teuthology.orchestra.run.smithi059:> test -f /run/.containerenv -o -f /.dockerenv 2022-04-28T19:27:58.955 DEBUG:teuthology.orchestra.run.smithi139:> test -f /run/.containerenv -o -f /.dockerenv 2022-04-28T19:27:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-04-28T19:27:58.961 DEBUG:teuthology.orchestra.run.smithi059:> uname -r 2022-04-28T19:27:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-04-28T19:27:58.963 DEBUG:teuthology.orchestra.run.smithi139:> uname -r 2022-04-28T19:27:59.010 INFO:teuthology.orchestra.run.smithi059.stdout:4.15.0-176-generic 2022-04-28T19:27:59.010 INFO:teuthology.task.kernel:Running kernel on smithi059: 4.15.0-176-generic 2022-04-28T19:27:59.010 DEBUG:teuthology.orchestra.run.smithi059:> sudo apt-get clean 2022-04-28T19:27:59.012 INFO:teuthology.orchestra.run.smithi139.stdout:4.15.0-176-generic 2022-04-28T19:27:59.013 INFO:teuthology.task.kernel:Running kernel on smithi139: 4.15.0-176-generic 2022-04-28T19:27:59.013 DEBUG:teuthology.orchestra.run.smithi139:> sudo apt-get clean 2022-04-28T19:27:59.133 DEBUG:teuthology.orchestra.run.smithi139:> sudo apt-get update 2022-04-28T19:27:59.158 DEBUG:teuthology.orchestra.run.smithi059:> sudo apt-get update 2022-04-28T19:27:59.301 INFO:teuthology.orchestra.run.smithi059.stdout:Hit:1 http://archive.ubuntu.com/ubuntu bionic InRelease 2022-04-28T19:27:59.301 INFO:teuthology.orchestra.run.smithi059.stdout:Get:2 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB] 2022-04-28T19:27:59.373 INFO:teuthology.orchestra.run.smithi059.stdout:Get:3 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB] 2022-04-28T19:27:59.400 INFO:teuthology.orchestra.run.smithi059.stdout:Get:4 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB] 2022-04-28T19:27:59.412 INFO:teuthology.orchestra.run.smithi139.stdout:Hit:1 http://archive.ubuntu.com/ubuntu bionic InRelease 2022-04-28T19:27:59.412 INFO:teuthology.orchestra.run.smithi139.stdout:Get:2 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB] 2022-04-28T19:27:59.475 INFO:teuthology.orchestra.run.smithi139.stdout:Get:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB] 2022-04-28T19:27:59.644 INFO:teuthology.orchestra.run.smithi059.stdout:Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,544 kB] 2022-04-28T19:27:59.781 INFO:teuthology.orchestra.run.smithi059.stdout:Get:6 http://archive.ubuntu.com/ubuntu bionic-updates/main i386 Packages [1,464 kB] 2022-04-28T19:27:59.790 INFO:teuthology.orchestra.run.smithi139.stdout:Get:4 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB] 2022-04-28T19:27:59.808 INFO:teuthology.orchestra.run.smithi059.stdout:Get:7 http://archive.ubuntu.com/ubuntu bionic-updates/main Translation-en [476 kB] 2022-04-28T19:27:59.813 INFO:teuthology.orchestra.run.smithi059.stdout:Get:8 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [734 kB] 2022-04-28T19:27:59.823 INFO:teuthology.orchestra.run.smithi059.stdout:Get:9 http://archive.ubuntu.com/ubuntu bionic-updates/restricted Translation-en [101 kB] 2022-04-28T19:27:59.824 INFO:teuthology.orchestra.run.smithi059.stdout:Get:10 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1,806 kB] 2022-04-28T19:27:59.850 INFO:teuthology.orchestra.run.smithi059.stdout:Get:11 http://archive.ubuntu.com/ubuntu bionic-updates/universe i386 Packages [1,606 kB] 2022-04-28T19:27:59.857 INFO:teuthology.orchestra.run.smithi139.stdout:Get:5 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,202 kB] 2022-04-28T19:27:59.867 INFO:teuthology.orchestra.run.smithi059.stdout:Get:12 http://archive.ubuntu.com/ubuntu bionic-updates/universe Translation-en [391 kB] 2022-04-28T19:27:59.947 INFO:teuthology.orchestra.run.smithi059.stdout:Get:13 http://security.ubuntu.com/ubuntu bionic-security/main i386 Packages [1,162 kB] 2022-04-28T19:27:59.995 INFO:teuthology.orchestra.run.smithi139.stdout:Get:6 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,544 kB] 2022-04-28T19:28:00.357 INFO:teuthology.orchestra.run.smithi059.stdout:Get:14 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,202 kB] 2022-04-28T19:28:00.369 INFO:teuthology.orchestra.run.smithi139.stdout:Get:7 http://security.ubuntu.com/ubuntu bionic-security/main i386 Packages [1,162 kB] 2022-04-28T19:28:00.399 INFO:teuthology.orchestra.run.smithi139.stdout:Get:8 http://security.ubuntu.com/ubuntu bionic-security/main Translation-en [386 kB] 2022-04-28T19:28:00.409 INFO:teuthology.orchestra.run.smithi139.stdout:Get:9 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [709 kB] 2022-04-28T19:28:00.449 INFO:teuthology.orchestra.run.smithi139.stdout:Get:10 http://security.ubuntu.com/ubuntu bionic-security/restricted Translation-en [97.1 kB] 2022-04-28T19:28:00.451 INFO:teuthology.orchestra.run.smithi139.stdout:Get:11 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,193 kB] 2022-04-28T19:28:00.482 INFO:teuthology.orchestra.run.smithi139.stdout:Get:12 http://security.ubuntu.com/ubuntu bionic-security/universe i386 Packages [1,016 kB] 2022-04-28T19:28:00.484 INFO:teuthology.orchestra.run.smithi059.stdout:Get:15 http://security.ubuntu.com/ubuntu bionic-security/main Translation-en [386 kB] 2022-04-28T19:28:00.485 INFO:teuthology.orchestra.run.smithi139.stdout:Get:13 http://archive.ubuntu.com/ubuntu bionic-updates/main i386 Packages [1,464 kB] 2022-04-28T19:28:00.500 INFO:teuthology.orchestra.run.smithi059.stdout:Get:16 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [709 kB] 2022-04-28T19:28:00.507 INFO:teuthology.orchestra.run.smithi139.stdout:Get:14 http://security.ubuntu.com/ubuntu bionic-security/universe Translation-en [274 kB] 2022-04-28T19:28:00.519 INFO:teuthology.orchestra.run.smithi139.stdout:Get:15 http://archive.ubuntu.com/ubuntu bionic-updates/main Translation-en [476 kB] 2022-04-28T19:28:00.524 INFO:teuthology.orchestra.run.smithi059.stdout:Get:17 http://security.ubuntu.com/ubuntu bionic-security/restricted Translation-en [97.1 kB] 2022-04-28T19:28:00.526 INFO:teuthology.orchestra.run.smithi059.stdout:Get:18 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,193 kB] 2022-04-28T19:28:00.529 INFO:teuthology.orchestra.run.smithi139.stdout:Get:16 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [734 kB] 2022-04-28T19:28:00.553 INFO:teuthology.orchestra.run.smithi059.stdout:Get:19 http://security.ubuntu.com/ubuntu bionic-security/universe i386 Packages [1,016 kB] 2022-04-28T19:28:00.567 INFO:teuthology.orchestra.run.smithi139.stdout:Get:17 http://archive.ubuntu.com/ubuntu bionic-updates/restricted Translation-en [101 kB] 2022-04-28T19:28:00.568 INFO:teuthology.orchestra.run.smithi139.stdout:Get:18 http://archive.ubuntu.com/ubuntu bionic-updates/universe i386 Packages [1,606 kB] 2022-04-28T19:28:00.591 INFO:teuthology.orchestra.run.smithi059.stdout:Get:20 http://security.ubuntu.com/ubuntu bionic-security/universe Translation-en [274 kB] 2022-04-28T19:28:00.610 INFO:teuthology.orchestra.run.smithi139.stdout:Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1,806 kB] 2022-04-28T19:28:00.665 INFO:teuthology.orchestra.run.smithi139.stdout:Get:20 http://archive.ubuntu.com/ubuntu bionic-updates/universe Translation-en [391 kB] 2022-04-28T19:28:02.110 INFO:teuthology.orchestra.run.smithi059.stdout:Fetched 16.4 MB in 3s (5,967 kB/s) 2022-04-28T19:28:02.491 INFO:teuthology.orchestra.run.smithi139.stdout:Fetched 16.4 MB in 3s (5,173 kB/s) 2022-04-28T19:28:03.669 INFO:teuthology.orchestra.run.smithi059.stdout:Reading package lists... 2022-04-28T19:28:03.719 DEBUG:teuthology.orchestra.run.smithi059:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2022-04-28T19:28:03.801 INFO:teuthology.orchestra.run.smithi059.stdout:Reading package lists... 2022-04-28T19:28:04.008 INFO:teuthology.orchestra.run.smithi139.stdout:Reading package lists... 2022-04-28T19:28:04.059 DEBUG:teuthology.orchestra.run.smithi139:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2022-04-28T19:28:04.084 INFO:teuthology.orchestra.run.smithi059.stdout:Building dependency tree... 2022-04-28T19:28:04.085 INFO:teuthology.orchestra.run.smithi059.stdout:Reading state information... 2022-04-28T19:28:04.139 INFO:teuthology.orchestra.run.smithi139.stdout:Reading package lists... 2022-04-28T19:28:04.340 INFO:teuthology.orchestra.run.smithi059.stdout:linux-image-generic is already the newest version (4.15.0.176.165). 2022-04-28T19:28:04.341 INFO:teuthology.orchestra.run.smithi059.stdout:linux-image-generic set to manually installed. 2022-04-28T19:28:04.341 INFO:teuthology.orchestra.run.smithi059.stdout:0 upgraded, 0 newly installed, 0 to remove and 9 not upgraded. 2022-04-28T19:28:04.342 DEBUG:teuthology.orchestra.run.smithi059:> dpkg -s linux-image-generic 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Package: linux-image-generic 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Status: install ok installed 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Priority: optional 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Section: kernel 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Installed-Size: 18 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Maintainer: Ubuntu Kernel Team 2022-04-28T19:28:04.366 INFO:teuthology.orchestra.run.smithi059.stdout:Architecture: amd64 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout:Source: linux-meta 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout:Version: 4.15.0.176.165 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout:Provides: wireguard-modules (= 1.0.20201112-1~18.04.1), zfs-modules (= 0.7.5-1ubuntu16.12) 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout:Depends: linux-image-4.15.0-176-generic, linux-modules-extra-4.15.0-176-generic, linux-firmware, intel-microcode, amd64-microcode 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout:Recommends: thermald 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout:Description: Generic Linux kernel image 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout: This package will always depend on the latest generic kernel image 2022-04-28T19:28:04.367 INFO:teuthology.orchestra.run.smithi059.stdout: available. 2022-04-28T19:28:04.367 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-04-28T19:28:04.368 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2022-04-28T19:28:04.368 DEBUG:teuthology.orchestra.run.smithi059:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-04-28T19:28:04.410 INFO:teuthology.orchestra.run.smithi139.stdout:Building dependency tree... 2022-04-28T19:28:04.411 INFO:teuthology.orchestra.run.smithi139.stdout:Reading state information... 2022-04-28T19:28:04.422 INFO:teuthology.orchestra.run.smithi059.stdout:ttyS1 2022-04-28T19:28:04.455 DEBUG:teuthology.parallel:result is None 2022-04-28T19:28:04.651 INFO:teuthology.orchestra.run.smithi139.stdout:linux-image-generic is already the newest version (4.15.0.176.165). 2022-04-28T19:28:04.652 INFO:teuthology.orchestra.run.smithi139.stdout:linux-image-generic set to manually installed. 2022-04-28T19:28:04.652 INFO:teuthology.orchestra.run.smithi139.stdout:0 upgraded, 0 newly installed, 0 to remove and 9 not upgraded. 2022-04-28T19:28:04.653 DEBUG:teuthology.orchestra.run.smithi139:> dpkg -s linux-image-generic 2022-04-28T19:28:04.676 INFO:teuthology.orchestra.run.smithi139.stdout:Package: linux-image-generic 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Status: install ok installed 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Priority: optional 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Section: kernel 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Installed-Size: 18 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Maintainer: Ubuntu Kernel Team 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Architecture: amd64 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Source: linux-meta 2022-04-28T19:28:04.677 INFO:teuthology.orchestra.run.smithi139.stdout:Version: 4.15.0.176.165 2022-04-28T19:28:04.678 INFO:teuthology.orchestra.run.smithi139.stdout:Provides: wireguard-modules (= 1.0.20201112-1~18.04.1), zfs-modules (= 0.7.5-1ubuntu16.12) 2022-04-28T19:28:04.678 INFO:teuthology.orchestra.run.smithi139.stdout:Depends: linux-image-4.15.0-176-generic, linux-modules-extra-4.15.0-176-generic, linux-firmware, intel-microcode, amd64-microcode 2022-04-28T19:28:04.678 INFO:teuthology.orchestra.run.smithi139.stdout:Recommends: thermald 2022-04-28T19:28:04.678 INFO:teuthology.orchestra.run.smithi139.stdout:Description: Generic Linux kernel image 2022-04-28T19:28:04.678 INFO:teuthology.orchestra.run.smithi139.stdout: This package will always depend on the latest generic kernel image 2022-04-28T19:28:04.678 INFO:teuthology.orchestra.run.smithi139.stdout: available. 2022-04-28T19:28:04.678 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2022-04-28T19:28:04.678 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2022-04-28T19:28:04.679 DEBUG:teuthology.orchestra.run.smithi139:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2022-04-28T19:28:04.734 INFO:teuthology.orchestra.run.smithi139.stdout:ttyS1 2022-04-28T19:28:04.767 DEBUG:teuthology.parallel:result is None 2022-04-28T19:28:04.767 INFO:teuthology.run_tasks:Running task internal.base... 2022-04-28T19:28:04.782 INFO:teuthology.task.internal:Creating test directory... 2022-04-28T19:28:04.782 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-04-28T19:28:04.785 DEBUG:teuthology.orchestra.run.smithi139:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2022-04-28T19:28:04.815 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2022-04-28T19:28:04.826 INFO:teuthology.run_tasks:Running task internal.archive... 2022-04-28T19:28:04.877 INFO:teuthology.task.internal:Creating archive directory... 2022-04-28T19:28:04.877 DEBUG:teuthology.orchestra.run.smithi059:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-04-28T19:28:04.880 DEBUG:teuthology.orchestra.run.smithi139:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2022-04-28T19:28:04.908 INFO:teuthology.run_tasks:Running task internal.coredump... 2022-04-28T19:28:04.937 INFO:teuthology.task.internal:Enabling coredump saving... 2022-04-28T19:28:04.938 DEBUG:teuthology.orchestra.run.smithi059:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-04-28T19:28:04.948 DEBUG:teuthology.orchestra.run.smithi139:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2022-04-28T19:28:04.965 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-04-28T19:28:04.975 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-04-28T19:28:04.975 INFO:teuthology.orchestra.run.smithi059.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-04-28T19:28:04.983 INFO:teuthology.orchestra.run.smithi139.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2022-04-28T19:28:04.986 INFO:teuthology.run_tasks:Running task internal.sudo... 2022-04-28T19:28:04.996 INFO:teuthology.task.internal:Configuring sudo... 2022-04-28T19:28:04.996 DEBUG:teuthology.orchestra.run.smithi059:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-04-28T19:28:05.022 DEBUG:teuthology.orchestra.run.smithi139:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2022-04-28T19:28:05.044 INFO:teuthology.run_tasks:Running task internal.syslog... 2022-04-28T19:28:05.056 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2022-04-28T19:28:05.056 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-04-28T19:28:05.078 DEBUG:teuthology.orchestra.run.smithi139:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2022-04-28T19:28:05.092 DEBUG:teuthology.orchestra.run.smithi059:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-04-28T19:28:05.132 DEBUG:teuthology.orchestra.run.smithi059:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-04-28T19:28:05.179 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:28:05.179 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-04-28T19:28:05.252 DEBUG:teuthology.orchestra.run.smithi139:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2022-04-28T19:28:05.259 DEBUG:teuthology.orchestra.run.smithi139:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2022-04-28T19:28:05.307 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:28:05.307 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2022-04-28T19:28:05.371 DEBUG:teuthology.orchestra.run.smithi059:> sudo service rsyslog restart 2022-04-28T19:28:05.374 DEBUG:teuthology.orchestra.run.smithi139:> sudo service rsyslog restart 2022-04-28T19:28:05.453 INFO:teuthology.run_tasks:Running task internal.timer... 2022-04-28T19:28:05.472 INFO:teuthology.task.internal:Starting timer... 2022-04-28T19:28:05.473 INFO:teuthology.run_tasks:Running task pcp... 2022-04-28T19:28:05.536 INFO:teuthology.run_tasks:Running task selinux... 2022-04-28T19:28:05.620 DEBUG:teuthology.task.selinux:Excluding smithi059: OS 'ubuntu' does not support SELinux 2022-04-28T19:28:05.620 DEBUG:teuthology.task.selinux:Excluding smithi139: OS 'ubuntu' does not support SELinux 2022-04-28T19:28:05.621 DEBUG:teuthology.task.selinux:Getting current SELinux state 2022-04-28T19:28:05.621 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2022-04-28T19:28:05.621 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2022-04-28T19:28:05.621 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2022-04-28T19:28:05.664 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master was just updated or references a specific commit; assuming it is current 2022-04-28T19:28:05.664 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master to origin/master 2022-04-28T19:28:05.687 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2022-04-28T19:28:05.688 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi059.front.sepia.ceph.com,smithi139.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/cephlab.yml 2022-04-28T19:32:00.299 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi059.front.sepia.ceph.com'), Remote(name='ubuntu@smithi139.front.sepia.ceph.com')] 2022-04-28T19:32:00.301 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi059.front.sepia.ceph.com' 2022-04-28T19:32:00.303 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi059.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-04-28T19:32:00.391 DEBUG:teuthology.orchestra.run.smithi059:> true 2022-04-28T19:32:00.512 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi139.front.sepia.ceph.com' 2022-04-28T19:32:00.513 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi139.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2022-04-28T19:32:00.595 DEBUG:teuthology.orchestra.run.smithi139:> true 2022-04-28T19:32:00.712 INFO:teuthology.run_tasks:Running task clock... 2022-04-28T19:32:00.727 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2022-04-28T19:32:00.727 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-04-28T19:32:00.729 DEBUG:teuthology.orchestra.run.smithi059:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-04-28T19:32:00.731 INFO:teuthology.orchestra.run:Running command with timeout 360 2022-04-28T19:32:00.732 DEBUG:teuthology.orchestra.run.smithi139:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2022-04-28T19:32:00.793 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: ntpd 4.2.8p10@1.3728-o (1): Starting 2022-04-28T19:32:00.793 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Command line: ntpd -gq 2022-04-28T19:32:00.794 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: proto: precision = 0.126 usec (-23) 2022-04-28T19:32:00.794 INFO:teuthology.orchestra.run.smithi059.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-04-28T19:32:00.795 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-04-28T19:32:00.795 INFO:teuthology.orchestra.run.smithi059.stderr:restrict ::: KOD does nothing without LIMITED. 2022-04-28T19:32:00.796 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: restrict ::: KOD does nothing without LIMITED. 2022-04-28T19:32:00.796 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listen and drop on 0 v6wildcard [::]:123 2022-04-28T19:32:00.796 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2022-04-28T19:32:00.797 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listen normally on 2 lo 127.0.0.1:123 2022-04-28T19:32:00.797 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listen normally on 3 ens1f0 172.21.15.59:123 2022-04-28T19:32:00.797 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listen normally on 4 lo [::1]:123 2022-04-28T19:32:00.797 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listen normally on 5 ens1f0 [fe80::ec4:7aff:febd:192a%3]:123 2022-04-28T19:32:00.797 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:00 ntpd[18905]: Listening on routing socket on fd #22 for interface updates 2022-04-28T19:32:00.814 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: ntpd 4.2.8p10@1.3728-o (1): Starting 2022-04-28T19:32:00.814 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Command line: ntpd -gq 2022-04-28T19:32:00.815 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: proto: precision = 0.123 usec (-23) 2022-04-28T19:32:00.816 INFO:teuthology.orchestra.run.smithi139.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-04-28T19:32:00.816 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2022-04-28T19:32:00.817 INFO:teuthology.orchestra.run.smithi139.stderr:restrict ::: KOD does nothing without LIMITED. 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: restrict ::: KOD does nothing without LIMITED. 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listen and drop on 0 v6wildcard [::]:123 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listen normally on 2 lo 127.0.0.1:123 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listen normally on 3 enp3s0f1 172.21.15.139:123 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listen normally on 4 lo [::1]:123 2022-04-28T19:32:00.818 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7f59%5]:123 2022-04-28T19:32:00.819 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:00 ntpd[18542]: Listening on routing socket on fd #22 for interface updates 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:01.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:01 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:01.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:01 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:01.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:01 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:01.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:01 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:01.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:01 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:02.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:02.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:02 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:03.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:03.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:03.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:03.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:03.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:03.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:03 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:03.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:03 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:03.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:03 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:03.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:03 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:03.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:03 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:04.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:04 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:04.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:04 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:04.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:04 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:04.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:04 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:04.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:04.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:04.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:04.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:04.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:04.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:04 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:05.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:05.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:05.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:05.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:05.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:05.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:05 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:05.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:05 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:05.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:05 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:05.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:05 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:05.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:05 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:06.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:06 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:06.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:06 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:06.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:06 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:06.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:06 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:06.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:06.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:06.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:06.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:06.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:06.818 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:06 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:07.794 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:07.795 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:07.796 INFO:teuthology.orchestra.run.smithi059.stdout:28 Apr 19:32:07 ntpd[18905]: ntpd: time slew +0.003127 s 2022-04-28T19:32:07.796 INFO:teuthology.orchestra.run.smithi059.stdout:ntpd: time slew +0.003127s 2022-04-28T19:32:07.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2022-04-28T19:32:07.796 INFO:teuthology.orchestra.run.smithi059.stderr:28 Apr 19:32:07 ntpd[18905]: can't open /var/log/ntpstats/loopstats.20220428: Permission denied 2022-04-28T19:32:07.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:07 ntpd[18542]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2022-04-28T19:32:07.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:07 ntpd[18542]: can't open /var/log/ntpstats/rawstats.20220428: Permission denied 2022-04-28T19:32:07.815 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:07 ntpd[18542]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2022-04-28T19:32:07.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:07 ntpd[18542]: can't open /var/log/ntpstats/peerstats.20220428: Permission denied 2022-04-28T19:32:07.816 INFO:teuthology.orchestra.run.smithi139.stdout:28 Apr 19:32:07 ntpd[18542]: ntpd: time slew -0.003931 s 2022-04-28T19:32:07.816 INFO:teuthology.orchestra.run.smithi139.stdout:ntpd: time slew -0.003931s 2022-04-28T19:32:07.816 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:07 ntpd[18542]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2022-04-28T19:32:07.817 INFO:teuthology.orchestra.run.smithi139.stderr:28 Apr 19:32:07 ntpd[18542]: can't open /var/log/ntpstats/loopstats.20220428: Permission denied 2022-04-28T19:32:07.848 INFO:teuthology.orchestra.run.smithi059.stdout: remote refid st t when poll reach delay offset jitter 2022-04-28T19:32:07.848 INFO:teuthology.orchestra.run.smithi059.stdout:============================================================================== 2022-04-28T19:32:07.848 INFO:teuthology.orchestra.run.smithi059.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.849 INFO:teuthology.orchestra.run.smithi059.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.849 INFO:teuthology.orchestra.run.smithi059.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.849 INFO:teuthology.orchestra.run.smithi059.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.886 INFO:teuthology.orchestra.run.smithi139.stdout: remote refid st t when poll reach delay offset jitter 2022-04-28T19:32:07.886 INFO:teuthology.orchestra.run.smithi139.stdout:============================================================================== 2022-04-28T19:32:07.886 INFO:teuthology.orchestra.run.smithi139.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.886 INFO:teuthology.orchestra.run.smithi139.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.886 INFO:teuthology.orchestra.run.smithi139.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.886 INFO:teuthology.orchestra.run.smithi139.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2022-04-28T19:32:07.887 INFO:teuthology.run_tasks:Running task install... 2022-04-28T19:32:07.899 DEBUG:teuthology.task.install:project ceph 2022-04-28T19:32:07.899 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '4fa079ba14503defa8dc257d7c2d506ebefcfe6d'}} 2022-04-28T19:32:07.899 DEBUG:teuthology.task.install:config {'branch': 'octopus', 'flavor': 'default', 'sha1': '4fa079ba14503defa8dc257d7c2d506ebefcfe6d'} 2022-04-28T19:32:07.899 INFO:teuthology.task.install:Using flavor: default 2022-04-28T19:32:07.905 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2022-04-28T19:32:07.905 INFO:teuthology.task.install:extra packages: [] 2022-04-28T19:32:07.906 DEBUG:teuthology.orchestra.run.smithi059:> sudo lsb_release -is 2022-04-28T19:32:07.959 INFO:teuthology.orchestra.run.smithi059.stdout:Ubuntu 2022-04-28T19:32:07.965 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-04-28T19:32:07.965 DEBUG:teuthology.orchestra.run.smithi139:> sudo lsb_release -is 2022-04-28T19:32:07.965 DEBUG:teuthology.orchestra.run.smithi059:> sudo apt-key list | grep Ceph 2022-04-28T19:32:08.019 INFO:teuthology.orchestra.run.smithi139.stdout:Ubuntu 2022-04-28T19:32:08.024 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-04-28T19:32:08.024 DEBUG:teuthology.orchestra.run.smithi139:> sudo apt-key list | grep Ceph 2022-04-28T19:32:08.054 INFO:teuthology.orchestra.run.smithi059.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2022-04-28T19:32:08.114 INFO:teuthology.orchestra.run.smithi059.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2022-04-28T19:32:08.114 INFO:teuthology.orchestra.run.smithi059.stdout:uid [ unknown] Ceph.com (release key) 2022-04-28T19:32:08.115 DEBUG:teuthology.orchestra.run.smithi059:> sudo lsb_release -is 2022-04-28T19:32:08.118 INFO:teuthology.orchestra.run.smithi139.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2022-04-28T19:32:08.168 INFO:teuthology.orchestra.run.smithi059.stdout:Ubuntu 2022-04-28T19:32:08.173 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-04-28T19:32:08.173 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2022-04-28T19:32:08.173 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-04-28T19:32:08.174 INFO:teuthology.packaging:ref: None 2022-04-28T19:32:08.174 INFO:teuthology.packaging:tag: None 2022-04-28T19:32:08.174 INFO:teuthology.packaging:branch: octopus 2022-04-28T19:32:08.174 INFO:teuthology.packaging:sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:32:08.174 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=octopus 2022-04-28T19:32:08.179 INFO:teuthology.orchestra.run.smithi139.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2022-04-28T19:32:08.179 INFO:teuthology.orchestra.run.smithi139.stdout:uid [ unknown] Ceph.com (release key) 2022-04-28T19:32:08.180 DEBUG:teuthology.orchestra.run.smithi139:> sudo lsb_release -is 2022-04-28T19:32:08.230 INFO:teuthology.orchestra.run.smithi139.stdout:Ubuntu 2022-04-28T19:32:08.235 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-04-28T19:32:08.235 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2022-04-28T19:32:08.235 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-04-28T19:32:08.236 INFO:teuthology.packaging:ref: None 2022-04-28T19:32:08.236 INFO:teuthology.packaging:tag: None 2022-04-28T19:32:08.236 INFO:teuthology.packaging:branch: octopus 2022-04-28T19:32:08.236 INFO:teuthology.packaging:sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:32:08.236 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=octopus 2022-04-28T19:32:08.426 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default/ 2022-04-28T19:32:08.426 INFO:teuthology.task.install.deb:Package version is 15.2.16-1bionic 2022-04-28T19:32:08.458 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:32:08.458 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2022-04-28T19:32:08.472 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default/ 2022-04-28T19:32:08.473 INFO:teuthology.task.install.deb:Package version is 15.2.16-1bionic 2022-04-28T19:32:08.478 DEBUG:teuthology.orchestra.run.smithi059:> sudo apt-get update 2022-04-28T19:32:08.499 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:32:08.500 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2022-04-28T19:32:08.515 DEBUG:teuthology.orchestra.run.smithi139:> sudo apt-get update 2022-04-28T19:32:08.650 INFO:teuthology.orchestra.run.smithi059.stdout:Ign:1 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic InRelease 2022-04-28T19:32:08.651 INFO:teuthology.orchestra.run.smithi059.stdout:Get:2 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic Release [7,690 B] 2022-04-28T19:32:08.654 INFO:teuthology.orchestra.run.smithi059.stdout:Ign:3 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic Release.gpg 2022-04-28T19:32:08.655 INFO:teuthology.orchestra.run.smithi059.stdout:Get:4 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 Packages [16.0 kB] 2022-04-28T19:32:08.662 INFO:teuthology.orchestra.run.smithi139.stdout:Ign:1 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic InRelease 2022-04-28T19:32:08.664 INFO:teuthology.orchestra.run.smithi139.stdout:Get:2 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic Release [7,690 B] 2022-04-28T19:32:08.666 INFO:teuthology.orchestra.run.smithi139.stdout:Ign:3 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic Release.gpg 2022-04-28T19:32:08.666 INFO:teuthology.orchestra.run.smithi059.stdout:Get:5 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main i386 Packages [3,277 B] 2022-04-28T19:32:08.667 INFO:teuthology.orchestra.run.smithi139.stdout:Get:4 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 Packages [16.0 kB] 2022-04-28T19:32:08.678 INFO:teuthology.orchestra.run.smithi139.stdout:Get:5 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main i386 Packages [3,277 B] 2022-04-28T19:32:08.785 INFO:teuthology.orchestra.run.smithi059.stdout:Hit:6 http://security.ubuntu.com/ubuntu bionic-security InRelease 2022-04-28T19:32:08.785 INFO:teuthology.orchestra.run.smithi059.stdout:Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease 2022-04-28T19:32:08.814 INFO:teuthology.orchestra.run.smithi139.stdout:Hit:6 http://security.ubuntu.com/ubuntu bionic-security InRelease 2022-04-28T19:32:08.814 INFO:teuthology.orchestra.run.smithi139.stdout:Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease 2022-04-28T19:32:08.854 INFO:teuthology.orchestra.run.smithi059.stdout:Hit:8 http://archive.ubuntu.com/ubuntu bionic-updates InRelease 2022-04-28T19:32:08.884 INFO:teuthology.orchestra.run.smithi139.stdout:Hit:8 http://archive.ubuntu.com/ubuntu bionic-updates InRelease 2022-04-28T19:32:08.934 INFO:teuthology.orchestra.run.smithi059.stdout:Hit:9 http://archive.ubuntu.com/ubuntu bionic-backports InRelease 2022-04-28T19:32:08.964 INFO:teuthology.orchestra.run.smithi139.stdout:Hit:9 http://archive.ubuntu.com/ubuntu bionic-backports InRelease 2022-04-28T19:32:09.406 INFO:teuthology.orchestra.run.smithi059.stdout:Fetched 26.9 kB in 1s (37.4 kB/s) 2022-04-28T19:32:09.423 INFO:teuthology.orchestra.run.smithi139.stdout:Fetched 26.9 kB in 1s (38.3 kB/s) 2022-04-28T19:32:10.811 INFO:teuthology.orchestra.run.smithi059.stdout:Reading package lists... 2022-04-28T19:32:10.862 DEBUG:teuthology.orchestra.run.smithi059:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=15.2.16-1bionic cephadm=15.2.16-1bionic ceph-mds=15.2.16-1bionic ceph-mgr=15.2.16-1bionic ceph-common=15.2.16-1bionic ceph-fuse=15.2.16-1bionic ceph-test=15.2.16-1bionic radosgw=15.2.16-1bionic python3-rados=15.2.16-1bionic python3-rgw=15.2.16-1bionic python3-cephfs=15.2.16-1bionic python3-rbd=15.2.16-1bionic libcephfs2=15.2.16-1bionic libcephfs-dev=15.2.16-1bionic librados2=15.2.16-1bionic librbd1=15.2.16-1bionic rbd-fuse=15.2.16-1bionic 2022-04-28T19:32:10.943 INFO:teuthology.orchestra.run.smithi059.stdout:Reading package lists... 2022-04-28T19:32:11.120 INFO:teuthology.orchestra.run.smithi139.stdout:Reading package lists... 2022-04-28T19:32:11.171 DEBUG:teuthology.orchestra.run.smithi139:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=15.2.16-1bionic cephadm=15.2.16-1bionic ceph-mds=15.2.16-1bionic ceph-mgr=15.2.16-1bionic ceph-common=15.2.16-1bionic ceph-fuse=15.2.16-1bionic ceph-test=15.2.16-1bionic radosgw=15.2.16-1bionic python3-rados=15.2.16-1bionic python3-rgw=15.2.16-1bionic python3-cephfs=15.2.16-1bionic python3-rbd=15.2.16-1bionic libcephfs2=15.2.16-1bionic libcephfs-dev=15.2.16-1bionic librados2=15.2.16-1bionic librbd1=15.2.16-1bionic rbd-fuse=15.2.16-1bionic 2022-04-28T19:32:11.199 INFO:teuthology.orchestra.run.smithi059.stdout:Building dependency tree... 2022-04-28T19:32:11.200 INFO:teuthology.orchestra.run.smithi059.stdout:Reading state information... 2022-04-28T19:32:11.248 INFO:teuthology.orchestra.run.smithi139.stdout:Reading package lists... 2022-04-28T19:32:11.397 INFO:teuthology.orchestra.run.smithi059.stdout:The following additional packages will be installed: 2022-04-28T19:32:11.397 INFO:teuthology.orchestra.run.smithi059.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-cloud 2022-04-28T19:32:11.397 INFO:teuthology.orchestra.run.smithi059.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: ceph-mon ceph-osd jq libjq1 libleveldb1v5 liblttng-ust0 liboath0 libonig4 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: librabbitmq4 libradosstriper1 librdkafka1 librgw2 libsnappy1v5 libxslt1.1 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: nvme-cli python-attr python-funcsigs python-pastedeploy-tpl python-pluggy 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: python-py python-pytest python3-asn1crypto python3-attr python3-bcrypt 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: python3-bs4 python3-ceph-argparse python3-ceph-common python3-cffi-backend 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: python3-cherrypy3 python3-cryptography python3-cycler python3-dateutil 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: python3-decorator python3-fasteners python3-html5lib python3-httplib2 2022-04-28T19:32:11.399 INFO:teuthology.orchestra.run.smithi059.stdout: python3-jinja2 python3-joblib python3-jwt python3-kubernetes 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-matplotlib python3-monotonic python3-numpy python3-oauth2client 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-olefile python3-openssl python3-paste python3-pastedeploy 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-pastescript python3-pecan python3-pil python3-pluggy 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-prettytable python3-py python3-pyasn1 python3-pyasn1-modules 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-pyinotify python3-pyparsing python3-pytest python3-repoze.lru 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-routes python3-rsa python3-scipy python3-setuptools 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-sklearn python3-sklearn-lib python3-sqlalchemy python3-tempita 2022-04-28T19:32:11.400 INFO:teuthology.orchestra.run.smithi059.stdout: python3-tk python3-tz python3-uritemplate python3-waitress 2022-04-28T19:32:11.401 INFO:teuthology.orchestra.run.smithi059.stdout: python3-webencodings python3-webob python3-websocket python3-webtest 2022-04-28T19:32:11.401 INFO:teuthology.orchestra.run.smithi059.stdout: python3-werkzeug socat xmlstarlet 2022-04-28T19:32:11.401 INFO:teuthology.orchestra.run.smithi059.stdout:Suggested packages: 2022-04-28T19:32:11.401 INFO:teuthology.orchestra.run.smithi059.stdout: python3-influxdb python-attr-doc python-funcsigs-doc subversion 2022-04-28T19:32:11.401 INFO:teuthology.orchestra.run.smithi059.stdout: python-pytest-xdist python-mock python-cryptography-doc 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python3-cryptography-vectors python-cycler-doc python3-genshi 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python-jinja2-doc python3-crypto python3-lxml-dbg python-lxml-doc 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python3-beaker python-mako-doc dvipng ffmpeg gir1.2-gtk-3.0 ghostscript 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: inkscape ipython3 python-matplotlib-doc python3-cairocffi python3-gi-cairo 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python3-gobject python3-pyqt4 python3-sip python3-tornado 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: texlive-extra-utils texlive-latex-extra ttf-staypuft python-numpy-doc 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python3-dev python3-numpy-dbg python-openssl-doc python3-openssl-dbg 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python-pecan-doc python-pil-doc python3-pil-dbg python-pyinotify-doc 2022-04-28T19:32:11.402 INFO:teuthology.orchestra.run.smithi059.stdout: python-pyparsing-doc python-scipy-doc python-setuptools-doc python3-dap 2022-04-28T19:32:11.403 INFO:teuthology.orchestra.run.smithi059.stdout: python-sklearn-doc python-sqlalchemy-doc python3-psycopg2 python3-pymysql 2022-04-28T19:32:11.403 INFO:teuthology.orchestra.run.smithi059.stdout: python3-fdb tix python3-tk-dbg python-waitress-doc python-webob-doc 2022-04-28T19:32:11.403 INFO:teuthology.orchestra.run.smithi059.stdout: python-webtest-doc python3-termcolor python3-watchdog python-werkzeug-doc 2022-04-28T19:32:11.497 INFO:teuthology.orchestra.run.smithi139.stdout:Building dependency tree... 2022-04-28T19:32:11.497 INFO:teuthology.orchestra.run.smithi139.stdout:Reading state information... 2022-04-28T19:32:11.516 INFO:teuthology.orchestra.run.smithi059.stdout:The following NEW packages will be installed: 2022-04-28T19:32:11.516 INFO:teuthology.orchestra.run.smithi059.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2022-04-28T19:32:11.516 INFO:teuthology.orchestra.run.smithi059.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-cloud 2022-04-28T19:32:11.516 INFO:teuthology.orchestra.run.smithi059.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-04-28T19:32:11.517 INFO:teuthology.orchestra.run.smithi059.stdout: ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev libcephfs2 libjq1 2022-04-28T19:32:11.517 INFO:teuthology.orchestra.run.smithi059.stdout: libleveldb1v5 liblttng-ust0 liboath0 libonig4 librabbitmq4 libradosstriper1 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: librdkafka1 librgw2 libsnappy1v5 libxslt1.1 nvme-cli python-attr 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: python-funcsigs python-pastedeploy-tpl python-pluggy python-py python-pytest 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: python3-asn1crypto python3-attr python3-bcrypt python3-bs4 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: python3-cffi-backend python3-cherrypy3 python3-cryptography python3-cycler 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: python3-dateutil python3-decorator python3-fasteners python3-html5lib 2022-04-28T19:32:11.519 INFO:teuthology.orchestra.run.smithi059.stdout: python3-httplib2 python3-jinja2 python3-joblib python3-jwt 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-markupsafe python3-matplotlib python3-monotonic python3-numpy 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-oauth2client python3-olefile python3-openssl python3-paste 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pil 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-pluggy python3-prettytable python3-py python3-pyasn1 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-pyasn1-modules python3-pyinotify python3-pyparsing python3-pytest 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-rados python3-rbd python3-repoze.lru python3-rgw python3-routes 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-rsa python3-scipy python3-setuptools python3-simplegeneric 2022-04-28T19:32:11.520 INFO:teuthology.orchestra.run.smithi059.stdout: python3-simplejson python3-singledispatch python3-sklearn 2022-04-28T19:32:11.521 INFO:teuthology.orchestra.run.smithi059.stdout: python3-sklearn-lib python3-sqlalchemy python3-tempita python3-tk python3-tz 2022-04-28T19:32:11.521 INFO:teuthology.orchestra.run.smithi059.stdout: python3-uritemplate python3-waitress python3-webencodings python3-webob 2022-04-28T19:32:11.521 INFO:teuthology.orchestra.run.smithi059.stdout: python3-websocket python3-webtest python3-werkzeug radosgw rbd-fuse socat 2022-04-28T19:32:11.521 INFO:teuthology.orchestra.run.smithi059.stdout: xmlstarlet 2022-04-28T19:32:11.521 INFO:teuthology.orchestra.run.smithi059.stdout:The following packages will be upgraded: 2022-04-28T19:32:11.521 INFO:teuthology.orchestra.run.smithi059.stdout: librados2 librbd1 2022-04-28T19:32:11.591 INFO:teuthology.orchestra.run.smithi059.stdout:2 upgraded, 108 newly installed, 0 to remove and 8 not upgraded. 2022-04-28T19:32:11.591 INFO:teuthology.orchestra.run.smithi059.stdout:Need to get 156 MB of archives. 2022-04-28T19:32:11.591 INFO:teuthology.orchestra.run.smithi059.stdout:After this operation, 724 MB of additional disk space will be used. 2022-04-28T19:32:11.592 INFO:teuthology.orchestra.run.smithi059.stdout:Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-uritemplate all 0.6-1ubuntu1 [5,126 B] 2022-04-28T19:32:11.612 INFO:teuthology.orchestra.run.smithi059.stdout:Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 liblttng-ust0 amd64 2.10.1-1 [154 kB] 2022-04-28T19:32:11.678 INFO:teuthology.orchestra.run.smithi139.stdout:The following additional packages will be installed: 2022-04-28T19:32:11.678 INFO:teuthology.orchestra.run.smithi139.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-cloud 2022-04-28T19:32:11.678 INFO:teuthology.orchestra.run.smithi139.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-04-28T19:32:11.679 INFO:teuthology.orchestra.run.smithi059.stdout:Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-prettytable all 0.7.2-3 [19.7 kB] 2022-04-28T19:32:11.680 INFO:teuthology.orchestra.run.smithi139.stdout: ceph-mon ceph-osd jq libjq1 libleveldb1v5 liblttng-ust0 liboath0 libonig4 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: librabbitmq4 libradosstriper1 librdkafka1 librgw2 libsnappy1v5 libxslt1.1 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: nvme-cli python-attr python-funcsigs python-pastedeploy-tpl python-pluggy 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: python-py python-pytest python3-asn1crypto python3-attr python3-bcrypt 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: python3-bs4 python3-ceph-argparse python3-ceph-common python3-cffi-backend 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: python3-cherrypy3 python3-cryptography python3-cycler python3-dateutil 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: python3-decorator python3-fasteners python3-html5lib python3-httplib2 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: python3-jinja2 python3-joblib python3-jwt python3-kubernetes 2022-04-28T19:32:11.681 INFO:teuthology.orchestra.run.smithi139.stdout: python3-logutils python3-lxml python3-mako python3-markupsafe 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-matplotlib python3-monotonic python3-numpy python3-oauth2client 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-olefile python3-openssl python3-paste python3-pastedeploy 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-pastescript python3-pecan python3-pil python3-pluggy 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-prettytable python3-py python3-pyasn1 python3-pyasn1-modules 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-pyinotify python3-pyparsing python3-pytest python3-repoze.lru 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-routes python3-rsa python3-scipy python3-setuptools 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-sklearn python3-sklearn-lib python3-sqlalchemy python3-tempita 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-tk python3-tz python3-uritemplate python3-waitress 2022-04-28T19:32:11.682 INFO:teuthology.orchestra.run.smithi139.stdout: python3-webencodings python3-webob python3-websocket python3-webtest 2022-04-28T19:32:11.683 INFO:teuthology.orchestra.run.smithi139.stdout: python3-werkzeug socat xmlstarlet 2022-04-28T19:32:11.683 INFO:teuthology.orchestra.run.smithi139.stdout:Suggested packages: 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python3-influxdb python-attr-doc python-funcsigs-doc subversion 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python-pytest-xdist python-mock python-cryptography-doc 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python3-cryptography-vectors python-cycler-doc python3-genshi 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python-jinja2-doc python3-crypto python3-lxml-dbg python-lxml-doc 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python3-beaker python-mako-doc dvipng ffmpeg gir1.2-gtk-3.0 ghostscript 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: inkscape ipython3 python-matplotlib-doc python3-cairocffi python3-gi-cairo 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python3-gobject python3-pyqt4 python3-sip python3-tornado 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: texlive-extra-utils texlive-latex-extra ttf-staypuft python-numpy-doc 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: python3-dev python3-numpy-dbg python-openssl-doc python3-openssl-dbg 2022-04-28T19:32:11.684 INFO:teuthology.orchestra.run.smithi139.stdout: httpd-wsgi libapache2-mod-python libapache2-mod-scgi libjs-mochikit 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi139.stdout: python-pecan-doc python-pil-doc python3-pil-dbg python-pyinotify-doc 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi139.stdout: python-pyparsing-doc python-scipy-doc python-setuptools-doc python3-dap 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi139.stdout: python-sklearn-doc python-sqlalchemy-doc python3-psycopg2 python3-pymysql 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi139.stdout: python3-fdb tix python3-tk-dbg python-waitress-doc python-webob-doc 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi139.stdout: python-webtest-doc python3-termcolor python3-watchdog python-werkzeug-doc 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi059.stdout:Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 librabbitmq4 amd64 0.8.0-1ubuntu0.18.04.2 [33.9 kB] 2022-04-28T19:32:11.685 INFO:teuthology.orchestra.run.smithi059.stdout:Get:5 http://archive.ubuntu.com/ubuntu bionic/universe amd64 librdkafka1 amd64 0.11.3-1build1 [293 kB] 2022-04-28T19:32:11.692 INFO:teuthology.orchestra.run.smithi059.stdout:Get:6 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 librbd1 amd64 15.2.16-1bionic [1,641 kB] 2022-04-28T19:32:11.719 INFO:teuthology.orchestra.run.smithi059.stdout:Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsnappy1v5 amd64 1.1.7-1 [16.0 kB] 2022-04-28T19:32:11.719 INFO:teuthology.orchestra.run.smithi059.stdout:Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 libleveldb1v5 amd64 1.20-2 [136 kB] 2022-04-28T19:32:11.728 INFO:teuthology.orchestra.run.smithi059.stdout:Get:9 http://archive.ubuntu.com/ubuntu bionic/universe amd64 liboath0 amd64 2.6.1-1 [44.7 kB] 2022-04-28T19:32:11.732 INFO:teuthology.orchestra.run.smithi059.stdout:Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-dateutil all 2.6.1-1 [52.3 kB] 2022-04-28T19:32:11.735 INFO:teuthology.orchestra.run.smithi059.stdout:Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-asn1crypto all 0.24.0-1 [72.8 kB] 2022-04-28T19:32:11.739 INFO:teuthology.orchestra.run.smithi059.stdout:Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-cffi-backend amd64 1.11.5-1 [64.6 kB] 2022-04-28T19:32:11.741 INFO:teuthology.orchestra.run.smithi059.stdout:Get:13 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-cryptography amd64 2.1.4-1ubuntu1.4 [220 kB] 2022-04-28T19:32:11.750 INFO:teuthology.orchestra.run.smithi059.stdout:Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-openssl all 17.5.0-1ubuntu1 [41.5 kB] 2022-04-28T19:32:11.751 INFO:teuthology.orchestra.run.smithi059.stdout:Get:15 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-bcrypt amd64 3.1.4-2 [29.9 kB] 2022-04-28T19:32:11.756 INFO:teuthology.orchestra.run.smithi059.stdout:Get:16 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-cherrypy3 all 8.9.1-2 [160 kB] 2022-04-28T19:32:11.760 INFO:teuthology.orchestra.run.smithi059.stdout:Get:17 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-jwt all 1.5.3+ds1-1 [15.9 kB] 2022-04-28T19:32:11.760 INFO:teuthology.orchestra.run.smithi059.stdout:Get:18 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-logutils all 0.3.3-5 [16.7 kB] 2022-04-28T19:32:11.776 INFO:teuthology.orchestra.run.smithi059.stdout:Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-markupsafe amd64 1.0-1build1 [13.5 kB] 2022-04-28T19:32:11.776 INFO:teuthology.orchestra.run.smithi059.stdout:Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-mako all 1.0.7+ds1-1 [59.3 kB] 2022-04-28T19:32:11.777 INFO:teuthology.orchestra.run.smithi059.stdout:Get:21 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-simplegeneric all 0.8.1-1 [11.5 kB] 2022-04-28T19:32:11.778 INFO:teuthology.orchestra.run.smithi059.stdout:Get:22 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-singledispatch all 3.4.0.3-2 [7,022 B] 2022-04-28T19:32:11.778 INFO:teuthology.orchestra.run.smithi059.stdout:Get:23 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-webob all 1:1.7.3-2fakesync1 [64.3 kB] 2022-04-28T19:32:11.779 INFO:teuthology.orchestra.run.smithi059.stdout:Get:24 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-bs4 all 4.6.0-1 [67.8 kB] 2022-04-28T19:32:11.781 INFO:teuthology.orchestra.run.smithi059.stdout:Get:25 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-waitress all 1.0.1-1 [53.4 kB] 2022-04-28T19:32:11.796 INFO:teuthology.orchestra.run.smithi059.stdout:Get:26 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-tempita all 0.5.2-2 [13.9 kB] 2022-04-28T19:32:11.796 INFO:teuthology.orchestra.run.smithi059.stdout:Get:27 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-paste all 2.0.3+dfsg-4ubuntu1 [456 kB] 2022-04-28T19:32:11.800 INFO:teuthology.orchestra.run.smithi139.stdout:The following NEW packages will be installed: 2022-04-28T19:32:11.800 INFO:teuthology.orchestra.run.smithi139.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2022-04-28T19:32:11.800 INFO:teuthology.orchestra.run.smithi139.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-cloud 2022-04-28T19:32:11.800 INFO:teuthology.orchestra.run.smithi139.stdout: ceph-mgr-diskprediction-local ceph-mgr-k8sevents ceph-mgr-modules-core 2022-04-28T19:32:11.801 INFO:teuthology.orchestra.run.smithi139.stdout: ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev libcephfs2 libjq1 2022-04-28T19:32:11.801 INFO:teuthology.orchestra.run.smithi139.stdout: libleveldb1v5 liblttng-ust0 liboath0 libonig4 librabbitmq4 libradosstriper1 2022-04-28T19:32:11.802 INFO:teuthology.orchestra.run.smithi139.stdout: librdkafka1 librgw2 libsnappy1v5 libxslt1.1 nvme-cli python-attr 2022-04-28T19:32:11.802 INFO:teuthology.orchestra.run.smithi139.stdout: python-funcsigs python-pastedeploy-tpl python-pluggy python-py python-pytest 2022-04-28T19:32:11.802 INFO:teuthology.orchestra.run.smithi139.stdout: python3-asn1crypto python3-attr python3-bcrypt python3-bs4 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-cffi-backend python3-cherrypy3 python3-cryptography python3-cycler 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-dateutil python3-decorator python3-fasteners python3-html5lib 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-httplib2 python3-jinja2 python3-joblib python3-jwt 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-markupsafe python3-matplotlib python3-monotonic python3-numpy 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-oauth2client python3-olefile python3-openssl python3-paste 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-pastedeploy python3-pastescript python3-pecan python3-pil 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-pluggy python3-prettytable python3-py python3-pyasn1 2022-04-28T19:32:11.803 INFO:teuthology.orchestra.run.smithi139.stdout: python3-pyasn1-modules python3-pyinotify python3-pyparsing python3-pytest 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: python3-rados python3-rbd python3-repoze.lru python3-rgw python3-routes 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: python3-rsa python3-scipy python3-setuptools python3-simplegeneric 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: python3-simplejson python3-singledispatch python3-sklearn 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: python3-sklearn-lib python3-sqlalchemy python3-tempita python3-tk python3-tz 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: python3-uritemplate python3-waitress python3-webencodings python3-webob 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: python3-websocket python3-webtest python3-werkzeug radosgw rbd-fuse socat 2022-04-28T19:32:11.804 INFO:teuthology.orchestra.run.smithi139.stdout: xmlstarlet 2022-04-28T19:32:11.805 INFO:teuthology.orchestra.run.smithi139.stdout:The following packages will be upgraded: 2022-04-28T19:32:11.805 INFO:teuthology.orchestra.run.smithi139.stdout: librados2 librbd1 2022-04-28T19:32:11.809 INFO:teuthology.orchestra.run.smithi059.stdout:Get:28 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-pastedeploy-tpl all 1.5.2-4 [4,796 B] 2022-04-28T19:32:11.816 INFO:teuthology.orchestra.run.smithi059.stdout:Get:29 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pastedeploy all 1.5.2-4 [13.4 kB] 2022-04-28T19:32:11.816 INFO:teuthology.orchestra.run.smithi059.stdout:Get:30 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-webtest all 2.0.28-1ubuntu1 [27.9 kB] 2022-04-28T19:32:11.817 INFO:teuthology.orchestra.run.smithi059.stdout:Get:31 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pecan all 1.2.1-2 [86.1 kB] 2022-04-28T19:32:11.821 INFO:teuthology.orchestra.run.smithi059.stdout:Get:32 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-werkzeug all 0.14.1+dfsg1-1ubuntu0.1 [174 kB] 2022-04-28T19:32:11.827 INFO:teuthology.orchestra.run.smithi059.stdout:Get:33 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-jinja2 all 2.10-1ubuntu0.18.04.1 [95.4 kB] 2022-04-28T19:32:11.830 INFO:teuthology.orchestra.run.smithi059.stdout:Get:34 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-repoze.lru all 0.7-1 [11.7 kB] 2022-04-28T19:32:11.830 INFO:teuthology.orchestra.run.smithi059.stdout:Get:35 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-routes all 2.4.1-1 [28.4 kB] 2022-04-28T19:32:11.838 INFO:teuthology.orchestra.run.smithi059.stdout:Get:36 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-numpy amd64 1:1.13.3-2ubuntu1 [1,943 kB] 2022-04-28T19:32:11.868 INFO:teuthology.orchestra.run.smithi059.stdout:Get:37 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-decorator all 4.1.2-1 [9,364 B] 2022-04-28T19:32:11.868 INFO:teuthology.orchestra.run.smithi059.stdout:Get:38 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-scipy amd64 0.19.1-2ubuntu1 [9,619 kB] 2022-04-28T19:32:11.877 INFO:teuthology.orchestra.run.smithi059.stdout:Get:39 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 librados2 amd64 15.2.16-1bionic [3,063 kB] 2022-04-28T19:32:11.941 INFO:teuthology.orchestra.run.smithi139.stdout:2 upgraded, 108 newly installed, 0 to remove and 8 not upgraded. 2022-04-28T19:32:11.941 INFO:teuthology.orchestra.run.smithi139.stdout:Need to get 156 MB of archives. 2022-04-28T19:32:11.941 INFO:teuthology.orchestra.run.smithi139.stdout:After this operation, 724 MB of additional disk space will be used. 2022-04-28T19:32:11.941 INFO:teuthology.orchestra.run.smithi139.stdout:Get:1 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 librbd1 amd64 15.2.16-1bionic [1,641 kB] 2022-04-28T19:32:11.979 INFO:teuthology.orchestra.run.smithi059.stdout:Get:40 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 libcephfs2 amd64 15.2.16-1bionic [436 kB] 2022-04-28T19:32:11.989 INFO:teuthology.orchestra.run.smithi059.stdout:Get:41 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-rados amd64 15.2.16-1bionic [345 kB] 2022-04-28T19:32:11.993 INFO:teuthology.orchestra.run.smithi059.stdout:Get:42 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-ceph-argparse all 15.2.16-1bionic [20.8 kB] 2022-04-28T19:32:11.993 INFO:teuthology.orchestra.run.smithi059.stdout:Get:43 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-cephfs amd64 15.2.16-1bionic [134 kB] 2022-04-28T19:32:11.994 INFO:teuthology.orchestra.run.smithi139.stdout:Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-uritemplate all 0.6-1ubuntu1 [5,126 B] 2022-04-28T19:32:11.999 INFO:teuthology.orchestra.run.smithi059.stdout:Get:44 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-sklearn-lib amd64 0.19.1-3 [1,429 kB] 2022-04-28T19:32:12.004 INFO:teuthology.orchestra.run.smithi059.stdout:Get:45 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-ceph-common all 15.2.16-1bionic [29.4 kB] 2022-04-28T19:32:12.004 INFO:teuthology.orchestra.run.smithi059.stdout:Get:46 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-rbd amd64 15.2.16-1bionic [325 kB] 2022-04-28T19:32:12.008 INFO:teuthology.orchestra.run.smithi059.stdout:Get:47 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 librgw2 amd64 15.2.16-1bionic [2,885 kB] 2022-04-28T19:32:12.015 INFO:teuthology.orchestra.run.smithi059.stdout:Get:48 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-joblib all 0.11-1 [109 kB] 2022-04-28T19:32:12.017 INFO:teuthology.orchestra.run.smithi059.stdout:Get:49 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-sklearn all 0.19.1-3 [1,373 kB] 2022-04-28T19:32:12.030 INFO:teuthology.orchestra.run.smithi059.stdout:Get:50 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-monotonic all 1.1-2 [5,332 B] 2022-04-28T19:32:12.031 INFO:teuthology.orchestra.run.smithi059.stdout:Get:51 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-fasteners all 0.12.0-3 [13.5 kB] 2022-04-28T19:32:12.031 INFO:teuthology.orchestra.run.smithi059.stdout:Get:52 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-httplib2 all 0.9.2+dfsg-1ubuntu0.3 [28.8 kB] 2022-04-28T19:32:12.031 INFO:teuthology.orchestra.run.smithi059.stdout:Get:53 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyasn1 all 0.4.2-3 [46.8 kB] 2022-04-28T19:32:12.032 INFO:teuthology.orchestra.run.smithi059.stdout:Get:54 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyasn1-modules all 0.2.1-0.2 [32.9 kB] 2022-04-28T19:32:12.032 INFO:teuthology.orchestra.run.smithi059.stdout:Get:55 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-rsa all 3.4.2-1 [29.0 kB] 2022-04-28T19:32:12.033 INFO:teuthology.orchestra.run.smithi059.stdout:Get:56 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-sqlalchemy all 1.1.11+ds1-1ubuntu1 [680 kB] 2022-04-28T19:32:12.051 INFO:teuthology.orchestra.run.smithi059.stdout:Get:57 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-oauth2client all 4.1.2-3 [61.5 kB] 2022-04-28T19:32:12.052 INFO:teuthology.orchestra.run.smithi059.stdout:Get:58 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-setuptools all 39.0.1-2 [248 kB] 2022-04-28T19:32:12.055 INFO:teuthology.orchestra.run.smithi139.stdout:Get:3 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 librados2 amd64 15.2.16-1bionic [3,063 kB] 2022-04-28T19:32:12.071 INFO:teuthology.orchestra.run.smithi059.stdout:Get:59 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-websocket all 0.44.0-0ubuntu2 [30.7 kB] 2022-04-28T19:32:12.072 INFO:teuthology.orchestra.run.smithi059.stdout:Get:60 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-kubernetes all 2.0.0-2ubuntu1 [181 kB] 2022-04-28T19:32:12.075 INFO:teuthology.orchestra.run.smithi139.stdout:Get:4 http://archive.ubuntu.com/ubuntu bionic/universe amd64 liblttng-ust0 amd64 2.10.1-1 [154 kB] 2022-04-28T19:32:12.076 INFO:teuthology.orchestra.run.smithi059.stdout:Get:61 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libonig4 amd64 6.7.0-1 [119 kB] 2022-04-28T19:32:12.128 INFO:teuthology.orchestra.run.smithi059.stdout:Get:62 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libjq1 amd64 1.5+dfsg-2 [111 kB] 2022-04-28T19:32:12.128 INFO:teuthology.orchestra.run.smithi059.stdout:Get:63 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-rgw amd64 15.2.16-1bionic [99.3 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:64 http://archive.ubuntu.com/ubuntu bionic/universe amd64 jq amd64 1.5+dfsg-2 [45.6 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:65 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 libradosstriper1 amd64 15.2.16-1bionic [338 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:66 http://archive.ubuntu.com/ubuntu bionic/main amd64 socat amd64 1.7.3.2-2ubuntu2 [342 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:67 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-common amd64 15.2.16-1bionic [18.8 MB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:68 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libxslt1.1 amd64 1.1.29-5ubuntu0.2 [150 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:69 http://archive.ubuntu.com/ubuntu bionic/universe amd64 xmlstarlet amd64 1.6.1-2 [382 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:70 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 nvme-cli amd64 1.5-1ubuntu1.2 [184 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:71 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-attr all 17.4.0-2 [23.7 kB] 2022-04-28T19:32:12.129 INFO:teuthology.orchestra.run.smithi059.stdout:Get:72 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-funcsigs all 1.0.2-4 [13.5 kB] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:73 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-pluggy all 0.6.0-1 [12.2 kB] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:74 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python-py all 1.5.2-1ubuntu0.1 [65.3 kB] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:75 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-pytest all 3.3.2-2 [142 kB] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:76 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-attr all 17.4.0-2 [23.8 kB] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:77 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-cycler all 0.10.0-1 [7,622 B] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:78 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-webencodings all 0.5-2 [10.4 kB] 2022-04-28T19:32:12.130 INFO:teuthology.orchestra.run.smithi059.stdout:Get:79 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-html5lib all 0.999999999-1 [81.9 kB] 2022-04-28T19:32:12.136 INFO:teuthology.orchestra.run.smithi059.stdout:Get:80 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-lxml amd64 4.2.1-1ubuntu0.6 [1,098 kB] 2022-04-28T19:32:12.156 INFO:teuthology.orchestra.run.smithi059.stdout:Get:81 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyparsing all 2.2.0+dfsg1-2 [52.2 kB] 2022-04-28T19:32:12.163 INFO:teuthology.orchestra.run.smithi059.stdout:Get:82 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-tz all 2018.3-2 [25.1 kB] 2022-04-28T19:32:12.164 INFO:teuthology.orchestra.run.smithi059.stdout:Get:83 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-matplotlib amd64 2.1.1-2ubuntu3 [3,907 kB] 2022-04-28T19:32:12.167 INFO:teuthology.orchestra.run.smithi139.stdout:Get:5 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 libcephfs2 amd64 15.2.16-1bionic [436 kB] 2022-04-28T19:32:12.183 INFO:teuthology.orchestra.run.smithi139.stdout:Get:6 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-rados amd64 15.2.16-1bionic [345 kB] 2022-04-28T19:32:12.190 INFO:teuthology.orchestra.run.smithi139.stdout:Get:7 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-ceph-argparse all 15.2.16-1bionic [20.8 kB] 2022-04-28T19:32:12.190 INFO:teuthology.orchestra.run.smithi139.stdout:Get:8 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-cephfs amd64 15.2.16-1bionic [134 kB] 2022-04-28T19:32:12.204 INFO:teuthology.orchestra.run.smithi139.stdout:Get:9 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-ceph-common all 15.2.16-1bionic [29.4 kB] 2022-04-28T19:32:12.204 INFO:teuthology.orchestra.run.smithi139.stdout:Get:10 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-rbd amd64 15.2.16-1bionic [325 kB] 2022-04-28T19:32:12.210 INFO:teuthology.orchestra.run.smithi139.stdout:Get:11 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 librgw2 amd64 15.2.16-1bionic [2,885 kB] 2022-04-28T19:32:12.219 INFO:teuthology.orchestra.run.smithi059.stdout:Get:84 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-olefile all 0.45.1-1 [33.3 kB] 2022-04-28T19:32:12.220 INFO:teuthology.orchestra.run.smithi059.stdout:Get:85 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pastescript all 2.0.2-2 [54.6 kB] 2022-04-28T19:32:12.220 INFO:teuthology.orchestra.run.smithi059.stdout:Get:86 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-pil amd64 5.1.0-1ubuntu0.7 [331 kB] 2022-04-28T19:32:12.225 INFO:teuthology.orchestra.run.smithi059.stdout:Get:87 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pluggy all 0.6.0-1 [12.3 kB] 2022-04-28T19:32:12.225 INFO:teuthology.orchestra.run.smithi059.stdout:Get:88 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-py all 1.5.2-1ubuntu0.1 [65.3 kB] 2022-04-28T19:32:12.226 INFO:teuthology.orchestra.run.smithi059.stdout:Get:89 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyinotify all 0.9.6-1 [24.7 kB] 2022-04-28T19:32:12.226 INFO:teuthology.orchestra.run.smithi059.stdout:Get:90 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pytest all 3.3.2-2 [142 kB] 2022-04-28T19:32:12.227 INFO:teuthology.orchestra.run.smithi059.stdout:Get:91 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-simplejson amd64 3.13.2-1 [49.6 kB] 2022-04-28T19:32:12.228 INFO:teuthology.orchestra.run.smithi059.stdout:Get:92 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-tk amd64 3.6.9-1~18.04 [119 kB] 2022-04-28T19:32:12.307 INFO:teuthology.orchestra.run.smithi139.stdout:Get:12 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 python3-rgw amd64 15.2.16-1bionic [99.3 kB] 2022-04-28T19:32:12.319 INFO:teuthology.orchestra.run.smithi139.stdout:Get:13 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 libradosstriper1 amd64 15.2.16-1bionic [338 kB] 2022-04-28T19:32:12.326 INFO:teuthology.orchestra.run.smithi139.stdout:Get:14 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-common amd64 15.2.16-1bionic [18.8 MB] 2022-04-28T19:32:12.352 INFO:teuthology.orchestra.run.smithi139.stdout:Get:15 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-prettytable all 0.7.2-3 [19.7 kB] 2022-04-28T19:32:12.364 INFO:teuthology.orchestra.run.smithi139.stdout:Get:16 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 librabbitmq4 amd64 0.8.0-1ubuntu0.18.04.2 [33.9 kB] 2022-04-28T19:32:12.387 INFO:teuthology.orchestra.run.smithi139.stdout:Get:17 http://archive.ubuntu.com/ubuntu bionic/universe amd64 librdkafka1 amd64 0.11.3-1build1 [293 kB] 2022-04-28T19:32:12.464 INFO:teuthology.orchestra.run.smithi059.stdout:Get:93 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-base amd64 15.2.16-1bionic [5,210 kB] 2022-04-28T19:32:12.489 INFO:teuthology.orchestra.run.smithi139.stdout:Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsnappy1v5 amd64 1.1.7-1 [16.0 kB] 2022-04-28T19:32:12.492 INFO:teuthology.orchestra.run.smithi139.stdout:Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 libleveldb1v5 amd64 1.20-2 [136 kB] 2022-04-28T19:32:12.520 INFO:teuthology.orchestra.run.smithi139.stdout:Get:20 http://archive.ubuntu.com/ubuntu bionic/universe amd64 liboath0 amd64 2.6.1-1 [44.7 kB] 2022-04-28T19:32:12.528 INFO:teuthology.orchestra.run.smithi139.stdout:Get:21 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-dateutil all 2.6.1-1 [52.3 kB] 2022-04-28T19:32:12.537 INFO:teuthology.orchestra.run.smithi139.stdout:Get:22 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-asn1crypto all 0.24.0-1 [72.8 kB] 2022-04-28T19:32:12.547 INFO:teuthology.orchestra.run.smithi139.stdout:Get:23 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-cffi-backend amd64 1.11.5-1 [64.6 kB] 2022-04-28T19:32:12.556 INFO:teuthology.orchestra.run.smithi139.stdout:Get:24 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-cryptography amd64 2.1.4-1ubuntu1.4 [220 kB] 2022-04-28T19:32:12.557 INFO:teuthology.orchestra.run.smithi059.stdout:Get:94 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-modules-core all 15.2.16-1bionic [162 kB] 2022-04-28T19:32:12.559 INFO:teuthology.orchestra.run.smithi059.stdout:Get:95 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr amd64 15.2.16-1bionic [1,310 kB] 2022-04-28T19:32:12.582 INFO:teuthology.orchestra.run.smithi139.stdout:Get:25 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-openssl all 17.5.0-1ubuntu1 [41.5 kB] 2022-04-28T19:32:12.585 INFO:teuthology.orchestra.run.smithi139.stdout:Get:26 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-bcrypt amd64 3.1.4-2 [29.9 kB] 2022-04-28T19:32:12.587 INFO:teuthology.orchestra.run.smithi059.stdout:Get:96 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mon amd64 15.2.16-1bionic [5,955 kB] 2022-04-28T19:32:12.596 INFO:teuthology.orchestra.run.smithi139.stdout:Get:27 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-cherrypy3 all 8.9.1-2 [160 kB] 2022-04-28T19:32:12.609 INFO:teuthology.orchestra.run.smithi139.stdout:Get:28 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-jwt all 1.5.3+ds1-1 [15.9 kB] 2022-04-28T19:32:12.676 INFO:teuthology.orchestra.run.smithi139.stdout:Get:29 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-logutils all 0.3.3-5 [16.7 kB] 2022-04-28T19:32:12.677 INFO:teuthology.orchestra.run.smithi139.stdout:Get:30 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-markupsafe amd64 1.0-1build1 [13.5 kB] 2022-04-28T19:32:12.678 INFO:teuthology.orchestra.run.smithi139.stdout:Get:31 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-mako all 1.0.7+ds1-1 [59.3 kB] 2022-04-28T19:32:12.680 INFO:teuthology.orchestra.run.smithi139.stdout:Get:32 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-simplegeneric all 0.8.1-1 [11.5 kB] 2022-04-28T19:32:12.681 INFO:teuthology.orchestra.run.smithi139.stdout:Get:33 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-singledispatch all 3.4.0.3-2 [7,022 B] 2022-04-28T19:32:12.681 INFO:teuthology.orchestra.run.smithi139.stdout:Get:34 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-webob all 1:1.7.3-2fakesync1 [64.3 kB] 2022-04-28T19:32:12.684 INFO:teuthology.orchestra.run.smithi139.stdout:Get:35 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-bs4 all 4.6.0-1 [67.8 kB] 2022-04-28T19:32:12.687 INFO:teuthology.orchestra.run.smithi139.stdout:Get:36 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-waitress all 1.0.1-1 [53.4 kB] 2022-04-28T19:32:12.695 INFO:teuthology.orchestra.run.smithi059.stdout:Get:97 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-osd amd64 15.2.16-1bionic [22.9 MB] 2022-04-28T19:32:12.755 INFO:teuthology.orchestra.run.smithi139.stdout:Get:37 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-tempita all 0.5.2-2 [13.9 kB] 2022-04-28T19:32:12.756 INFO:teuthology.orchestra.run.smithi139.stdout:Get:38 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-paste all 2.0.3+dfsg-4ubuntu1 [456 kB] 2022-04-28T19:32:12.835 INFO:teuthology.orchestra.run.smithi139.stdout:Get:39 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-pastedeploy-tpl all 1.5.2-4 [4,796 B] 2022-04-28T19:32:12.836 INFO:teuthology.orchestra.run.smithi139.stdout:Get:40 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pastedeploy all 1.5.2-4 [13.4 kB] 2022-04-28T19:32:12.836 INFO:teuthology.orchestra.run.smithi139.stdout:Get:41 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-webtest all 2.0.28-1ubuntu1 [27.9 kB] 2022-04-28T19:32:12.837 INFO:teuthology.orchestra.run.smithi139.stdout:Get:42 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pecan all 1.2.1-2 [86.1 kB] 2022-04-28T19:32:12.841 INFO:teuthology.orchestra.run.smithi139.stdout:Get:43 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-werkzeug all 0.14.1+dfsg1-1ubuntu0.1 [174 kB] 2022-04-28T19:32:12.849 INFO:teuthology.orchestra.run.smithi139.stdout:Get:44 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-jinja2 all 2.10-1ubuntu0.18.04.1 [95.4 kB] 2022-04-28T19:32:12.853 INFO:teuthology.orchestra.run.smithi139.stdout:Get:45 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-repoze.lru all 0.7-1 [11.7 kB] 2022-04-28T19:32:12.854 INFO:teuthology.orchestra.run.smithi139.stdout:Get:46 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-routes all 2.4.1-1 [28.4 kB] 2022-04-28T19:32:12.896 INFO:teuthology.orchestra.run.smithi139.stdout:Get:47 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-base amd64 15.2.16-1bionic [5,210 kB] 2022-04-28T19:32:12.916 INFO:teuthology.orchestra.run.smithi139.stdout:Get:48 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-numpy amd64 1:1.13.3-2ubuntu1 [1,943 kB] 2022-04-28T19:32:13.000 INFO:teuthology.orchestra.run.smithi139.stdout:Get:49 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-decorator all 4.1.2-1 [9,364 B] 2022-04-28T19:32:13.001 INFO:teuthology.orchestra.run.smithi139.stdout:Get:50 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-scipy amd64 0.19.1-2ubuntu1 [9,619 kB] 2022-04-28T19:32:13.043 INFO:teuthology.orchestra.run.smithi139.stdout:Get:51 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-modules-core all 15.2.16-1bionic [162 kB] 2022-04-28T19:32:13.044 INFO:teuthology.orchestra.run.smithi139.stdout:Get:52 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr amd64 15.2.16-1bionic [1,310 kB] 2022-04-28T19:32:13.080 INFO:teuthology.orchestra.run.smithi059.stdout:Get:98 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph amd64 15.2.16-1bionic [3,988 B] 2022-04-28T19:32:13.081 INFO:teuthology.orchestra.run.smithi059.stdout:Get:99 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-fuse amd64 15.2.16-1bionic [517 kB] 2022-04-28T19:32:13.086 INFO:teuthology.orchestra.run.smithi139.stdout:Get:53 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mon amd64 15.2.16-1bionic [5,955 kB] 2022-04-28T19:32:13.088 INFO:teuthology.orchestra.run.smithi059.stdout:Get:100 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mds amd64 15.2.16-1bionic [1,855 kB] 2022-04-28T19:32:13.124 INFO:teuthology.orchestra.run.smithi059.stdout:Get:101 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 cephadm amd64 15.2.16-1bionic [53.2 kB] 2022-04-28T19:32:13.124 INFO:teuthology.orchestra.run.smithi059.stdout:Get:102 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-cephadm all 15.2.16-1bionic [58.2 kB] 2022-04-28T19:32:13.125 INFO:teuthology.orchestra.run.smithi059.stdout:Get:103 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-dashboard all 15.2.16-1bionic [3,217 kB] 2022-04-28T19:32:13.175 INFO:teuthology.orchestra.run.smithi059.stdout:Get:104 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-diskprediction-cloud all 15.2.16-1bionic [34.3 kB] 2022-04-28T19:32:13.175 INFO:teuthology.orchestra.run.smithi059.stdout:Get:105 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-diskprediction-local all 15.2.16-1bionic [7,878 kB] 2022-04-28T19:32:13.206 INFO:teuthology.orchestra.run.smithi139.stdout:Get:54 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-osd amd64 15.2.16-1bionic [22.9 MB] 2022-04-28T19:32:13.290 INFO:teuthology.orchestra.run.smithi139.stdout:Get:55 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-sklearn-lib amd64 0.19.1-3 [1,429 kB] 2022-04-28T19:32:13.308 INFO:teuthology.orchestra.run.smithi059.stdout:Get:106 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-k8sevents all 15.2.16-1bionic [4,108 B] 2022-04-28T19:32:13.308 INFO:teuthology.orchestra.run.smithi059.stdout:Get:107 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-test amd64 15.2.16-1bionic [44.0 MB] 2022-04-28T19:32:13.323 INFO:teuthology.orchestra.run.smithi139.stdout:Get:56 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-joblib all 0.11-1 [109 kB] 2022-04-28T19:32:13.328 INFO:teuthology.orchestra.run.smithi139.stdout:Get:57 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-sklearn all 0.19.1-3 [1,373 kB] 2022-04-28T19:32:13.366 INFO:teuthology.orchestra.run.smithi139.stdout:Get:58 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-monotonic all 1.1-2 [5,332 B] 2022-04-28T19:32:13.366 INFO:teuthology.orchestra.run.smithi139.stdout:Get:59 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-fasteners all 0.12.0-3 [13.5 kB] 2022-04-28T19:32:13.367 INFO:teuthology.orchestra.run.smithi139.stdout:Get:60 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-httplib2 all 0.9.2+dfsg-1ubuntu0.3 [28.8 kB] 2022-04-28T19:32:13.367 INFO:teuthology.orchestra.run.smithi139.stdout:Get:61 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyasn1 all 0.4.2-3 [46.8 kB] 2022-04-28T19:32:13.368 INFO:teuthology.orchestra.run.smithi139.stdout:Get:62 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyasn1-modules all 0.2.1-0.2 [32.9 kB] 2022-04-28T19:32:13.369 INFO:teuthology.orchestra.run.smithi139.stdout:Get:63 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-rsa all 3.4.2-1 [29.0 kB] 2022-04-28T19:32:13.370 INFO:teuthology.orchestra.run.smithi139.stdout:Get:64 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-sqlalchemy all 1.1.11+ds1-1ubuntu1 [680 kB] 2022-04-28T19:32:13.450 INFO:teuthology.orchestra.run.smithi139.stdout:Get:65 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-oauth2client all 4.1.2-3 [61.5 kB] 2022-04-28T19:32:13.451 INFO:teuthology.orchestra.run.smithi139.stdout:Get:66 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-setuptools all 39.0.1-2 [248 kB] 2022-04-28T19:32:13.453 INFO:teuthology.orchestra.run.smithi139.stdout:Get:67 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-websocket all 0.44.0-0ubuntu2 [30.7 kB] 2022-04-28T19:32:13.454 INFO:teuthology.orchestra.run.smithi139.stdout:Get:68 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-kubernetes all 2.0.0-2ubuntu1 [181 kB] 2022-04-28T19:32:13.455 INFO:teuthology.orchestra.run.smithi139.stdout:Get:69 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libonig4 amd64 6.7.0-1 [119 kB] 2022-04-28T19:32:13.457 INFO:teuthology.orchestra.run.smithi139.stdout:Get:70 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libjq1 amd64 1.5+dfsg-2 [111 kB] 2022-04-28T19:32:13.458 INFO:teuthology.orchestra.run.smithi139.stdout:Get:71 http://archive.ubuntu.com/ubuntu bionic/universe amd64 jq amd64 1.5+dfsg-2 [45.6 kB] 2022-04-28T19:32:13.530 INFO:teuthology.orchestra.run.smithi139.stdout:Get:72 http://archive.ubuntu.com/ubuntu bionic/main amd64 socat amd64 1.7.3.2-2ubuntu2 [342 kB] 2022-04-28T19:32:13.538 INFO:teuthology.orchestra.run.smithi139.stdout:Get:73 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libxslt1.1 amd64 1.1.29-5ubuntu0.2 [150 kB] 2022-04-28T19:32:13.543 INFO:teuthology.orchestra.run.smithi139.stdout:Get:74 http://archive.ubuntu.com/ubuntu bionic/universe amd64 xmlstarlet amd64 1.6.1-2 [382 kB] 2022-04-28T19:32:13.610 INFO:teuthology.orchestra.run.smithi139.stdout:Get:75 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 nvme-cli amd64 1.5-1ubuntu1.2 [184 kB] 2022-04-28T19:32:13.615 INFO:teuthology.orchestra.run.smithi139.stdout:Get:76 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-attr all 17.4.0-2 [23.7 kB] 2022-04-28T19:32:13.616 INFO:teuthology.orchestra.run.smithi139.stdout:Get:77 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-funcsigs all 1.0.2-4 [13.5 kB] 2022-04-28T19:32:13.616 INFO:teuthology.orchestra.run.smithi139.stdout:Get:78 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-pluggy all 0.6.0-1 [12.2 kB] 2022-04-28T19:32:13.617 INFO:teuthology.orchestra.run.smithi139.stdout:Get:79 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python-py all 1.5.2-1ubuntu0.1 [65.3 kB] 2022-04-28T19:32:13.619 INFO:teuthology.orchestra.run.smithi139.stdout:Get:80 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-pytest all 3.3.2-2 [142 kB] 2022-04-28T19:32:13.622 INFO:teuthology.orchestra.run.smithi139.stdout:Get:81 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-attr all 17.4.0-2 [23.8 kB] 2022-04-28T19:32:13.658 INFO:teuthology.orchestra.run.smithi139.stdout:Get:82 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph amd64 15.2.16-1bionic [3,988 B] 2022-04-28T19:32:13.659 INFO:teuthology.orchestra.run.smithi139.stdout:Get:83 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-fuse amd64 15.2.16-1bionic [517 kB] 2022-04-28T19:32:13.667 INFO:teuthology.orchestra.run.smithi139.stdout:Get:84 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mds amd64 15.2.16-1bionic [1,855 kB] 2022-04-28T19:32:13.689 INFO:teuthology.orchestra.run.smithi139.stdout:Get:85 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-cycler all 0.10.0-1 [7,622 B] 2022-04-28T19:32:13.690 INFO:teuthology.orchestra.run.smithi139.stdout:Get:86 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-webencodings all 0.5-2 [10.4 kB] 2022-04-28T19:32:13.691 INFO:teuthology.orchestra.run.smithi139.stdout:Get:87 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-html5lib all 0.999999999-1 [81.9 kB] 2022-04-28T19:32:13.694 INFO:teuthology.orchestra.run.smithi139.stdout:Get:88 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 cephadm amd64 15.2.16-1bionic [53.2 kB] 2022-04-28T19:32:13.695 INFO:teuthology.orchestra.run.smithi139.stdout:Get:89 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-cephadm all 15.2.16-1bionic [58.2 kB] 2022-04-28T19:32:13.696 INFO:teuthology.orchestra.run.smithi139.stdout:Get:90 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-dashboard all 15.2.16-1bionic [3,217 kB] 2022-04-28T19:32:13.755 INFO:teuthology.orchestra.run.smithi139.stdout:Get:91 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-diskprediction-cloud all 15.2.16-1bionic [34.3 kB] 2022-04-28T19:32:13.756 INFO:teuthology.orchestra.run.smithi139.stdout:Get:92 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-diskprediction-local all 15.2.16-1bionic [7,878 kB] 2022-04-28T19:32:13.769 INFO:teuthology.orchestra.run.smithi139.stdout:Get:93 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-lxml amd64 4.2.1-1ubuntu0.6 [1,098 kB] 2022-04-28T19:32:13.792 INFO:teuthology.orchestra.run.smithi139.stdout:Get:94 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyparsing all 2.2.0+dfsg1-2 [52.2 kB] 2022-04-28T19:32:13.793 INFO:teuthology.orchestra.run.smithi139.stdout:Get:95 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-tz all 2018.3-2 [25.1 kB] 2022-04-28T19:32:13.793 INFO:teuthology.orchestra.run.smithi139.stdout:Get:96 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-matplotlib amd64 2.1.1-2ubuntu3 [3,907 kB] 2022-04-28T19:32:13.907 INFO:teuthology.orchestra.run.smithi139.stdout:Get:97 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-mgr-k8sevents all 15.2.16-1bionic [4,108 B] 2022-04-28T19:32:13.907 INFO:teuthology.orchestra.run.smithi139.stdout:Get:98 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 ceph-test amd64 15.2.16-1bionic [44.0 MB] 2022-04-28T19:32:13.930 INFO:teuthology.orchestra.run.smithi139.stdout:Get:99 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-olefile all 0.45.1-1 [33.3 kB] 2022-04-28T19:32:13.931 INFO:teuthology.orchestra.run.smithi139.stdout:Get:100 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pastescript all 2.0.2-2 [54.6 kB] 2022-04-28T19:32:13.931 INFO:teuthology.orchestra.run.smithi139.stdout:Get:101 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-pil amd64 5.1.0-1ubuntu0.7 [331 kB] 2022-04-28T19:32:13.939 INFO:teuthology.orchestra.run.smithi139.stdout:Get:102 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pluggy all 0.6.0-1 [12.3 kB] 2022-04-28T19:32:13.939 INFO:teuthology.orchestra.run.smithi139.stdout:Get:103 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-py all 1.5.2-1ubuntu0.1 [65.3 kB] 2022-04-28T19:32:13.940 INFO:teuthology.orchestra.run.smithi139.stdout:Get:104 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pyinotify all 0.9.6-1 [24.7 kB] 2022-04-28T19:32:13.940 INFO:teuthology.orchestra.run.smithi139.stdout:Get:105 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-pytest all 3.3.2-2 [142 kB] 2022-04-28T19:32:13.942 INFO:teuthology.orchestra.run.smithi139.stdout:Get:106 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-simplejson amd64 3.13.2-1 [49.6 kB] 2022-04-28T19:32:13.942 INFO:teuthology.orchestra.run.smithi139.stdout:Get:107 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-tk amd64 3.6.9-1~18.04 [119 kB] 2022-04-28T19:32:14.131 INFO:teuthology.orchestra.run.smithi059.stdout:Get:108 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 libcephfs-dev amd64 15.2.16-1bionic [18.1 kB] 2022-04-28T19:32:14.131 INFO:teuthology.orchestra.run.smithi059.stdout:Get:109 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 radosgw amd64 15.2.16-1bionic [8,745 kB] 2022-04-28T19:32:14.408 INFO:teuthology.orchestra.run.smithi059.stdout:Get:110 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 rbd-fuse amd64 15.2.16-1bionic [63.1 kB] 2022-04-28T19:32:14.654 INFO:teuthology.orchestra.run.smithi139.stdout:Get:108 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 libcephfs-dev amd64 15.2.16-1bionic [18.1 kB] 2022-04-28T19:32:14.654 INFO:teuthology.orchestra.run.smithi139.stdout:Get:109 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 radosgw amd64 15.2.16-1bionic [8,745 kB] 2022-04-28T19:32:14.741 INFO:teuthology.orchestra.run.smithi139.stdout:Get:110 https://chacra.ceph.com/r/ceph/octopus/d46a73d6d0a67a79558054a3a5a72cb561724974/ubuntu/bionic/flavors/default bionic/main amd64 rbd-fuse amd64 15.2.16-1bionic [63.1 kB] 2022-04-28T19:32:15.165 INFO:teuthology.orchestra.run.smithi059.stdout:Fetched 156 MB in 3s (54.6 MB/s) 2022-04-28T19:32:15.284 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-uritemplate. 2022-04-28T19:32:15.323 INFO:teuthology.orchestra.run.smithi059.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 100349 files and directories currently installed.) 2022-04-28T19:32:15.325 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../000-python3-uritemplate_0.6-1ubuntu1_all.deb ... 2022-04-28T19:32:15.358 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-uritemplate (0.6-1ubuntu1) ... 2022-04-28T19:32:15.458 INFO:teuthology.orchestra.run.smithi139.stdout:Fetched 156 MB in 3s (53.7 MB/s) 2022-04-28T19:32:15.599 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-uritemplate. 2022-04-28T19:32:15.632 INFO:teuthology.orchestra.run.smithi139.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 100349 files and directories currently installed.) 2022-04-28T19:32:15.635 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../000-python3-uritemplate_0.6-1ubuntu1_all.deb ... 2022-04-28T19:32:15.664 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-uritemplate (0.6-1ubuntu1) ... 2022-04-28T19:32:15.677 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2022-04-28T19:32:15.689 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../001-liblttng-ust0_2.10.1-1_amd64.deb ... 2022-04-28T19:32:15.769 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking liblttng-ust0:amd64 (2.10.1-1) ... 2022-04-28T19:32:15.975 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2022-04-28T19:32:15.987 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../001-liblttng-ust0_2.10.1-1_amd64.deb ... 2022-04-28T19:32:16.067 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking liblttng-ust0:amd64 (2.10.1-1) ... 2022-04-28T19:32:16.110 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../002-librbd1_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:16.231 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking librbd1 (15.2.16-1bionic) over (12.2.13-0ubuntu0.18.04.10) ... 2022-04-28T19:32:16.398 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../002-librbd1_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:16.521 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking librbd1 (15.2.16-1bionic) over (12.2.13-0ubuntu0.18.04.10) ... 2022-04-28T19:32:16.862 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../003-librados2_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:16.977 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking librados2 (15.2.16-1bionic) over (12.2.13-0ubuntu0.18.04.10) ... 2022-04-28T19:32:17.044 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../003-librados2_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:17.158 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking librados2 (15.2.16-1bionic) over (12.2.13-0ubuntu0.18.04.10) ... 2022-04-28T19:32:17.622 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libcephfs2. 2022-04-28T19:32:17.634 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../004-libcephfs2_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:17.672 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libcephfs2 (15.2.16-1bionic) ... 2022-04-28T19:32:17.786 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libcephfs2. 2022-04-28T19:32:17.798 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../004-libcephfs2_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:17.836 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libcephfs2 (15.2.16-1bionic) ... 2022-04-28T19:32:17.975 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-rados. 2022-04-28T19:32:17.986 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../005-python3-rados_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:18.024 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-rados (15.2.16-1bionic) ... 2022-04-28T19:32:18.164 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-rados. 2022-04-28T19:32:18.175 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../005-python3-rados_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:18.213 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-rados (15.2.16-1bionic) ... 2022-04-28T19:32:18.352 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-ceph-argparse. 2022-04-28T19:32:18.364 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../006-python3-ceph-argparse_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:18.402 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-ceph-argparse (15.2.16-1bionic) ... 2022-04-28T19:32:18.549 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-ceph-argparse. 2022-04-28T19:32:18.561 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../006-python3-ceph-argparse_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:18.599 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-ceph-argparse (15.2.16-1bionic) ... 2022-04-28T19:32:18.705 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-cephfs. 2022-04-28T19:32:18.716 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../007-python3-cephfs_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:18.755 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-cephfs (15.2.16-1bionic) ... 2022-04-28T19:32:18.894 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-cephfs. 2022-04-28T19:32:18.905 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../007-python3-cephfs_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:18.943 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-cephfs (15.2.16-1bionic) ... 2022-04-28T19:32:19.066 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-ceph-common. 2022-04-28T19:32:19.078 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../008-python3-ceph-common_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:19.116 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-ceph-common (15.2.16-1bionic) ... 2022-04-28T19:32:19.246 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-ceph-common. 2022-04-28T19:32:19.259 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../008-python3-ceph-common_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:19.297 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-ceph-common (15.2.16-1bionic) ... 2022-04-28T19:32:19.427 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-prettytable. 2022-04-28T19:32:19.439 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../009-python3-prettytable_0.7.2-3_all.deb ... 2022-04-28T19:32:19.469 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-prettytable (0.7.2-3) ... 2022-04-28T19:32:19.632 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-prettytable. 2022-04-28T19:32:19.644 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../009-python3-prettytable_0.7.2-3_all.deb ... 2022-04-28T19:32:19.682 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-prettytable (0.7.2-3) ... 2022-04-28T19:32:19.696 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-rbd. 2022-04-28T19:32:19.708 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../010-python3-rbd_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:19.746 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-rbd (15.2.16-1bionic) ... 2022-04-28T19:32:19.926 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-rbd. 2022-04-28T19:32:19.939 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../010-python3-rbd_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:19.976 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-rbd (15.2.16-1bionic) ... 2022-04-28T19:32:20.108 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package librabbitmq4:amd64. 2022-04-28T19:32:20.119 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../011-librabbitmq4_0.8.0-1ubuntu0.18.04.2_amd64.deb ... 2022-04-28T19:32:20.174 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... 2022-04-28T19:32:20.329 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package librabbitmq4:amd64. 2022-04-28T19:32:20.341 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../011-librabbitmq4_0.8.0-1ubuntu0.18.04.2_amd64.deb ... 2022-04-28T19:32:20.396 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... 2022-04-28T19:32:20.510 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package librdkafka1:amd64. 2022-04-28T19:32:20.522 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../012-librdkafka1_0.11.3-1build1_amd64.deb ... 2022-04-28T19:32:20.560 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking librdkafka1:amd64 (0.11.3-1build1) ... 2022-04-28T19:32:20.715 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package librdkafka1:amd64. 2022-04-28T19:32:20.727 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../012-librdkafka1_0.11.3-1build1_amd64.deb ... 2022-04-28T19:32:20.765 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking librdkafka1:amd64 (0.11.3-1build1) ... 2022-04-28T19:32:20.871 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package librgw2. 2022-04-28T19:32:20.883 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../013-librgw2_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:20.913 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking librgw2 (15.2.16-1bionic) ... 2022-04-28T19:32:21.134 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package librgw2. 2022-04-28T19:32:21.146 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../013-librgw2_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:21.184 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking librgw2 (15.2.16-1bionic) ... 2022-04-28T19:32:21.432 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-rgw. 2022-04-28T19:32:21.445 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../014-python3-rgw_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:21.482 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-rgw (15.2.16-1bionic) ... 2022-04-28T19:32:21.695 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-rgw. 2022-04-28T19:32:21.706 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../014-python3-rgw_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:21.736 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-rgw (15.2.16-1bionic) ... 2022-04-28T19:32:21.801 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libsnappy1v5:amd64. 2022-04-28T19:32:21.814 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../015-libsnappy1v5_1.1.7-1_amd64.deb ... 2022-04-28T19:32:21.851 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libsnappy1v5:amd64 (1.1.7-1) ... 2022-04-28T19:32:22.039 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libsnappy1v5:amd64. 2022-04-28T19:32:22.052 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../015-libsnappy1v5_1.1.7-1_amd64.deb ... 2022-04-28T19:32:22.089 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libsnappy1v5:amd64 (1.1.7-1) ... 2022-04-28T19:32:22.154 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libleveldb1v5:amd64. 2022-04-28T19:32:22.165 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../016-libleveldb1v5_1.20-2_amd64.deb ... 2022-04-28T19:32:22.196 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libleveldb1v5:amd64 (1.20-2) ... 2022-04-28T19:32:22.391 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libleveldb1v5:amd64. 2022-04-28T19:32:22.403 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../016-libleveldb1v5_1.20-2_amd64.deb ... 2022-04-28T19:32:22.466 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libleveldb1v5:amd64 (1.20-2) ... 2022-04-28T19:32:22.549 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package liboath0. 2022-04-28T19:32:22.560 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../017-liboath0_2.6.1-1_amd64.deb ... 2022-04-28T19:32:22.599 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking liboath0 (2.6.1-1) ... 2022-04-28T19:32:22.836 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package liboath0. 2022-04-28T19:32:22.847 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../017-liboath0_2.6.1-1_amd64.deb ... 2022-04-28T19:32:22.886 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking liboath0 (2.6.1-1) ... 2022-04-28T19:32:22.910 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libradosstriper1. 2022-04-28T19:32:22.921 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../018-libradosstriper1_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:22.960 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libradosstriper1 (15.2.16-1bionic) ... 2022-04-28T19:32:23.155 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libradosstriper1. 2022-04-28T19:32:23.166 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../018-libradosstriper1_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:23.197 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libradosstriper1 (15.2.16-1bionic) ... 2022-04-28T19:32:23.346 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-common. 2022-04-28T19:32:23.359 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../019-ceph-common_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:23.388 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-common (15.2.16-1bionic) ... 2022-04-28T19:32:23.633 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-common. 2022-04-28T19:32:23.645 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../019-ceph-common_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:23.691 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-common (15.2.16-1bionic) ... 2022-04-28T19:32:25.268 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-base. 2022-04-28T19:32:25.281 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../020-ceph-base_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:25.327 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-base (15.2.16-1bionic) ... 2022-04-28T19:32:25.438 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-base. 2022-04-28T19:32:25.451 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../020-ceph-base_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:25.497 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-base (15.2.16-1bionic) ... 2022-04-28T19:32:26.140 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-dateutil. 2022-04-28T19:32:26.151 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../021-python3-dateutil_2.6.1-1_all.deb ... 2022-04-28T19:32:26.181 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-dateutil (2.6.1-1) ... 2022-04-28T19:32:26.301 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-dateutil. 2022-04-28T19:32:26.313 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../021-python3-dateutil_2.6.1-1_all.deb ... 2022-04-28T19:32:26.351 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-dateutil (2.6.1-1) ... 2022-04-28T19:32:26.518 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-asn1crypto. 2022-04-28T19:32:26.560 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../022-python3-asn1crypto_0.24.0-1_all.deb ... 2022-04-28T19:32:26.568 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-asn1crypto (0.24.0-1) ... 2022-04-28T19:32:26.654 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-asn1crypto. 2022-04-28T19:32:26.666 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../022-python3-asn1crypto_0.24.0-1_all.deb ... 2022-04-28T19:32:26.704 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-asn1crypto (0.24.0-1) ... 2022-04-28T19:32:26.820 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-cffi-backend. 2022-04-28T19:32:26.833 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../023-python3-cffi-backend_1.11.5-1_amd64.deb ... 2022-04-28T19:32:26.870 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-cffi-backend (1.11.5-1) ... 2022-04-28T19:32:26.973 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-cffi-backend. 2022-04-28T19:32:26.983 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../023-python3-cffi-backend_1.11.5-1_amd64.deb ... 2022-04-28T19:32:27.023 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-cffi-backend (1.11.5-1) ... 2022-04-28T19:32:27.165 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-cryptography. 2022-04-28T19:32:27.176 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../024-python3-cryptography_2.1.4-1ubuntu1.4_amd64.deb ... 2022-04-28T19:32:27.215 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-cryptography (2.1.4-1ubuntu1.4) ... 2022-04-28T19:32:27.342 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-cryptography. 2022-04-28T19:32:27.354 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../024-python3-cryptography_2.1.4-1ubuntu1.4_amd64.deb ... 2022-04-28T19:32:27.392 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-cryptography (2.1.4-1ubuntu1.4) ... 2022-04-28T19:32:27.551 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-openssl. 2022-04-28T19:32:27.562 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../025-python3-openssl_17.5.0-1ubuntu1_all.deb ... 2022-04-28T19:32:27.601 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-openssl (17.5.0-1ubuntu1) ... 2022-04-28T19:32:27.762 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-openssl. 2022-04-28T19:32:27.775 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../025-python3-openssl_17.5.0-1ubuntu1_all.deb ... 2022-04-28T19:32:27.812 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-openssl (17.5.0-1ubuntu1) ... 2022-04-28T19:32:27.879 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2022-04-28T19:32:27.889 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:27.929 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr-modules-core (15.2.16-1bionic) ... 2022-04-28T19:32:28.173 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2022-04-28T19:32:28.185 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:28.240 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr-modules-core (15.2.16-1bionic) ... 2022-04-28T19:32:28.425 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-bcrypt. 2022-04-28T19:32:28.436 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../027-python3-bcrypt_3.1.4-2_amd64.deb ... 2022-04-28T19:32:28.475 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-bcrypt (3.1.4-2) ... 2022-04-28T19:32:28.585 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-bcrypt. 2022-04-28T19:32:28.598 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../027-python3-bcrypt_3.1.4-2_amd64.deb ... 2022-04-28T19:32:28.635 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-bcrypt (3.1.4-2) ... 2022-04-28T19:32:28.778 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-cherrypy3. 2022-04-28T19:32:28.790 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../028-python3-cherrypy3_8.9.1-2_all.deb ... 2022-04-28T19:32:28.828 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-cherrypy3 (8.9.1-2) ... 2022-04-28T19:32:28.938 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-cherrypy3. 2022-04-28T19:32:28.950 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../028-python3-cherrypy3_8.9.1-2_all.deb ... 2022-04-28T19:32:28.988 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-cherrypy3 (8.9.1-2) ... 2022-04-28T19:32:29.147 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-jwt. 2022-04-28T19:32:29.160 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../029-python3-jwt_1.5.3+ds1-1_all.deb ... 2022-04-28T19:32:29.231 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-jwt (1.5.3+ds1-1) ... 2022-04-28T19:32:29.316 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-jwt. 2022-04-28T19:32:29.328 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../029-python3-jwt_1.5.3+ds1-1_all.deb ... 2022-04-28T19:32:29.366 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-jwt (1.5.3+ds1-1) ... 2022-04-28T19:32:29.542 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-logutils. 2022-04-28T19:32:29.554 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../030-python3-logutils_0.3.3-5_all.deb ... 2022-04-28T19:32:29.592 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-logutils (0.3.3-5) ... 2022-04-28T19:32:29.686 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-logutils. 2022-04-28T19:32:29.698 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../030-python3-logutils_0.3.3-5_all.deb ... 2022-04-28T19:32:29.736 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-logutils (0.3.3-5) ... 2022-04-28T19:32:29.837 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-markupsafe. 2022-04-28T19:32:29.850 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../031-python3-markupsafe_1.0-1build1_amd64.deb ... 2022-04-28T19:32:29.878 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-markupsafe (1.0-1build1) ... 2022-04-28T19:32:29.980 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-markupsafe. 2022-04-28T19:32:29.991 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../031-python3-markupsafe_1.0-1build1_amd64.deb ... 2022-04-28T19:32:30.021 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-markupsafe (1.0-1build1) ... 2022-04-28T19:32:30.148 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-mako. 2022-04-28T19:32:30.161 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../032-python3-mako_1.0.7+ds1-1_all.deb ... 2022-04-28T19:32:30.206 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-mako (1.0.7+ds1-1) ... 2022-04-28T19:32:30.324 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-mako. 2022-04-28T19:32:30.335 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../032-python3-mako_1.0.7+ds1-1_all.deb ... 2022-04-28T19:32:30.374 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-mako (1.0.7+ds1-1) ... 2022-04-28T19:32:30.517 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-simplegeneric. 2022-04-28T19:32:30.529 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../033-python3-simplegeneric_0.8.1-1_all.deb ... 2022-04-28T19:32:30.589 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-simplegeneric (0.8.1-1) ... 2022-04-28T19:32:30.685 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-simplegeneric. 2022-04-28T19:32:30.697 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../033-python3-simplegeneric_0.8.1-1_all.deb ... 2022-04-28T19:32:30.735 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-simplegeneric (0.8.1-1) ... 2022-04-28T19:32:30.862 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-singledispatch. 2022-04-28T19:32:30.873 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../034-python3-singledispatch_3.4.0.3-2_all.deb ... 2022-04-28T19:32:30.903 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-singledispatch (3.4.0.3-2) ... 2022-04-28T19:32:31.030 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-singledispatch. 2022-04-28T19:32:31.042 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../034-python3-singledispatch_3.4.0.3-2_all.deb ... 2022-04-28T19:32:31.096 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-singledispatch (3.4.0.3-2) ... 2022-04-28T19:32:31.198 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-webob. 2022-04-28T19:32:31.209 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../035-python3-webob_1%3a1.7.3-2fakesync1_all.deb ... 2022-04-28T19:32:31.250 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-webob (1:1.7.3-2fakesync1) ... 2022-04-28T19:32:31.340 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-webob. 2022-04-28T19:32:31.351 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../035-python3-webob_1%3a1.7.3-2fakesync1_all.deb ... 2022-04-28T19:32:31.382 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-webob (1:1.7.3-2fakesync1) ... 2022-04-28T19:32:31.569 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-bs4. 2022-04-28T19:32:31.581 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../036-python3-bs4_4.6.0-1_all.deb ... 2022-04-28T19:32:31.619 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-bs4 (4.6.0-1) ... 2022-04-28T19:32:31.668 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-bs4. 2022-04-28T19:32:31.681 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../036-python3-bs4_4.6.0-1_all.deb ... 2022-04-28T19:32:31.718 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-bs4 (4.6.0-1) ... 2022-04-28T19:32:31.938 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-waitress. 2022-04-28T19:32:31.950 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../037-python3-waitress_1.0.1-1_all.deb ... 2022-04-28T19:32:32.013 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-waitress (1.0.1-1) ... 2022-04-28T19:32:32.047 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-waitress. 2022-04-28T19:32:32.058 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../037-python3-waitress_1.0.1-1_all.deb ... 2022-04-28T19:32:32.096 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-waitress (1.0.1-1) ... 2022-04-28T19:32:32.316 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-tempita. 2022-04-28T19:32:32.327 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../038-python3-tempita_0.5.2-2_all.deb ... 2022-04-28T19:32:32.367 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-tempita (0.5.2-2) ... 2022-04-28T19:32:32.424 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-tempita. 2022-04-28T19:32:32.435 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../038-python3-tempita_0.5.2-2_all.deb ... 2022-04-28T19:32:32.474 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-tempita (0.5.2-2) ... 2022-04-28T19:32:32.619 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-paste. 2022-04-28T19:32:32.632 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../039-python3-paste_2.0.3+dfsg-4ubuntu1_all.deb ... 2022-04-28T19:32:32.669 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-paste (2.0.3+dfsg-4ubuntu1) ... 2022-04-28T19:32:32.810 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-paste. 2022-04-28T19:32:32.822 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../039-python3-paste_2.0.3+dfsg-4ubuntu1_all.deb ... 2022-04-28T19:32:32.860 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-paste (2.0.3+dfsg-4ubuntu1) ... 2022-04-28T19:32:32.972 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2022-04-28T19:32:32.984 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../040-python-pastedeploy-tpl_1.5.2-4_all.deb ... 2022-04-28T19:32:33.022 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python-pastedeploy-tpl (1.5.2-4) ... 2022-04-28T19:32:33.171 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2022-04-28T19:32:33.180 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../040-python-pastedeploy-tpl_1.5.2-4_all.deb ... 2022-04-28T19:32:33.221 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python-pastedeploy-tpl (1.5.2-4) ... 2022-04-28T19:32:33.275 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pastedeploy. 2022-04-28T19:32:33.288 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../041-python3-pastedeploy_1.5.2-4_all.deb ... 2022-04-28T19:32:33.325 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pastedeploy (1.5.2-4) ... 2022-04-28T19:32:33.507 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pastedeploy. 2022-04-28T19:32:33.520 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../041-python3-pastedeploy_1.5.2-4_all.deb ... 2022-04-28T19:32:33.557 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pastedeploy (1.5.2-4) ... 2022-04-28T19:32:33.653 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-webtest. 2022-04-28T19:32:33.665 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../042-python3-webtest_2.0.28-1ubuntu1_all.deb ... 2022-04-28T19:32:33.702 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-webtest (2.0.28-1ubuntu1) ... 2022-04-28T19:32:33.818 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-webtest. 2022-04-28T19:32:33.831 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../042-python3-webtest_2.0.28-1ubuntu1_all.deb ... 2022-04-28T19:32:33.859 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-webtest (2.0.28-1ubuntu1) ... 2022-04-28T19:32:34.064 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pecan. 2022-04-28T19:32:34.075 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../043-python3-pecan_1.2.1-2_all.deb ... 2022-04-28T19:32:34.114 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pecan (1.2.1-2) ... 2022-04-28T19:32:34.154 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pecan. 2022-04-28T19:32:34.167 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../043-python3-pecan_1.2.1-2_all.deb ... 2022-04-28T19:32:34.204 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pecan (1.2.1-2) ... 2022-04-28T19:32:34.468 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-werkzeug. 2022-04-28T19:32:34.482 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../044-python3-werkzeug_0.14.1+dfsg1-1ubuntu0.1_all.deb ... 2022-04-28T19:32:34.518 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... 2022-04-28T19:32:34.542 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-werkzeug. 2022-04-28T19:32:34.554 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../044-python3-werkzeug_0.14.1+dfsg1-1ubuntu0.1_all.deb ... 2022-04-28T19:32:34.591 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... 2022-04-28T19:32:34.887 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr. 2022-04-28T19:32:34.898 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../045-ceph-mgr_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:34.904 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr. 2022-04-28T19:32:34.917 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../045-ceph-mgr_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:34.936 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr (15.2.16-1bionic) ... 2022-04-28T19:32:34.954 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr (15.2.16-1bionic) ... 2022-04-28T19:32:35.366 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mon. 2022-04-28T19:32:35.376 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../046-ceph-mon_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:35.415 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mon (15.2.16-1bionic) ... 2022-04-28T19:32:35.419 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mon. 2022-04-28T19:32:35.431 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../046-ceph-mon_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:35.472 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mon (15.2.16-1bionic) ... 2022-04-28T19:32:36.326 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-osd. 2022-04-28T19:32:36.338 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../047-ceph-osd_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:36.352 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-osd. 2022-04-28T19:32:36.365 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../047-ceph-osd_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:36.376 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-osd (15.2.16-1bionic) ... 2022-04-28T19:32:36.403 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-osd (15.2.16-1bionic) ... 2022-04-28T19:32:38.256 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph. 2022-04-28T19:32:38.268 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../048-ceph_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:38.299 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph. 2022-04-28T19:32:38.312 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../048-ceph_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:38.331 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph (15.2.16-1bionic) ... 2022-04-28T19:32:38.349 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph (15.2.16-1bionic) ... 2022-04-28T19:32:38.635 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-fuse. 2022-04-28T19:32:38.647 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../049-ceph-fuse_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:38.650 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-fuse. 2022-04-28T19:32:38.662 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../049-ceph-fuse_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:38.685 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-fuse (15.2.16-1bionic) ... 2022-04-28T19:32:38.700 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-fuse (15.2.16-1bionic) ... 2022-04-28T19:32:39.045 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mds. 2022-04-28T19:32:39.056 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../050-ceph-mds_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:39.071 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mds. 2022-04-28T19:32:39.083 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../050-ceph-mds_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:39.095 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mds (15.2.16-1bionic) ... 2022-04-28T19:32:39.138 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mds (15.2.16-1bionic) ... 2022-04-28T19:32:39.556 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package cephadm. 2022-04-28T19:32:39.568 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../051-cephadm_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:39.606 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking cephadm (15.2.16-1bionic) ... 2022-04-28T19:32:39.732 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package cephadm. 2022-04-28T19:32:39.745 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../051-cephadm_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:39.782 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking cephadm (15.2.16-1bionic) ... 2022-04-28T19:32:39.951 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-jinja2. 2022-04-28T19:32:39.962 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../052-python3-jinja2_2.10-1ubuntu0.18.04.1_all.deb ... 2022-04-28T19:32:40.000 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-jinja2 (2.10-1ubuntu0.18.04.1) ... 2022-04-28T19:32:40.077 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-jinja2. 2022-04-28T19:32:40.090 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../052-python3-jinja2_2.10-1ubuntu0.18.04.1_all.deb ... 2022-04-28T19:32:40.127 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-jinja2 (2.10-1ubuntu0.18.04.1) ... 2022-04-28T19:32:40.320 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2022-04-28T19:32:40.331 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../053-ceph-mgr-cephadm_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:40.370 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr-cephadm (15.2.16-1bionic) ... 2022-04-28T19:32:40.422 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2022-04-28T19:32:40.434 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../053-ceph-mgr-cephadm_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:40.471 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr-cephadm (15.2.16-1bionic) ... 2022-04-28T19:32:40.656 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-repoze.lru. 2022-04-28T19:32:40.668 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../054-python3-repoze.lru_0.7-1_all.deb ... 2022-04-28T19:32:40.698 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-repoze.lru (0.7-1) ... 2022-04-28T19:32:40.833 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-repoze.lru. 2022-04-28T19:32:40.844 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../054-python3-repoze.lru_0.7-1_all.deb ... 2022-04-28T19:32:40.883 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-repoze.lru (0.7-1) ... 2022-04-28T19:32:40.976 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-routes. 2022-04-28T19:32:40.987 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../055-python3-routes_2.4.1-1_all.deb ... 2022-04-28T19:32:41.017 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-routes (2.4.1-1) ... 2022-04-28T19:32:41.194 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-routes. 2022-04-28T19:32:41.206 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../055-python3-routes_2.4.1-1_all.deb ... 2022-04-28T19:32:41.244 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-routes (2.4.1-1) ... 2022-04-28T19:32:41.287 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2022-04-28T19:32:41.299 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../056-ceph-mgr-dashboard_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:41.336 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr-dashboard (15.2.16-1bionic) ... 2022-04-28T19:32:41.530 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2022-04-28T19:32:41.541 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../056-ceph-mgr-dashboard_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:41.580 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr-dashboard (15.2.16-1bionic) ... 2022-04-28T19:32:42.185 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr-diskprediction-cloud. 2022-04-28T19:32:42.199 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../057-ceph-mgr-diskprediction-cloud_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:42.243 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr-diskprediction-cloud (15.2.16-1bionic) ... 2022-04-28T19:32:42.495 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr-diskprediction-cloud. 2022-04-28T19:32:42.508 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../057-ceph-mgr-diskprediction-cloud_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:42.521 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-numpy. 2022-04-28T19:32:42.532 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../058-python3-numpy_1%3a1.13.3-2ubuntu1_amd64.deb ... 2022-04-28T19:32:42.545 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr-diskprediction-cloud (15.2.16-1bionic) ... 2022-04-28T19:32:42.571 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-numpy (1:1.13.3-2ubuntu1) ... 2022-04-28T19:32:42.865 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-numpy. 2022-04-28T19:32:42.876 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../058-python3-numpy_1%3a1.13.3-2ubuntu1_amd64.deb ... 2022-04-28T19:32:42.915 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-numpy (1:1.13.3-2ubuntu1) ... 2022-04-28T19:32:43.220 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-decorator. 2022-04-28T19:32:43.233 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../059-python3-decorator_4.1.2-1_all.deb ... 2022-04-28T19:32:43.270 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-decorator (4.1.2-1) ... 2022-04-28T19:32:44.187 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-decorator. 2022-04-28T19:32:44.200 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../059-python3-decorator_4.1.2-1_all.deb ... 2022-04-28T19:32:44.229 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-decorator (4.1.2-1) ... 2022-04-28T19:32:44.398 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-scipy. 2022-04-28T19:32:44.411 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../060-python3-scipy_0.19.1-2ubuntu1_amd64.deb ... 2022-04-28T19:32:44.448 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-scipy (0.19.1-2ubuntu1) ... 2022-04-28T19:32:44.523 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-scipy. 2022-04-28T19:32:44.535 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../060-python3-scipy_0.19.1-2ubuntu1_amd64.deb ... 2022-04-28T19:32:44.573 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-scipy (0.19.1-2ubuntu1) ... 2022-04-28T19:32:45.597 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-sklearn-lib. 2022-04-28T19:32:45.608 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../061-python3-sklearn-lib_0.19.1-3_amd64.deb ... 2022-04-28T19:32:45.630 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-sklearn-lib. 2022-04-28T19:32:45.643 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../061-python3-sklearn-lib_0.19.1-3_amd64.deb ... 2022-04-28T19:32:45.647 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-sklearn-lib (0.19.1-3) ... 2022-04-28T19:32:45.672 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-sklearn-lib (0.19.1-3) ... 2022-04-28T19:32:46.084 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-joblib. 2022-04-28T19:32:46.097 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../062-python3-joblib_0.11-1_all.deb ... 2022-04-28T19:32:46.109 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-joblib. 2022-04-28T19:32:46.121 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../062-python3-joblib_0.11-1_all.deb ... 2022-04-28T19:32:46.133 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-joblib (0.11-1) ... 2022-04-28T19:32:46.159 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-joblib (0.11-1) ... 2022-04-28T19:32:46.504 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-sklearn. 2022-04-28T19:32:46.512 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-sklearn. 2022-04-28T19:32:46.516 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../063-python3-sklearn_0.19.1-3_all.deb ... 2022-04-28T19:32:46.525 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../063-python3-sklearn_0.19.1-3_all.deb ... 2022-04-28T19:32:46.553 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-sklearn (0.19.1-3) ... 2022-04-28T19:32:46.562 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-sklearn (0.19.1-3) ... 2022-04-28T19:32:47.068 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2022-04-28T19:32:47.082 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../064-ceph-mgr-diskprediction-local_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:47.126 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr-diskprediction-local (15.2.16-1bionic) ... 2022-04-28T19:32:47.135 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2022-04-28T19:32:47.147 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../064-ceph-mgr-diskprediction-local_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:47.185 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr-diskprediction-local (15.2.16-1bionic) ... 2022-04-28T19:32:48.188 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-monotonic. 2022-04-28T19:32:48.199 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../065-python3-monotonic_1.1-2_all.deb ... 2022-04-28T19:32:48.229 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-monotonic (1.1-2) ... 2022-04-28T19:32:48.355 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-monotonic. 2022-04-28T19:32:48.367 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../065-python3-monotonic_1.1-2_all.deb ... 2022-04-28T19:32:48.405 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-monotonic (1.1-2) ... 2022-04-28T19:32:48.540 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-fasteners. 2022-04-28T19:32:48.552 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../066-python3-fasteners_0.12.0-3_all.deb ... 2022-04-28T19:32:48.615 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-fasteners (0.12.0-3) ... 2022-04-28T19:32:48.783 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-fasteners. 2022-04-28T19:32:48.794 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../066-python3-fasteners_0.12.0-3_all.deb ... 2022-04-28T19:32:48.832 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-fasteners (0.12.0-3) ... 2022-04-28T19:32:48.927 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-httplib2. 2022-04-28T19:32:48.939 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../067-python3-httplib2_0.9.2+dfsg-1ubuntu0.3_all.deb ... 2022-04-28T19:32:48.977 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-httplib2 (0.9.2+dfsg-1ubuntu0.3) ... 2022-04-28T19:32:49.119 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-httplib2. 2022-04-28T19:32:49.131 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../067-python3-httplib2_0.9.2+dfsg-1ubuntu0.3_all.deb ... 2022-04-28T19:32:49.169 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-httplib2 (0.9.2+dfsg-1ubuntu0.3) ... 2022-04-28T19:32:49.296 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pyasn1. 2022-04-28T19:32:49.308 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../068-python3-pyasn1_0.4.2-3_all.deb ... 2022-04-28T19:32:49.346 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pyasn1 (0.4.2-3) ... 2022-04-28T19:32:49.480 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pyasn1. 2022-04-28T19:32:49.493 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../068-python3-pyasn1_0.4.2-3_all.deb ... 2022-04-28T19:32:49.530 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pyasn1 (0.4.2-3) ... 2022-04-28T19:32:49.640 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pyasn1-modules. 2022-04-28T19:32:49.654 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../069-python3-pyasn1-modules_0.2.1-0.2_all.deb ... 2022-04-28T19:32:49.691 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2) ... 2022-04-28T19:32:49.850 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pyasn1-modules. 2022-04-28T19:32:49.862 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../069-python3-pyasn1-modules_0.2.1-0.2_all.deb ... 2022-04-28T19:32:49.899 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2) ... 2022-04-28T19:32:50.010 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-rsa. 2022-04-28T19:32:50.023 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../070-python3-rsa_3.4.2-1_all.deb ... 2022-04-28T19:32:50.060 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-rsa (3.4.2-1) ... 2022-04-28T19:32:50.236 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-rsa. 2022-04-28T19:32:50.249 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../070-python3-rsa_3.4.2-1_all.deb ... 2022-04-28T19:32:50.285 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-rsa (3.4.2-1) ... 2022-04-28T19:32:50.355 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-sqlalchemy. 2022-04-28T19:32:50.368 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../071-python3-sqlalchemy_1.1.11+ds1-1ubuntu1_all.deb ... 2022-04-28T19:32:50.413 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-sqlalchemy (1.1.11+ds1-1ubuntu1) ... 2022-04-28T19:32:50.597 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-sqlalchemy. 2022-04-28T19:32:50.610 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../071-python3-sqlalchemy_1.1.11+ds1-1ubuntu1_all.deb ... 2022-04-28T19:32:50.647 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-sqlalchemy (1.1.11+ds1-1ubuntu1) ... 2022-04-28T19:32:50.892 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-oauth2client. 2022-04-28T19:32:50.904 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../072-python3-oauth2client_4.1.2-3_all.deb ... 2022-04-28T19:32:50.942 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-oauth2client (4.1.2-3) ... 2022-04-28T19:32:51.076 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-oauth2client. 2022-04-28T19:32:51.089 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../072-python3-oauth2client_4.1.2-3_all.deb ... 2022-04-28T19:32:51.118 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-oauth2client (4.1.2-3) ... 2022-04-28T19:32:51.270 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-setuptools. 2022-04-28T19:32:51.283 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../073-python3-setuptools_39.0.1-2_all.deb ... 2022-04-28T19:32:51.320 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-setuptools (39.0.1-2) ... 2022-04-28T19:32:51.437 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-setuptools. 2022-04-28T19:32:51.451 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../073-python3-setuptools_39.0.1-2_all.deb ... 2022-04-28T19:32:51.498 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-setuptools (39.0.1-2) ... 2022-04-28T19:32:51.698 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-websocket. 2022-04-28T19:32:51.711 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../074-python3-websocket_0.44.0-0ubuntu2_all.deb ... 2022-04-28T19:32:51.748 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-websocket (0.44.0-0ubuntu2) ... 2022-04-28T19:32:51.849 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-websocket. 2022-04-28T19:32:51.863 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../074-python3-websocket_0.44.0-0ubuntu2_all.deb ... 2022-04-28T19:32:51.907 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-websocket (0.44.0-0ubuntu2) ... 2022-04-28T19:32:52.092 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-kubernetes. 2022-04-28T19:32:52.106 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../075-python3-kubernetes_2.0.0-2ubuntu1_all.deb ... 2022-04-28T19:32:52.142 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-kubernetes (2.0.0-2ubuntu1) ... 2022-04-28T19:32:52.193 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-kubernetes. 2022-04-28T19:32:52.206 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../075-python3-kubernetes_2.0.0-2ubuntu1_all.deb ... 2022-04-28T19:32:52.244 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-kubernetes (2.0.0-2ubuntu1) ... 2022-04-28T19:32:52.538 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2022-04-28T19:32:52.552 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../076-ceph-mgr-k8sevents_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:52.596 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-mgr-k8sevents (15.2.16-1bionic) ... 2022-04-28T19:32:52.631 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2022-04-28T19:32:52.645 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../076-ceph-mgr-k8sevents_15.2.16-1bionic_all.deb ... 2022-04-28T19:32:52.689 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-mgr-k8sevents (15.2.16-1bionic) ... 2022-04-28T19:32:52.891 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libonig4:amd64. 2022-04-28T19:32:52.905 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../077-libonig4_6.7.0-1_amd64.deb ... 2022-04-28T19:32:52.940 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libonig4:amd64 (6.7.0-1) ... 2022-04-28T19:32:53.033 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libonig4:amd64. 2022-04-28T19:32:53.046 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../077-libonig4_6.7.0-1_amd64.deb ... 2022-04-28T19:32:53.083 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libonig4:amd64 (6.7.0-1) ... 2022-04-28T19:32:53.302 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libjq1:amd64. 2022-04-28T19:32:53.315 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../078-libjq1_1.5+dfsg-2_amd64.deb ... 2022-04-28T19:32:53.352 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libjq1:amd64 (1.5+dfsg-2) ... 2022-04-28T19:32:53.394 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libjq1:amd64. 2022-04-28T19:32:53.408 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../078-libjq1_1.5+dfsg-2_amd64.deb ... 2022-04-28T19:32:53.461 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libjq1:amd64 (1.5+dfsg-2) ... 2022-04-28T19:32:53.654 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package jq. 2022-04-28T19:32:53.667 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../079-jq_1.5+dfsg-2_amd64.deb ... 2022-04-28T19:32:53.704 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking jq (1.5+dfsg-2) ... 2022-04-28T19:32:53.731 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package jq. 2022-04-28T19:32:53.742 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../079-jq_1.5+dfsg-2_amd64.deb ... 2022-04-28T19:32:53.780 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking jq (1.5+dfsg-2) ... 2022-04-28T19:32:53.965 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package socat. 2022-04-28T19:32:53.978 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../080-socat_1.7.3.2-2ubuntu2_amd64.deb ... 2022-04-28T19:32:54.041 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package socat. 2022-04-28T19:32:54.052 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../080-socat_1.7.3.2-2ubuntu2_amd64.deb ... 2022-04-28T19:32:54.100 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking socat (1.7.3.2-2ubuntu2) ... 2022-04-28T19:32:54.176 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking socat (1.7.3.2-2ubuntu2) ... 2022-04-28T19:32:54.454 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libxslt1.1:amd64. 2022-04-28T19:32:54.467 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../081-libxslt1.1_1.1.29-5ubuntu0.2_amd64.deb ... 2022-04-28T19:32:54.504 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libxslt1.1:amd64 (1.1.29-5ubuntu0.2) ... 2022-04-28T19:32:54.555 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libxslt1.1:amd64. 2022-04-28T19:32:54.567 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../081-libxslt1.1_1.1.29-5ubuntu0.2_amd64.deb ... 2022-04-28T19:32:54.605 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libxslt1.1:amd64 (1.1.29-5ubuntu0.2) ... 2022-04-28T19:32:54.781 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package xmlstarlet. 2022-04-28T19:32:54.793 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../082-xmlstarlet_1.6.1-2_amd64.deb ... 2022-04-28T19:32:54.832 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking xmlstarlet (1.6.1-2) ... 2022-04-28T19:32:54.891 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package xmlstarlet. 2022-04-28T19:32:54.903 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../082-xmlstarlet_1.6.1-2_amd64.deb ... 2022-04-28T19:32:54.941 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking xmlstarlet (1.6.1-2) ... 2022-04-28T19:32:55.185 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package ceph-test. 2022-04-28T19:32:55.198 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../083-ceph-test_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:55.235 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking ceph-test (15.2.16-1bionic) ... 2022-04-28T19:32:55.278 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package ceph-test. 2022-04-28T19:32:55.292 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../083-ceph-test_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:55.345 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking ceph-test (15.2.16-1bionic) ... 2022-04-28T19:32:58.540 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package libcephfs-dev. 2022-04-28T19:32:58.551 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../084-libcephfs-dev_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:58.589 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package libcephfs-dev. 2022-04-28T19:32:58.602 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../084-libcephfs-dev_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:32:58.606 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking libcephfs-dev (15.2.16-1bionic) ... 2022-04-28T19:32:58.647 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking libcephfs-dev (15.2.16-1bionic) ... 2022-04-28T19:32:59.159 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package nvme-cli. 2022-04-28T19:32:59.172 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../085-nvme-cli_1.5-1ubuntu1.2_amd64.deb ... 2022-04-28T19:32:59.209 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking nvme-cli (1.5-1ubuntu1.2) ... 2022-04-28T19:32:59.216 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package nvme-cli. 2022-04-28T19:32:59.227 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../085-nvme-cli_1.5-1ubuntu1.2_amd64.deb ... 2022-04-28T19:32:59.266 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking nvme-cli (1.5-1ubuntu1.2) ... 2022-04-28T19:32:59.563 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python-attr. 2022-04-28T19:32:59.576 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../086-python-attr_17.4.0-2_all.deb ... 2022-04-28T19:32:59.612 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python-attr. 2022-04-28T19:32:59.613 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python-attr (17.4.0-2) ... 2022-04-28T19:32:59.623 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../086-python-attr_17.4.0-2_all.deb ... 2022-04-28T19:32:59.661 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python-attr (17.4.0-2) ... 2022-04-28T19:32:59.892 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python-funcsigs. 2022-04-28T19:32:59.905 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../087-python-funcsigs_1.0.2-4_all.deb ... 2022-04-28T19:32:59.941 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python-funcsigs (1.0.2-4) ... 2022-04-28T19:32:59.964 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python-funcsigs. 2022-04-28T19:32:59.978 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../087-python-funcsigs_1.0.2-4_all.deb ... 2022-04-28T19:33:00.014 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python-funcsigs (1.0.2-4) ... 2022-04-28T19:33:00.269 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python-pluggy. 2022-04-28T19:33:00.283 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../088-python-pluggy_0.6.0-1_all.deb ... 2022-04-28T19:33:00.317 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python-pluggy. 2022-04-28T19:33:00.327 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python-pluggy (0.6.0-1) ... 2022-04-28T19:33:00.329 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../088-python-pluggy_0.6.0-1_all.deb ... 2022-04-28T19:33:00.367 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python-pluggy (0.6.0-1) ... 2022-04-28T19:33:00.614 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python-py. 2022-04-28T19:33:00.627 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../089-python-py_1.5.2-1ubuntu0.1_all.deb ... 2022-04-28T19:33:00.670 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python-py. 2022-04-28T19:33:00.672 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:00.683 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../089-python-py_1.5.2-1ubuntu0.1_all.deb ... 2022-04-28T19:33:00.720 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:00.991 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python-pytest. 2022-04-28T19:33:01.005 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../090-python-pytest_3.3.2-2_all.deb ... 2022-04-28T19:33:01.023 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python-pytest. 2022-04-28T19:33:01.037 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../090-python-pytest_3.3.2-2_all.deb ... 2022-04-28T19:33:01.050 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python-pytest (3.3.2-2) ... 2022-04-28T19:33:01.081 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python-pytest (3.3.2-2) ... 2022-04-28T19:33:01.376 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-attr. 2022-04-28T19:33:01.388 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../091-python3-attr_17.4.0-2_all.deb ... 2022-04-28T19:33:01.403 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-attr. 2022-04-28T19:33:01.416 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../091-python3-attr_17.4.0-2_all.deb ... 2022-04-28T19:33:01.417 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-attr (17.4.0-2) ... 2022-04-28T19:33:01.470 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-attr (17.4.0-2) ... 2022-04-28T19:33:01.720 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-cycler. 2022-04-28T19:33:01.734 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../092-python3-cycler_0.10.0-1_all.deb ... 2022-04-28T19:33:01.770 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-cycler (0.10.0-1) ... 2022-04-28T19:33:01.772 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-cycler. 2022-04-28T19:33:01.784 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../092-python3-cycler_0.10.0-1_all.deb ... 2022-04-28T19:33:01.822 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-cycler (0.10.0-1) ... 2022-04-28T19:33:02.083 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-webencodings. 2022-04-28T19:33:02.097 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../093-python3-webencodings_0.5-2_all.deb ... 2022-04-28T19:33:02.098 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-webencodings. 2022-04-28T19:33:02.111 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../093-python3-webencodings_0.5-2_all.deb ... 2022-04-28T19:33:02.133 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-webencodings (0.5-2) ... 2022-04-28T19:33:02.148 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-webencodings (0.5-2) ... 2022-04-28T19:33:02.426 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-html5lib. 2022-04-28T19:33:02.436 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-html5lib. 2022-04-28T19:33:02.437 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../094-python3-html5lib_0.999999999-1_all.deb ... 2022-04-28T19:33:02.449 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../094-python3-html5lib_0.999999999-1_all.deb ... 2022-04-28T19:33:02.475 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-html5lib (0.999999999-1) ... 2022-04-28T19:33:02.486 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-html5lib (0.999999999-1) ... 2022-04-28T19:33:02.812 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-lxml:amd64. 2022-04-28T19:33:02.814 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-lxml:amd64. 2022-04-28T19:33:02.826 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../095-python3-lxml_4.2.1-1ubuntu0.6_amd64.deb ... 2022-04-28T19:33:02.828 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../095-python3-lxml_4.2.1-1ubuntu0.6_amd64.deb ... 2022-04-28T19:33:02.862 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-lxml:amd64 (4.2.1-1ubuntu0.6) ... 2022-04-28T19:33:02.872 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-lxml:amd64 (4.2.1-1ubuntu0.6) ... 2022-04-28T19:33:03.276 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pyparsing. 2022-04-28T19:33:03.289 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../096-python3-pyparsing_2.2.0+dfsg1-2_all.deb ... 2022-04-28T19:33:03.290 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pyparsing. 2022-04-28T19:33:03.304 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../096-python3-pyparsing_2.2.0+dfsg1-2_all.deb ... 2022-04-28T19:33:03.326 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pyparsing (2.2.0+dfsg1-2) ... 2022-04-28T19:33:03.348 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pyparsing (2.2.0+dfsg1-2) ... 2022-04-28T19:33:03.629 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-tz. 2022-04-28T19:33:03.641 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../097-python3-tz_2018.3-2_all.deb ... 2022-04-28T19:33:03.668 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-tz. 2022-04-28T19:33:03.678 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-tz (2018.3-2) ... 2022-04-28T19:33:03.681 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../097-python3-tz_2018.3-2_all.deb ... 2022-04-28T19:33:03.726 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-tz (2018.3-2) ... 2022-04-28T19:33:03.990 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-matplotlib. 2022-04-28T19:33:04.002 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../098-python3-matplotlib_2.1.1-2ubuntu3_amd64.deb ... 2022-04-28T19:33:04.029 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-matplotlib. 2022-04-28T19:33:04.039 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-matplotlib (2.1.1-2ubuntu3) ... 2022-04-28T19:33:04.042 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../098-python3-matplotlib_2.1.1-2ubuntu3_amd64.deb ... 2022-04-28T19:33:04.079 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-matplotlib (2.1.1-2ubuntu3) ... 2022-04-28T19:33:04.726 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-olefile. 2022-04-28T19:33:04.739 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../099-python3-olefile_0.45.1-1_all.deb ... 2022-04-28T19:33:04.745 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-olefile. 2022-04-28T19:33:04.757 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../099-python3-olefile_0.45.1-1_all.deb ... 2022-04-28T19:33:04.787 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-olefile (0.45.1-1) ... 2022-04-28T19:33:04.792 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-olefile (0.45.1-1) ... 2022-04-28T19:33:05.090 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pastescript. 2022-04-28T19:33:05.102 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../100-python3-pastescript_2.0.2-2_all.deb ... 2022-04-28T19:33:05.112 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pastescript. 2022-04-28T19:33:05.124 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../100-python3-pastescript_2.0.2-2_all.deb ... 2022-04-28T19:33:05.139 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pastescript (2.0.2-2) ... 2022-04-28T19:33:05.162 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pastescript (2.0.2-2) ... 2022-04-28T19:33:05.449 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pil:amd64. 2022-04-28T19:33:05.460 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pil:amd64. 2022-04-28T19:33:05.463 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../101-python3-pil_5.1.0-1ubuntu0.7_amd64.deb ... 2022-04-28T19:33:05.473 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../101-python3-pil_5.1.0-1ubuntu0.7_amd64.deb ... 2022-04-28T19:33:05.507 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pil:amd64 (5.1.0-1ubuntu0.7) ... 2022-04-28T19:33:05.518 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pil:amd64 (5.1.0-1ubuntu0.7) ... 2022-04-28T19:33:05.893 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pluggy. 2022-04-28T19:33:05.906 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../102-python3-pluggy_0.6.0-1_all.deb ... 2022-04-28T19:33:05.938 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pluggy. 2022-04-28T19:33:05.943 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pluggy (0.6.0-1) ... 2022-04-28T19:33:05.950 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../102-python3-pluggy_0.6.0-1_all.deb ... 2022-04-28T19:33:05.988 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pluggy (0.6.0-1) ... 2022-04-28T19:33:06.255 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-py. 2022-04-28T19:33:06.268 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../103-python3-py_1.5.2-1ubuntu0.1_all.deb ... 2022-04-28T19:33:06.274 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-py. 2022-04-28T19:33:06.288 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../103-python3-py_1.5.2-1ubuntu0.1_all.deb ... 2022-04-28T19:33:06.313 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:06.332 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:06.633 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pyinotify. 2022-04-28T19:33:06.646 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1_all.deb ... 2022-04-28T19:33:06.660 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pyinotify. 2022-04-28T19:33:06.673 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1_all.deb ... 2022-04-28T19:33:06.682 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pyinotify (0.9.6-1) ... 2022-04-28T19:33:06.710 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pyinotify (0.9.6-1) ... 2022-04-28T19:33:06.943 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-pytest. 2022-04-28T19:33:06.957 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../105-python3-pytest_3.3.2-2_all.deb ... 2022-04-28T19:33:06.988 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-pytest. 2022-04-28T19:33:07.000 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../105-python3-pytest_3.3.2-2_all.deb ... 2022-04-28T19:33:07.001 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-pytest (3.3.2-2) ... 2022-04-28T19:33:07.029 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-pytest (3.3.2-2) ... 2022-04-28T19:33:07.280 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-simplejson. 2022-04-28T19:33:07.294 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../106-python3-simplejson_3.13.2-1_amd64.deb ... 2022-04-28T19:33:07.338 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-simplejson (3.13.2-1) ... 2022-04-28T19:33:07.375 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-simplejson. 2022-04-28T19:33:07.387 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../106-python3-simplejson_3.13.2-1_amd64.deb ... 2022-04-28T19:33:07.424 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-simplejson (3.13.2-1) ... 2022-04-28T19:33:07.666 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package python3-tk:amd64. 2022-04-28T19:33:07.680 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../107-python3-tk_3.6.9-1~18.04_amd64.deb ... 2022-04-28T19:33:07.724 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking python3-tk:amd64 (3.6.9-1~18.04) ... 2022-04-28T19:33:07.736 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package python3-tk:amd64. 2022-04-28T19:33:07.749 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../107-python3-tk_3.6.9-1~18.04_amd64.deb ... 2022-04-28T19:33:07.777 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking python3-tk:amd64 (3.6.9-1~18.04) ... 2022-04-28T19:33:08.195 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package radosgw. 2022-04-28T19:33:08.207 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../108-radosgw_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:33:08.223 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package radosgw. 2022-04-28T19:33:08.236 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../108-radosgw_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:33:08.245 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking radosgw (15.2.16-1bionic) ... 2022-04-28T19:33:08.273 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking radosgw (15.2.16-1bionic) ... 2022-04-28T19:33:09.139 INFO:teuthology.orchestra.run.smithi059.stdout:Selecting previously unselected package rbd-fuse. 2022-04-28T19:33:09.151 INFO:teuthology.orchestra.run.smithi059.stdout:Preparing to unpack .../109-rbd-fuse_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:33:09.181 INFO:teuthology.orchestra.run.smithi059.stdout:Unpacking rbd-fuse (15.2.16-1bionic) ... 2022-04-28T19:33:09.226 INFO:teuthology.orchestra.run.smithi139.stdout:Selecting previously unselected package rbd-fuse. 2022-04-28T19:33:09.238 INFO:teuthology.orchestra.run.smithi139.stdout:Preparing to unpack .../109-rbd-fuse_15.2.16-1bionic_amd64.deb ... 2022-04-28T19:33:09.276 INFO:teuthology.orchestra.run.smithi139.stdout:Unpacking rbd-fuse (15.2.16-1bionic) ... 2022-04-28T19:33:09.468 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-logutils (0.3.3-5) ... 2022-04-28T19:33:09.562 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-logutils (0.3.3-5) ... 2022-04-28T19:33:09.805 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... 2022-04-28T19:33:09.906 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-werkzeug (0.14.1+dfsg1-1ubuntu0.1) ... 2022-04-28T19:33:10.156 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up librdkafka1:amd64 (0.11.3-1build1) ... 2022-04-28T19:33:10.266 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up librdkafka1:amd64 (0.11.3-1build1) ... 2022-04-28T19:33:10.273 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-sqlalchemy (1.1.11+ds1-1ubuntu1) ... 2022-04-28T19:33:10.392 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-sqlalchemy (1.1.11+ds1-1ubuntu1) ... 2022-04-28T19:33:10.876 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pil:amd64 (5.1.0-1ubuntu0.7) ... 2022-04-28T19:33:10.986 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pil:amd64 (5.1.0-1ubuntu0.7) ... 2022-04-28T19:33:11.277 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-simplegeneric (0.8.1-1) ... 2022-04-28T19:33:11.346 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-simplegeneric (0.8.1-1) ... 2022-04-28T19:33:11.622 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-webencodings (0.5-2) ... 2022-04-28T19:33:11.628 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-webencodings (0.5-2) ... 2022-04-28T19:33:11.906 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python-attr (17.4.0-2) ... 2022-04-28T19:33:11.929 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python-attr (17.4.0-2) ... 2022-04-28T19:33:12.100 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-cffi-backend (1.11.5-1) ... 2022-04-28T19:33:12.114 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-cffi-backend (1.11.5-1) ... 2022-04-28T19:33:12.225 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:12.239 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:12.552 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-waitress (1.0.1-1) ... 2022-04-28T19:33:12.631 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-waitress (1.0.1-1) ... 2022-04-28T19:33:12.905 INFO:teuthology.orchestra.run.smithi139.stdout:update-alternatives: using /usr/bin/waitress-serve-python3 to provide /usr/bin/waitress-serve (waitress-serve) in auto mode 2022-04-28T19:33:12.936 INFO:teuthology.orchestra.run.smithi059.stdout:update-alternatives: using /usr/bin/waitress-serve-python3 to provide /usr/bin/waitress-serve (waitress-serve) in auto mode 2022-04-28T19:33:13.196 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-olefile (0.45.1-1) ... 2022-04-28T19:33:13.244 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-olefile (0.45.1-1) ... 2022-04-28T19:33:13.497 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-ceph-argparse (15.2.16-1bionic) ... 2022-04-28T19:33:13.553 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-ceph-argparse (15.2.16-1bionic) ... 2022-04-28T19:33:13.798 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-tempita (0.5.2-2) ... 2022-04-28T19:33:13.837 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-tempita (0.5.2-2) ... 2022-04-28T19:33:14.099 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up socat (1.7.3.2-2ubuntu2) ... 2022-04-28T19:33:14.130 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up socat (1.7.3.2-2ubuntu2) ... 2022-04-28T19:33:14.225 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-numpy (1:1.13.3-2ubuntu1) ... 2022-04-28T19:33:14.231 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-numpy (1:1.13.3-2ubuntu1) ... 2022-04-28T19:33:15.153 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-webob (1:1.7.3-2fakesync1) ... 2022-04-28T19:33:15.159 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-webob (1:1.7.3-2fakesync1) ... 2022-04-28T19:33:15.471 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-bcrypt (3.1.4-2) ... 2022-04-28T19:33:15.476 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-bcrypt (3.1.4-2) ... 2022-04-28T19:33:15.756 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:15.761 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python-py (1.5.2-1ubuntu0.1) ... 2022-04-28T19:33:15.987 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-simplejson (3.13.2-1) ... 2022-04-28T19:33:15.990 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-simplejson (3.13.2-1) ... 2022-04-28T19:33:16.314 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libonig4:amd64 (6.7.0-1) ... 2022-04-28T19:33:16.325 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libonig4:amd64 (6.7.0-1) ... 2022-04-28T19:33:16.423 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python-pluggy (0.6.0-1) ... 2022-04-28T19:33:16.451 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python-pluggy (0.6.0-1) ... 2022-04-28T19:33:16.607 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-singledispatch (3.4.0.3-2) ... 2022-04-28T19:33:16.644 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-singledispatch (3.4.0.3-2) ... 2022-04-28T19:33:16.908 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pyparsing (2.2.0+dfsg1-2) ... 2022-04-28T19:33:16.953 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pyparsing (2.2.0+dfsg1-2) ... 2022-04-28T19:33:17.226 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-cycler (0.10.0-1) ... 2022-04-28T19:33:17.271 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-cycler (0.10.0-1) ... 2022-04-28T19:33:17.543 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-cherrypy3 (8.9.1-2) ... 2022-04-28T19:33:17.580 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-cherrypy3 (8.9.1-2) ... 2022-04-28T19:33:17.920 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-bs4 (4.6.0-1) ... 2022-04-28T19:33:17.940 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-bs4 (4.6.0-1) ... 2022-04-28T19:33:18.241 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pluggy (0.6.0-1) ... 2022-04-28T19:33:18.254 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pluggy (0.6.0-1) ... 2022-04-28T19:33:18.522 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up cephadm (15.2.16-1bionic) ... 2022-04-28T19:33:18.543 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up cephadm (15.2.16-1bionic) ... 2022-04-28T19:33:19.123 INFO:teuthology.orchestra.run.smithi059.stdout:Adding system user cephadm....done 2022-04-28T19:33:19.234 INFO:teuthology.orchestra.run.smithi139.stdout:Adding system user cephadm....done 2022-04-28T19:33:19.295 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-markupsafe (1.0-1build1) ... 2022-04-28T19:33:19.365 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-markupsafe (1.0-1build1) ... 2022-04-28T19:33:19.605 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libxslt1.1:amd64 (1.1.29-5ubuntu0.2) ... 2022-04-28T19:33:19.650 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libxslt1.1:amd64 (1.1.29-5ubuntu0.2) ... 2022-04-28T19:33:19.722 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up nvme-cli (1.5-1ubuntu1.2) ... 2022-04-28T19:33:19.767 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up nvme-cli (1.5-1ubuntu1.2) ... 2022-04-28T19:33:19.831 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-asn1crypto (0.24.0-1) ... 2022-04-28T19:33:19.902 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-asn1crypto (0.24.0-1) ... 2022-04-28T19:33:20.166 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-prettytable (0.7.2-3) ... 2022-04-28T19:33:20.245 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-prettytable (0.7.2-3) ... 2022-04-28T19:33:20.475 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libjq1:amd64 (1.5+dfsg-2) ... 2022-04-28T19:33:20.546 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libjq1:amd64 (1.5+dfsg-2) ... 2022-04-28T19:33:20.601 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pyasn1 (0.4.2-3) ... 2022-04-28T19:33:20.672 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pyasn1 (0.4.2-3) ... 2022-04-28T19:33:20.944 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-ceph-common (15.2.16-1bionic) ... 2022-04-28T19:33:21.015 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-ceph-common (15.2.16-1bionic) ... 2022-04-28T19:33:21.270 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-monotonic (1.1-2) ... 2022-04-28T19:33:21.325 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-monotonic (1.1-2) ... 2022-04-28T19:33:21.580 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libsnappy1v5:amd64 (1.1.7-1) ... 2022-04-28T19:33:21.601 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libsnappy1v5:amd64 (1.1.7-1) ... 2022-04-28T19:33:21.689 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-jinja2 (2.10-1ubuntu0.18.04.1) ... 2022-04-28T19:33:21.743 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-jinja2 (2.10-1ubuntu0.18.04.1) ... 2022-04-28T19:33:22.040 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-html5lib (0.999999999-1) ... 2022-04-28T19:33:22.095 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-html5lib (0.999999999-1) ... 2022-04-28T19:33:22.367 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-repoze.lru (0.7-1) ... 2022-04-28T19:33:22.448 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-repoze.lru (0.7-1) ... 2022-04-28T19:33:22.668 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-httplib2 (0.9.2+dfsg-1ubuntu0.3) ... 2022-04-28T19:33:22.747 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-httplib2 (0.9.2+dfsg-1ubuntu0.3) ... 2022-04-28T19:33:22.977 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2) ... 2022-04-28T19:33:23.082 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2) ... 2022-04-28T19:33:23.320 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-attr (17.4.0-2) ... 2022-04-28T19:33:23.408 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-attr (17.4.0-2) ... 2022-04-28T19:33:23.646 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python-funcsigs (1.0.2-4) ... 2022-04-28T19:33:23.692 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python-funcsigs (1.0.2-4) ... 2022-04-28T19:33:23.847 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-paste (2.0.3+dfsg-4ubuntu1) ... 2022-04-28T19:33:23.885 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-paste (2.0.3+dfsg-4ubuntu1) ... 2022-04-28T19:33:24.232 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-websocket (0.44.0-0ubuntu2) ... 2022-04-28T19:33:24.304 INFO:teuthology.orchestra.run.smithi059.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2022-04-28T19:33:24.312 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-websocket (0.44.0-0ubuntu2) ... 2022-04-28T19:33:24.401 INFO:teuthology.orchestra.run.smithi139.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2022-04-28T19:33:24.534 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up liboath0 (2.6.1-1) ... 2022-04-28T19:33:24.638 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up liboath0 (2.6.1-1) ... 2022-04-28T19:33:24.735 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... 2022-04-28T19:33:24.806 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up librabbitmq4:amd64 (0.8.0-1ubuntu0.18.04.2) ... 2022-04-28T19:33:24.861 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-tk:amd64 (3.6.9-1~18.04) ... 2022-04-28T19:33:24.907 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-tk:amd64 (3.6.9-1~18.04) ... 2022-04-28T19:33:25.120 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python-pastedeploy-tpl (1.5.2-4) ... 2022-04-28T19:33:25.133 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python-pastedeploy-tpl (1.5.2-4) ... 2022-04-28T19:33:25.251 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-setuptools (39.0.1-2) ... 2022-04-28T19:33:25.271 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-setuptools (39.0.1-2) ... 2022-04-28T19:33:25.652 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up liblttng-ust0:amd64 (2.10.1-1) ... 2022-04-28T19:33:25.673 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up liblttng-ust0:amd64 (2.10.1-1) ... 2022-04-28T19:33:25.774 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-decorator (4.1.2-1) ... 2022-04-28T19:33:25.787 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-decorator (4.1.2-1) ... 2022-04-28T19:33:26.041 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-tz (2018.3-2) ... 2022-04-28T19:33:26.080 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-tz (2018.3-2) ... 2022-04-28T19:33:26.334 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-uritemplate (0.6-1ubuntu1) ... 2022-04-28T19:33:26.389 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-uritemplate (0.6-1ubuntu1) ... 2022-04-28T19:33:26.635 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-jwt (1.5.3+ds1-1) ... 2022-04-28T19:33:26.682 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-jwt (1.5.3+ds1-1) ... 2022-04-28T19:33:26.945 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-joblib (0.11-1) ... 2022-04-28T19:33:26.975 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-joblib (0.11-1) ... 2022-04-28T19:33:27.309 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-routes (2.4.1-1) ... 2022-04-28T19:33:27.329 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-routes (2.4.1-1) ... 2022-04-28T19:33:27.652 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pyinotify (0.9.6-1) ... 2022-04-28T19:33:27.672 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pyinotify (0.9.6-1) ... 2022-04-28T19:33:27.970 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-dateutil (2.6.1-1) ... 2022-04-28T19:33:27.973 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-dateutil (2.6.1-1) ... 2022-04-28T19:33:28.274 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-cryptography (2.1.4-1ubuntu1.4) ... 2022-04-28T19:33:28.288 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-cryptography (2.1.4-1ubuntu1.4) ... 2022-04-28T19:33:28.617 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-fasteners (0.12.0-3) ... 2022-04-28T19:33:28.656 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-fasteners (0.12.0-3) ... 2022-04-28T19:33:28.927 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up jq (1.5+dfsg-2) ... 2022-04-28T19:33:28.940 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up jq (1.5+dfsg-2) ... 2022-04-28T19:33:29.041 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-scipy (0.19.1-2ubuntu1) ... 2022-04-28T19:33:29.053 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-scipy (0.19.1-2ubuntu1) ... 2022-04-28T19:33:30.188 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-lxml:amd64 (4.2.1-1ubuntu0.6) ... 2022-04-28T19:33:30.283 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-lxml:amd64 (4.2.1-1ubuntu0.6) ... 2022-04-28T19:33:30.523 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-mako (1.0.7+ds1-1) ... 2022-04-28T19:33:30.592 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-mako (1.0.7+ds1-1) ... 2022-04-28T19:33:30.832 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-sklearn-lib (0.19.1-3) ... 2022-04-28T19:33:30.919 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-sklearn-lib (0.19.1-3) ... 2022-04-28T19:33:30.933 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pytest (3.3.2-2) ... 2022-04-28T19:33:31.044 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pytest (3.3.2-2) ... 2022-04-28T19:33:31.293 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python-pytest (3.3.2-2) ... 2022-04-28T19:33:31.404 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python-pytest (3.3.2-2) ... 2022-04-28T19:33:31.546 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libleveldb1v5:amd64 (1.20-2) ... 2022-04-28T19:33:31.672 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up xmlstarlet (1.6.1-2) ... 2022-04-28T19:33:31.690 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libleveldb1v5:amd64 (1.20-2) ... 2022-04-28T19:33:31.814 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up xmlstarlet (1.6.1-2) ... 2022-04-28T19:33:31.823 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up librados2 (15.2.16-1bionic) ... 2022-04-28T19:33:31.915 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up librados2 (15.2.16-1bionic) ... 2022-04-28T19:33:31.924 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-rsa (3.4.2-1) ... 2022-04-28T19:33:32.016 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-rsa (3.4.2-1) ... 2022-04-28T19:33:32.284 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libcephfs2 (15.2.16-1bionic) ... 2022-04-28T19:33:32.317 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libcephfs2 (15.2.16-1bionic) ... 2022-04-28T19:33:32.401 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libcephfs-dev (15.2.16-1bionic) ... 2022-04-28T19:33:32.443 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libcephfs-dev (15.2.16-1bionic) ... 2022-04-28T19:33:32.530 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-rados (15.2.16-1bionic) ... 2022-04-28T19:33:32.570 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-rados (15.2.16-1bionic) ... 2022-04-28T19:33:32.653 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-openssl (17.5.0-1ubuntu1) ... 2022-04-28T19:33:32.695 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-openssl (17.5.0-1ubuntu1) ... 2022-04-28T19:33:32.971 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pastedeploy (1.5.2-4) ... 2022-04-28T19:33:33.013 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pastedeploy (1.5.2-4) ... 2022-04-28T19:33:33.289 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-oauth2client (4.1.2-3) ... 2022-04-28T19:33:33.339 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-oauth2client (4.1.2-3) ... 2022-04-28T19:33:33.615 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-matplotlib (2.1.1-2ubuntu3) ... 2022-04-28T19:33:33.682 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-matplotlib (2.1.1-2ubuntu3) ... 2022-04-28T19:33:34.459 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-sklearn (0.19.1-3) ... 2022-04-28T19:33:34.526 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-sklearn (0.19.1-3) ... 2022-04-28T19:33:35.372 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-webtest (2.0.28-1ubuntu1) ... 2022-04-28T19:33:35.414 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-webtest (2.0.28-1ubuntu1) ... 2022-04-28T19:33:35.750 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up libradosstriper1 (15.2.16-1bionic) ... 2022-04-28T19:33:35.774 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up libradosstriper1 (15.2.16-1bionic) ... 2022-04-28T19:33:35.859 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-cephfs (15.2.16-1bionic) ... 2022-04-28T19:33:35.875 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-cephfs (15.2.16-1bionic) ... 2022-04-28T19:33:36.168 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up librgw2 (15.2.16-1bionic) ... 2022-04-28T19:33:36.194 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up librgw2 (15.2.16-1bionic) ... 2022-04-28T19:33:36.294 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-fuse (15.2.16-1bionic) ... 2022-04-28T19:33:36.319 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-fuse (15.2.16-1bionic) ... 2022-04-28T19:33:36.510 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-04-28T19:33:36.511 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-04-28T19:33:36.541 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-04-28T19:33:36.541 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2022-04-28T19:33:36.820 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up librbd1 (15.2.16-1bionic) ... 2022-04-28T19:33:36.854 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up librbd1 (15.2.16-1bionic) ... 2022-04-28T19:33:36.947 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pecan (1.2.1-2) ... 2022-04-28T19:33:36.989 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pecan (1.2.1-2) ... 2022-04-28T19:33:37.036 INFO:teuthology.orchestra.run.smithi139.stdout:update-alternatives: using /usr/bin/python3-pecan to provide /usr/bin/pecan (pecan) in auto mode 2022-04-28T19:33:37.060 INFO:teuthology.orchestra.run.smithi139.stdout:update-alternatives: using /usr/bin/python3-gunicorn_pecan to provide /usr/bin/gunicorn_pecan (gunicorn_pecan) in auto mode 2022-04-28T19:33:37.078 INFO:teuthology.orchestra.run.smithi059.stdout:update-alternatives: using /usr/bin/python3-pecan to provide /usr/bin/pecan (pecan) in auto mode 2022-04-28T19:33:37.102 INFO:teuthology.orchestra.run.smithi059.stdout:update-alternatives: using /usr/bin/python3-gunicorn_pecan to provide /usr/bin/gunicorn_pecan (gunicorn_pecan) in auto mode 2022-04-28T19:33:37.350 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-kubernetes (2.0.0-2ubuntu1) ... 2022-04-28T19:33:37.417 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-kubernetes (2.0.0-2ubuntu1) ... 2022-04-28T19:33:37.994 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr-modules-core (15.2.16-1bionic) ... 2022-04-28T19:33:38.077 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr-modules-core (15.2.16-1bionic) ... 2022-04-28T19:33:38.119 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-pastescript (2.0.2-2) ... 2022-04-28T19:33:38.203 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-pastescript (2.0.2-2) ... 2022-04-28T19:33:38.421 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-rgw (15.2.16-1bionic) ... 2022-04-28T19:33:38.530 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-rgw (15.2.16-1bionic) ... 2022-04-28T19:33:38.547 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up rbd-fuse (15.2.16-1bionic) ... 2022-04-28T19:33:38.655 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up rbd-fuse (15.2.16-1bionic) ... 2022-04-28T19:33:38.673 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up python3-rbd (15.2.16-1bionic) ... 2022-04-28T19:33:38.782 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-common (15.2.16-1bionic) ... 2022-04-28T19:33:38.807 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up python3-rbd (15.2.16-1bionic) ... 2022-04-28T19:33:38.916 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-common (15.2.16-1bionic) ... 2022-04-28T19:33:39.300 INFO:teuthology.orchestra.run.smithi139.stdout:Adding group ceph....done 2022-04-28T19:33:39.392 INFO:teuthology.orchestra.run.smithi059.stdout:Adding group ceph....done 2022-04-28T19:33:39.694 INFO:teuthology.orchestra.run.smithi139.stdout:Adding system user ceph....done 2022-04-28T19:33:39.802 INFO:teuthology.orchestra.run.smithi059.stdout:Adding system user ceph....done 2022-04-28T19:33:39.903 INFO:teuthology.orchestra.run.smithi139.stdout:Setting system user ceph properties....done 2022-04-28T19:33:39.920 INFO:teuthology.orchestra.run.smithi139.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2022-04-28T19:33:39.986 INFO:teuthology.orchestra.run.smithi059.stdout:Setting system user ceph properties....done 2022-04-28T19:33:40.004 INFO:teuthology.orchestra.run.smithi059.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2022-04-28T19:33:40.063 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2022-04-28T19:33:40.144 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2022-04-28T19:33:40.322 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2022-04-28T19:33:40.394 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2022-04-28T19:33:40.667 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up radosgw (15.2.16-1bionic) ... 2022-04-28T19:33:40.717 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up radosgw (15.2.16-1bionic) ... 2022-04-28T19:33:40.907 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-04-28T19:33:40.908 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-04-28T19:33:40.960 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-04-28T19:33:40.960 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2022-04-28T19:33:41.319 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-test (15.2.16-1bionic) ... 2022-04-28T19:33:41.377 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-test (15.2.16-1bionic) ... 2022-04-28T19:33:41.444 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-base (15.2.16-1bionic) ... 2022-04-28T19:33:41.503 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-base (15.2.16-1bionic) ... 2022-04-28T19:33:41.835 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2022-04-28T19:33:41.900 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2022-04-28T19:33:42.147 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr (15.2.16-1bionic) ... 2022-04-28T19:33:42.247 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr (15.2.16-1bionic) ... 2022-04-28T19:33:42.364 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-04-28T19:33:42.364 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-04-28T19:33:42.465 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-04-28T19:33:42.465 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2022-04-28T19:33:42.681 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr-diskprediction-local (15.2.16-1bionic) ... 2022-04-28T19:33:42.789 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr-diskprediction-local (15.2.16-1bionic) ... 2022-04-28T19:33:42.824 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-osd (15.2.16-1bionic) ... 2022-04-28T19:33:42.932 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-osd (15.2.16-1bionic) ... 2022-04-28T19:33:43.228 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-04-28T19:33:43.228 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-04-28T19:33:43.405 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-04-28T19:33:43.405 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2022-04-28T19:33:43.526 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr-cephadm (15.2.16-1bionic) ... 2022-04-28T19:33:43.652 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr-diskprediction-cloud (15.2.16-1bionic) ... 2022-04-28T19:33:43.692 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr-cephadm (15.2.16-1bionic) ... 2022-04-28T19:33:43.794 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mds (15.2.16-1bionic) ... 2022-04-28T19:33:43.801 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr-diskprediction-cloud (15.2.16-1bionic) ... 2022-04-28T19:33:43.944 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mds (15.2.16-1bionic) ... 2022-04-28T19:33:44.014 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-04-28T19:33:44.015 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-04-28T19:33:44.162 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-04-28T19:33:44.162 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2022-04-28T19:33:44.337 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mon (15.2.16-1bionic) ... 2022-04-28T19:33:44.487 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mon (15.2.16-1bionic) ... 2022-04-28T19:33:44.556 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-04-28T19:33:44.556 INFO:teuthology.orchestra.run.smithi139.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-04-28T19:33:44.717 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-04-28T19:33:44.717 INFO:teuthology.orchestra.run.smithi059.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2022-04-28T19:33:44.973 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr-k8sevents (15.2.16-1bionic) ... 2022-04-28T19:33:45.099 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph-mgr-dashboard (15.2.16-1bionic) ... 2022-04-28T19:33:45.164 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr-k8sevents (15.2.16-1bionic) ... 2022-04-28T19:33:45.225 INFO:teuthology.orchestra.run.smithi139.stdout:Setting up ceph (15.2.16-1bionic) ... 2022-04-28T19:33:45.290 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph-mgr-dashboard (15.2.16-1bionic) ... 2022-04-28T19:33:45.326 INFO:teuthology.orchestra.run.smithi139.stdout:Processing triggers for systemd (237-3ubuntu10.53) ... 2022-04-28T19:33:45.425 INFO:teuthology.orchestra.run.smithi059.stdout:Setting up ceph (15.2.16-1bionic) ... 2022-04-28T19:33:45.502 INFO:teuthology.orchestra.run.smithi139.stdout:Processing triggers for man-db (2.8.3-2ubuntu0.1) ... 2022-04-28T19:33:45.526 INFO:teuthology.orchestra.run.smithi059.stdout:Processing triggers for systemd (237-3ubuntu10.53) ... 2022-04-28T19:33:45.727 INFO:teuthology.orchestra.run.smithi059.stdout:Processing triggers for man-db (2.8.3-2ubuntu0.1) ... 2022-04-28T19:33:47.229 INFO:teuthology.orchestra.run.smithi139.stdout:Processing triggers for ureadahead (0.100.0-21) ... 2022-04-28T19:33:47.272 INFO:teuthology.orchestra.run.smithi139.stdout:ureadahead will be reprofiled on next reboot 2022-04-28T19:33:47.330 INFO:teuthology.orchestra.run.smithi139.stdout:Processing triggers for libc-bin (2.27-3ubuntu1.5) ... 2022-04-28T19:33:47.463 INFO:teuthology.orchestra.run.smithi059.stdout:Processing triggers for ureadahead (0.100.0-21) ... 2022-04-28T19:33:47.505 INFO:teuthology.orchestra.run.smithi059.stdout:ureadahead will be reprofiled on next reboot 2022-04-28T19:33:47.563 INFO:teuthology.orchestra.run.smithi059.stdout:Processing triggers for libc-bin (2.27-3ubuntu1.5) ... 2022-04-28T19:33:47.794 INFO:teuthology.orchestra.run.smithi139.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2022-04-28T19:33:47.797 DEBUG:teuthology.parallel:result is None 2022-04-28T19:33:48.052 INFO:teuthology.orchestra.run.smithi059.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2022-04-28T19:33:48.054 DEBUG:teuthology.parallel:result is None 2022-04-28T19:33:48.055 DEBUG:teuthology.orchestra.run.smithi059:> sudo lsb_release -is 2022-04-28T19:33:48.099 INFO:teuthology.orchestra.run.smithi059.stdout:Ubuntu 2022-04-28T19:33:48.104 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-04-28T19:33:48.104 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-04-28T19:33:48.104 INFO:teuthology.packaging:ref: None 2022-04-28T19:33:48.104 INFO:teuthology.packaging:tag: None 2022-04-28T19:33:48.105 INFO:teuthology.packaging:branch: octopus 2022-04-28T19:33:48.105 INFO:teuthology.packaging:sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:33:48.105 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=octopus 2022-04-28T19:33:48.319 DEBUG:teuthology.orchestra.run.smithi059:> dpkg-query -W -f '${Version}' ceph 2022-04-28T19:33:48.341 INFO:teuthology.orchestra.run.smithi059.stdout:15.2.16-1bionic 2022-04-28T19:33:48.342 INFO:teuthology.packaging:The installed version of ceph is 15.2.16-1bionic 2022-04-28T19:33:48.342 INFO:teuthology.task.install:The correct ceph version 15.2.16-1bionic is installed. 2022-04-28T19:33:48.342 DEBUG:teuthology.orchestra.run.smithi139:> sudo lsb_release -is 2022-04-28T19:33:48.396 INFO:teuthology.orchestra.run.smithi139.stdout:Ubuntu 2022-04-28T19:33:48.401 DEBUG:teuthology.misc:System to be installed: Ubuntu 2022-04-28T19:33:48.402 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2022-04-28T19:33:48.402 INFO:teuthology.packaging:ref: None 2022-04-28T19:33:48.402 INFO:teuthology.packaging:tag: None 2022-04-28T19:33:48.402 INFO:teuthology.packaging:branch: octopus 2022-04-28T19:33:48.402 INFO:teuthology.packaging:sha1: 4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:33:48.402 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&ref=octopus 2022-04-28T19:33:48.622 DEBUG:teuthology.orchestra.run.smithi139:> dpkg-query -W -f '${Version}' ceph 2022-04-28T19:33:48.647 INFO:teuthology.orchestra.run.smithi139.stdout:15.2.16-1bionic 2022-04-28T19:33:48.647 INFO:teuthology.packaging:The installed version of ceph is 15.2.16-1bionic 2022-04-28T19:33:48.648 INFO:teuthology.task.install:The correct ceph version 15.2.16-1bionic is installed. 2022-04-28T19:33:48.648 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2022-04-28T19:33:48.648 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:33:48.648 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-04-28T19:33:48.662 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:33:48.662 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2022-04-28T19:33:48.704 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2022-04-28T19:33:48.704 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:33:48.705 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/usr/bin/daemon-helper 2022-04-28T19:33:48.720 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-04-28T19:33:48.777 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:33:48.777 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/usr/bin/daemon-helper 2022-04-28T19:33:48.793 DEBUG:teuthology.orchestra.run.smithi139:> sudo chmod a=rx -- /usr/bin/daemon-helper 2022-04-28T19:33:48.851 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2022-04-28T19:33:48.852 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:33:48.852 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/usr/bin/adjust-ulimits 2022-04-28T19:33:48.866 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-04-28T19:33:48.918 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:33:48.918 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/usr/bin/adjust-ulimits 2022-04-28T19:33:48.933 DEBUG:teuthology.orchestra.run.smithi139:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2022-04-28T19:33:48.992 INFO:teuthology.run_tasks:Running task print... 2022-04-28T19:33:49.007 INFO:teuthology.task.print:**** done install task... 2022-04-28T19:33:49.007 INFO:teuthology.run_tasks:Running task print... 2022-04-28T19:33:49.039 INFO:teuthology.task.print:**** done start installing octopus cephadm ... 2022-04-28T19:33:49.039 INFO:teuthology.run_tasks:Running task cephadm... 2022-04-28T19:33:49.161 INFO:tasks.cephadm:Config: {'add_mons_via_daemon_add': True, 'avoid_pacific_features': True, 'cephadm_branch': 'octopus', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:octopus', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-whitelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '4fa079ba14503defa8dc257d7c2d506ebefcfe6d'} 2022-04-28T19:33:49.161 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:octopus 2022-04-28T19:33:49.162 INFO:tasks.cephadm:Cluster fsid is 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:33:49.162 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2022-04-28T19:33:49.162 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.59', 'mon.c': '[v2:172.21.15.59:3301,v1:172.21.15.59:6790]', 'mon.b': '172.21.15.139'} 2022-04-28T19:33:49.162 INFO:tasks.cephadm:First mon is mon.a on smithi059 2022-04-28T19:33:49.162 INFO:tasks.cephadm:First mgr is y 2022-04-28T19:33:49.163 INFO:tasks.cephadm:Normalizing hostnames... 2022-04-28T19:33:49.163 DEBUG:teuthology.orchestra.run.smithi059:> sudo hostname $(hostname -s) 2022-04-28T19:33:49.180 DEBUG:teuthology.orchestra.run.smithi139:> sudo hostname $(hostname -s) 2022-04-28T19:33:49.195 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref octopus)... 2022-04-28T19:33:49.196 DEBUG:teuthology.orchestra.run.smithi059:> curl --silent https://raw.githubusercontent.com/ceph/ceph/octopus/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2022-04-28T19:33:49.445 INFO:teuthology.orchestra.run.smithi059.stdout:-rw-rw-r-- 1 ubuntu ubuntu 223468 Apr 28 19:33 /home/ubuntu/cephtest/cephadm 2022-04-28T19:33:49.446 DEBUG:teuthology.orchestra.run.smithi139:> curl --silent https://raw.githubusercontent.com/ceph/ceph/octopus/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2022-04-28T19:33:49.586 INFO:teuthology.orchestra.run.smithi139.stdout:-rw-rw-r-- 1 ubuntu ubuntu 223468 Apr 28 19:33 /home/ubuntu/cephtest/cephadm 2022-04-28T19:33:49.587 DEBUG:teuthology.orchestra.run.smithi059:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2022-04-28T19:33:49.599 DEBUG:teuthology.orchestra.run.smithi139:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2022-04-28T19:33:49.621 DEBUG:teuthology.orchestra.run.smithi059:> sudo mkdir -p /etc/ceph 2022-04-28T19:33:49.657 DEBUG:teuthology.orchestra.run.smithi139:> sudo mkdir -p /etc/ceph 2022-04-28T19:33:49.673 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod 777 /etc/ceph 2022-04-28T19:33:49.713 DEBUG:teuthology.orchestra.run.smithi139:> sudo chmod 777 /etc/ceph 2022-04-28T19:33:49.731 INFO:tasks.cephadm:Writing seed config... 2022-04-28T19:33:49.732 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2022-04-28T19:33:49.732 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2022-04-28T19:33:49.732 INFO:tasks.cephadm: override: [osd] debug ms = 1 2022-04-28T19:33:49.732 INFO:tasks.cephadm: override: [osd] debug osd = 20 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [mon] debug mon = 20 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [mon] debug ms = 1 2022-04-28T19:33:49.733 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2022-04-28T19:33:49.734 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:33:49.734 DEBUG:teuthology.orchestra.run.smithi059:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2022-04-28T19:33:49.761 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 20a1061c-c72a-11ec-8c39-001a4aab830c mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2022-04-28T19:33:49.761 DEBUG:teuthology.orchestra.run.smithi059:mon.a> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a.service 2022-04-28T19:33:49.805 DEBUG:teuthology.orchestra.run.smithi059:mgr.y> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y.service 2022-04-28T19:33:49.848 INFO:tasks.cephadm:Bootstrapping... 2022-04-28T19:33:49.848 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus -v bootstrap --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.59 && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2022-04-28T19:33:49.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:33:49.872 INFO:journalctl@ceph.mgr.y.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:33:50.115 INFO:teuthology.orchestra.run.smithi059.stderr:Could not locate podman: podman not found 2022-04-28T19:33:50.116 INFO:teuthology.orchestra.run.smithi059.stderr:container_init=False 2022-04-28T19:33:50.116 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying podman|docker is present... 2022-04-28T19:33:50.116 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying lvm2 is present... 2022-04-28T19:33:50.116 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying time synchronization is in place... 2022-04-28T19:33:50.117 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled chrony.service 2022-04-28T19:33:50.120 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2022-04-28T19:33:50.120 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active chrony.service 2022-04-28T19:33:50.128 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.128 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled chronyd.service 2022-04-28T19:33:50.136 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2022-04-28T19:33:50.136 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active chronyd.service 2022-04-28T19:33:50.142 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.143 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled systemd-timesyncd.service 2022-04-28T19:33:50.151 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout enabled 2022-04-28T19:33:50.152 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active systemd-timesyncd.service 2022-04-28T19:33:50.159 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.159 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled ntpd.service 2022-04-28T19:33:50.166 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2022-04-28T19:33:50.167 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active ntpd.service 2022-04-28T19:33:50.174 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.175 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled ntp.service 2022-04-28T19:33:50.183 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout enabled 2022-04-28T19:33:50.184 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active ntp.service 2022-04-28T19:33:50.188 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout active 2022-04-28T19:33:50.189 INFO:teuthology.orchestra.run.smithi059.stderr:Unit ntp.service is enabled and running 2022-04-28T19:33:50.189 INFO:teuthology.orchestra.run.smithi059.stderr:Repeating the final host check... 2022-04-28T19:33:50.189 INFO:teuthology.orchestra.run.smithi059.stderr:Could not locate podman: podman not found 2022-04-28T19:33:50.190 INFO:teuthology.orchestra.run.smithi059.stderr:podman|docker (/usr/bin/docker) is present 2022-04-28T19:33:50.190 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl is present 2022-04-28T19:33:50.190 INFO:teuthology.orchestra.run.smithi059.stderr:lvcreate is present 2022-04-28T19:33:50.190 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled chrony.service 2022-04-28T19:33:50.196 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2022-04-28T19:33:50.197 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active chrony.service 2022-04-28T19:33:50.205 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.205 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled chronyd.service 2022-04-28T19:33:50.212 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2022-04-28T19:33:50.212 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active chronyd.service 2022-04-28T19:33:50.218 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.219 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled systemd-timesyncd.service 2022-04-28T19:33:50.226 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout enabled 2022-04-28T19:33:50.226 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active systemd-timesyncd.service 2022-04-28T19:33:50.234 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.234 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled ntpd.service 2022-04-28T19:33:50.242 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2022-04-28T19:33:50.243 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active ntpd.service 2022-04-28T19:33:50.248 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout inactive 2022-04-28T19:33:50.249 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-enabled ntp.service 2022-04-28T19:33:50.255 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout enabled 2022-04-28T19:33:50.256 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl is-active ntp.service 2022-04-28T19:33:50.261 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stdout active 2022-04-28T19:33:50.262 INFO:teuthology.orchestra.run.smithi059.stderr:Unit ntp.service is enabled and running 2022-04-28T19:33:50.262 INFO:teuthology.orchestra.run.smithi059.stderr:Host looks OK 2022-04-28T19:33:50.262 INFO:teuthology.orchestra.run.smithi059.stderr:Cluster fsid: 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:33:50.262 INFO:teuthology.orchestra.run.smithi059.stderr:Acquiring lock 139765807533304 on /run/cephadm/20a1061c-c72a-11ec-8c39-001a4aab830c.lock 2022-04-28T19:33:50.262 INFO:teuthology.orchestra.run.smithi059.stderr:Lock 139765807533304 acquired on /run/cephadm/20a1061c-c72a-11ec-8c39-001a4aab830c.lock 2022-04-28T19:33:50.263 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying IP 172.21.15.59 port 3300 ... 2022-04-28T19:33:50.263 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying IP 172.21.15.59 port 6789 ... 2022-04-28T19:33:50.263 INFO:teuthology.orchestra.run.smithi059.stderr:Base mon IP is 172.21.15.59, final addrv is [v2:172.21.15.59:3300,v1:172.21.15.59:6789] 2022-04-28T19:33:50.264 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /sbin/ip route ls 2022-04-28T19:33:50.266 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 2022-04-28T19:33:50.267 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.59 metric 100 2022-04-28T19:33:50.267 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2022-04-28T19:33:50.267 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.59 2022-04-28T19:33:50.267 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout 172.21.15.254 dev ens1f0 proto dhcp scope link src 172.21.15.59 metric 100 2022-04-28T19:33:50.268 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /sbin/ip -6 route ls 2022-04-28T19:33:50.271 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2022-04-28T19:33:50.271 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2022-04-28T19:33:50.271 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /sbin/ip -6 addr ls 2022-04-28T19:33:50.274 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2022-04-28T19:33:50.274 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout inet6 ::1/128 scope host 2022-04-28T19:33:50.274 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout valid_lft forever preferred_lft forever 2022-04-28T19:33:50.274 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout 3: ens1f0: mtu 1500 state UP qlen 1000 2022-04-28T19:33:50.275 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:192a/64 scope link 2022-04-28T19:33:50.275 INFO:teuthology.orchestra.run.smithi059.stderr:/sbin/ip: stdout valid_lft forever preferred_lft forever 2022-04-28T19:33:50.275 INFO:teuthology.orchestra.run.smithi059.stderr:Mon IP 172.21.15.59 is in CIDR network 172.21.0.0/20 2022-04-28T19:33:50.277 INFO:teuthology.orchestra.run.smithi059.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:octopus... 2022-04-28T19:33:50.277 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker pull quay.ceph.io/ceph-ci/ceph:octopus 2022-04-28T19:33:51.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout octopus: Pulling from ceph-ci/ceph 2022-04-28T19:33:51.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout f0a2109a2528: Pulling fs layer 2022-04-28T19:33:51.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout 9f45cf1cd9f2: Pulling fs layer 2022-04-28T19:33:51.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout bfd1401568a8: Pulling fs layer 2022-04-28T19:33:51.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout fee18ea417d6: Pulling fs layer 2022-04-28T19:33:51.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout e7ab4750ce65: Pulling fs layer 2022-04-28T19:33:51.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout fee18ea417d6: Waiting 2022-04-28T19:33:51.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout e7ab4750ce65: Waiting 2022-04-28T19:33:51.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout 9f45cf1cd9f2: Verifying Checksum 2022-04-28T19:33:51.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout 9f45cf1cd9f2: Download complete 2022-04-28T19:33:52.188 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout bfd1401568a8: Verifying Checksum 2022-04-28T19:33:52.188 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout bfd1401568a8: Download complete 2022-04-28T19:33:53.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout fee18ea417d6: Verifying Checksum 2022-04-28T19:33:53.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout fee18ea417d6: Download complete 2022-04-28T19:33:53.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout f0a2109a2528: Verifying Checksum 2022-04-28T19:33:53.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout f0a2109a2528: Download complete 2022-04-28T19:33:55.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout e7ab4750ce65: Verifying Checksum 2022-04-28T19:33:55.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout e7ab4750ce65: Download complete 2022-04-28T19:33:57.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout f0a2109a2528: Pull complete 2022-04-28T19:33:57.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout 9f45cf1cd9f2: Pull complete 2022-04-28T19:33:58.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout bfd1401568a8: Pull complete 2022-04-28T19:34:00.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout fee18ea417d6: Pull complete 2022-04-28T19:34:10.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout e7ab4750ce65: Pull complete 2022-04-28T19:34:10.178 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout Digest: sha256:2c31462ccc670466649347b80186aee8600ff2a54fe91e872e7daf513bd648cb 2022-04-28T19:34:10.230 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout Status: Downloaded newer image for quay.ceph.io/ceph-ci/ceph:octopus 2022-04-28T19:34:10.261 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:octopus 2022-04-28T19:34:10.265 INFO:teuthology.orchestra.run.smithi059.stderr:Extracting ceph user uid/gid from container image... 2022-04-28T19:34:10.265 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint stat -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 quay.ceph.io/ceph-ci/ceph:octopus -c %u %g /var/lib/ceph 2022-04-28T19:34:20.275 INFO:teuthology.orchestra.run.smithi059.stderr:stat: stdout 167 167 2022-04-28T19:34:20.888 INFO:teuthology.orchestra.run.smithi059.stderr:Creating initial keys... 2022-04-28T19:34:20.888 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph-authtool -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 quay.ceph.io/ceph-ci/ceph:octopus --gen-print-key 2022-04-28T19:34:22.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-authtool: stdout AQA+7GpiO0c/AhAARLckR9ZIivu5pe8rL3e52g== 2022-04-28T19:34:22.541 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph-authtool -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 quay.ceph.io/ceph-ci/ceph:octopus --gen-print-key 2022-04-28T19:34:23.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-authtool: stdout AQA/7Gpig3dXGxAAcWUPpDsoYCGteCh+EGb9bA== 2022-04-28T19:34:23.930 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph-authtool -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 quay.ceph.io/ceph-ci/ceph:octopus --gen-print-key 2022-04-28T19:34:24.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-authtool: stdout AQBA7GpiDZcwNhAASgrlFmNM+XhSgOzrSh8eGw== 2022-04-28T19:34:25.452 INFO:teuthology.orchestra.run.smithi059.stderr:Creating initial monmap... 2022-04-28T19:34:25.453 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/monmaptool -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /tmp/ceph-tmpcsp4oh4y:/tmp/monmap:z quay.ceph.io/ceph-ci/ceph:octopus --create --clobber --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c --addv a [v2:172.21.15.59:3300,v1:172.21.15.59:6789] /tmp/monmap 2022-04-28T19:34:26.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2022-04-28T19:34:26.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:26.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2022-04-28T19:34:26.925 INFO:teuthology.orchestra.run.smithi059.stderr:Creating mon... 2022-04-28T19:34:26.925 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph-mon -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a:/var/lib/ceph/mon/ceph-a:z -v /tmp/ceph-tmp6guqbmqu:/tmp/keyring:z -v /tmp/ceph-tmpcsp4oh4y:/tmp/monmap:z quay.ceph.io/ceph-ci/ceph:octopus --mkfs -i a --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -c /dev/null --monmap /tmp/monmap --keyring /tmp/keyring --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true 2022-04-28T19:34:27.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:34:27.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 1 imported monmap: 2022-04-28T19:34:27.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr epoch 0 2022-04-28T19:34:27.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:27.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr last_changed 2022-04-28T19:34:26.452116+0000 2022-04-28T19:34:27.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr created 2022-04-28T19:34:26.452116+0000 2022-04-28T19:34:27.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr min_mon_release 0 (unknown) 2022-04-28T19:34:27.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon.a 2022-04-28T19:34:27.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:27.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 /usr/bin/ceph-mon: set fsid to 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:27.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:34:27.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:34:27.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:34:27.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: RocksDB version: 6.1.2 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Compile date Mar 25 2022 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: DB SUMMARY 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2022-04-28T19:34:27.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.create_if_missing: 1 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.env: 0x55991c7c91c0 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.info_log: 0x55991e383420 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:34:28.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:34:28.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.db_log_dir: 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:34:28.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.write_buffer_manager: 0x55991e4067b0 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:34:28.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:34:28.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.row_cache: None 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.wal_filter: None 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:34:28.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:34:28.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:34:28.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:34:28.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:34:28.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:34:28.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:34:28.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:34:28.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:34:28.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:34:28.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:34:28.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:34:28.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:34:28.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:34:28.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:34:28.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:34:28.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:34:28.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Compression algorithms supported: 2022-04-28T19:34:28.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:34:28.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kZSTD supported: 0 2022-04-28T19:34:28.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:34:28.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:34:28.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:34:28.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:34:28.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:34:28.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:34:28.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:34:28.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.969+0000 7f0fa2eed6c0 4 rocksdb: [db/db_impl_open.cc:228] Creating manifest 1 2022-04-28T19:34:28.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2022-04-28T19:34:28.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2022-04-28T19:34:28.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:34:28.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.merge_operator: 2022-04-28T19:34:28.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55991e31ba88) 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 1 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2022-04-28T19:34:28.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr index_type: 0 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr checksum: 1 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr no_block_cache: 0 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_cache: 0x55991e352510 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_cache_options: 2022-04-28T19:34:28.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr capacity : 536870912 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_size: 4096 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2022-04-28T19:34:28.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr partition_filters: 0 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr verify_compression: 0 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr format_version: 2 2022-04-28T19:34:28.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr block_align: 0 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.num_levels: 7 2022-04-28T19:34:28.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:34:28.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:34:28.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:34:28.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:34:28.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:34:28.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: Options.ttl: 0 2022-04-28T19:34:28.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.973+0000 7f0fa2eed6c0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.977+0000 7f0fa2eed6c0 4 rocksdb: DB pointer 0x55991ef80400 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.977+0000 7f0f8bdbe700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.977+0000 7f0f8bdbe700 4 rocksdb: [db/db_impl.cc:778] 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** DB Stats ** 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:34:28.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:34:28.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:28.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2022-04-28T19:34:28.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:34:28.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:34:28.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr 2022-04-28T19:34:28.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.981+0000 7f0fa2eed6c0 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work 2022-04-28T19:34:28.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.981+0000 7f0fa2eed6c0 4 rocksdb: [db/db_impl.cc:563] Shutdown complete 2022-04-28T19:34:28.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph-mon: stderr debug 2022-04-28T19:34:27.981+0000 7f0fa2eed6c0 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2022-04-28T19:34:28.513 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: install -d -m0770 -o 167 -g 167 /var/run/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:28.520 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl disable ceph.target 2022-04-28T19:34:28.529 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2022-04-28T19:34:28.632 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl enable ceph.target 2022-04-28T19:34:28.638 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2022-04-28T19:34:28.746 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl start ceph.target 2022-04-28T19:34:28.753 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl enable ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target 2022-04-28T19:34:28.759 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target -> /etc/systemd/system/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target. 2022-04-28T19:34:28.759 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target -> /etc/systemd/system/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target. 2022-04-28T19:34:28.880 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl start ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target 2022-04-28T19:34:28.887 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl daemon-reload 2022-04-28T19:34:29.014 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl stop ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a 2022-04-28T19:34:29.022 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl reset-failed ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a 2022-04-28T19:34:29.027 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to reset failed state of unit ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a.service: Unit ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a.service is not loaded. 2022-04-28T19:34:29.028 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl enable ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a 2022-04-28T19:34:29.034 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Created symlink /etc/systemd/system/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target.wants/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a.service -> /etc/systemd/system/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@.service. 2022-04-28T19:34:29.154 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl start ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a 2022-04-28T19:34:29.164 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld does not appear to be present 2022-04-28T19:34:29.164 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to enable service . firewalld.service is not available 2022-04-28T19:34:29.164 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to open ports <[]>. firewalld.service is not available 2022-04-28T19:34:29.165 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mon to start... 2022-04-28T19:34:29.165 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mon... 2022-04-28T19:34:29.166 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a:/var/lib/ceph/mon/ceph-a:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus status 2022-04-28T19:34:29.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:29 smithi059 systemd[1]: Started Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:34:30.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:30 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.427900+0000 mon.a (mon.0) 0 : [INF] mkfs 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:30.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:30 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.413927+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2022-04-28T19:34:30.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout cluster: 2022-04-28T19:34:30.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout id: 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:30.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout health: HEALTH_OK 2022-04-28T19:34:30.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:30.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout services: 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.35171s) 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mgr: no daemons active 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout data: 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2022-04-28T19:34:30.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2022-04-28T19:34:30.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2022-04-28T19:34:30.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout pgs: 2022-04-28T19:34:30.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:31.313 INFO:teuthology.orchestra.run.smithi059.stderr:mon is available 2022-04-28T19:34:31.313 INFO:teuthology.orchestra.run.smithi059.stderr:Assimilating anything we can from ceph.conf... 2022-04-28T19:34:31.313 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z -v /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a:/var/lib/ceph/mon/ceph-a:z quay.ceph.io/ceph-ci/ceph:octopus config assimilate-conf -i /var/lib/ceph/mon/ceph-a/config 2022-04-28T19:34:31.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:31 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.423357+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2022-04-28T19:34:31.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:31 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.424843+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0]} 2022-04-28T19:34:31.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:31 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.434131+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2022-04-28T19:34:31.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:31 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.445254+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2022-04-28T19:34:31.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:31 smithi059 bash[23211]: cluster 2022-04-28T19:34:30.446300+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2022-04-28T19:34:31.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:31 smithi059 bash[23211]: audit 2022-04-28T19:34:30.774997+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.59:0/2561016572' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T19:34:32.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:32.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout [global] 2022-04-28T19:34:32.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout fsid = 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:32.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mon_election_default_strategy = 1 2022-04-28T19:34:32.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.59:3300,v1:172.21.15.59:6789] 2022-04-28T19:34:32.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2022-04-28T19:34:32.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2022-04-28T19:34:32.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2022-04-28T19:34:32.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout [mgr] 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout [osd] 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout osd_map_max_advance = 10 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout osd_memory_target_autotune = true 2022-04-28T19:34:32.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout osd_sloppy_crc = true 2022-04-28T19:34:33.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:32 smithi059 bash[23211]: audit 2022-04-28T19:34:32.582827+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.59:0/2537570944' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2022-04-28T19:34:33.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:32 smithi059 bash[23211]: audit 2022-04-28T19:34:32.587236+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.59:0/2537570944' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2022-04-28T19:34:33.103 INFO:teuthology.orchestra.run.smithi059.stderr:Generating new minimal ceph.conf... 2022-04-28T19:34:33.103 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z -v /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a:/var/lib/ceph/mon/ceph-a:z quay.ceph.io/ceph-ci/ceph:octopus config generate-minimal-conf -o /var/lib/ceph/mon/ceph-a/config 2022-04-28T19:34:34.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:34 smithi059 bash[23211]: audit 2022-04-28T19:34:34.340805+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.59:0/3555912825' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:34:34.901 INFO:teuthology.orchestra.run.smithi059.stderr:Restarting the monitor... 2022-04-28T19:34:34.901 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl restart ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.a 2022-04-28T19:34:35.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:34 smithi059 systemd[1]: Stopping Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:34:35.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:35 smithi059 bash[23211]: debug 2022-04-28T19:34:35.017+0000 7fc6f396e700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:34:35.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:35 smithi059 bash[23211]: debug 2022-04-28T19:34:35.017+0000 7fc6f396e700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2022-04-28T19:34:35.616 INFO:teuthology.orchestra.run.smithi059.stderr:Setting mon public_network... 2022-04-28T19:34:35.617 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus config set mon public_network 172.21.0.0/20 2022-04-28T19:34:36.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:35 smithi059 docker[23737]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.a 2022-04-28T19:34:36.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:35 smithi059 systemd[1]: Stopped Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:34:36.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:35 smithi059 systemd[1]: Started Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:34:37.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.925+0000 7f643725c6c0 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:34:37.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.925+0000 7f643725c6c0 0 ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable), process ceph-mon, pid 1 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.925+0000 7f643725c6c0 0 pidfile_write: ignore empty --pid-file 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 load: jerasure load: lrc load: isa 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:34:37.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:34:37.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:34:37.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 1 rocksdb: do_open column families: [default] 2022-04-28T19:34:37.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: RocksDB version: 6.1.2 2022-04-28T19:34:37.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:34:37.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Compile date Mar 25 2022 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: DB SUMMARY 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: CURRENT file: CURRENT 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: IDENTITY file: IDENTITY 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 109 Bytes 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 70795 ; 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.create_if_missing: 0 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:34:37.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.env: 0x55e1973f51c0 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.info_log: 0x55e197e32520 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:34:37.009 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:34:37.010 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:34:37.010 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:34:37.010 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:34:37.010 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:34:37.010 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:34:37.010 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:34:37.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.db_log_dir: 2022-04-28T19:34:37.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2022-04-28T19:34:37.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:34:37.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:34:37.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:34:37.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.write_buffer_manager: 0x55e197e39530 2022-04-28T19:34:37.012 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:34:37.013 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.row_cache: None 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.wal_filter: None 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:34:37.014 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:34:37.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:34:37.017 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:34:37.017 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:34:37.017 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:34:37.017 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:34:37.017 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Compression algorithms supported: 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kZSTD supported: 0 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:34:37.018 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:34:37.019 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:34:37.019 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:34:37.019 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.933+0000 7f643725c6c0 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:34:37.019 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2022-04-28T19:34:37.019 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2022-04-28T19:34:37.019 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:34:37.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.merge_operator: 2022-04-28T19:34:37.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:34:37.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:34:37.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:34:37.021 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:34:37.021 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e197d4da80) 2022-04-28T19:34:37.021 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: cache_index_and_filter_blocks: 1 2022-04-28T19:34:37.021 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: cache_index_and_filter_blocks_with_high_priority: 1 2022-04-28T19:34:37.021 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: pin_top_level_index_and_filter: 1 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: index_type: 0 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: data_block_index_type: 0 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: hash_index_allow_collision: 1 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: checksum: 1 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: no_block_cache: 0 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_cache: 0x55e197d84610 2022-04-28T19:34:37.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_cache_name: BinnedLRUCache 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_cache_options: 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: capacity : 536870912 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: num_shard_bits : 4 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: strict_capacity_limit : 0 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: high_pri_pool_ratio: 0.000 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_cache_compressed: (nil) 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: persistent_cache: (nil) 2022-04-28T19:34:37.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_size: 4096 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_size_deviation: 10 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_restart_interval: 16 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: index_block_restart_interval: 1 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: metadata_block_size: 4096 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: partition_filters: 0 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: use_delta_encoding: 1 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: whole_key_filtering: 1 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: verify_compression: 0 2022-04-28T19:34:37.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: read_amp_bytes_per_bit: 0 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: format_version: 2 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: enable_index_compression: 1 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: block_align: 0 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:34:37.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.num_levels: 7 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:34:37.026 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:34:37.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:34:37.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:34:37.029 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:34:37.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:34:37.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: Options.ttl: 0 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 4 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174476943167, "job": 1, "event": "recovery_started", "log_files": [6]} 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.937+0000 7f643725c6c0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #6 mode 2 2022-04-28T19:34:37.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.941+0000 7f643725c6c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174476944854, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 67563, "table_properties": {"data_size": 65868, "index_size": 300, "filter_size": 581, "raw_key_size": 9349, "raw_average_key_size": 47, "raw_value_size": 60542, "raw_average_value_size": 310, "num_data_blocks": 10, "num_entries": 195, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.941+0000 7f643725c6c0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 8 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.941+0000 7f643725c6c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174476945816, "job": 1, "event": "recovery_finished"} 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 4 rocksdb: DB pointer 0x55e1989b2400 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 0 starting mon.a rank 0 at public addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] at bind addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 1 mon.a@-1(???) e1 preinit fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 0 mon.a@-1(???).mds e1 new map 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 0 mon.a@-1(???).mds e1 print_map 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: e1 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: enable_multiple, ever_enabled_multiple: 0,0 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:34:37.033 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: legacy client fscid: -1 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: No filesystems configured 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: audit 2022-04-28T19:34:34.340805+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.59:0/3555912825' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f643725c6c0 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f641e0b8700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2022-04-28T19:34:37.034 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.945+0000 7f641e0b8700 4 rocksdb: [db/db_impl.cc:778] 2022-04-28T19:34:37.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** DB Stats ** 2022-04-28T19:34:37.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:34:37.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:34:37.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:34:37.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:34:37.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** Compaction Stats [default] ** 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: L0 2/0 67.60 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Sum 2/0 67.60 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** Compaction Stats [default] ** 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:34:37.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Cumulative compaction: 0.00 GB write, 5.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Interval compaction: 0.00 GB write, 5.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** Compaction Stats [default] ** 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:37.038 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: L0 2/0 67.60 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Sum 2/0 67.60 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** Compaction Stats [default] ** 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 60.1 0.00 0.00 1 0.001 0 0 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:34:37.042 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Cumulative compaction: 0.00 GB write, 5.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expand map: {default=false} 2022-04-28T19:34:37.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta from 'false' to 'false' 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expanded map: {default=false} 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expand map: {default=info} 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta from 'info' to 'info' 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expanded map: {default=info} 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expand map: {default=daemon} 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta from 'daemon' to 'daemon' 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expanded map: {default=daemon} 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expand map: {default=debug} 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta from 'debug' to 'debug' 2022-04-28T19:34:37.044 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:36 smithi059 bash[23786]: debug 2022-04-28T19:34:36.949+0000 7f643725c6c0 20 expand_channel_meta expanded map: {default=debug} 2022-04-28T19:34:37.045 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:37 smithi059 bash[23786]: cluster 2022-04-28T19:34:36.956608+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2022-04-28T19:34:37.045 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:37 smithi059 bash[23786]: cluster 2022-04-28T19:34:36.956676+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0]} 2022-04-28T19:34:37.045 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:37 smithi059 bash[23786]: cluster 2022-04-28T19:34:36.956747+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2022-04-28T19:34:37.045 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:37 smithi059 bash[23786]: cluster 2022-04-28T19:34:36.956776+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2022-04-28T19:34:37.045 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:37 smithi059 bash[23786]: cluster 2022-04-28T19:34:36.957168+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2022-04-28T19:34:37.812 INFO:teuthology.orchestra.run.smithi059.stderr:Creating mgr... 2022-04-28T19:34:37.813 INFO:teuthology.orchestra.run.smithi059.stderr:Verifying port 9283 ... 2022-04-28T19:34:37.815 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl daemon-reload 2022-04-28T19:34:37.954 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl stop ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y 2022-04-28T19:34:37.964 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl reset-failed ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y 2022-04-28T19:34:37.971 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Failed to reset failed state of unit ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y.service: Unit ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y.service is not loaded. 2022-04-28T19:34:37.971 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl enable ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y 2022-04-28T19:34:37.978 INFO:teuthology.orchestra.run.smithi059.stderr:systemctl: stderr Created symlink /etc/systemd/system/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c.target.wants/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y.service -> /etc/systemd/system/ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@.service. 2022-04-28T19:34:38.084 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: systemctl start ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y 2022-04-28T19:34:38.094 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld does not appear to be present 2022-04-28T19:34:38.094 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to enable service . firewalld.service is not available 2022-04-28T19:34:38.094 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to open ports <[]>. firewalld.service is not available 2022-04-28T19:34:38.094 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld does not appear to be present 2022-04-28T19:34:38.095 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2022-04-28T19:34:38.095 INFO:teuthology.orchestra.run.smithi059.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2022-04-28T19:34:38.095 INFO:teuthology.orchestra.run.smithi059.stderr:Wrote config to /etc/ceph/ceph.conf 2022-04-28T19:34:38.095 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mgr to start... 2022-04-28T19:34:38.095 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for mgr... 2022-04-28T19:34:38.095 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus status -f json-pretty 2022-04-28T19:34:38.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:38 smithi059 bash[23786]: audit 2022-04-28T19:34:37.202731+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.59:0/1187605777' entity='client.admin' 2022-04-28T19:34:38.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:38 smithi059 systemd[1]: Started Ceph mgr.y for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:34:39.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:39.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:39.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "fsid": "20a1061c-c72a-11ec-8c39-001a4aab830c", 2022-04-28T19:34:39.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "health": { 2022-04-28T19:34:39.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2022-04-28T19:34:39.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "checks": {}, 2022-04-28T19:34:39.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mutes": [] 2022-04-28T19:34:39.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "election_epoch": 5, 2022-04-28T19:34:39.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum": [ 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 0 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum_names": [ 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "a" 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum_age": 2, 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "monmap": { 2022-04-28T19:34:39.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_mon_release_name": "octopus", 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_mons": 1 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osdmap": { 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_osds": 0, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_up_osds": 0, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osd_up_since": 0, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_in_osds": 0, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osd_in_since": 0, 2022-04-28T19:34:39.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pgmap": { 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pgs_by_state": [], 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_pgs": 0, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_pools": 0, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_objects": 0, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "data_bytes": 0, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_used": 0, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_avail": 0, 2022-04-28T19:34:39.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_total": 0 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "fsmap": { 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "by_rank": [], 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "up:standby": 0 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mgrmap": { 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available": false, 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_standbys": 0, 2022-04-28T19:34:39.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modules": [ 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "iostat", 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "restful" 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {} 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "servicemap": { 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modified": "2022-04-28T19:34:30.428786+0000", 2022-04-28T19:34:39.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {} 2022-04-28T19:34:39.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:39.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress_events": {} 2022-04-28T19:34:39.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:39.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:39 smithi059 bash[23786]: audit 2022-04-28T19:34:39.754651+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.59:0/520350399' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2022-04-28T19:34:39.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:39 smithi059 bash[24088]: debug 2022-04-28T19:34:39.581+0000 7f7f99978700 1 -- 172.21.15.59:0/3631250330 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a8d87324e0 con 0x55a8d8818800 2022-04-28T19:34:40.295 INFO:teuthology.orchestra.run.smithi059.stderr:mgr not available, waiting (1/10)... 2022-04-28T19:34:41.297 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus status -f json-pretty 2022-04-28T19:34:42.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:42.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:42.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "fsid": "20a1061c-c72a-11ec-8c39-001a4aab830c", 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "health": { 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "checks": {}, 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mutes": [] 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "election_epoch": 5, 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum": [ 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 0 2022-04-28T19:34:42.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:42.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum_names": [ 2022-04-28T19:34:42.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "a" 2022-04-28T19:34:42.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:42.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum_age": 5, 2022-04-28T19:34:42.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "monmap": { 2022-04-28T19:34:42.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:42.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_mon_release_name": "octopus", 2022-04-28T19:34:42.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_mons": 1 2022-04-28T19:34:42.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osdmap": { 2022-04-28T19:34:42.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:42.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_osds": 0, 2022-04-28T19:34:42.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_up_osds": 0, 2022-04-28T19:34:42.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osd_up_since": 0, 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_in_osds": 0, 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osd_in_since": 0, 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pgmap": { 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pgs_by_state": [], 2022-04-28T19:34:42.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_pgs": 0, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_pools": 0, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_objects": 0, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "data_bytes": 0, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_used": 0, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_avail": 0, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_total": 0 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "fsmap": { 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "by_rank": [], 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "up:standby": 0 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mgrmap": { 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available": false, 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_standbys": 0, 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modules": [ 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "iostat", 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "restful" 2022-04-28T19:34:42.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {} 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "servicemap": { 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modified": "2022-04-28T19:34:30.428786+0000", 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {} 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress_events": {} 2022-04-28T19:34:42.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:42.992 INFO:teuthology.orchestra.run.smithi059.stderr:mgr not available, waiting (2/10)... 2022-04-28T19:34:43.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:42 smithi059 bash[23786]: audit 2022-04-28T19:34:42.444793+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.59:0/2585413236' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2022-04-28T19:34:43.993 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus status -f json-pretty 2022-04-28T19:34:44.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: cluster 2022-04-28T19:34:44.161449+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2022-04-28T19:34:44.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: cluster 2022-04-28T19:34:44.167253+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00597093s) 2022-04-28T19:34:44.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.168890+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.169427+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.170064+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.170499+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.170891+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.171461+0000 mon.a (mon.0) 16 : audit [DBG] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: cluster 2022-04-28T19:34:44.175983+0000 mon.a (mon.0) 17 : cluster [INF] Manager daemon y is now available 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.187101+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' 2022-04-28T19:34:44.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.193354+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:34:44.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.195793+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:34:44.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.204909+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' 2022-04-28T19:34:44.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:44 smithi059 bash[23786]: audit 2022-04-28T19:34:44.208166+0000 mon.a (mon.0) 22 : audit [INF] from='mgr.14100 172.21.15.59:0/507645597' entity='mgr.y' 2022-04-28T19:34:45.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 2022-04-28T19:34:45.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:45.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "fsid": "20a1061c-c72a-11ec-8c39-001a4aab830c", 2022-04-28T19:34:45.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "health": { 2022-04-28T19:34:45.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "checks": {}, 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mutes": [] 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "election_epoch": 5, 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum": [ 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 0 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum_names": [ 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "a" 2022-04-28T19:34:45.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "quorum_age": 8, 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "monmap": { 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_mon_release_name": "octopus", 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_mons": 1 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osdmap": { 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_osds": 0, 2022-04-28T19:34:45.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_up_osds": 0, 2022-04-28T19:34:45.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osd_up_since": 0, 2022-04-28T19:34:45.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_in_osds": 0, 2022-04-28T19:34:45.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "osd_in_since": 0, 2022-04-28T19:34:45.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2022-04-28T19:34:45.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pgmap": { 2022-04-28T19:34:45.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pgs_by_state": [], 2022-04-28T19:34:45.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_pgs": 0, 2022-04-28T19:34:45.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_pools": 0, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_objects": 0, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "data_bytes": 0, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_used": 0, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_avail": 0, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "bytes_total": 0 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "fsmap": { 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "by_rank": [], 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "up:standby": 0 2022-04-28T19:34:45.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mgrmap": { 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available": true, 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_standbys": 0, 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modules": [ 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "iostat", 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "restful" 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {} 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "servicemap": { 2022-04-28T19:34:45.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 1, 2022-04-28T19:34:45.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modified": "2022-04-28T19:34:30.428786+0000", 2022-04-28T19:34:45.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {} 2022-04-28T19:34:45.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:45.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress_events": {} 2022-04-28T19:34:45.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:46.124 INFO:teuthology.orchestra.run.smithi059.stderr:mgr is available 2022-04-28T19:34:46.124 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus config-key set mgr/cephadm/ssh_user root 2022-04-28T19:34:46.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:46 smithi059 bash[23786]: cluster 2022-04-28T19:34:45.170187+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e3: y(active, since 1.00892s) 2022-04-28T19:34:46.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:46 smithi059 bash[23786]: audit 2022-04-28T19:34:45.636386+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.59:0/3135880390' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2022-04-28T19:34:47.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:47 smithi059 bash[23786]: cluster 2022-04-28T19:34:46.174092+0000 mon.a (mon.0) 25 : cluster [DBG] mgrmap e4: y(active, since 2s) 2022-04-28T19:34:47.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stderr set mgr/cephadm/ssh_user 2022-04-28T19:34:48.097 INFO:teuthology.orchestra.run.smithi059.stderr:Enabling cephadm module... 2022-04-28T19:34:48.097 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus mgr module enable cephadm 2022-04-28T19:34:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:48 smithi059 bash[23786]: audit 2022-04-28T19:34:47.594548+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.59:0/4076817678' entity='client.admin' 2022-04-28T19:34:50.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:49 smithi059 bash[23786]: audit 2022-04-28T19:34:49.533672+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.59:0/1057595933' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2022-04-28T19:34:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:49 smithi059 bash[24088]: ignoring --setuser ceph since I am not root 2022-04-28T19:34:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:49 smithi059 bash[24088]: ignoring --setgroup ceph since I am not root 2022-04-28T19:34:50.086 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:49 smithi059 bash[24088]: debug 2022-04-28T19:34:49.637+0000 7f6d182cc700 1 -- 172.21.15.59:0/1562013662 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5591290084e0 con 0x5591290f0800 2022-04-28T19:34:50.119 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus mgr dump 2022-04-28T19:34:50.878 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:50 smithi059 bash[23786]: audit 2022-04-28T19:34:49.603370+0000 mon.a (mon.0) 28 : audit [INF] from='client.? 172.21.15.59:0/1057595933' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2022-04-28T19:34:50.878 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:50 smithi059 bash[23786]: cluster 2022-04-28T19:34:49.603465+0000 mon.a (mon.0) 29 : cluster [DBG] mgrmap e5: y(active, since 5s) 2022-04-28T19:34:51.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 5, 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_gid": 14100, 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_name": "y", 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_addrs": { 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addrvec": [ 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v2", 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:6800", 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 1632668962 2022-04-28T19:34:51.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v1", 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:6801", 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 1632668962 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_addr": "172.21.15.59:6801/1632668962", 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_change": "2022-04-28T19:34:44.161250+0000", 2022-04-28T19:34:51.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_mgr_features": 4540138292840890367, 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available": true, 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "standbys": [], 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modules": [ 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "cephadm", 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "iostat", 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "restful" 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available_modules": [ 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:51.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "alerts", 2022-04-28T19:34:51.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:51.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:51.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:51.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:34:51.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:51.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "How frequently to reexamine health status", 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:51.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:51.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:51.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:51.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:51.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_destination": { 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_destination", 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Email address to send alerts to", 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_from_name": { 2022-04-28T19:34:52.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_from_name", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "Ceph", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Email From: name", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_host": { 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_host", 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "SMTP server", 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_password": { 2022-04-28T19:34:52.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_password", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Password to authenticate with", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_port": { 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_port", 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "465", 2022-04-28T19:34:52.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "SMTP port", 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_sender": { 2022-04-28T19:34:52.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_sender", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "SMTP envelope sender", 2022-04-28T19:34:52.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_ssl": { 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_ssl", 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Use SSL to connect to SMTP server", 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_user": { 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_user", 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "User to authenticate as", 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "balancer", 2022-04-28T19:34:52.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active": { 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "active", 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "automatically balance PGs across cluster", 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "begin_time": { 2022-04-28T19:34:52.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "begin_time", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0000", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "beginning time of day to automatically balance", 2022-04-28T19:34:52.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "This is a time of day in the format HHMM.", 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "begin_weekday": { 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "begin_weekday", 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "0", 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "7", 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Restrict automatic balancing to this day of the week or later", 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "0 or 7 = Sunday, 1 = Monday, etc.", 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush_compat_max_iterations": { 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crush_compat_max_iterations", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "25", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "1", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "250", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "maximum number of iterations to attempt optimization", 2022-04-28T19:34:52.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush_compat_metrics": { 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crush_compat_metrics", 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "pgs,objects,bytes", 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "metrics with which to calculate OSD utilization", 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush_compat_step": { 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crush_compat_step", 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "float", 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0.5", 2022-04-28T19:34:52.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "0.001", 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "0.999", 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "aggressiveness of optimization", 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": ".99 is very aggressive, .01 is less aggressive", 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "end_time": { 2022-04-28T19:34:52.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "end_time", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "2400", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "ending time of day to automatically balance", 2022-04-28T19:34:52.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "This is a time of day in the format HHMM.", 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "end_weekday": { 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "end_weekday", 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:34:52.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "7", 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "0", 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "7", 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Restrict automatic balancing to days of the week earlier than this", 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "0 or 7 = Sunday, 1 = Monday, etc.", 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_score": { 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "min_score", 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "float", 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "minimum score, below which no optimization is attempted", 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mode": { 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mode", 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "none", 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush-compat", 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "none", 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "upmap" 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Balancer mode", 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pool_ids": { 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pool_ids", 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "pools which the automatic balancing will be limited to", 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:34:52.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to wake up and attempt optimization", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "upmap_max_deviation": { 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "upmap_max_deviation", 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "1", 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "deviation below which no optimization is attempted", 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "upmap_max_optimizations": { 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "upmap_max_optimizations", 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "maximum upmap optimizations to make per attempt", 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "cephadm", 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "allow_ptrace": { 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "allow_ptrace", 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "allow SYS_PTRACE capability on ceph containers", 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "config_dashboard": { 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "config_dashboard", 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "manage configs like API endpoints in Dashboard.", 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_alertmanager": { 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_alertmanager", 2022-04-28T19:34:52.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/prometheus/alertmanager:v0.20.0", 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:34:52.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_base": { 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_base", 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/ceph/ceph", 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Container image name, without the tag", 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_grafana": { 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_grafana", 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/ceph/ceph-grafana:6.7.4", 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_node_exporter": { 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_node_exporter", 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/prometheus/node-exporter:v0.18.1", 2022-04-28T19:34:52.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_prometheus": { 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_prometheus", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/prometheus/prometheus:v2.18.1", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:34:52.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_init": { 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_init", 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Run podman/docker with `--init`", 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "daemon_cache_timeout": { 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "daemon_cache_timeout", 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "seconds to cache service (daemon) inventory", 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "device_cache_timeout": { 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "device_cache_timeout", 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "1800", 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "seconds to cache device inventory", 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "facts_cache_timeout": { 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "facts_cache_timeout", 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "seconds to cache host facts data", 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "host_check_interval": { 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "host_check_interval", 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to perform a host check", 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "log to the \"cephadm\" cluster log channel\"", 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "manage_etc_ceph_ceph_conf": { 2022-04-28T19:34:52.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "manage_etc_ceph_ceph_conf", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2022-04-28T19:34:52.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "migration_current": { 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "migration_current", 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "internal - do not modify", 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mode": { 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mode", 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "root", 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "cephadm-package", 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "root" 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "mode for remote execution of cephadm", 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "prometheus_alerts_path": { 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "prometheus_alerts_path", 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "location of alerts to include in prometheus deployments", 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "registry_password": { 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "registry_password", 2022-04-28T19:34:52.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Custom repository password", 2022-04-28T19:34:52.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "registry_url": { 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "registry_url", 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Custom repository url", 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "registry_username": { 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "registry_username", 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Custom repository username", 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssh_config_file": { 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssh_config_file", 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "customized SSH config file to connect to managed hosts", 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "use_repo_digest": { 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "use_repo_digest", 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_on_failed_host_check": { 2022-04-28T19:34:52.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_on_failed_host_check", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise a health warning if the host check fails", 2022-04-28T19:34:52.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_on_stray_daemons": { 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_on_stray_daemons", 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_on_stray_hosts": { 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_on_stray_hosts", 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crash", 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "retain_interval": { 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "retain_interval", 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "31536000", 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how long to retain crashes before pruning them", 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_recent_interval": { 2022-04-28T19:34:52.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_recent_interval", 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "1209600", 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "time interval in which to warn about recent crashes", 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "dashboard", 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ACCOUNT_LOCKOUT_ATTEMPTS": { 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2022-04-28T19:34:52.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ALERTMANAGER_API_HOST": { 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ALERTMANAGER_API_HOST", 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ALERTMANAGER_API_SSL_VERIFY": { 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ALERTMANAGER_API_SSL_VERIFY", 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "AUDIT_API_ENABLED": { 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "AUDIT_API_ENABLED", 2022-04-28T19:34:52.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "AUDIT_API_LOG_PAYLOAD": { 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "AUDIT_API_LOG_PAYLOAD", 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ENABLE_BROWSABLE_API": { 2022-04-28T19:34:52.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ENABLE_BROWSABLE_API", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_cephfs": { 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_cephfs", 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_iscsi": { 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_iscsi", 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_mirroring": { 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_mirroring", 2022-04-28T19:34:52.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_nfs": { 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_nfs", 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_rbd": { 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_rbd", 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_rgw": { 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_rgw", 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.273 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.273 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.274 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_PASSWORD": { 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_PASSWORD", 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.275 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "admin", 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.276 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.277 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_SSL_VERIFY": { 2022-04-28T19:34:52.277 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_SSL_VERIFY", 2022-04-28T19:34:52.277 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.277 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.277 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.277 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.280 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.281 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_URL": { 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_URL", 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.282 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.283 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_USERNAME": { 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_USERNAME", 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "admin", 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.284 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_FRONTEND_API_URL": { 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_FRONTEND_API_URL", 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.285 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.286 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_UPDATE_DASHBOARDS": { 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_UPDATE_DASHBOARDS", 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.287 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.288 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ISCSI_API_SSL_VERIFICATION": { 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ISCSI_API_SSL_VERIFICATION", 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.289 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.290 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PROMETHEUS_API_HOST": { 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PROMETHEUS_API_HOST", 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.291 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PROMETHEUS_API_SSL_VERIFY": { 2022-04-28T19:34:52.292 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PROMETHEUS_API_SSL_VERIFY", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.293 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.294 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.295 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.296 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.297 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.298 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.299 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.300 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.301 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2022-04-28T19:34:52.302 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.303 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.304 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.305 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_ENABLED": { 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_ENABLED", 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.306 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_EXCLUSION_LIST": { 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_EXCLUSION_LIST", 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.307 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.308 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_MIN_COMPLEXITY": { 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_MIN_COMPLEXITY", 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.309 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_MIN_LENGTH": { 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_MIN_LENGTH", 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.310 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8", 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.311 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "REST_REQUESTS_TIMEOUT": { 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "REST_REQUESTS_TIMEOUT", 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "45", 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.312 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_ACCESS_KEY": { 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_ACCESS_KEY", 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.313 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.314 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_ADMIN_RESOURCE": { 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_ADMIN_RESOURCE", 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "admin", 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.315 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_HOST": { 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_HOST", 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.316 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.317 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_PORT": { 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_PORT", 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "80", 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.318 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_SCHEME": { 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_SCHEME", 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.319 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "http", 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.320 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_SECRET_KEY": { 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_SECRET_KEY", 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.321 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_SSL_VERIFY": { 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_SSL_VERIFY", 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.322 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.323 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_USER_ID": { 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_USER_ID", 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.324 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "USER_PWD_EXPIRATION_SPAN": { 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "USER_PWD_EXPIRATION_SPAN", 2022-04-28T19:34:52.325 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.326 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "USER_PWD_EXPIRATION_WARNING_1": { 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "USER_PWD_EXPIRATION_WARNING_1", 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:34:52.327 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "USER_PWD_EXPIRATION_WARNING_2": { 2022-04-28T19:34:52.328 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "USER_PWD_EXPIRATION_WARNING_2", 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.329 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crt_file": { 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crt_file", 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.330 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.331 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug": { 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "debug", 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.332 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Enable/disable debug options", 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "jwt_token_ttl": { 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "jwt_token_ttl", 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.333 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "28800", 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.334 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "key_file": { 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "key_file", 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.335 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.336 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.337 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.338 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.339 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.340 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.341 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.342 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.343 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "motd": { 2022-04-28T19:34:52.344 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "motd", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "The message of the day", 2022-04-28T19:34:52.345 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_addr": { 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_addr", 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.346 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "::", 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.347 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_port": { 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_port", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8080", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.348 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssl": { 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssl", 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.349 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.350 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssl_server_port": { 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssl_server_port", 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8443", 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.351 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "standby_behaviour": { 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "standby_behaviour", 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.352 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "redirect", 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "redirect" 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.353 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "standby_error_status_code": { 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "standby_error_status_code", 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.354 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "500", 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "400", 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "599", 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.355 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "url_prefix": { 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "url_prefix", 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.356 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.357 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "devicehealth", 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enable_monitoring": { 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "enable_monitoring", 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.358 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "monitor device health metrics", 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.359 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.360 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.361 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.362 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.363 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.364 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.365 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.366 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.367 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mark_out_threshold": { 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mark_out_threshold", 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "2419200", 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.368 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "automatically mark OSD if it may fail before this long", 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pool_name": { 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pool_name", 2022-04-28T19:34:52.369 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "device_health_metrics", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "name of pool in which to store device health metrics", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.370 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "retention_period": { 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "retention_period", 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "15552000", 2022-04-28T19:34:52.371 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how long to retain device health metrics", 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "scrape_frequency": { 2022-04-28T19:34:52.372 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "scrape_frequency", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "86400", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to scrape device health metrics", 2022-04-28T19:34:52.373 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "self_heal": { 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "self_heal", 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.374 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "preemptively heal cluster around devices that may fail", 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.376 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to wake up and check device health", 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_threshold": { 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_threshold", 2022-04-28T19:34:52.377 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "7257600", 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise health warning if OSD may fail before this long", 2022-04-28T19:34:52.378 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "diskprediction_local", 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.379 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.380 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.381 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.382 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.383 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.384 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.385 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.386 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.387 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "predict_interval": { 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "predict_interval", 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.388 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "86400", 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.389 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "predictor_model": { 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "predictor_model", 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "prophetstor", 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.390 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.391 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.392 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "influx", 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": false, 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "influxdb python module not found", 2022-04-28T19:34:52.393 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "batch_size": { 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "batch_size", 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5000", 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.394 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "database": { 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "database", 2022-04-28T19:34:52.395 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "ceph", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.396 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "hostname": { 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "hostname", 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.397 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.398 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "30", 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.399 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.400 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.401 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.402 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.403 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.404 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.405 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.406 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.407 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "password": { 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "password", 2022-04-28T19:34:52.408 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.409 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "port": { 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "port", 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8086", 2022-04-28T19:34:52.410 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.411 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssl": { 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssl", 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "false", 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.412 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "threads": { 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "threads", 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.413 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.414 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "username": { 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "username", 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.415 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "verify_ssl": { 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "verify_ssl", 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.416 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "true", 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.417 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "insights", 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.418 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.419 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.420 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.421 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.422 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.423 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.424 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.425 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.426 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "iostat", 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.427 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.428 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.429 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.430 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.431 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.432 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.432 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.432 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.432 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.432 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.432 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.433 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.433 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.433 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.433 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.433 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.433 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.434 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.435 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "k8sevents", 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ceph_event_retention_days": { 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ceph_event_retention_days", 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "7", 2022-04-28T19:34:52.436 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Days to hold ceph event information within local cache", 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "config_check_secs": { 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "config_check_secs", 2022-04-28T19:34:52.437 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "10", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "interval (secs) to check for cluster configuration changes", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.438 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.439 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.440 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.441 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.442 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.443 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.444 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "localpool", 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.445 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "failure_domain": { 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "failure_domain", 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "host", 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.446 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "failure domain for any created local pool", 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.447 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.448 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.449 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.450 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.451 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.452 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.453 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_size": { 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "min_size", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "default min_size for any created local pool", 2022-04-28T19:34:52.454 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_rep": { 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "num_rep", 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "3", 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.455 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "default replica count for any created local pool", 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pg_num": { 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pg_num", 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.456 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "128", 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "default pg_num for any created local pool", 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "prefix": { 2022-04-28T19:34:52.457 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "prefix", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "name prefix for any created local pool", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.458 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "subtree": { 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "subtree", 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "rack", 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.459 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "CRUSH level for which to create a local pool", 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "orchestrator", 2022-04-28T19:34:52.460 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.461 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.462 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.463 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.464 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.465 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.466 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.467 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "orchestrator": { 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "orchestrator", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "cephadm", 2022-04-28T19:34:52.468 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rook", 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "test_orchestrator" 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Orchestrator backend", 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.469 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "osd_support", 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.470 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.471 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.472 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.473 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.474 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.475 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.476 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pg_autoscaler", 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.477 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.478 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.479 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.480 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.481 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.482 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.483 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.484 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.485 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "progress", 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enabled": { 2022-04-28T19:34:52.486 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "enabled", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.487 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.488 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.489 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.490 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.491 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.492 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.493 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.494 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max_completed_events": { 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "max_completed_events", 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "50", 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.495 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "number of past completed events to remember", 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "persist_interval": { 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "persist_interval", 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.496 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to persist completed events", 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.497 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "prometheus", 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.498 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.499 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.500 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.501 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.502 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.503 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.504 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_stats_pools": { 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rbd_stats_pools", 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.505 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_stats_pools_refresh_interval": { 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rbd_stats_pools_refresh_interval", 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.506 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "300", 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.507 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "scrape_interval": { 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "scrape_interval", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.508 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_addr": { 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_addr", 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.509 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_port": { 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_port", 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.510 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "stale_cache_strategy": { 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "stale_cache_strategy", 2022-04-28T19:34:52.511 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.512 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rbd_support", 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.513 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.514 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.515 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.516 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.517 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.518 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.519 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max_concurrent_snap_create": { 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "max_concurrent_snap_create", 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.520 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mirror_snapshot_schedule": { 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mirror_snapshot_schedule", 2022-04-28T19:34:52.521 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.522 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "trash_purge_schedule": { 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "trash_purge_schedule", 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.523 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "restful", 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.524 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "key_file": { 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "key_file", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.525 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.526 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.527 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.528 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.529 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.530 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.531 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.532 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_addr": { 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_addr", 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.533 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_port": { 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_port", 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.534 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.535 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rook", 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.536 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.537 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.538 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.539 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.540 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.541 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.542 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "selftest", 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.543 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.544 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.545 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.546 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.547 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.548 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.549 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.550 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "roption1": { 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "roption1", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.551 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "roption2": { 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "roption2", 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "xyz", 2022-04-28T19:34:52.552 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption1": { 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption1", 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.553 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.554 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption2": { 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption2", 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.555 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption3": { 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption3", 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "float", 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.556 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption4": { 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption4", 2022-04-28T19:34:52.557 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.558 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption5": { 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption5", 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.559 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption6": { 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption6", 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.560 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.561 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "testkey": { 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "testkey", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.562 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "testlkey": { 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "testlkey", 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.563 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "testnewline": { 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "testnewline", 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.564 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.565 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "status", 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.566 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.567 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.568 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.569 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.570 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.571 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.572 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "telegraf", 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "address": { 2022-04-28T19:34:52.573 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "address", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "unixgram:///tmp/telegraf.sock", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.574 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "15", 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.575 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.576 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.577 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.578 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.579 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.580 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.581 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.582 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "telemetry", 2022-04-28T19:34:52.583 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_basic": { 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_basic", 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.584 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Share basic cluster information (size, version)", 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_crash": { 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_crash", 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.585 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.586 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_device": { 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_device", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.587 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_ident": { 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_ident", 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.588 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "contact": { 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "contact", 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.589 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.590 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "description": { 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "description", 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.591 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "device_url": { 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "device_url", 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.592 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "https://telemetry.ceph.com/device", 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enabled": { 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "enabled", 2022-04-28T19:34:52.593 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.594 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.597 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "24", 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "8", 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.598 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "last_opt_revision": { 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "last_opt_revision", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "1", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.599 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "leaderboard": { 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "leaderboard", 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.600 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.601 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.602 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.603 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.604 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.605 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.606 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.607 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "organization": { 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "organization", 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.608 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.609 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.609 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.609 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "proxy": { 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "proxy", 2022-04-28T19:34:52.610 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.611 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "url": { 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "url", 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "https://telemetry.ceph.com/report", 2022-04-28T19:34:52.612 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.613 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.614 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.614 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.614 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.614 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.615 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.615 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.615 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.615 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.616 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.616 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "test_orchestrator", 2022-04-28T19:34:52.616 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.616 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.617 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.618 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.618 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.618 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.618 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.618 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.618 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.619 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.620 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.620 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.620 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.620 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.620 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.620 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.621 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.622 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.623 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.624 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.625 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "volumes", 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.626 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.627 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.628 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.629 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.630 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.631 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.632 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max_concurrent_clones": { 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "max_concurrent_clones", 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.633 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "4", 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Number of asynchronous cloner threads", 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "snapshot_clone_delay": { 2022-04-28T19:34:52.634 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "snapshot_clone_delay", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.635 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix", 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "discovery_interval": { 2022-04-28T19:34:52.636 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "discovery_interval", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "100", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.637 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "identifier": { 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "identifier", 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.638 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:34:52.639 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.640 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.641 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.642 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.643 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:34:52.644 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.645 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:34:52.646 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "zabbix_host": { 2022-04-28T19:34:52.647 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix_host", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "zabbix_port": { 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix_port", 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10051", 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.649 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "zabbix_sender": { 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix_sender", 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:34:52.650 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "/usr/bin/zabbix_sender", 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:34:52.651 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {}, 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "always_on_modules": { 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nautilus": [ 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "balancer", 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crash", 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "devicehealth", 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "orchestrator_cli", 2022-04-28T19:34:52.652 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_support", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "volumes" 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "octopus": [ 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "balancer", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crash", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "devicehealth", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "orchestrator", 2022-04-28T19:34:52.653 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pg_autoscaler", 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress", 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_support", 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status", 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "telemetry", 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "volumes" 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "last_failure_osd_epoch": 0, 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_clients": [ 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.654 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addrvec": [ 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v2", 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:0", 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 1269652554 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addrvec": [ 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:52.655 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v2", 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:0", 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 3662111215 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.656 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:52.657 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:51 smithi059 bash[23786]: audit 2022-04-28T19:34:51.508920+0000 mon.a (mon.0) 30 : audit [DBG] from='client.? 172.21.15.59:0/3795601956' entity='client.admin' cmd=[{"prefix": "mgr dump"}]: dispatch 2022-04-28T19:34:52.657 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for the mgr to restart... 2022-04-28T19:34:52.657 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for Mgr epoch 5... 2022-04-28T19:34:52.658 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus tell mgr mgr_status 2022-04-28T19:34:55.269 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:34:55.270 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2022-04-28T19:34:55.270 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "initialized": true 2022-04-28T19:34:55.270 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:34:55.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: cluster 2022-04-28T19:34:54.197197+0000 mon.a (mon.0) 31 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:34:55.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: cluster 2022-04-28T19:34:54.198026+0000 mon.a (mon.0) 32 : cluster [INF] Activating manager daemon y 2022-04-28T19:34:55.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: cluster 2022-04-28T19:34:54.201564+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2022-04-28T19:34:55.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: cluster 2022-04-28T19:34:54.253263+0000 mon.a (mon.0) 34 : cluster [DBG] mgrmap e6: y(active, starting, since 0.0553859s) 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.254886+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.256710+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.258131+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.258567+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.258941+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.259760+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.261572+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.262430+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.263241+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:34:55.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.263941+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: cluster 2022-04-28T19:34:54.267914+0000 mon.a (mon.0) 45 : cluster [INF] Manager daemon y is now available 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.288690+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.295563+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.302079+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.444150+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.453391+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.463419+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:34:55.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:55 smithi059 bash[23786]: audit 2022-04-28T19:34:54.466142+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:34:55.842 INFO:teuthology.orchestra.run.smithi059.stderr:Mgr epoch 5 is available 2022-04-28T19:34:55.843 INFO:teuthology.orchestra.run.smithi059.stderr:Setting orchestrator backend to cephadm... 2022-04-28T19:34:55.843 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus orch set backend cephadm 2022-04-28T19:34:56.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:56 smithi059 bash[23786]: audit 2022-04-28T19:34:55.256462+0000 mgr.y (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2022-04-28T19:34:56.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:56 smithi059 bash[23786]: cluster 2022-04-28T19:34:55.258165+0000 mon.a (mon.0) 53 : cluster [DBG] mgrmap e7: y(active, since 1.06027s) 2022-04-28T19:34:56.561 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:56 smithi059 bash[23786]: audit 2022-04-28T19:34:55.273473+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2022-04-28T19:34:57.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:57 smithi059 bash[23786]: cluster 2022-04-28T19:34:56.262776+0000 mon.a (mon.0) 54 : cluster [DBG] mgrmap e8: y(active, since 2s) 2022-04-28T19:34:57.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:57 smithi059 bash[23786]: audit 2022-04-28T19:34:57.205096+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:57.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:57 smithi059 bash[23786]: audit 2022-04-28T19:34:57.229952+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:34:57.724 INFO:teuthology.orchestra.run.smithi059.stderr:Generating ssh key... 2022-04-28T19:34:57.747 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus cephadm generate-key 2022-04-28T19:34:58.539 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:34:58 smithi059 bash[23786]: audit 2022-04-28T19:34:57.196570+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:34:59.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: Generating public/private rsa key pair. 2022-04-28T19:34:59.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: Your identification has been saved in /tmp/tmpcl3bhnfe/key. 2022-04-28T19:34:59.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: Your public key has been saved in /tmp/tmpcl3bhnfe/key.pub. 2022-04-28T19:34:59.836 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: The key fingerprint is: 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: SHA256:vAeKyeqA//HG9MSIOJLa2d/d3D1R/cqKzM2OPckr7LM ceph-20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: The key's randomart image is: 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: +---[RSA 3072]----+ 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: | | 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: | .| 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: | . o| 2022-04-28T19:34:59.837 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: | . . . oS o| 2022-04-28T19:34:59.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: |+ o..oo.oo ..| 2022-04-28T19:34:59.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: |+o +=o.o.... o ..| 2022-04-28T19:34:59.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: |.oo..oo..=+O+.oo | 2022-04-28T19:34:59.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: | .+..oo ..E=X+. o| 2022-04-28T19:34:59.838 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:34:59 smithi059 bash[24088]: +----[SHA256]-----+ 2022-04-28T19:35:00.030 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus cephadm get-pub-key 2022-04-28T19:35:00.807 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:00 smithi059 bash[23786]: cephadm 2022-04-28T19:34:59.027595+0000 mgr.y (mgr.14116) 5 : cephadm [INF] Generating ssh key... 2022-04-28T19:35:00.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:00 smithi059 bash[23786]: audit 2022-04-28T19:34:59.494164+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:35:00.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:00 smithi059 bash[23786]: audit 2022-04-28T19:34:59.497945+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:35:01.375 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChdH5x6EhnSgLvKjYFnwih45ryDMf7mZm+fEnJ8JChTWnyxy2LvPhoeNrZfwGlq1e4i0jXIDf67NWc2/hQKxRNxPlP7AZn+yzK31bgygcfEbe8tEhgTiwepgn7rNpaoKW3VPRVAHc9tGLVPeCBbsXGlhCaOttK7l4wxCnOPzNoQjymVW/laC/X9Fp/rBzStK9/5XDYoXtWr2V4SkTGjq0mHzPFwVJWehJ8/6WmytOOUxZ0N3B9lkXAsxpTphbTv1HhyseMJ/guJSg62S1TODlfYNsIN7laY1nxOeHJGAjjldlCfBDYMsrwttpVr+OksGhJoSVxH3COfLsMc3rBpWHFuKbXUyLqLTpbeH7mqBPh6VcoRKLRcCUXeyXjl0oik0Y+YJ2GgC3z3aZ+hqXeTo426Zxw/AdLprCMJlldNqDiWX0xKnJSvXpxIJInhSt/aqUWPJOJXt7m9QM0EZBYBbiDsp0eGN4xZ9NgtKFNTd3W3RfIp6GjNsESwi/7LcOFI/E= ceph-20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:35:01.928 INFO:teuthology.orchestra.run.smithi059.stderr:Wrote public SSH key to to /home/ubuntu/cephtest/ceph.pub 2022-04-28T19:35:01.928 INFO:teuthology.orchestra.run.smithi059.stderr:Adding key to root@localhost's authorized_keys... 2022-04-28T19:35:01.929 INFO:teuthology.orchestra.run.smithi059.stderr:Adding host smithi059... 2022-04-28T19:35:01.929 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus orch host add smithi059 2022-04-28T19:35:02.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:01 smithi059 bash[23786]: audit 2022-04-28T19:35:01.379300+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:35:03.835 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:35:03 smithi059 bash[24088]: Warning: Permanently added 'smithi059,172.21.15.59' (ECDSA) to the list of known hosts. 2022-04-28T19:35:03.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:03 smithi059 bash[23786]: audit 2022-04-28T19:35:03.451913+0000 mgr.y (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi059", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:35:04.182 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout Added host 'smithi059' 2022-04-28T19:35:05.033 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus config set mgr mgr/cephadm/container_init False --force 2022-04-28T19:35:05.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:05 smithi059 bash[23786]: audit 2022-04-28T19:35:04.185549+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:35:05.588 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:05 smithi059 bash[23786]: cephadm 2022-04-28T19:35:04.186208+0000 mgr.y (mgr.14116) 8 : cephadm [INF] Added host smithi059 2022-04-28T19:35:05.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:05 smithi059 bash[23786]: audit 2022-04-28T19:35:04.582195+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:35:05.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:05 smithi059 bash[23786]: audit 2022-04-28T19:35:04.583644+0000 mon.a (mon.0) 61 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:35:06.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:06 smithi059 bash[23786]: audit 2022-04-28T19:35:05.214847+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' 2022-04-28T19:35:06.584 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:06 smithi059 bash[23786]: audit 2022-04-28T19:35:05.215960+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.14116 172.21.15.59:0/79231820' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:35:07.689 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus config set mgr mgr/dashboard/ssl_server_port 8443 2022-04-28T19:35:08.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:07 smithi059 bash[23786]: audit 2022-04-28T19:35:06.944895+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.59:0/1043698047' entity='client.admin' 2022-04-28T19:35:09.703 INFO:teuthology.orchestra.run.smithi059.stderr:Enabling the dashboard module... 2022-04-28T19:35:09.703 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus mgr module enable dashboard 2022-04-28T19:35:10.467 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:10 smithi059 bash[23786]: audit 2022-04-28T19:35:09.193707+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.59:0/3417869460' entity='client.admin' 2022-04-28T19:35:11.585 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:11 smithi059 bash[23786]: audit 2022-04-28T19:35:11.114051+0000 mon.a (mon.0) 66 : audit [INF] from='client.? 172.21.15.59:0/1836264567' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2022-04-28T19:35:11.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:35:11 smithi059 bash[24088]: ignoring --setuser ceph since I am not root 2022-04-28T19:35:11.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:35:11 smithi059 bash[24088]: ignoring --setgroup ceph since I am not root 2022-04-28T19:35:11.586 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:35:11 smithi059 bash[24088]: debug 2022-04-28T19:35:11.229+0000 7f6282805700 1 -- 172.21.15.59:0/1409848246 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fab32cc4e0 con 0x55fab33b4800 2022-04-28T19:35:11.735 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus mgr dump 2022-04-28T19:35:12.541 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:12 smithi059 bash[23786]: audit 2022-04-28T19:35:11.195876+0000 mon.a (mon.0) 67 : audit [INF] from='client.? 172.21.15.59:0/1836264567' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2022-04-28T19:35:12.541 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:12 smithi059 bash[23786]: cluster 2022-04-28T19:35:11.195969+0000 mon.a (mon.0) 68 : cluster [DBG] mgrmap e9: y(active, since 16s) 2022-04-28T19:35:13.195 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.195 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "epoch": 9, 2022-04-28T19:35:13.196 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_gid": 14116, 2022-04-28T19:35:13.196 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_name": "y", 2022-04-28T19:35:13.196 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_addrs": { 2022-04-28T19:35:13.196 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addrvec": [ 2022-04-28T19:35:13.196 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.196 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v2", 2022-04-28T19:35:13.198 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:6800", 2022-04-28T19:35:13.198 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 2735512980 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v1", 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:6801", 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 2735512980 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_addr": "172.21.15.59:6801/2735512980", 2022-04-28T19:35:13.199 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_change": "2022-04-28T19:34:54.197856+0000", 2022-04-28T19:35:13.200 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_mgr_features": 4540138292840890367, 2022-04-28T19:35:13.200 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available": true, 2022-04-28T19:35:13.200 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "standbys": [], 2022-04-28T19:35:13.200 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "modules": [ 2022-04-28T19:35:13.200 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "cephadm", 2022-04-28T19:35:13.200 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "dashboard", 2022-04-28T19:35:13.681 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "iostat", 2022-04-28T19:35:13.681 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "restful" 2022-04-28T19:35:13.681 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.681 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "available_modules": [ 2022-04-28T19:35:13.681 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "alerts", 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.682 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "How frequently to reexamine health status", 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.683 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.684 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.685 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.686 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.687 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.688 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.689 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_destination": { 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_destination", 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.690 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Email address to send alerts to", 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_from_name": { 2022-04-28T19:35:13.691 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_from_name", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "Ceph", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Email From: name", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.692 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_host": { 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_host", 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.693 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "SMTP server", 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_password": { 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_password", 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.694 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Password to authenticate with", 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_port": { 2022-04-28T19:35:13.695 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_port", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "465", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "SMTP port", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.696 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_sender": { 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_sender", 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.697 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "SMTP envelope sender", 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_ssl": { 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_ssl", 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.698 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Use SSL to connect to SMTP server", 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "smtp_user": { 2022-04-28T19:35:13.699 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "smtp_user", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "User to authenticate as", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.700 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "balancer", 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active": { 2022-04-28T19:35:13.701 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "active", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "automatically balance PGs across cluster", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.702 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "begin_time": { 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "begin_time", 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0000", 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.703 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "beginning time of day to automatically balance", 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "This is a time of day in the format HHMM.", 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "begin_weekday": { 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "begin_weekday", 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.704 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "0", 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "7", 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Restrict automatic balancing to this day of the week or later", 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "0 or 7 = Sunday, 1 = Monday, etc.", 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush_compat_max_iterations": { 2022-04-28T19:35:13.705 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crush_compat_max_iterations", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "25", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "1", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "250", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "maximum number of iterations to attempt optimization", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.706 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush_compat_metrics": { 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crush_compat_metrics", 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "pgs,objects,bytes", 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.707 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "metrics with which to calculate OSD utilization", 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush_compat_step": { 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crush_compat_step", 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "float", 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.708 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0.5", 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "0.001", 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "0.999", 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "aggressiveness of optimization", 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": ".99 is very aggressive, .01 is less aggressive", 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "end_time": { 2022-04-28T19:35:13.709 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "end_time", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "2400", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "ending time of day to automatically balance", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "This is a time of day in the format HHMM.", 2022-04-28T19:35:13.710 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "end_weekday": { 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "end_weekday", 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "7", 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "0", 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "7", 2022-04-28T19:35:13.711 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Restrict automatic balancing to days of the week earlier than this", 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "0 or 7 = Sunday, 1 = Monday, etc.", 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.712 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.713 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.714 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.715 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.716 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.717 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.718 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_score": { 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "min_score", 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "float", 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.719 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "minimum score, below which no optimization is attempted", 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mode": { 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mode", 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.720 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "none", 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crush-compat", 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "none", 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "upmap" 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Balancer mode", 2022-04-28T19:35:13.721 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pool_ids": { 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pool_ids", 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.722 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "pools which the automatic balancing will be limited to", 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.723 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to wake up and attempt optimization", 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.724 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "upmap_max_deviation": { 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "upmap_max_deviation", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "1", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "deviation below which no optimization is attempted", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2022-04-28T19:35:13.725 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "upmap_max_optimizations": { 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "upmap_max_optimizations", 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "uint", 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.726 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "maximum upmap optimizations to make per attempt", 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "cephadm", 2022-04-28T19:35:13.727 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "allow_ptrace": { 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "allow_ptrace", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.728 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "allow SYS_PTRACE capability on ceph containers", 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "config_dashboard": { 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "config_dashboard", 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.729 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "manage configs like API endpoints in Dashboard.", 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_alertmanager": { 2022-04-28T19:35:13.730 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_alertmanager", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/prometheus/alertmanager:v0.20.0", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.731 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_base": { 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_base", 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/ceph/ceph", 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.732 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Container image name, without the tag", 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_grafana": { 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_grafana", 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.733 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/ceph/ceph-grafana:6.7.4", 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_node_exporter": { 2022-04-28T19:35:13.734 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_node_exporter", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/prometheus/node-exporter:v0.18.1", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.735 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_image_prometheus": { 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_image_prometheus", 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "quay.io/prometheus/prometheus:v2.18.1", 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.736 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Prometheus container image", 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "container_init": { 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "container_init", 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.737 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Run podman/docker with `--init`", 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.738 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "daemon_cache_timeout": { 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "daemon_cache_timeout", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "seconds to cache service (daemon) inventory", 2022-04-28T19:35:13.739 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "device_cache_timeout": { 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "device_cache_timeout", 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "1800", 2022-04-28T19:35:13.740 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "seconds to cache device inventory", 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "facts_cache_timeout": { 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "facts_cache_timeout", 2022-04-28T19:35:13.741 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "seconds to cache host facts data", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.742 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "host_check_interval": { 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "host_check_interval", 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.743 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to perform a host check", 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.744 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.745 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.746 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "log to the \"cephadm\" cluster log channel\"", 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.747 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.748 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.749 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.750 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "manage_etc_ceph_ceph_conf": { 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "manage_etc_ceph_ceph_conf", 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.751 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "migration_current": { 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "migration_current", 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.752 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "internal - do not modify", 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mode": { 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mode", 2022-04-28T19:35:13.753 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "root", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "cephadm-package", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "root" 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "mode for remote execution of cephadm", 2022-04-28T19:35:13.754 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "prometheus_alerts_path": { 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "prometheus_alerts_path", 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.755 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "location of alerts to include in prometheus deployments", 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "registry_password": { 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "registry_password", 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.756 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Custom repository password", 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.757 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "registry_url": { 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "registry_url", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Custom repository url", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.758 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "registry_username": { 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "registry_username", 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.759 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Custom repository username", 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssh_config_file": { 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssh_config_file", 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.760 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "customized SSH config file to connect to managed hosts", 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.761 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "use_repo_digest": { 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "use_repo_digest", 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.762 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_on_failed_host_check": { 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_on_failed_host_check", 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.763 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise a health warning if the host check fails", 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_on_stray_daemons": { 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_on_stray_daemons", 2022-04-28T19:35:13.764 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.765 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_on_stray_hosts": { 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_on_stray_hosts", 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.766 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crash", 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.767 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.768 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.769 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.770 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.771 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.772 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.773 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "retain_interval": { 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "retain_interval", 2022-04-28T19:35:13.774 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "31536000", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how long to retain crashes before pruning them", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.775 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_recent_interval": { 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_recent_interval", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "1209600", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "time interval in which to warn about recent crashes", 2022-04-28T19:35:13.776 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "dashboard", 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.777 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ACCOUNT_LOCKOUT_ATTEMPTS": { 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.778 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ALERTMANAGER_API_HOST": { 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ALERTMANAGER_API_HOST", 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.779 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ALERTMANAGER_API_SSL_VERIFY": { 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ALERTMANAGER_API_SSL_VERIFY", 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.780 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.781 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "AUDIT_API_ENABLED": { 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "AUDIT_API_ENABLED", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.782 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "AUDIT_API_LOG_PAYLOAD": { 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "AUDIT_API_LOG_PAYLOAD", 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.783 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ENABLE_BROWSABLE_API": { 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ENABLE_BROWSABLE_API", 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.784 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_cephfs": { 2022-04-28T19:35:13.785 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_cephfs", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.786 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_iscsi": { 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_iscsi", 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.787 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_mirroring": { 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_mirroring", 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.788 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_nfs": { 2022-04-28T19:35:13.789 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_nfs", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.790 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_rbd": { 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_rbd", 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.791 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "FEATURE_TOGGLE_rgw": { 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "FEATURE_TOGGLE_rgw", 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.792 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2022-04-28T19:35:13.793 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.794 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_PASSWORD": { 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_PASSWORD", 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "admin", 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.795 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_SSL_VERIFY": { 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_SSL_VERIFY", 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.796 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_URL": { 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_URL", 2022-04-28T19:35:13.797 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.798 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_API_USERNAME": { 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_API_USERNAME", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "admin", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.799 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_FRONTEND_API_URL": { 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_FRONTEND_API_URL", 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.800 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "GRAFANA_UPDATE_DASHBOARDS": { 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "GRAFANA_UPDATE_DASHBOARDS", 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.802 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ISCSI_API_SSL_VERIFICATION": { 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ISCSI_API_SSL_VERIFICATION", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.803 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PROMETHEUS_API_HOST": { 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PROMETHEUS_API_HOST", 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.804 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PROMETHEUS_API_SSL_VERIFY": { 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PROMETHEUS_API_SSL_VERIFY", 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.805 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.806 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.807 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.808 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.809 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2022-04-28T19:35:13.810 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.811 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.812 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2022-04-28T19:35:13.813 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.814 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.815 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_ENABLED": { 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_ENABLED", 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.816 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_EXCLUSION_LIST": { 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_EXCLUSION_LIST", 2022-04-28T19:35:13.817 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.818 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_MIN_COMPLEXITY": { 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_MIN_COMPLEXITY", 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.819 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "PWD_POLICY_MIN_LENGTH": { 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "PWD_POLICY_MIN_LENGTH", 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.820 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8", 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "REST_REQUESTS_TIMEOUT": { 2022-04-28T19:35:13.821 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "REST_REQUESTS_TIMEOUT", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "45", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.822 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_ACCESS_KEY": { 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_ACCESS_KEY", 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.823 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_ADMIN_RESOURCE": { 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_ADMIN_RESOURCE", 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.824 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "admin", 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_HOST": { 2022-04-28T19:35:13.825 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_HOST", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.826 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_PORT": { 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_PORT", 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "80", 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.827 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_SCHEME": { 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_SCHEME", 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.828 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "http", 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_SECRET_KEY": { 2022-04-28T19:35:13.829 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_SECRET_KEY", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.830 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_SSL_VERIFY": { 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_SSL_VERIFY", 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.831 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "RGW_API_USER_ID": { 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "RGW_API_USER_ID", 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.832 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "USER_PWD_EXPIRATION_SPAN": { 2022-04-28T19:35:13.833 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "USER_PWD_EXPIRATION_SPAN", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.834 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "USER_PWD_EXPIRATION_WARNING_1": { 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "USER_PWD_EXPIRATION_WARNING_1", 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.835 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "USER_PWD_EXPIRATION_WARNING_2": { 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "USER_PWD_EXPIRATION_WARNING_2", 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.836 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crt_file": { 2022-04-28T19:35:13.837 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "crt_file", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.838 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug": { 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "debug", 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.839 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Enable/disable debug options", 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "jwt_token_ttl": { 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "jwt_token_ttl", 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.840 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "28800", 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.841 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "key_file": { 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "key_file", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.842 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.843 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.844 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.845 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.846 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.847 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.848 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.849 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "motd": { 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "motd", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "The message of the day", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.850 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_addr": { 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_addr", 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "::", 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.851 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_port": { 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_port", 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.852 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8080", 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssl": { 2022-04-28T19:35:13.853 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssl", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.854 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssl_server_port": { 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssl_server_port", 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8443", 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.855 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "standby_behaviour": { 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "standby_behaviour", 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.856 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "redirect", 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "redirect" 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.857 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "standby_error_status_code": { 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "standby_error_status_code", 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "500", 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "400", 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "599", 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.858 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "url_prefix": { 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "url_prefix", 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.859 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.860 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "devicehealth", 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enable_monitoring": { 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "enable_monitoring", 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.861 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "monitor device health metrics", 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.862 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.863 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.864 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.865 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.866 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.867 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.868 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mark_out_threshold": { 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mark_out_threshold", 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "2419200", 2022-04-28T19:35:13.869 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "automatically mark OSD if it may fail before this long", 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pool_name": { 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pool_name", 2022-04-28T19:35:13.870 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "device_health_metrics", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "name of pool in which to store device health metrics", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.871 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "retention_period": { 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "retention_period", 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "15552000", 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.872 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how long to retain device health metrics", 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "scrape_frequency": { 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "scrape_frequency", 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "86400", 2022-04-28T19:35:13.873 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to scrape device health metrics", 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "self_heal": { 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "self_heal", 2022-04-28T19:35:13.874 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "preemptively heal cluster around devices that may fail", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.875 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to wake up and check device health", 2022-04-28T19:35:13.876 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warn_threshold": { 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "warn_threshold", 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "7257600", 2022-04-28T19:35:13.877 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "raise health warning if OSD may fail before this long", 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.878 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "diskprediction_local", 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.879 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.881 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.882 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.883 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.884 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.885 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "predict_interval": { 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "predict_interval", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "86400", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.886 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "predictor_model": { 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "predictor_model", 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "prophetstor", 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.887 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.888 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "600", 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.889 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "influx", 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": false, 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "influxdb python module not found", 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "batch_size": { 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "batch_size", 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.890 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5000", 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.891 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "database": { 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "database", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "ceph", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.892 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "hostname": { 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "hostname", 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.893 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.894 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "30", 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.895 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.896 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.897 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.898 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.899 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.900 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.901 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "password": { 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "password", 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.902 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.903 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "port": { 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "port", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "8086", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.904 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ssl": { 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ssl", 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "false", 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.905 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "threads": { 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "threads", 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.906 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.907 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "username": { 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "username", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.908 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "verify_ssl": { 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "verify_ssl", 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "true", 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.909 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.910 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "insights", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.911 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.913 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.914 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.915 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.916 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.917 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "iostat", 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.918 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.919 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.920 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.921 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.922 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.923 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.924 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "k8sevents", 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.925 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "ceph_event_retention_days": { 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "ceph_event_retention_days", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "7", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Days to hold ceph event information within local cache", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.926 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "config_check_secs": { 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "config_check_secs", 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "10", 2022-04-28T19:35:13.927 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "interval (secs) to check for cluster configuration changes", 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.928 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.929 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.930 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.931 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.932 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.933 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.934 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "localpool", 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "failure_domain": { 2022-04-28T19:35:13.935 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "failure_domain", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "host", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "failure domain for any created local pool", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.936 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.937 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.938 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.939 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.940 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.941 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.942 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.943 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min_size": { 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "min_size", 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.944 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "default min_size for any created local pool", 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "num_rep": { 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "num_rep", 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.945 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "3", 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "default replica count for any created local pool", 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.946 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pg_num": { 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pg_num", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "128", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "default pg_num for any created local pool", 2022-04-28T19:35:13.947 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "prefix": { 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "prefix", 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.948 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "name prefix for any created local pool", 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "subtree": { 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "subtree", 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.949 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "rack", 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "CRUSH level for which to create a local pool", 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.950 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "orchestrator", 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.951 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.952 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.953 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.954 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.955 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.956 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.957 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "orchestrator": { 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "orchestrator", 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.958 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "cephadm", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rook", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "test_orchestrator" 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Orchestrator backend", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.959 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "osd_support", 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.960 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.961 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.962 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.963 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.964 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.965 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.966 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "pg_autoscaler", 2022-04-28T19:35:13.967 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.968 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.969 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.970 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.971 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.972 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.973 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.974 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "sleep_interval": { 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "sleep_interval", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.975 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "progress", 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.976 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enabled": { 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "enabled", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.977 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.978 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.979 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.980 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.981 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.982 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.983 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max_completed_events": { 2022-04-28T19:35:13.984 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "max_completed_events", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "50", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "number of past completed events to remember", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.985 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "persist_interval": { 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "persist_interval", 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "5", 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "how frequently to persist completed events", 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "prometheus", 2022-04-28T19:35:13.987 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.988 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.989 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.990 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.991 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:13.992 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:13.993 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.994 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_stats_pools": { 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rbd_stats_pools", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.995 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_stats_pools_refresh_interval": { 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rbd_stats_pools_refresh_interval", 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "300", 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.996 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "scrape_interval": { 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "scrape_interval", 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.997 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:13.998 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_addr": { 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_addr", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:13.999 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_port": { 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_port", 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.000 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "stale_cache_strategy": { 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "stale_cache_strategy", 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.001 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.002 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rbd_support", 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.003 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.004 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.005 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.006 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.007 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.008 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.009 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max_concurrent_snap_create": { 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "max_concurrent_snap_create", 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10", 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.010 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mirror_snapshot_schedule": { 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "mirror_snapshot_schedule", 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.011 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.012 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "trash_purge_schedule": { 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "trash_purge_schedule", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.013 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "restful", 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "key_file": { 2022-04-28T19:35:14.014 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "key_file", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.015 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.016 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.017 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.018 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.019 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.020 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.021 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.022 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_addr": { 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_addr", 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.023 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "server_port": { 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "server_port", 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.024 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.025 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rook", 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.026 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.027 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.028 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.029 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.030 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.031 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.032 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "selftest", 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.033 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.034 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.035 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.036 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.037 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.038 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.039 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "roption1": { 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "roption1", 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.040 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.041 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "roption2": { 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "roption2", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "xyz", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.042 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption1": { 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption1", 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.043 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption2": { 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption2", 2022-04-28T19:35:14.044 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.045 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption3": { 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption3", 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "float", 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.046 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption4": { 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption4", 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.047 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption5": { 2022-04-28T19:35:14.048 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption5", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.049 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rwoption6": { 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "rwoption6", 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.050 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "testkey": { 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "testkey", 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.051 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "testlkey": { 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "testlkey", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.052 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "testnewline": { 2022-04-28T19:35:14.053 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "testnewline", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.054 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "status", 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.055 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.056 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.057 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.058 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.059 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.060 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.061 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "telegraf", 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.062 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "address": { 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "address", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "unixgram:///tmp/telegraf.sock", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.063 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:14.064 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "15", 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.065 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.066 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.067 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.068 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.069 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.070 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.071 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.072 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "telemetry", 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_basic": { 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_basic", 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.073 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Share basic cluster information (size, version)", 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_crash": { 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_crash", 2022-04-28T19:35:14.074 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.075 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_device": { 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_device", 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "True", 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.076 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "channel_ident": { 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "channel_ident", 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.077 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "contact": { 2022-04-28T19:35:14.078 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "contact", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.079 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "description": { 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "description", 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.080 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "device_url": { 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "device_url", 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.081 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "https://telemetry.ceph.com/device", 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enabled": { 2022-04-28T19:35:14.082 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "enabled", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.083 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "24", 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "8", 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.084 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "last_opt_revision": { 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "last_opt_revision", 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.085 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "1", 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "leaderboard": { 2022-04-28T19:35:14.086 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "leaderboard", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.087 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.088 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.089 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.090 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.091 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.092 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.093 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "organization": { 2022-04-28T19:35:14.094 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "organization", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.095 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "proxy": { 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "proxy", 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.096 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "url": { 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "url", 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.097 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "https://telemetry.ceph.com/report", 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.098 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "test_orchestrator", 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.099 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.100 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.101 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.102 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.103 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.104 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.105 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "volumes", 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.106 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.107 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.108 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.109 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.110 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.111 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.112 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max_concurrent_clones": { 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "max_concurrent_clones", 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.113 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "4", 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Number of asynchronous cloner threads", 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "snapshot_clone_delay": { 2022-04-28T19:35:14.114 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "snapshot_clone_delay", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "0", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.115 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix", 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "can_run": true, 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error_string": "", 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "module_options": { 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "discovery_interval": { 2022-04-28T19:35:14.116 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "discovery_interval", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "100", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.117 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "identifier": { 2022-04-28T19:35:14.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "identifier", 2022-04-28T19:35:14.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.118 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.119 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.120 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "interval": { 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "interval", 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "secs", 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "60", 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.121 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_level": { 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_level", 2022-04-28T19:35:14.122 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "", 2022-04-28T19:35:14.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.123 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.124 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.125 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster": { 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster", 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.126 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.127 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_cluster_level": { 2022-04-28T19:35:14.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_cluster_level", 2022-04-28T19:35:14.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.128 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "info", 2022-04-28T19:35:14.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.129 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [ 2022-04-28T19:35:14.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "", 2022-04-28T19:35:14.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "critical", 2022-04-28T19:35:14.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "debug", 2022-04-28T19:35:14.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "error", 2022-04-28T19:35:14.130 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "info", 2022-04-28T19:35:14.131 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "warning" 2022-04-28T19:35:14.131 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.131 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.131 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.131 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.131 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.132 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.132 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "log_to_file": { 2022-04-28T19:35:14.132 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "log_to_file", 2022-04-28T19:35:14.132 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "bool", 2022-04-28T19:35:14.132 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.133 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 1, 2022-04-28T19:35:14.133 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "False", 2022-04-28T19:35:14.133 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.133 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.133 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.134 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "zabbix_host": { 2022-04-28T19:35:14.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix_host", 2022-04-28T19:35:14.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.135 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "None", 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.136 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "zabbix_port": { 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix_port", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "int", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "10051", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.137 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "zabbix_sender": { 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "name": "zabbix_sender", 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "str", 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "level": "advanced", 2022-04-28T19:35:14.138 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "flags": 0, 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "default_value": "/usr/bin/zabbix_sender", 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "min": "", 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "max": "", 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "enum_allowed": [], 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "desc": "", 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "long_desc": "", 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "tags": [], 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "see_also": [] 2022-04-28T19:35:14.139 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "services": {}, 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "always_on_modules": { 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nautilus": [ 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "balancer", 2022-04-28T19:35:14.140 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crash", 2022-04-28T19:35:14.141 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "devicehealth", 2022-04-28T19:35:14.141 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "orchestrator_cli", 2022-04-28T19:35:14.141 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress", 2022-04-28T19:35:14.141 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_support", 2022-04-28T19:35:14.141 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status", 2022-04-28T19:35:14.141 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "volumes" 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "octopus": [ 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "balancer", 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "crash", 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "devicehealth", 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "orchestrator", 2022-04-28T19:35:14.142 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "pg_autoscaler", 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "progress", 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "rbd_support", 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "status", 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "telemetry", 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "volumes" 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ], 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "last_failure_osd_epoch": 2, 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "active_clients": [ 2022-04-28T19:35:14.143 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addrvec": [ 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v2", 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:0", 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 2161102763 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.144 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:35:14.145 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout }, 2022-04-28T19:35:14.145 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.145 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addrvec": [ 2022-04-28T19:35:14.145 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:14.145 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "type": "v2", 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "addr": "172.21.15.59:0", 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "nonce": 1235412511 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout ] 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.146 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:14.147 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:13 smithi059 bash[23786]: audit 2022-04-28T19:35:13.192638+0000 mon.a (mon.0) 69 : audit [DBG] from='client.? 172.21.15.59:0/98680675' entity='client.admin' cmd=[{"prefix": "mgr dump"}]: dispatch 2022-04-28T19:35:14.147 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for the mgr to restart... 2022-04-28T19:35:14.147 INFO:teuthology.orchestra.run.smithi059.stderr:Waiting for Mgr epoch 9... 2022-04-28T19:35:14.148 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus tell mgr mgr_status 2022-04-28T19:35:16.585 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[24088]: Warning: Permanently added 'smithi059,172.21.15.59' (ECDSA) to the list of known hosts. 2022-04-28T19:35:16.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout { 2022-04-28T19:35:16.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2022-04-28T19:35:16.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout "initialized": true 2022-04-28T19:35:16.912 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout } 2022-04-28T19:35:17.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: cluster 2022-04-28T19:35:15.837289+0000 mon.a (mon.0) 70 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:35:17.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: cluster 2022-04-28T19:35:15.837939+0000 mon.a (mon.0) 71 : cluster [INF] Activating manager daemon y 2022-04-28T19:35:17.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: cluster 2022-04-28T19:35:15.841135+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2022-04-28T19:35:17.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: cluster 2022-04-28T19:35:15.892846+0000 mon.a (mon.0) 73 : cluster [DBG] mgrmap e10: y(active, starting, since 0.0550452s) 2022-04-28T19:35:17.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.894846+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:35:17.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.896346+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:35:17.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.897578+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:35:17.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.898049+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:35:17.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.898423+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:35:17.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.899172+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:35:17.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.901024+0000 mon.a (mon.0) 80 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:35:17.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.901725+0000 mon.a (mon.0) 81 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:35:17.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.902198+0000 mon.a (mon.0) 82 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:35:17.088 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.902659+0000 mon.a (mon.0) 83 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:35:17.089 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.903147+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:35:17.089 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.903624+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:35:17.089 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.904091+0000 mon.a (mon.0) 86 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:35:17.089 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.904540+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:35:17.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:15.905001+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:35:17.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: cluster 2022-04-28T19:35:15.909277+0000 mon.a (mon.0) 89 : cluster [INF] Manager daemon y is now available 2022-04-28T19:35:17.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:16.101243+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:35:17.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:16.103994+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:35:17.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:16 smithi059 bash[23786]: audit 2022-04-28T19:35:16.345758+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:35:17.810 INFO:teuthology.orchestra.run.smithi059.stderr:Mgr epoch 9 is available 2022-04-28T19:35:17.810 INFO:teuthology.orchestra.run.smithi059.stderr:Generating a dashboard self-signed certificate... 2022-04-28T19:35:17.810 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus dashboard create-self-signed-cert 2022-04-28T19:35:18.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:17 smithi059 bash[23786]: audit 2022-04-28T19:35:16.896441+0000 mgr.y (mgr.14140) 1 : audit [DBG] from='client.14144 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2022-04-28T19:35:18.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:17 smithi059 bash[23786]: cluster 2022-04-28T19:35:16.898096+0000 mon.a (mon.0) 93 : cluster [DBG] mgrmap e11: y(active, since 1.0603s) 2022-04-28T19:35:18.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:17 smithi059 bash[23786]: audit 2022-04-28T19:35:16.916232+0000 mgr.y (mgr.14140) 2 : audit [DBG] from='client.14144 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2022-04-28T19:35:18.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:17 smithi059 bash[23786]: audit 2022-04-28T19:35:16.983974+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:18.336 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:17 smithi059 bash[23786]: audit 2022-04-28T19:35:16.984685+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:35:19.085 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:18 smithi059 bash[23786]: cluster 2022-04-28T19:35:17.988723+0000 mon.a (mon.0) 96 : cluster [DBG] mgrmap e12: y(active, since 2s) 2022-04-28T19:35:19.648 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout Self-signed certificate created 2022-04-28T19:35:20.203 INFO:teuthology.orchestra.run.smithi059.stderr:Creating initial admin user... 2022-04-28T19:35:20.204 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmp30kusth3:/tmp/dashboard.pw:z quay.ceph.io/ceph-ci/ceph:octopus dashboard ac-user-create admin -i /tmp/dashboard.pw administrator --force-password --pwd-update-required 2022-04-28T19:35:20.973 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:20 smithi059 bash[23786]: audit 2022-04-28T19:35:19.597124+0000 mgr.y (mgr.14140) 3 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:35:20.973 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:20 smithi059 bash[23786]: audit 2022-04-28T19:35:19.650009+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:20.973 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:20 smithi059 bash[23786]: audit 2022-04-28T19:35:19.652749+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:21.880 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$u0tVzg7inz5YLya1i9tWHOpQTBuvPbkk9mOX1zdJt7ckMEKuHBbiS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1651174521, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2022-04-28T19:35:22.429 INFO:teuthology.orchestra.run.smithi059.stderr:Fetching dashboard port number... 2022-04-28T19:35:22.429 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/docker run --rm --ipc=host --net=host --entrypoint /usr/bin/ceph -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph:octopus -e NODE_NAME=smithi059 -v /var/log/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c:/var/log/ceph:z -v /tmp/ceph-tmpbh9z95a9:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpfo9repq_:/etc/ceph/ceph.conf:z quay.ceph.io/ceph-ci/ceph:octopus config get mgr mgr/dashboard/ssl_server_port 2022-04-28T19:35:23.190 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:22 smithi059 bash[23786]: audit 2022-04-28T19:35:21.622662+0000 mgr.y (mgr.14140) 4 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:35:23.190 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:22 smithi059 bash[23786]: audit 2022-04-28T19:35:21.884526+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:23.190 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:22 smithi059 bash[23786]: cluster 2022-04-28T19:35:21.896532+0000 mon.a (mon.0) 100 : cluster [DBG] mgrmap e13: y(active, since 6s) 2022-04-28T19:35:23.801 INFO:teuthology.orchestra.run.smithi059.stderr:/usr/bin/ceph: stdout 8443 2022-04-28T19:35:24.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:23 smithi059 bash[23786]: audit 2022-04-28T19:35:23.805606+0000 mon.a (mon.0) 101 : audit [DBG] from='client.? 172.21.15.59:0/3810349923' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2022-04-28T19:35:24.348 INFO:teuthology.orchestra.run.smithi059.stderr:firewalld does not appear to be present 2022-04-28T19:35:24.349 INFO:teuthology.orchestra.run.smithi059.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2022-04-28T19:35:24.353 INFO:teuthology.orchestra.run.smithi059.stderr:Ceph Dashboard is now available at: 2022-04-28T19:35:24.353 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.353 INFO:teuthology.orchestra.run.smithi059.stderr: URL: https://smithi059:8443/ 2022-04-28T19:35:24.353 INFO:teuthology.orchestra.run.smithi059.stderr: User: admin 2022-04-28T19:35:24.353 INFO:teuthology.orchestra.run.smithi059.stderr: Password: mmoqftkl1q 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr:You can access the Ceph CLI with: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr:Please consider enabling telemetry to help improve Ceph: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: ceph telemetry on 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr:For more information see: 2022-04-28T19:35:24.354 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.355 INFO:teuthology.orchestra.run.smithi059.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2022-04-28T19:35:24.355 INFO:teuthology.orchestra.run.smithi059.stderr: 2022-04-28T19:35:24.355 INFO:teuthology.orchestra.run.smithi059.stderr:Bootstrap complete. 2022-04-28T19:35:24.355 INFO:teuthology.orchestra.run.smithi059.stderr:Releasing lock 139765807533304 on /run/cephadm/20a1061c-c72a-11ec-8c39-001a4aab830c.lock 2022-04-28T19:35:24.355 INFO:teuthology.orchestra.run.smithi059.stderr:Lock 139765807533304 released on /run/cephadm/20a1061c-c72a-11ec-8c39-001a4aab830c.lock 2022-04-28T19:35:24.386 INFO:tasks.cephadm:Fetching config... 2022-04-28T19:35:24.387 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:35:24.387 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2022-04-28T19:35:24.394 INFO:tasks.cephadm:Fetching client.admin keyring... 2022-04-28T19:35:24.394 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:35:24.394 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2022-04-28T19:35:24.445 INFO:tasks.cephadm:Fetching mon keyring... 2022-04-28T19:35:24.445 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:35:24.445 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd if=/var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/keyring of=/dev/stdout 2022-04-28T19:35:24.504 INFO:tasks.cephadm:Fetching pub ssh key... 2022-04-28T19:35:24.505 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:35:24.505 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2022-04-28T19:35:24.551 INFO:tasks.cephadm:Installing pub ssh key for root users... 2022-04-28T19:35:24.552 DEBUG:teuthology.orchestra.run.smithi059:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChdH5x6EhnSgLvKjYFnwih45ryDMf7mZm+fEnJ8JChTWnyxy2LvPhoeNrZfwGlq1e4i0jXIDf67NWc2/hQKxRNxPlP7AZn+yzK31bgygcfEbe8tEhgTiwepgn7rNpaoKW3VPRVAHc9tGLVPeCBbsXGlhCaOttK7l4wxCnOPzNoQjymVW/laC/X9Fp/rBzStK9/5XDYoXtWr2V4SkTGjq0mHzPFwVJWehJ8/6WmytOOUxZ0N3B9lkXAsxpTphbTv1HhyseMJ/guJSg62S1TODlfYNsIN7laY1nxOeHJGAjjldlCfBDYMsrwttpVr+OksGhJoSVxH3COfLsMc3rBpWHFuKbXUyLqLTpbeH7mqBPh6VcoRKLRcCUXeyXjl0oik0Y+YJ2GgC3z3aZ+hqXeTo426Zxw/AdLprCMJlldNqDiWX0xKnJSvXpxIJInhSt/aqUWPJOJXt7m9QM0EZBYBbiDsp0eGN4xZ9NgtKFNTd3W3RfIp6GjNsESwi/7LcOFI/E= ceph-20a1061c-c72a-11ec-8c39-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2022-04-28T19:35:24.623 INFO:teuthology.orchestra.run.smithi059.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChdH5x6EhnSgLvKjYFnwih45ryDMf7mZm+fEnJ8JChTWnyxy2LvPhoeNrZfwGlq1e4i0jXIDf67NWc2/hQKxRNxPlP7AZn+yzK31bgygcfEbe8tEhgTiwepgn7rNpaoKW3VPRVAHc9tGLVPeCBbsXGlhCaOttK7l4wxCnOPzNoQjymVW/laC/X9Fp/rBzStK9/5XDYoXtWr2V4SkTGjq0mHzPFwVJWehJ8/6WmytOOUxZ0N3B9lkXAsxpTphbTv1HhyseMJ/guJSg62S1TODlfYNsIN7laY1nxOeHJGAjjldlCfBDYMsrwttpVr+OksGhJoSVxH3COfLsMc3rBpWHFuKbXUyLqLTpbeH7mqBPh6VcoRKLRcCUXeyXjl0oik0Y+YJ2GgC3z3aZ+hqXeTo426Zxw/AdLprCMJlldNqDiWX0xKnJSvXpxIJInhSt/aqUWPJOJXt7m9QM0EZBYBbiDsp0eGN4xZ9NgtKFNTd3W3RfIp6GjNsESwi/7LcOFI/E= ceph-20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:35:24.634 DEBUG:teuthology.orchestra.run.smithi139:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChdH5x6EhnSgLvKjYFnwih45ryDMf7mZm+fEnJ8JChTWnyxy2LvPhoeNrZfwGlq1e4i0jXIDf67NWc2/hQKxRNxPlP7AZn+yzK31bgygcfEbe8tEhgTiwepgn7rNpaoKW3VPRVAHc9tGLVPeCBbsXGlhCaOttK7l4wxCnOPzNoQjymVW/laC/X9Fp/rBzStK9/5XDYoXtWr2V4SkTGjq0mHzPFwVJWehJ8/6WmytOOUxZ0N3B9lkXAsxpTphbTv1HhyseMJ/guJSg62S1TODlfYNsIN7laY1nxOeHJGAjjldlCfBDYMsrwttpVr+OksGhJoSVxH3COfLsMc3rBpWHFuKbXUyLqLTpbeH7mqBPh6VcoRKLRcCUXeyXjl0oik0Y+YJ2GgC3z3aZ+hqXeTo426Zxw/AdLprCMJlldNqDiWX0xKnJSvXpxIJInhSt/aqUWPJOJXt7m9QM0EZBYBbiDsp0eGN4xZ9NgtKFNTd3W3RfIp6GjNsESwi/7LcOFI/E= ceph-20a1061c-c72a-11ec-8c39-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2022-04-28T19:35:24.655 INFO:teuthology.orchestra.run.smithi139.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChdH5x6EhnSgLvKjYFnwih45ryDMf7mZm+fEnJ8JChTWnyxy2LvPhoeNrZfwGlq1e4i0jXIDf67NWc2/hQKxRNxPlP7AZn+yzK31bgygcfEbe8tEhgTiwepgn7rNpaoKW3VPRVAHc9tGLVPeCBbsXGlhCaOttK7l4wxCnOPzNoQjymVW/laC/X9Fp/rBzStK9/5XDYoXtWr2V4SkTGjq0mHzPFwVJWehJ8/6WmytOOUxZ0N3B9lkXAsxpTphbTv1HhyseMJ/guJSg62S1TODlfYNsIN7laY1nxOeHJGAjjldlCfBDYMsrwttpVr+OksGhJoSVxH3COfLsMc3rBpWHFuKbXUyLqLTpbeH7mqBPh6VcoRKLRcCUXeyXjl0oik0Y+YJ2GgC3z3aZ+hqXeTo426Zxw/AdLprCMJlldNqDiWX0xKnJSvXpxIJInhSt/aqUWPJOJXt7m9QM0EZBYBbiDsp0eGN4xZ9NgtKFNTd3W3RfIp6GjNsESwi/7LcOFI/E= ceph-20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:35:24.666 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph config set mgr mgr/cephadm/allow_ptrace true 2022-04-28T19:35:28.213 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:27 smithi059 bash[23786]: audit 2022-04-28T19:35:27.881846+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:35:29.256 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi139 2022-04-28T19:35:29.256 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:35:29.256 DEBUG:teuthology.orchestra.run.smithi139:> dd of=/etc/ceph/ceph.conf 2022-04-28T19:35:29.263 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:35:29.263 DEBUG:teuthology.orchestra.run.smithi139:> dd of=/etc/ceph/ceph.client.admin.keyring 2022-04-28T19:35:29.311 INFO:tasks.cephadm:Adding host smithi139 to orchestrator... 2022-04-28T19:35:29.311 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch host add smithi139 2022-04-28T19:35:29.586 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: audit 2022-04-28T19:35:28.186511+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:29.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: cephadm 2022-04-28T19:35:28.187393+0000 mgr.y (mgr.14140) 5 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:35:29.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: audit 2022-04-28T19:35:28.619650+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:29.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: audit 2022-04-28T19:35:28.624117+0000 mon.a (mon.0) 105 : audit [INF] from='client.? 172.21.15.59:0/3049619019' entity='client.admin' 2022-04-28T19:35:29.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: audit 2022-04-28T19:35:28.624385+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:35:29.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: audit 2022-04-28T19:35:28.627668+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:35:29.587 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:29 smithi059 bash[23786]: audit 2022-04-28T19:35:28.629393+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:35:30.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:30 smithi059 bash[23786]: cephadm 2022-04-28T19:35:28.620308+0000 mgr.y (mgr.14140) 6 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2022-04-28T19:35:30.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:30 smithi059 bash[23786]: cephadm 2022-04-28T19:35:28.628731+0000 mgr.y (mgr.14140) 7 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:35:33.674 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:33 smithi059 bash[23786]: audit 2022-04-28T19:35:32.376548+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:33.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:33 smithi059 bash[23786]: cephadm 2022-04-28T19:35:32.377580+0000 mgr.y (mgr.14140) 8 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2022-04-28T19:35:33.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:33 smithi059 bash[23786]: audit 2022-04-28T19:35:32.378220+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.y"}]: dispatch 2022-04-28T19:35:33.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:33 smithi059 bash[23786]: audit 2022-04-28T19:35:32.379822+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:35:33.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:33 smithi059 bash[23786]: cephadm 2022-04-28T19:35:32.381086+0000 mgr.y (mgr.14140) 9 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:35:33.714 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:33 smithi059 bash[23786]: audit 2022-04-28T19:35:32.382025+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.y", "key": "container_image"}]: dispatch 2022-04-28T19:35:35.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:35 smithi059 bash[23786]: audit 2022-04-28T19:35:34.195144+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:35.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:35 smithi059 bash[23786]: audit 2022-04-28T19:35:34.198509+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:35:35.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:35 smithi059 bash[23786]: audit 2022-04-28T19:35:34.800713+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:35.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:35 smithi059 bash[23786]: audit 2022-04-28T19:35:34.806932+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:35:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:36 smithi059 bash[23786]: cluster 2022-04-28T19:35:35.899054+0000 mgr.y (mgr.14140) 10 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:38 smithi059 bash[23786]: cluster 2022-04-28T19:35:37.899462+0000 mgr.y (mgr.14140) 11 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:40 smithi059 bash[23786]: cluster 2022-04-28T19:35:39.899882+0000 mgr.y (mgr.14140) 12 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:43.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:42 smithi059 bash[23786]: cluster 2022-04-28T19:35:41.900292+0000 mgr.y (mgr.14140) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:45.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:44 smithi059 bash[23786]: cluster 2022-04-28T19:35:43.900678+0000 mgr.y (mgr.14140) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:47.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:46 smithi059 bash[23786]: cluster 2022-04-28T19:35:45.901056+0000 mgr.y (mgr.14140) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:49.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:48 smithi059 bash[23786]: cluster 2022-04-28T19:35:47.901428+0000 mgr.y (mgr.14140) 16 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:50 smithi059 bash[23786]: cluster 2022-04-28T19:35:49.901796+0000 mgr.y (mgr.14140) 17 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:52 smithi059 bash[23786]: cluster 2022-04-28T19:35:51.902160+0000 mgr.y (mgr.14140) 18 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:55.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:54 smithi059 bash[23786]: cluster 2022-04-28T19:35:53.902538+0000 mgr.y (mgr.14140) 19 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:57.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:56 smithi059 bash[23786]: cluster 2022-04-28T19:35:55.902862+0000 mgr.y (mgr.14140) 20 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:35:59.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:35:58 smithi059 bash[23786]: cluster 2022-04-28T19:35:57.903241+0000 mgr.y (mgr.14140) 21 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:00.343 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:35:59 smithi059 bash[24088]: Warning: Permanently added 'smithi139,172.21.15.139' (ECDSA) to the list of known hosts. 2022-04-28T19:36:00.508 INFO:teuthology.orchestra.run.smithi139.stdout:Added host 'smithi139' 2022-04-28T19:36:01.329 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch host ls --format=json 2022-04-28T19:36:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:00 smithi059 bash[23786]: audit 2022-04-28T19:35:59.795438+0000 mgr.y (mgr.14140) 22 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi139", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:00 smithi059 bash[23786]: cluster 2022-04-28T19:35:59.903871+0000 mgr.y (mgr.14140) 23 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:00 smithi059 bash[23786]: audit 2022-04-28T19:36:00.502820+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:00 smithi059 bash[23786]: audit 2022-04-28T19:36:00.541083+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:00 smithi059 bash[23786]: audit 2022-04-28T19:36:00.928370+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:01.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:00 smithi059 bash[23786]: audit 2022-04-28T19:36:00.929651+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:02 smithi059 bash[23786]: cephadm 2022-04-28T19:36:00.503733+0000 mgr.y (mgr.14140) 24 : cephadm [INF] Added host smithi139 2022-04-28T19:36:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:02 smithi059 bash[23786]: audit 2022-04-28T19:36:01.164457+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:02 smithi059 bash[23786]: audit 2022-04-28T19:36:01.219312+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:02 smithi059 bash[23786]: audit 2022-04-28T19:36:01.220611+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:36:03.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:03 smithi059 bash[23786]: cluster 2022-04-28T19:36:01.904367+0000 mgr.y (mgr.14140) 25 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:05.235 INFO:teuthology.orchestra.run.smithi139.stdout: 2022-04-28T19:36:05.235 INFO:teuthology.orchestra.run.smithi139.stdout:[{"addr": "smithi059", "hostname": "smithi059", "labels": [], "status": ""}, {"addr": "smithi139", "hostname": "smithi139", "labels": [], "status": ""}] 2022-04-28T19:36:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:05 smithi059 bash[23786]: cluster 2022-04-28T19:36:03.904769+0000 mgr.y (mgr.14140) 26 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:05.982 INFO:tasks.cephadm:Setting crush tunables to default 2022-04-28T19:36:05.982 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd crush tunables default 2022-04-28T19:36:06.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:06 smithi059 bash[23786]: audit 2022-04-28T19:36:05.231253+0000 mgr.y (mgr.14140) 27 : audit [DBG] from='client.14160 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:36:07.319 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:07 smithi059 bash[23786]: cluster 2022-04-28T19:36:05.905156+0000 mgr.y (mgr.14140) 28 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:07.319 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:07 smithi059 bash[23786]: audit 2022-04-28T19:36:06.499733+0000 mon.a (mon.0) 124 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:07.319 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:07 smithi059 bash[23786]: audit 2022-04-28T19:36:06.815634+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:07.319 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:07 smithi059 bash[23786]: audit 2022-04-28T19:36:07.224972+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:08.521 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:08 smithi059 bash[23786]: cephadm 2022-04-28T19:36:06.816276+0000 mgr.y (mgr.14140) 29 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:36:08.522 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:08 smithi059 bash[23786]: audit 2022-04-28T19:36:07.230383+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:09.249 INFO:teuthology.orchestra.run.smithi059.stderr:adjusted tunables profile to default 2022-04-28T19:36:09.785 INFO:tasks.cephadm:Adding mon.c on smithi059 2022-04-28T19:36:09.785 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add mon 'smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c' 2022-04-28T19:36:10.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:09 smithi059 bash[23786]: cluster 2022-04-28T19:36:07.905615+0000 mgr.y (mgr.14140) 30 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:10.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:09 smithi059 bash[23786]: audit 2022-04-28T19:36:08.898410+0000 mon.a (mon.0) 128 : audit [INF] from='client.? 172.21.15.59:0/874560301' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2022-04-28T19:36:11.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: audit 2022-04-28T19:36:09.241819+0000 mon.a (mon.0) 129 : audit [INF] from='client.? 172.21.15.59:0/874560301' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2022-04-28T19:36:11.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: cluster 2022-04-28T19:36:09.241952+0000 mon.a (mon.0) 130 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2022-04-28T19:36:11.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: audit 2022-04-28T19:36:09.242866+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2022-04-28T19:36:11.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: cluster 2022-04-28T19:36:09.906056+0000 mgr.y (mgr.14140) 31 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:11.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: audit 2022-04-28T19:36:10.243026+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2022-04-28T19:36:11.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: cluster 2022-04-28T19:36:10.243184+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2022-04-28T19:36:11.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:10 smithi059 bash[23786]: audit 2022-04-28T19:36:10.244807+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:36:12.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:12 smithi059 bash[23786]: audit 2022-04-28T19:36:11.242507+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:36:12.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:12 smithi059 bash[23786]: cluster 2022-04-28T19:36:11.242577+0000 mon.a (mon.0) 136 : cluster [DBG] osdmap e6: 0 total, 0 up, 0 in 2022-04-28T19:36:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:13 smithi059 bash[23786]: cluster 2022-04-28T19:36:11.906508+0000 mgr.y (mgr.14140) 32 : cluster [DBG] pgmap v24: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:13 smithi059 bash[23786]: audit 2022-04-28T19:36:12.841852+0000 mon.a (mon.0) 137 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:13 smithi059 bash[23786]: audit 2022-04-28T19:36:12.843626+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:13 smithi059 bash[23786]: audit 2022-04-28T19:36:12.845498+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.c", "key": "container_image"}]: dispatch 2022-04-28T19:36:14.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:14 smithi059 bash[23786]: audit 2022-04-28T19:36:12.839229+0000 mgr.y (mgr.14140) 33 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add", "daemon_type": "mon", "placement": "smithi059:[v2:172.21.15.59:3301,v1:172.21.15.59:6790]=c", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:14.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:14 smithi059 bash[23786]: cephadm 2022-04-28T19:36:12.844841+0000 mgr.y (mgr.14140) 34 : cephadm [INF] Deploying daemon mon.c on smithi059 2022-04-28T19:36:15.530 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:15 smithi059 bash[23786]: cluster 2022-04-28T19:36:13.907066+0000 mgr.y (mgr.14140) 35 : cluster [DBG] pgmap v25: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:16 smithi059 bash[23786]: cluster 2022-04-28T19:36:15.907726+0000 mgr.y (mgr.14140) 36 : cluster [DBG] pgmap v26: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:16 smithi059 bash[23786]: audit 2022-04-28T19:36:16.103405+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:36:16.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:16 smithi059 bash[23786]: audit 2022-04-28T19:36:16.105866+0000 mon.a (mon.0) 141 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:36:16.569 INFO:teuthology.orchestra.run.smithi059.stdout:Deployed mon.c on host 'smithi059' 2022-04-28T19:36:17.639 DEBUG:teuthology.orchestra.run.smithi059:mon.c> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.c.service 2022-04-28T19:36:17.641 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2022-04-28T19:36:17.641 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph mon dump -f json 2022-04-28T19:36:17.665 INFO:journalctl@ceph.mon.c.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:36:17.823 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[23786]: audit 2022-04-28T19:36:16.565445+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:17.823 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[23786]: audit 2022-04-28T19:36:16.571590+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:17.823 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[23786]: audit 2022-04-28T19:36:17.284056+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:17.823 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[23786]: audit 2022-04-28T19:36:17.287041+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.929+0000 7f518dae86c0 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.929+0000 7f518dae86c0 0 ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable), process ceph-mon, pid 1 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.929+0000 7f518dae86c0 0 pidfile_write: ignore empty --pid-file 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 load: jerasure load: lrc load: isa 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:36:18.112 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 1 rocksdb: do_open column families: [default] 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: RocksDB version: 6.1.2 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Compile date Mar 25 2022 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: DB SUMMARY 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: CURRENT file: CURRENT 2022-04-28T19:36:18.113 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: IDENTITY file: IDENTITY 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000003.log size: 489 ; 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.create_if_missing: 0 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.env: 0x558729dc61c0 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.info_log: 0x55872b844520 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:36:18.114 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:36:18.115 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:36:18.115 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:36:18.115 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:36:18.115 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:36:18.115 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:36:18.115 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.db_log_dir: 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:36:18.116 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.write_buffer_manager: 0x55872b84b560 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:36:18.117 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:36:18.119 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:36:18.119 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:36:18.119 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:36:18.119 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.row_cache: None 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.wal_filter: None 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:36:18.120 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:36:18.121 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Compression algorithms supported: 2022-04-28T19:36:18.122 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kZSTD supported: 0 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:36:18.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.merge_operator: 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:36:18.124 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:36:18.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:36:18.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:36:18.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55872b75fa80) 2022-04-28T19:36:18.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: cache_index_and_filter_blocks: 1 2022-04-28T19:36:18.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: cache_index_and_filter_blocks_with_high_priority: 1 2022-04-28T19:36:18.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: pin_top_level_index_and_filter: 1 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: index_type: 0 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: data_block_index_type: 0 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: hash_index_allow_collision: 1 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: checksum: 1 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: no_block_cache: 0 2022-04-28T19:36:18.126 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_cache: 0x55872b796610 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_cache_name: BinnedLRUCache 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_cache_options: 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: capacity : 536870912 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: num_shard_bits : 4 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: strict_capacity_limit : 0 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: high_pri_pool_ratio: 0.000 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_cache_compressed: (nil) 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: persistent_cache: (nil) 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_size: 4096 2022-04-28T19:36:18.127 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_size_deviation: 10 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_restart_interval: 16 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: index_block_restart_interval: 1 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: metadata_block_size: 4096 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: partition_filters: 0 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: use_delta_encoding: 1 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: whole_key_filtering: 1 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: verify_compression: 0 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: read_amp_bytes_per_bit: 0 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: format_version: 2 2022-04-28T19:36:18.128 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: enable_index_compression: 1 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: block_align: 0 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.num_levels: 7 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:36:18.129 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:36:18.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:36:18.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:36:18.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:36:18.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:36:18.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:36:18.130 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:36:18.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:36:18.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:36:18.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:36:18.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:36:18.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:36:18.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:36:18.132 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:36:18.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:36:18.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:36:18.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:36:18.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:36:18.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:36:18.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:36:18.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:36:18.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:36:18.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:36:18.135 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:36:18.135 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:36:18.135 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:36:18.135 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:36:18.135 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:36:18.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:36:18.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:36:18.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:36:18.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:36:18.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:36:18.136 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:36:18.137 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:36:18.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:36:18.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:36:18.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:36:18.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:36:18.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:36:18.138 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:36:18.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:36:18.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:36:18.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:36:18.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:36:18.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.933+0000 7f518dae86c0 4 rocksdb: Options.ttl: 0 2022-04-28T19:36:18.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.937+0000 7f518dae86c0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.937+0000 7f518dae86c0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.937+0000 7f518dae86c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174577941136, "job": 1, "event": "recovery_started", "log_files": [3]} 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.937+0000 7f518dae86c0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 2 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.937+0000 7f518dae86c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174577942896, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1403, "table_properties": {"data_size": 501, "index_size": 28, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 379, "raw_average_value_size": 75, "num_data_blocks": 1, "num_entries": 5, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.937+0000 7f518dae86c0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 5 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.941+0000 7f518dae86c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174577943966, "job": 1, "event": "recovery_finished"} 2022-04-28T19:36:18.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.941+0000 7f518dae86c0 4 rocksdb: DB pointer 0x55872c3c4400 2022-04-28T19:36:18.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.941+0000 7f5174944700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2022-04-28T19:36:18.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.941+0000 7f5174944700 4 rocksdb: [db/db_impl.cc:778] 2022-04-28T19:36:18.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** DB Stats ** 2022-04-28T19:36:18.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:36:18.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:36:18.141 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** Compaction Stats [default] ** 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: L0 1/0 1.37 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Sum 1/0 1.37 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.142 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** Compaction Stats [default] ** 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:36:18.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Cumulative compaction: 0.00 GB write, 0.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Interval compaction: 0.00 GB write, 0.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** Compaction Stats [default] ** 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: L0 1/0 1.37 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Sum 1/0 1.37 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:36:18.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** Compaction Stats [default] ** 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Cumulative compaction: 0.00 GB write, 0.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:18.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.941+0000 7f518dae86c0 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.941+0000 7f518dae86c0 0 using public_addrv [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.945+0000 7f518dae86c0 0 starting mon.c rank -1 at public addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] at bind addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.945+0000 7f518dae86c0 1 mon.c@-1(???) e0 preinit fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 0 mon.c@-1(synchronizing).mds e1 new map 2022-04-28T19:36:18.146 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 0 mon.c@-1(synchronizing).mds e1 print_map 2022-04-28T19:36:18.147 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: e1 2022-04-28T19:36:18.147 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: enable_multiple, ever_enabled_multiple: 0,0 2022-04-28T19:36:18.147 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:36:18.147 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: legacy client fscid: -1 2022-04-28T19:36:18.147 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: 2022-04-28T19:36:18.147 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: No filesystems configured 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2022-04-28T19:36:18.148 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2022-04-28T19:36:18.149 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).osd e6 e6: 0 total, 0 up, 0 in 2022-04-28T19:36:18.149 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 0 mon.c@-1(synchronizing).osd e6 crush map has features 288514051259236352, adjusting msgr requires 2022-04-28T19:36:18.149 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 0 mon.c@-1(synchronizing).osd e6 crush map has features 3314933000852226048, adjusting msgr requires 2022-04-28T19:36:18.149 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 0 mon.c@-1(synchronizing).osd e6 crush map has features 288514051259236352, adjusting msgr requires 2022-04-28T19:36:18.149 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: audit 2022-04-28T19:36:16.565445+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:18.150 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: audit 2022-04-28T19:36:16.571590+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:18.150 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: audit 2022-04-28T19:36:17.284056+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:18.150 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: audit 2022-04-28T19:36:17.287041+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:18.150 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.969+0000 7f5177149700 1 mon.c@-1(synchronizing).paxosservice(auth 1..4) refresh upgraded, format 0 -> 3 2022-04-28T19:36:18.150 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 10 mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2022-04-28T19:36:18.150 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expand map: {default=false} 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta from 'false' to 'false' 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expanded map: {default=false} 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expand map: {default=info} 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta from 'info' to 'info' 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expanded map: {default=info} 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expand map: {default=daemon} 2022-04-28T19:36:18.151 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta from 'daemon' to 'daemon' 2022-04-28T19:36:18.152 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expanded map: {default=daemon} 2022-04-28T19:36:18.152 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expand map: {default=debug} 2022-04-28T19:36:18.152 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta from 'debug' to 'debug' 2022-04-28T19:36:18.152 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:17 smithi059 bash[29114]: debug 2022-04-28T19:36:17.973+0000 7f5177149700 20 expand_channel_meta expanded map: {default=debug} 2022-04-28T19:36:23.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:17.908240+0000 mgr.y (mgr.14140) 37 : cluster [DBG] pgmap v27: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:23.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:17.984726+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:36:23.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:17.985036+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2022-04-28T19:36:23.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:17.986901+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:18.981142+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:19.908729+0000 mgr.y (mgr.14140) 38 : cluster [DBG] pgmap v28: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:23.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:19.981082+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:19.983312+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-04-28T19:36:23.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:20.981158+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:21.909200+0000 mgr.y (mgr.14140) 39 : cluster [DBG] pgmap v29: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:23.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:21.981154+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: audit 2022-04-28T19:36:22.981345+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:22.989995+0000 mon.a (mon.0) 155 : cluster [INF] mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-04-28T19:36:23.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:22.996632+0000 mon.a (mon.0) 156 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:36:23.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:22.996723+0000 mon.a (mon.0) 157 : cluster [DBG] fsmap 2022-04-28T19:36:23.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:22.996766+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e6: 0 total, 0 up, 0 in 2022-04-28T19:36:23.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:22.997195+0000 mon.a (mon.0) 159 : cluster [DBG] mgrmap e13: y(active, since 67s) 2022-04-28T19:36:23.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[23786]: cluster 2022-04-28T19:36:23.003360+0000 mon.a (mon.0) 160 : cluster [INF] overall HEALTH_OK 2022-04-28T19:36:23.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:17.908240+0000 mgr.y (mgr.14140) 37 : cluster [DBG] pgmap v27: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:23.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:17.984726+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:36:23.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:17.985036+0000 mon.a (mon.0) 148 : cluster [INF] mon.a calling monitor election 2022-04-28T19:36:23.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:17.986901+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:18.981142+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:19.908729+0000 mgr.y (mgr.14140) 38 : cluster [DBG] pgmap v28: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:19.981082+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:19.983312+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:20.981158+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:21.909200+0000 mgr.y (mgr.14140) 39 : cluster [DBG] pgmap v29: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:23.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:21.981154+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: audit 2022-04-28T19:36:22.981345+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:23.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:22.989995+0000 mon.a (mon.0) 155 : cluster [INF] mon.a is new leader, mons a,c in quorum (ranks 0,1) 2022-04-28T19:36:23.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:22.996632+0000 mon.a (mon.0) 156 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:36:23.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:22.996723+0000 mon.a (mon.0) 157 : cluster [DBG] fsmap 2022-04-28T19:36:23.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:22.996766+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e6: 0 total, 0 up, 0 in 2022-04-28T19:36:23.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:22.997195+0000 mon.a (mon.0) 159 : cluster [DBG] mgrmap e13: y(active, since 67s) 2022-04-28T19:36:23.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:23 smithi059 bash[29114]: cluster 2022-04-28T19:36:23.003360+0000 mon.a (mon.0) 160 : cluster [INF] overall HEALTH_OK 2022-04-28T19:36:23.831 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:36:23.831 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":2,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","modified":"2022-04-28T19:36:17.981600Z","created":"2022-04-28T19:34:26.452116Z","min_mon_release":15,"min_mon_release_name":"octopus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3300","nonce":0},{"type":"v1","addr":"172.21.15.59:6789","nonce":0}]},"addr":"172.21.15.59:6789/0","public_addr":"172.21.15.59:6789/0","priority":0,"weight":0},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3301","nonce":0},{"type":"v1","addr":"172.21.15.59:6790","nonce":0}]},"addr":"172.21.15.59:6790/0","public_addr":"172.21.15.59:6790/0","priority":0,"weight":0}],"quorum":[0,1]} 2022-04-28T19:36:23.833 INFO:teuthology.orchestra.run.smithi059.stderr:dumped monmap epoch 2 2022-04-28T19:36:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:24 smithi059 bash[23786]: audit 2022-04-28T19:36:23.828498+0000 mon.a (mon.0) 161 : audit [DBG] from='client.? 172.21.15.59:0/4063527849' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-04-28T19:36:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:24 smithi059 bash[23786]: audit 2022-04-28T19:36:23.981336+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:24.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:24 smithi059 bash[29114]: audit 2022-04-28T19:36:23.828498+0000 mon.a (mon.0) 161 : audit [DBG] from='client.? 172.21.15.59:0/4063527849' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-04-28T19:36:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:24 smithi059 bash[29114]: audit 2022-04-28T19:36:23.981336+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:24.409 INFO:tasks.cephadm:Adding mon.b on smithi139 2022-04-28T19:36:24.410 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add mon smithi139:172.21.15.139=b 2022-04-28T19:36:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:25 smithi059 bash[23786]: cluster 2022-04-28T19:36:23.909544+0000 mgr.y (mgr.14140) 40 : cluster [DBG] pgmap v30: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:25.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:25 smithi059 bash[29114]: cluster 2022-04-28T19:36:23.909544+0000 mgr.y (mgr.14140) 40 : cluster [DBG] pgmap v30: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:27 smithi059 bash[23786]: cluster 2022-04-28T19:36:25.910000+0000 mgr.y (mgr.14140) 41 : cluster [DBG] pgmap v31: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:27 smithi059 bash[29114]: cluster 2022-04-28T19:36:25.910000+0000 mgr.y (mgr.14140) 41 : cluster [DBG] pgmap v31: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:28 smithi059 bash[23786]: audit 2022-04-28T19:36:27.326681+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:28 smithi059 bash[23786]: audit 2022-04-28T19:36:27.328323+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:28 smithi059 bash[23786]: audit 2022-04-28T19:36:27.330109+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.b", "key": "container_image"}]: dispatch 2022-04-28T19:36:28.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:28 smithi059 bash[29114]: audit 2022-04-28T19:36:27.326681+0000 mon.a (mon.0) 163 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:28.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:28 smithi059 bash[29114]: audit 2022-04-28T19:36:27.328323+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:28.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:28 smithi059 bash[29114]: audit 2022-04-28T19:36:27.330109+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.b", "key": "container_image"}]: dispatch 2022-04-28T19:36:29.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:29 smithi059 bash[23786]: audit 2022-04-28T19:36:27.324320+0000 mgr.y (mgr.14140) 42 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add", "daemon_type": "mon", "placement": "smithi139:172.21.15.139=b", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:29.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:29 smithi059 bash[23786]: cephadm 2022-04-28T19:36:27.329563+0000 mgr.y (mgr.14140) 43 : cephadm [INF] Deploying daemon mon.b on smithi139 2022-04-28T19:36:29.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:29 smithi059 bash[23786]: cluster 2022-04-28T19:36:27.910478+0000 mgr.y (mgr.14140) 44 : cluster [DBG] pgmap v32: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:29.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:29 smithi059 bash[29114]: audit 2022-04-28T19:36:27.324320+0000 mgr.y (mgr.14140) 42 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add", "daemon_type": "mon", "placement": "smithi139:172.21.15.139=b", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:29.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:29 smithi059 bash[29114]: cephadm 2022-04-28T19:36:27.329563+0000 mgr.y (mgr.14140) 43 : cephadm [INF] Deploying daemon mon.b on smithi139 2022-04-28T19:36:29.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:29 smithi059 bash[29114]: cluster 2022-04-28T19:36:27.910478+0000 mgr.y (mgr.14140) 44 : cluster [DBG] pgmap v32: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:31.585 INFO:teuthology.orchestra.run.smithi139.stdout:Deployed mon.b on host 'smithi139' 2022-04-28T19:36:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:31 smithi059 bash[23786]: cluster 2022-04-28T19:36:29.910947+0000 mgr.y (mgr.14140) 45 : cluster [DBG] pgmap v33: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:31 smithi059 bash[29114]: cluster 2022-04-28T19:36:29.910947+0000 mgr.y (mgr.14140) 45 : cluster [DBG] pgmap v33: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:32.699 DEBUG:teuthology.orchestra.run.smithi139:mon.b> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mon.b.service 2022-04-28T19:36:32.702 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2022-04-28T19:36:32.702 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph mon dump -f json 2022-04-28T19:36:32.776 INFO:journalctl@ceph.mon.b.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:36:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:31.581502+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: cephadm 2022-04-28T19:36:31.591974+0000 mgr.y (mgr.14140) 46 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:36:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:31.592920+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: cluster 2022-04-28T19:36:31.911542+0000 mgr.y (mgr.14140) 47 : cluster [DBG] pgmap v34: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:32.167527+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:32.174111+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: cephadm 2022-04-28T19:36:32.175015+0000 mgr.y (mgr.14140) 48 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:32.175670+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:32.177430+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: cephadm 2022-04-28T19:36:32.178780+0000 mgr.y (mgr.14140) 49 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[23786]: audit 2022-04-28T19:36:32.180190+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:31.581502+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: cephadm 2022-04-28T19:36:31.591974+0000 mgr.y (mgr.14140) 46 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:31.592920+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: cluster 2022-04-28T19:36:31.911542+0000 mgr.y (mgr.14140) 47 : cluster [DBG] pgmap v34: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:32.167527+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:32.174111+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: cephadm 2022-04-28T19:36:32.175015+0000 mgr.y (mgr.14140) 48 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:32.175670+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:32.177430+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: cephadm 2022-04-28T19:36:32.178780+0000 mgr.y (mgr.14140) 49 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:36:32.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:32 smithi059 bash[29114]: audit 2022-04-28T19:36:32.180190+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:36:33.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:32 smithi139 bash[23439]: debug 2022-04-28T19:36:32.993+0000 7f25f4c356c0 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:36:33.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:32 smithi139 bash[23439]: debug 2022-04-28T19:36:32.993+0000 7f25f4c356c0 0 ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable), process ceph-mon, pid 1 2022-04-28T19:36:33.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:32 smithi139 bash[23439]: debug 2022-04-28T19:36:32.993+0000 7f25f4c356c0 0 pidfile_write: ignore empty --pid-file 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 load: jerasure load: lrc load: isa 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 set rocksdb option compression = kNoCompression 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 set rocksdb option level_compaction_dynamic_level_bytes = true 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 0 set rocksdb option write_buffer_size = 33554432 2022-04-28T19:36:33.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 1 rocksdb: do_open column families: [default] 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: RocksDB version: 6.1.2 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Compile date Mar 25 2022 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: DB SUMMARY 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: CURRENT file: CURRENT 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: IDENTITY file: IDENTITY 2022-04-28T19:36:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2022-04-28T19:36:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2022-04-28T19:36:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 610 ; 2022-04-28T19:36:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:36:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.create_if_missing: 0 2022-04-28T19:36:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.env: 0x56206d2541c0 2022-04-28T19:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.info_log: 0x56206e66e520 2022-04-28T19:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:36:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:36:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:36:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:36:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:36:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.db_log_dir: 2022-04-28T19:36:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2022-04-28T19:36:33.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:36:33.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:36:33.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:36:33.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:36:33.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:36:33.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:36:33.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:36:33.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:36:33.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:36:33.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.write_buffer_manager: 0x56206e6754d0 2022-04-28T19:36:33.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:36:33.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:36:33.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:36:33.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:36:33.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:36:33.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:36:33.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:36:33.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:36:33.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.row_cache: None 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.wal_filter: None 2022-04-28T19:36:33.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:36:33.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:36:33.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:36:33.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:36:33.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:36:33.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:36:33.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:36:33.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:36:33.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:36:33.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:36:33.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:36:33.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:36:33.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:36:33.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:36:33.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:36:33.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Compression algorithms supported: 2022-04-28T19:36:33.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:36:33.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kZSTD supported: 0 2022-04-28T19:36:33.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:36:33.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:36:33.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:36:33.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:36:33.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:36:33.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:36:33.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:36:33.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2022-04-28T19:36:33.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: 2022-04-28T19:36:33.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:36:33.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.merge_operator: 2022-04-28T19:36:33.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:36:33.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:36:33.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:36:33.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:36:33.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56206e589a80) 2022-04-28T19:36:33.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: cache_index_and_filter_blocks: 1 2022-04-28T19:36:33.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: cache_index_and_filter_blocks_with_high_priority: 1 2022-04-28T19:36:33.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:36:33.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: pin_top_level_index_and_filter: 1 2022-04-28T19:36:33.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: index_type: 0 2022-04-28T19:36:33.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: data_block_index_type: 0 2022-04-28T19:36:33.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:36:33.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: hash_index_allow_collision: 1 2022-04-28T19:36:33.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: checksum: 1 2022-04-28T19:36:33.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: no_block_cache: 0 2022-04-28T19:36:33.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_cache: 0x56206e5c0610 2022-04-28T19:36:33.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_cache_name: BinnedLRUCache 2022-04-28T19:36:33.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_cache_options: 2022-04-28T19:36:33.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: capacity : 536870912 2022-04-28T19:36:33.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: num_shard_bits : 4 2022-04-28T19:36:33.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: strict_capacity_limit : 0 2022-04-28T19:36:33.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: high_pri_pool_ratio: 0.000 2022-04-28T19:36:33.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_cache_compressed: (nil) 2022-04-28T19:36:33.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: persistent_cache: (nil) 2022-04-28T19:36:33.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_size: 4096 2022-04-28T19:36:33.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_size_deviation: 10 2022-04-28T19:36:33.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_restart_interval: 16 2022-04-28T19:36:33.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: index_block_restart_interval: 1 2022-04-28T19:36:33.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: metadata_block_size: 4096 2022-04-28T19:36:33.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: partition_filters: 0 2022-04-28T19:36:33.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: use_delta_encoding: 1 2022-04-28T19:36:33.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:36:33.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: whole_key_filtering: 1 2022-04-28T19:36:33.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: verify_compression: 0 2022-04-28T19:36:33.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: read_amp_bytes_per_bit: 0 2022-04-28T19:36:33.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: format_version: 2 2022-04-28T19:36:33.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: enable_index_compression: 1 2022-04-28T19:36:33.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: block_align: 0 2022-04-28T19:36:33.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:36:33.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:36:33.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:36:33.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:36:33.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:36:33.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:36:33.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.num_levels: 7 2022-04-28T19:36:33.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:36:33.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:36:33.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:36:33.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:36:33.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:36:33.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:36:33.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:36:33.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:36:33.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:36:33.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:36:33.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:36:33.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:36:33.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:36:33.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:36:33.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:36:33.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:36:33.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:36:33.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:36:33.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:36:33.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:36:33.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:36:33.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:36:33.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:36:33.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:36:33.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:36:33.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:36:33.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:36:33.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:36:33.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:36:33.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:36:33.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:36:33.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:36:33.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:36:33.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:36:33.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:36:33.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:36:33.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:36:33.399 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:36:33.399 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:36:33.399 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:36:33.399 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:36:33.399 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:36:33.399 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:36:33.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:36:33.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:36:33.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:36:33.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:36:33.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:36:33.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:36:33.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:36:33.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:36:33.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:36:33.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: Options.ttl: 0 2022-04-28T19:36:33.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:36:33.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 0 2022-04-28T19:36:33.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174593004691, "job": 1, "event": "recovery_started", "log_files": [3]} 2022-04-28T19:36:33.403 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:32.997+0000 7f25f4c356c0 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #3 mode 2 2022-04-28T19:36:33.403 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25f4c356c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174593005628, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1524, "table_properties": {"data_size": 622, "index_size": 28, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 500, "raw_average_value_size": 100, "num_data_blocks": 1, "num_entries": 5, "filter_policy_name": "rocksdb.BuiltinBloomFilter"}} 2022-04-28T19:36:33.403 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25f4c356c0 4 rocksdb: [db/version_set.cc:3036] Creating manifest 5 2022-04-28T19:36:33.403 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25f4c356c0 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651174593006505, "job": 1, "event": "recovery_finished"} 2022-04-28T19:36:33.403 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25f4c356c0 4 rocksdb: DB pointer 0x56206f1ee400 2022-04-28T19:36:33.403 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25dba91700 4 rocksdb: [db/db_impl.cc:777] ------- DUMPING STATS ------- 2022-04-28T19:36:33.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25dba91700 4 rocksdb: [db/db_impl.cc:778] 2022-04-28T19:36:33.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** DB Stats ** 2022-04-28T19:36:33.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:36:33.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:36:33.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:36:33.405 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:36:33.405 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:36:33.405 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:36:33.405 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:36:33.405 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** Compaction Stats [default] ** 2022-04-28T19:36:33.405 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:33.406 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:33.406 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: L0 1/0 1.49 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.406 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Sum 1/0 1.49 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.406 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.406 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** Compaction Stats [default] ** 2022-04-28T19:36:33.406 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:33.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:33.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:36:33.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:33.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:33.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:36:33.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:36:33.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:36:33.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Cumulative compaction: 0.00 GB write, 0.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:33.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Interval compaction: 0.00 GB write, 0.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:33.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:36:33.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:36:33.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** Compaction Stats [default] ** 2022-04-28T19:36:33.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:33.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:33.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: L0 1/0 1.49 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Sum 1/0 1.49 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.410 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:36:33.410 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** Compaction Stats [default] ** 2022-04-28T19:36:33.410 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:36:33.410 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:36:33.410 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.6 0.00 0.00 1 0.001 0 0 2022-04-28T19:36:33.411 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:36:33.411 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:33.411 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:36:33.411 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:36:33.411 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:36:33.411 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:36:33.412 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Cumulative compaction: 0.00 GB write, 0.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:33.412 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:36:33.412 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:36:33.412 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:36:33.412 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25f4c356c0 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2022-04-28T19:36:33.412 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.001+0000 7f25f4c356c0 0 using public_addr v2:172.21.15.139:0/0 -> [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] 2022-04-28T19:36:33.413 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.005+0000 7f25f4c356c0 0 starting mon.b rank -1 at public addrs [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] at bind addrs [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:36:33.413 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.005+0000 7f25f4c356c0 1 mon.b@-1(???) e0 preinit fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:36:33.413 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 0 mon.b@-1(synchronizing).mds e1 new map 2022-04-28T19:36:33.413 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 0 mon.b@-1(synchronizing).mds e1 print_map 2022-04-28T19:36:33.413 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: e1 2022-04-28T19:36:33.414 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: enable_multiple, ever_enabled_multiple: 0,0 2022-04-28T19:36:33.414 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:36:33.414 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: legacy client fscid: -1 2022-04-28T19:36:33.414 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: 2022-04-28T19:36:33.414 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: No filesystems configured 2022-04-28T19:36:33.414 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2022-04-28T19:36:33.415 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2022-04-28T19:36:33.415 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2022-04-28T19:36:33.415 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2022-04-28T19:36:33.415 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2022-04-28T19:36:33.415 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2022-04-28T19:36:33.415 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2022-04-28T19:36:33.416 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).osd e6 e6: 0 total, 0 up, 0 in 2022-04-28T19:36:33.416 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 0 mon.b@-1(synchronizing).osd e6 crush map has features 288514051259236352, adjusting msgr requires 2022-04-28T19:36:33.416 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 0 mon.b@-1(synchronizing).osd e6 crush map has features 3314933000852226048, adjusting msgr requires 2022-04-28T19:36:33.416 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 0 mon.b@-1(synchronizing).osd e6 crush map has features 288514051259236352, adjusting msgr requires 2022-04-28T19:36:33.416 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:31.581502+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:33.416 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: cephadm 2022-04-28T19:36:31.591974+0000 mgr.y (mgr.14140) 46 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:36:33.417 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:31.592920+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:33.417 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: cluster 2022-04-28T19:36:31.911542+0000 mgr.y (mgr.14140) 47 : cluster [DBG] pgmap v34: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:33.417 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:32.167527+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:33.417 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:32.174111+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:33.417 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: cephadm 2022-04-28T19:36:32.175015+0000 mgr.y (mgr.14140) 48 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:36:33.418 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:32.175670+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:33.418 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:32.177430+0000 mon.a (mon.0) 171 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:33.418 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: cephadm 2022-04-28T19:36:32.178780+0000 mgr.y (mgr.14140) 49 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:36:33.418 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: audit 2022-04-28T19:36:32.180190+0000 mon.a (mon.0) 172 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:36:33.418 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.037+0000 7f25de296700 1 mon.b@-1(synchronizing).paxosservice(auth 1..5) refresh upgraded, format 0 -> 3 2022-04-28T19:36:33.418 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 10 mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2022-04-28T19:36:33.419 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expand map: {default=false} 2022-04-28T19:36:33.419 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta from 'false' to 'false' 2022-04-28T19:36:33.419 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expanded map: {default=false} 2022-04-28T19:36:33.419 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expand map: {default=info} 2022-04-28T19:36:33.419 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta from 'info' to 'info' 2022-04-28T19:36:33.420 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expanded map: {default=info} 2022-04-28T19:36:33.420 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expand map: {default=daemon} 2022-04-28T19:36:33.420 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta from 'daemon' to 'daemon' 2022-04-28T19:36:33.420 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expanded map: {default=daemon} 2022-04-28T19:36:33.420 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expand map: {default=debug} 2022-04-28T19:36:33.420 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta from 'debug' to 'debug' 2022-04-28T19:36:33.421 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:33 smithi139 bash[23439]: debug 2022-04-28T19:36:33.041+0000 7f25de296700 20 expand_channel_meta expanded map: {default=debug} 2022-04-28T19:36:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:33.054361+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:36:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:33.054860+0000 mon.a (mon.0) 175 : cluster [INF] mon.a calling monitor election 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:33.058031+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:33.058309+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:33.060370+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:33.911982+0000 mgr.y (mgr.14140) 50 : cluster [DBG] pgmap v35: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:34.047306+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:35.047598+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:35.052148+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:35.912442+0000 mgr.y (mgr.14140) 51 : cluster [DBG] pgmap v36: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:36.047326+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:37.047466+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:38.047611+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:38.080042+0000 mon.a (mon.0) 183 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:38.089449+0000 mon.a (mon.0) 184 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:38.089562+0000 mon.a (mon.0) 185 : cluster [DBG] fsmap 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:38.089621+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e6: 0 total, 0 up, 0 in 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:38.090012+0000 mon.a (mon.0) 187 : cluster [DBG] mgrmap e13: y(active, since 82s) 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: cluster 2022-04-28T19:36:38.097620+0000 mon.a (mon.0) 188 : cluster [INF] overall HEALTH_OK 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:38.104351+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:38.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:38.105997+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.y"}]: dispatch 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:38.107393+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[23786]: audit 2022-04-28T19:36:38.109492+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.y", "key": "container_image"}]: dispatch 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:33.054361+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:33.054860+0000 mon.a (mon.0) 175 : cluster [INF] mon.a calling monitor election 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:33.058031+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:33.058309+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:33.060370+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-04-28T19:36:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:33.911982+0000 mgr.y (mgr.14140) 50 : cluster [DBG] pgmap v35: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:34.047306+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:35.047598+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:35.052148+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:35.912442+0000 mgr.y (mgr.14140) 51 : cluster [DBG] pgmap v36: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:36.047326+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:37.047466+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:38.047611+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:38.080042+0000 mon.a (mon.0) 183 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:38.089449+0000 mon.a (mon.0) 184 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:36:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:38.089562+0000 mon.a (mon.0) 185 : cluster [DBG] fsmap 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:38.089621+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e6: 0 total, 0 up, 0 in 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:38.090012+0000 mon.a (mon.0) 187 : cluster [DBG] mgrmap e13: y(active, since 82s) 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: cluster 2022-04-28T19:36:38.097620+0000 mon.a (mon.0) 188 : cluster [INF] overall HEALTH_OK 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:38.104351+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:38.105997+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.y"}]: dispatch 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:38.107393+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:38.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:38 smithi059 bash[29114]: audit 2022-04-28T19:36:38.109492+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.y", "key": "container_image"}]: dispatch 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:33.054361+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:33.054860+0000 mon.a (mon.0) 175 : cluster [INF] mon.a calling monitor election 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:33.058031+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:33.058309+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:33.060370+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:33.911982+0000 mgr.y (mgr.14140) 50 : cluster [DBG] pgmap v35: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:38.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:34.047306+0000 mon.a (mon.0) 178 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:35.047598+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:35.052148+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:35.912442+0000 mgr.y (mgr.14140) 51 : cluster [DBG] pgmap v36: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:36.047326+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:37.047466+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:38.047611+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:38.080042+0000 mon.a (mon.0) 183 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:38.089449+0000 mon.a (mon.0) 184 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:38.089562+0000 mon.a (mon.0) 185 : cluster [DBG] fsmap 2022-04-28T19:36:38.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:38.089621+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e6: 0 total, 0 up, 0 in 2022-04-28T19:36:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:38.090012+0000 mon.a (mon.0) 187 : cluster [DBG] mgrmap e13: y(active, since 82s) 2022-04-28T19:36:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: cluster 2022-04-28T19:36:38.097620+0000 mon.a (mon.0) 188 : cluster [INF] overall HEALTH_OK 2022-04-28T19:36:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:38.104351+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:38.105997+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.y"}]: dispatch 2022-04-28T19:36:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:38.107393+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:38 smithi139 bash[23439]: audit 2022-04-28T19:36:38.109492+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.y", "key": "container_image"}]: dispatch 2022-04-28T19:36:39.021 INFO:teuthology.orchestra.run.smithi139.stdout: 2022-04-28T19:36:39.022 INFO:teuthology.orchestra.run.smithi139.stdout:{"epoch":3,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","modified":"2022-04-28T19:36:33.046630Z","created":"2022-04-28T19:34:26.452116Z","min_mon_release":15,"min_mon_release_name":"octopus","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3300","nonce":0},{"type":"v1","addr":"172.21.15.59:6789","nonce":0}]},"addr":"172.21.15.59:6789/0","public_addr":"172.21.15.59:6789/0","priority":0,"weight":0},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:3301","nonce":0},{"type":"v1","addr":"172.21.15.59:6790","nonce":0}]},"addr":"172.21.15.59:6790/0","public_addr":"172.21.15.59:6790/0","priority":0,"weight":0},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:3300","nonce":0},{"type":"v1","addr":"172.21.15.139:6789","nonce":0}]},"addr":"172.21.15.139:6789/0","public_addr":"172.21.15.139:6789/0","priority":0,"weight":0}],"quorum":[0,1,2]} 2022-04-28T19:36:39.024 INFO:teuthology.orchestra.run.smithi139.stderr:dumped monmap epoch 3 2022-04-28T19:36:39.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:39 smithi139 bash[23439]: cluster 2022-04-28T19:36:37.912855+0000 mgr.y (mgr.14140) 52 : cluster [DBG] pgmap v37: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:39.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:39 smithi139 bash[23439]: cephadm 2022-04-28T19:36:38.105427+0000 mgr.y (mgr.14140) 53 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2022-04-28T19:36:39.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:39 smithi139 bash[23439]: cephadm 2022-04-28T19:36:38.108614+0000 mgr.y (mgr.14140) 54 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:36:39.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:39 smithi139 bash[23439]: audit 2022-04-28T19:36:39.019195+0000 mon.a (mon.0) 193 : audit [DBG] from='client.? 172.21.15.139:0/147725908' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-04-28T19:36:39.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:39 smithi139 bash[23439]: audit 2022-04-28T19:36:39.047888+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:39.462 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[23786]: cluster 2022-04-28T19:36:37.912855+0000 mgr.y (mgr.14140) 52 : cluster [DBG] pgmap v37: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:39.462 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[23786]: cephadm 2022-04-28T19:36:38.105427+0000 mgr.y (mgr.14140) 53 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2022-04-28T19:36:39.462 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[23786]: cephadm 2022-04-28T19:36:38.108614+0000 mgr.y (mgr.14140) 54 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:36:39.462 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[23786]: audit 2022-04-28T19:36:39.019195+0000 mon.a (mon.0) 193 : audit [DBG] from='client.? 172.21.15.139:0/147725908' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-04-28T19:36:39.462 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[23786]: audit 2022-04-28T19:36:39.047888+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:39.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[29114]: cluster 2022-04-28T19:36:37.912855+0000 mgr.y (mgr.14140) 52 : cluster [DBG] pgmap v37: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:39.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[29114]: cephadm 2022-04-28T19:36:38.105427+0000 mgr.y (mgr.14140) 53 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2022-04-28T19:36:39.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[29114]: cephadm 2022-04-28T19:36:38.108614+0000 mgr.y (mgr.14140) 54 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:36:39.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[29114]: audit 2022-04-28T19:36:39.019195+0000 mon.a (mon.0) 193 : audit [DBG] from='client.? 172.21.15.139:0/147725908' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2022-04-28T19:36:39.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:39 smithi059 bash[29114]: audit 2022-04-28T19:36:39.047888+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:36:39.650 INFO:tasks.cephadm:Generating final ceph.conf file... 2022-04-28T19:36:39.650 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph config generate-minimal-conf 2022-04-28T19:36:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[23786]: cluster 2022-04-28T19:36:39.913356+0000 mgr.y (mgr.14140) 55 : cluster [DBG] pgmap v38: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[23786]: audit 2022-04-28T19:36:40.473614+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[23786]: audit 2022-04-28T19:36:40.474911+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[23786]: audit 2022-04-28T19:36:40.477102+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[23786]: audit 2022-04-28T19:36:40.479881+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.c", "key": "container_image"}]: dispatch 2022-04-28T19:36:41.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[29114]: cluster 2022-04-28T19:36:39.913356+0000 mgr.y (mgr.14140) 55 : cluster [DBG] pgmap v38: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:41.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[29114]: audit 2022-04-28T19:36:40.473614+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:41.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[29114]: audit 2022-04-28T19:36:40.474911+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:41.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[29114]: audit 2022-04-28T19:36:40.477102+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:41.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:41 smithi059 bash[29114]: audit 2022-04-28T19:36:40.479881+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.c", "key": "container_image"}]: dispatch 2022-04-28T19:36:41.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:41 smithi139 bash[23439]: cluster 2022-04-28T19:36:39.913356+0000 mgr.y (mgr.14140) 55 : cluster [DBG] pgmap v38: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:41.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:41 smithi139 bash[23439]: audit 2022-04-28T19:36:40.473614+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:41.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:41 smithi139 bash[23439]: audit 2022-04-28T19:36:40.474911+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:41.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:41 smithi139 bash[23439]: audit 2022-04-28T19:36:40.477102+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:41.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:41 smithi139 bash[23439]: audit 2022-04-28T19:36:40.479881+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.c", "key": "container_image"}]: dispatch 2022-04-28T19:36:42.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:42 smithi139 bash[23439]: cephadm 2022-04-28T19:36:40.474476+0000 mgr.y (mgr.14140) 56 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-04-28T19:36:42.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:42 smithi139 bash[23439]: cephadm 2022-04-28T19:36:40.478841+0000 mgr.y (mgr.14140) 57 : cephadm [INF] Reconfiguring daemon mon.c on smithi059 2022-04-28T19:36:42.629 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:42 smithi059 bash[23786]: cephadm 2022-04-28T19:36:40.474476+0000 mgr.y (mgr.14140) 56 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-04-28T19:36:42.629 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:42 smithi059 bash[23786]: cephadm 2022-04-28T19:36:40.478841+0000 mgr.y (mgr.14140) 57 : cephadm [INF] Reconfiguring daemon mon.c on smithi059 2022-04-28T19:36:42.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:42 smithi059 bash[29114]: cephadm 2022-04-28T19:36:40.474476+0000 mgr.y (mgr.14140) 56 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-04-28T19:36:42.630 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:42 smithi059 bash[29114]: cephadm 2022-04-28T19:36:40.478841+0000 mgr.y (mgr.14140) 57 : cephadm [INF] Reconfiguring daemon mon.c on smithi059 2022-04-28T19:36:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:43 smithi059 bash[23786]: cluster 2022-04-28T19:36:41.913847+0000 mgr.y (mgr.14140) 58 : cluster [DBG] pgmap v39: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:43.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:43 smithi059 bash[29114]: cluster 2022-04-28T19:36:41.913847+0000 mgr.y (mgr.14140) 58 : cluster [DBG] pgmap v39: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:43.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:43 smithi139 bash[23439]: cluster 2022-04-28T19:36:41.913847+0000 mgr.y (mgr.14140) 58 : cluster [DBG] pgmap v39: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:43.711 INFO:teuthology.orchestra.run.smithi059.stdout:# minimal ceph.conf for 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:36:43.711 INFO:teuthology.orchestra.run.smithi059.stdout:[global] 2022-04-28T19:36:43.711 INFO:teuthology.orchestra.run.smithi059.stdout: fsid = 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:36:43.711 INFO:teuthology.orchestra.run.smithi059.stdout: mon_host = [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] 2022-04-28T19:36:44.271 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2022-04-28T19:36:44.272 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:36:44.272 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/ceph/ceph.conf 2022-04-28T19:36:44.288 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:36:44.288 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2022-04-28T19:36:44.340 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:36:44.340 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/ceph/ceph.conf 2022-04-28T19:36:44.356 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:36:44.356 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2022-04-28T19:36:44.414 INFO:tasks.cephadm:Adding mgr.y on smithi059 2022-04-28T19:36:44.414 INFO:tasks.cephadm:Adding mgr.x on smithi139 2022-04-28T19:36:44.414 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch apply mgr '2;smithi059=y;smithi139=x' 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: audit 2022-04-28T19:36:43.320816+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: cephadm 2022-04-28T19:36:43.321621+0000 mgr.y (mgr.14140) 59 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: audit 2022-04-28T19:36:43.322104+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: audit 2022-04-28T19:36:43.323654+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: cephadm 2022-04-28T19:36:43.324647+0000 mgr.y (mgr.14140) 60 : cephadm [INF] Reconfiguring daemon mon.b on smithi139 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: audit 2022-04-28T19:36:43.325117+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.b", "key": "container_image"}]: dispatch 2022-04-28T19:36:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: audit 2022-04-28T19:36:43.708864+0000 mon.a (mon.0) 203 : audit [DBG] from='client.? 172.21.15.59:0/1270857291' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[23786]: cluster 2022-04-28T19:36:43.914404+0000 mgr.y (mgr.14140) 61 : cluster [DBG] pgmap v40: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: audit 2022-04-28T19:36:43.320816+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: cephadm 2022-04-28T19:36:43.321621+0000 mgr.y (mgr.14140) 59 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: audit 2022-04-28T19:36:43.322104+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: audit 2022-04-28T19:36:43.323654+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: cephadm 2022-04-28T19:36:43.324647+0000 mgr.y (mgr.14140) 60 : cephadm [INF] Reconfiguring daemon mon.b on smithi139 2022-04-28T19:36:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: audit 2022-04-28T19:36:43.325117+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.b", "key": "container_image"}]: dispatch 2022-04-28T19:36:44.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: audit 2022-04-28T19:36:43.708864+0000 mon.a (mon.0) 203 : audit [DBG] from='client.? 172.21.15.59:0/1270857291' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:44.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:44 smithi059 bash[29114]: cluster 2022-04-28T19:36:43.914404+0000 mgr.y (mgr.14140) 61 : cluster [DBG] pgmap v40: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:44.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: audit 2022-04-28T19:36:43.320816+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:44.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: cephadm 2022-04-28T19:36:43.321621+0000 mgr.y (mgr.14140) 59 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-04-28T19:36:44.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: audit 2022-04-28T19:36:43.322104+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:44.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: audit 2022-04-28T19:36:43.323654+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:44.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: cephadm 2022-04-28T19:36:43.324647+0000 mgr.y (mgr.14140) 60 : cephadm [INF] Reconfiguring daemon mon.b on smithi139 2022-04-28T19:36:44.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: audit 2022-04-28T19:36:43.325117+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.b", "key": "container_image"}]: dispatch 2022-04-28T19:36:44.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: audit 2022-04-28T19:36:43.708864+0000 mon.a (mon.0) 203 : audit [DBG] from='client.? 172.21.15.59:0/1270857291' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:44.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:44 smithi139 bash[23439]: cluster 2022-04-28T19:36:43.914404+0000 mgr.y (mgr.14140) 61 : cluster [DBG] pgmap v40: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:45.788420+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:45.790838+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:45.791408+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: cluster 2022-04-28T19:36:45.914830+0000 mgr.y (mgr.14140) 62 : cluster [DBG] pgmap v41: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:46.350617+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:46.477356+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:46.481762+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:46.482781+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:46.484222+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[23786]: audit 2022-04-28T19:36:46.486248+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:45.788420+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:45.790838+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:45.791408+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: cluster 2022-04-28T19:36:45.914830+0000 mgr.y (mgr.14140) 62 : cluster [DBG] pgmap v41: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:46.350617+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:46.477356+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:46.481762+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:46.482781+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:46.484222+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:47.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:46 smithi059 bash[29114]: audit 2022-04-28T19:36:46.486248+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.115 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:45.788420+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.115 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:45.790838+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.115 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:45.791408+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: cluster 2022-04-28T19:36:45.914830+0000 mgr.y (mgr.14140) 62 : cluster [DBG] pgmap v41: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:46.350617+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:46.477356+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:46.481762+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:46.482781+0000 mon.a (mon.0) 210 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:46.484222+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:47.116 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:46 smithi139 bash[23439]: audit 2022-04-28T19:36:46.486248+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon.a", "key": "container_image"}]: dispatch 2022-04-28T19:36:48.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:47 smithi059 bash[23786]: cephadm 2022-04-28T19:36:46.482350+0000 mgr.y (mgr.14140) 63 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:36:48.060 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:47 smithi059 bash[23786]: cephadm 2022-04-28T19:36:46.485311+0000 mgr.y (mgr.14140) 64 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:36:48.061 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:47 smithi059 bash[29114]: cephadm 2022-04-28T19:36:46.482350+0000 mgr.y (mgr.14140) 63 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:36:48.061 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:47 smithi059 bash[29114]: cephadm 2022-04-28T19:36:46.485311+0000 mgr.y (mgr.14140) 64 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:36:48.077 INFO:teuthology.orchestra.run.smithi139.stdout:Scheduled mgr update... 2022-04-28T19:36:48.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:47 smithi139 bash[23439]: cephadm 2022-04-28T19:36:46.482350+0000 mgr.y (mgr.14140) 63 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:36:48.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:47 smithi139 bash[23439]: cephadm 2022-04-28T19:36:46.485311+0000 mgr.y (mgr.14140) 64 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:36:48.630 DEBUG:teuthology.orchestra.run.smithi139:mgr.x> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.x.service 2022-04-28T19:36:48.633 INFO:tasks.cephadm:Deploying OSDs... 2022-04-28T19:36:48.633 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:36:48.633 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/scratch_devs of=/dev/stdout 2022-04-28T19:36:48.640 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-04-28T19:36:48.640 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-04-28T19:36:48.670 INFO:journalctl@ceph.mgr.x.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 6h/6d Inode: 344 Links: 1 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2022-04-28 19:31:30.214374716 +0000 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2022-04-28 19:31:29.986374716 +0000 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2022-04-28 19:31:29.986374716 +0000 2022-04-28T19:36:48.687 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: - 2022-04-28T19:36:48.696 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2022-04-28T19:36:48.696 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2022-04-28T19:36:48.696 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000290017 s, 1.8 MB/s 2022-04-28T19:36:48.699 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 6h/6d Inode: 353 Links: 1 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2022-04-28 19:31:30.574374716 +0000 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2022-04-28 19:31:30.382374716 +0000 2022-04-28T19:36:48.747 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2022-04-28 19:31:30.382374716 +0000 2022-04-28T19:36:48.748 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: - 2022-04-28T19:36:48.756 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2022-04-28T19:36:48.756 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2022-04-28T19:36:48.756 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000355854 s, 1.4 MB/s 2022-04-28T19:36:48.759 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 6h/6d Inode: 359 Links: 1 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2022-04-28 19:31:30.934374716 +0000 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2022-04-28 19:31:30.694374716 +0000 2022-04-28T19:36:48.807 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2022-04-28 19:31:30.694374716 +0000 2022-04-28T19:36:48.808 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: - 2022-04-28T19:36:48.814 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2022-04-28T19:36:48.814 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2022-04-28T19:36:48.814 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000371753 s, 1.4 MB/s 2022-04-28T19:36:48.817 DEBUG:teuthology.orchestra.run.smithi059:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout:Device: 6h/6d Inode: 365 Links: 1 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout:Access: 2022-04-28 19:31:31.282374716 +0000 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout:Modify: 2022-04-28 19:31:31.054374716 +0000 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout:Change: 2022-04-28 19:31:31.054374716 +0000 2022-04-28T19:36:48.866 INFO:teuthology.orchestra.run.smithi059.stdout: Birth: - 2022-04-28T19:36:48.873 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records in 2022-04-28T19:36:48.874 INFO:teuthology.orchestra.run.smithi059.stderr:1+0 records out 2022-04-28T19:36:48.874 INFO:teuthology.orchestra.run.smithi059.stderr:512 bytes copied, 0.000303103 s, 1.7 MB/s 2022-04-28T19:36:48.876 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:36:48.876 DEBUG:teuthology.orchestra.run.smithi139:> dd if=/scratch_devs of=/dev/stdout 2022-04-28T19:36:48.883 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2022-04-28T19:36:48.883 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_1 && sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_1 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 6h/6d Inode: 343 Links: 1 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2022-04-28 19:31:29.723118648 +0000 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2022-04-28 19:31:29.507118650 +0000 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2022-04-28 19:31:29.507118650 +0000 2022-04-28T19:36:48.934 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: - 2022-04-28T19:36:48.943 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2022-04-28T19:36:48.943 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2022-04-28T19:36:48.943 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000385016 s, 1.3 MB/s 2022-04-28T19:36:48.947 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_2 && sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_2 2022-04-28T19:36:48.997 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 6h/6d Inode: 349 Links: 1 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2022-04-28 19:31:30.087118644 +0000 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2022-04-28 19:31:29.875118646 +0000 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2022-04-28 19:31:29.875118646 +0000 2022-04-28T19:36:48.998 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: - 2022-04-28T19:36:49.007 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2022-04-28T19:36:49.007 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2022-04-28T19:36:49.007 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000365557 s, 1.4 MB/s 2022-04-28T19:36:49.012 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_3 && sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_3 2022-04-28T19:36:49.061 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2022-04-28T19:36:49.061 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:49.062 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 6h/6d Inode: 356 Links: 1 2022-04-28T19:36:49.062 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:49.062 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2022-04-28 19:31:30.431118640 +0000 2022-04-28T19:36:49.062 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2022-04-28 19:31:30.219118643 +0000 2022-04-28T19:36:49.062 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2022-04-28 19:31:30.219118643 +0000 2022-04-28T19:36:49.062 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: - 2022-04-28T19:36:49.071 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2022-04-28T19:36:49.071 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2022-04-28T19:36:49.071 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000304317 s, 1.7 MB/s 2022-04-28T19:36:49.075 DEBUG:teuthology.orchestra.run.smithi139:> stat /dev/vg_nvme/lv_4 && sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 && ! mount | grep -q /dev/vg_nvme/lv_4 2022-04-28T19:36:49.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[23786]: cluster 2022-04-28T19:36:47.915312+0000 mgr.y (mgr.14140) 65 : cluster [DBG] pgmap v42: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:49.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[23786]: audit 2022-04-28T19:36:48.064627+0000 mgr.y (mgr.14140) 66 : audit [DBG] from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi059=y;smithi139=x", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:49.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[23786]: cephadm 2022-04-28T19:36:48.066393+0000 mgr.y (mgr.14140) 67 : cephadm [INF] Saving service mgr spec with placement smithi059=y;smithi139=x;count:2 2022-04-28T19:36:49.077 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[23786]: audit 2022-04-28T19:36:48.073588+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:49.077 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[23786]: audit 2022-04-28T19:36:48.531796+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:49.129 INFO:teuthology.orchestra.run.smithi139.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout:Device: 6h/6d Inode: 362 Links: 1 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout:Access: 2022-04-28 19:31:30.783118637 +0000 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout:Modify: 2022-04-28 19:31:30.559118639 +0000 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout:Change: 2022-04-28 19:31:30.559118639 +0000 2022-04-28T19:36:49.130 INFO:teuthology.orchestra.run.smithi139.stdout: Birth: - 2022-04-28T19:36:49.139 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records in 2022-04-28T19:36:49.139 INFO:teuthology.orchestra.run.smithi139.stderr:1+0 records out 2022-04-28T19:36:49.139 INFO:teuthology.orchestra.run.smithi139.stderr:512 bytes copied, 0.000380046 s, 1.3 MB/s 2022-04-28T19:36:49.144 INFO:tasks.cephadm:Deploying osd.0 on smithi059 with /dev/vg_nvme/lv_4... 2022-04-28T19:36:49.145 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2022-04-28T19:36:49.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[23786]: audit 2022-04-28T19:36:48.534088+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:49.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[29114]: cluster 2022-04-28T19:36:47.915312+0000 mgr.y (mgr.14140) 65 : cluster [DBG] pgmap v42: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:49.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[29114]: audit 2022-04-28T19:36:48.064627+0000 mgr.y (mgr.14140) 66 : audit [DBG] from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi059=y;smithi139=x", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:49.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[29114]: cephadm 2022-04-28T19:36:48.066393+0000 mgr.y (mgr.14140) 67 : cephadm [INF] Saving service mgr spec with placement smithi059=y;smithi139=x;count:2 2022-04-28T19:36:49.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[29114]: audit 2022-04-28T19:36:48.073588+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:49.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[29114]: audit 2022-04-28T19:36:48.531796+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:49.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:49 smithi059 bash[29114]: audit 2022-04-28T19:36:48.534088+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:49.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:49 smithi139 bash[23439]: cluster 2022-04-28T19:36:47.915312+0000 mgr.y (mgr.14140) 65 : cluster [DBG] pgmap v42: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:49.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:49 smithi139 bash[23439]: audit 2022-04-28T19:36:48.064627+0000 mgr.y (mgr.14140) 66 : audit [DBG] from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi059=y;smithi139=x", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:49.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:49 smithi139 bash[23439]: cephadm 2022-04-28T19:36:48.066393+0000 mgr.y (mgr.14140) 67 : cephadm [INF] Saving service mgr spec with placement smithi059=y;smithi139=x;count:2 2022-04-28T19:36:49.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:49 smithi139 bash[23439]: audit 2022-04-28T19:36:48.073588+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:49.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:49 smithi139 bash[23439]: audit 2022-04-28T19:36:48.531796+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:49.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:49 smithi139 bash[23439]: audit 2022-04-28T19:36:48.534088+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:50.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.205299+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:50.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.210932+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:50.432 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.212861+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:36:50.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.217912+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2022-04-28T19:36:50.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.219167+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:36:50.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.220174+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:50.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: cephadm 2022-04-28T19:36:49.220820+0000 mgr.y (mgr.14140) 68 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:36:50.434 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[23786]: audit 2022-04-28T19:36:49.221412+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.x", "key": "container_image"}]: dispatch 2022-04-28T19:36:50.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.205299+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:50.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.210932+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:50.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.212861+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:36:50.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.217912+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2022-04-28T19:36:50.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.219167+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:36:50.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.220174+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:50.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: cephadm 2022-04-28T19:36:49.220820+0000 mgr.y (mgr.14140) 68 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:36:50.436 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:50 smithi059 bash[29114]: audit 2022-04-28T19:36:49.221412+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.x", "key": "container_image"}]: dispatch 2022-04-28T19:36:50.472 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.205299+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:50.472 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.210932+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:50.473 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.212861+0000 mon.a (mon.0) 218 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:36:50.473 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.217912+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2022-04-28T19:36:50.473 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.219167+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:36:50.473 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.220174+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:50.473 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: cephadm 2022-04-28T19:36:49.220820+0000 mgr.y (mgr.14140) 68 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:36:50.473 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:50 smithi139 bash[23439]: audit 2022-04-28T19:36:49.221412+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.x", "key": "container_image"}]: dispatch 2022-04-28T19:36:51.588 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:51 smithi139 bash[23439]: cluster 2022-04-28T19:36:49.915860+0000 mgr.y (mgr.14140) 69 : cluster [DBG] pgmap v43: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:51 smithi059 bash[23786]: cluster 2022-04-28T19:36:49.915860+0000 mgr.y (mgr.14140) 69 : cluster [DBG] pgmap v43: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:51 smithi059 bash[29114]: cluster 2022-04-28T19:36:49.915860+0000 mgr.y (mgr.14140) 69 : cluster [DBG] pgmap v43: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:51.865 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:36:51 smithi139 systemd[1]: Started Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:36:52.166 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_4 2022-04-28T19:36:52.166 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2022-04-28T19:36:52.166 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2022-04-28T19:36:52.167 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2022-04-28T19:36:52.167 INFO:teuthology.orchestra.run.smithi059.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0150316 s, 698 MB/s 2022-04-28T19:36:52.167 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 2022-04-28T19:36:52.167 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2022-04-28T19:36:52.785 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi059:vg_nvme/lv_4 2022-04-28T19:36:52.865 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[24478]: debug 2022-04-28T19:36:52.749+0000 7fc7ac304700 1 -- 172.21.15.139:0/151301538 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5595c071c4e0 con 0x5595c0803000 2022-04-28T19:36:52.865 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:51.616330+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:51.618451+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:51.618994+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: cluster 2022-04-28T19:36:51.916235+0000 mgr.y (mgr.14140) 70 : cluster [DBG] pgmap v44: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:52.276938+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:52.312900+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:52.317536+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:52.866 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:52 smithi139 bash[23439]: audit 2022-04-28T19:36:52.320556+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:53.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:51.616330+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:51.618451+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:51.618994+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: cluster 2022-04-28T19:36:51.916235+0000 mgr.y (mgr.14140) 70 : cluster [DBG] pgmap v44: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:52.276938+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:52.312900+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:52.317536+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[29114]: audit 2022-04-28T19:36:52.320556+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:51.616330+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:51.618451+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:51.618994+0000 mon.a (mon.0) 225 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: cluster 2022-04-28T19:36:51.916235+0000 mgr.y (mgr.14140) 70 : cluster [DBG] pgmap v44: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:52.276938+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:52.312900+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:52.317536+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:52 smithi059 bash[23786]: audit 2022-04-28T19:36:52.320556+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:36:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:54 smithi059 bash[29114]: audit 2022-04-28T19:36:53.018062+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:54 smithi059 bash[29114]: audit 2022-04-28T19:36:53.023664+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:54.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:54 smithi059 bash[23786]: audit 2022-04-28T19:36:53.018062+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:54.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:54 smithi059 bash[23786]: audit 2022-04-28T19:36:53.023664+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:54.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:54 smithi139 bash[23439]: audit 2022-04-28T19:36:53.018062+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:54.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:54 smithi139 bash[23439]: audit 2022-04-28T19:36:53.023664+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:36:55.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:55 smithi059 bash[23786]: cluster 2022-04-28T19:36:53.916747+0000 mgr.y (mgr.14140) 71 : cluster [DBG] pgmap v45: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:55.289 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:55 smithi059 bash[29114]: cluster 2022-04-28T19:36:53.916747+0000 mgr.y (mgr.14140) 71 : cluster [DBG] pgmap v45: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:55.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:55 smithi139 bash[23439]: cluster 2022-04-28T19:36:53.916747+0000 mgr.y (mgr.14140) 71 : cluster [DBG] pgmap v45: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:56.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[29114]: audit 2022-04-28T19:36:55.693561+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[29114]: audit 2022-04-28T19:36:55.697846+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[29114]: audit 2022-04-28T19:36:55.699369+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[29114]: audit 2022-04-28T19:36:55.701053+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[23786]: audit 2022-04-28T19:36:55.693561+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[23786]: audit 2022-04-28T19:36:55.697846+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[23786]: audit 2022-04-28T19:36:55.699369+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:56.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:56 smithi059 bash[23786]: audit 2022-04-28T19:36:55.701053+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:36:56.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:56 smithi139 bash[23439]: audit 2022-04-28T19:36:55.693561+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:36:56.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:56 smithi139 bash[23439]: audit 2022-04-28T19:36:55.697846+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:36:56.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:56 smithi139 bash[23439]: audit 2022-04-28T19:36:55.699369+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:36:56.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:56 smithi139 bash[23439]: audit 2022-04-28T19:36:55.701053+0000 mon.a (mon.0) 235 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:36:57.183 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:57 smithi059 bash[29114]: audit 2022-04-28T19:36:55.691717+0000 mgr.y (mgr.14140) 72 : audit [DBG] from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:57.183 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:57 smithi059 bash[29114]: cephadm 2022-04-28T19:36:55.697317+0000 mgr.y (mgr.14140) 73 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:36:57.183 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:57 smithi059 bash[29114]: cluster 2022-04-28T19:36:55.917222+0000 mgr.y (mgr.14140) 74 : cluster [DBG] pgmap v46: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:57.183 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:57 smithi059 bash[23786]: audit 2022-04-28T19:36:55.691717+0000 mgr.y (mgr.14140) 72 : audit [DBG] from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:57.184 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:57 smithi059 bash[23786]: cephadm 2022-04-28T19:36:55.697317+0000 mgr.y (mgr.14140) 73 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:36:57.184 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:57 smithi059 bash[23786]: cluster 2022-04-28T19:36:55.917222+0000 mgr.y (mgr.14140) 74 : cluster [DBG] pgmap v46: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:57.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:57 smithi139 bash[23439]: audit 2022-04-28T19:36:55.691717+0000 mgr.y (mgr.14140) 72 : audit [DBG] from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:36:57.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:57 smithi139 bash[23439]: cephadm 2022-04-28T19:36:55.697317+0000 mgr.y (mgr.14140) 73 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:36:57.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:57 smithi139 bash[23439]: cluster 2022-04-28T19:36:55.917222+0000 mgr.y (mgr.14140) 74 : cluster [DBG] pgmap v46: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[23786]: cluster 2022-04-28T19:36:57.105291+0000 mon.a (mon.0) 236 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:36:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[23786]: audit 2022-04-28T19:36:57.111044+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:36:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[23786]: audit 2022-04-28T19:36:57.111798+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:36:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[23786]: audit 2022-04-28T19:36:57.113772+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:36:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[23786]: audit 2022-04-28T19:36:57.114433+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:36:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[29114]: cluster 2022-04-28T19:36:57.105291+0000 mon.a (mon.0) 236 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:36:58.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[29114]: audit 2022-04-28T19:36:57.111044+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:36:58.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[29114]: audit 2022-04-28T19:36:57.111798+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:36:58.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[29114]: audit 2022-04-28T19:36:57.113772+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:36:58.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:58 smithi059 bash[29114]: audit 2022-04-28T19:36:57.114433+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:36:58.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:58 smithi139 bash[23439]: cluster 2022-04-28T19:36:57.105291+0000 mon.a (mon.0) 236 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:36:58.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:58 smithi139 bash[23439]: audit 2022-04-28T19:36:57.111044+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:36:58.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:58 smithi139 bash[23439]: audit 2022-04-28T19:36:57.111798+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:36:58.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:58 smithi139 bash[23439]: audit 2022-04-28T19:36:57.113772+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:36:58.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:58 smithi139 bash[23439]: audit 2022-04-28T19:36:57.114433+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.139:0/3464940690' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:36:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:59 smithi059 bash[23786]: cluster 2022-04-28T19:36:57.917711+0000 mgr.y (mgr.14140) 75 : cluster [DBG] pgmap v47: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:59 smithi059 bash[23786]: cluster 2022-04-28T19:36:58.054736+0000 mon.a (mon.0) 237 : cluster [DBG] mgrmap e14: y(active, since 102s), standbys: x 2022-04-28T19:36:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:36:59 smithi059 bash[23786]: audit 2022-04-28T19:36:58.055243+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:36:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:59 smithi059 bash[29114]: cluster 2022-04-28T19:36:57.917711+0000 mgr.y (mgr.14140) 75 : cluster [DBG] pgmap v47: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:59 smithi059 bash[29114]: cluster 2022-04-28T19:36:58.054736+0000 mon.a (mon.0) 237 : cluster [DBG] mgrmap e14: y(active, since 102s), standbys: x 2022-04-28T19:36:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:36:59 smithi059 bash[29114]: audit 2022-04-28T19:36:58.055243+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:36:59.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:59 smithi139 bash[23439]: cluster 2022-04-28T19:36:57.917711+0000 mgr.y (mgr.14140) 75 : cluster [DBG] pgmap v47: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:36:59.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:59 smithi139 bash[23439]: cluster 2022-04-28T19:36:58.054736+0000 mon.a (mon.0) 237 : cluster [DBG] mgrmap e14: y(active, since 102s), standbys: x 2022-04-28T19:36:59.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:36:59 smithi139 bash[23439]: audit 2022-04-28T19:36:58.055243+0000 mon.a (mon.0) 238 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:37:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[23786]: audit 2022-04-28T19:36:59.261604+0000 mon.c (mon.1) 3 : audit [INF] from='client.? 172.21.15.59:0/3501645720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]: dispatch 2022-04-28T19:37:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[23786]: audit 2022-04-28T19:36:59.262194+0000 mon.a (mon.0) 239 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]: dispatch 2022-04-28T19:37:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[23786]: audit 2022-04-28T19:36:59.270816+0000 mon.a (mon.0) 240 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]': finished 2022-04-28T19:37:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[23786]: cluster 2022-04-28T19:36:59.270963+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e7: 1 total, 0 up, 0 in 2022-04-28T19:37:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[23786]: audit 2022-04-28T19:36:59.271158+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[23786]: audit 2022-04-28T19:36:59.784934+0000 mon.a (mon.0) 243 : audit [DBG] from='client.? 172.21.15.59:0/1640575608' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:00.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[29114]: audit 2022-04-28T19:36:59.261604+0000 mon.c (mon.1) 3 : audit [INF] from='client.? 172.21.15.59:0/3501645720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]: dispatch 2022-04-28T19:37:00.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[29114]: audit 2022-04-28T19:36:59.262194+0000 mon.a (mon.0) 239 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]: dispatch 2022-04-28T19:37:00.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[29114]: audit 2022-04-28T19:36:59.270816+0000 mon.a (mon.0) 240 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]': finished 2022-04-28T19:37:00.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[29114]: cluster 2022-04-28T19:36:59.270963+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e7: 1 total, 0 up, 0 in 2022-04-28T19:37:00.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[29114]: audit 2022-04-28T19:36:59.271158+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:00.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:00 smithi059 bash[29114]: audit 2022-04-28T19:36:59.784934+0000 mon.a (mon.0) 243 : audit [DBG] from='client.? 172.21.15.59:0/1640575608' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:00.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:00 smithi139 bash[23439]: audit 2022-04-28T19:36:59.261604+0000 mon.c (mon.1) 3 : audit [INF] from='client.? 172.21.15.59:0/3501645720' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]: dispatch 2022-04-28T19:37:00.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:00 smithi139 bash[23439]: audit 2022-04-28T19:36:59.262194+0000 mon.a (mon.0) 239 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]: dispatch 2022-04-28T19:37:00.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:00 smithi139 bash[23439]: audit 2022-04-28T19:36:59.270816+0000 mon.a (mon.0) 240 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "923b6dc7-5112-4753-8e1f-7f260c42bcb7"}]': finished 2022-04-28T19:37:00.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:00 smithi139 bash[23439]: cluster 2022-04-28T19:36:59.270963+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e7: 1 total, 0 up, 0 in 2022-04-28T19:37:00.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:00 smithi139 bash[23439]: audit 2022-04-28T19:36:59.271158+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:00.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:00 smithi139 bash[23439]: audit 2022-04-28T19:36:59.784934+0000 mon.a (mon.0) 243 : audit [DBG] from='client.? 172.21.15.59:0/1640575608' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:01 smithi059 bash[23786]: cluster 2022-04-28T19:36:59.918210+0000 mgr.y (mgr.14140) 76 : cluster [DBG] pgmap v49: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:01.434 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:01 smithi059 bash[29114]: cluster 2022-04-28T19:36:59.918210+0000 mgr.y (mgr.14140) 76 : cluster [DBG] pgmap v49: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:01.435 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:01 smithi139 bash[23439]: cluster 2022-04-28T19:36:59.918210+0000 mgr.y (mgr.14140) 76 : cluster [DBG] pgmap v49: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:03 smithi059 bash[23786]: cluster 2022-04-28T19:37:01.918661+0000 mgr.y (mgr.14140) 77 : cluster [DBG] pgmap v50: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:03.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:03 smithi059 bash[23786]: audit 2022-04-28T19:37:02.601700+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:03 smithi059 bash[29114]: cluster 2022-04-28T19:37:01.918661+0000 mgr.y (mgr.14140) 77 : cluster [DBG] pgmap v50: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:03 smithi059 bash[29114]: audit 2022-04-28T19:37:02.601700+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:03.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:03 smithi139 bash[23439]: cluster 2022-04-28T19:37:01.918661+0000 mgr.y (mgr.14140) 77 : cluster [DBG] pgmap v50: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:03.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:03 smithi139 bash[23439]: audit 2022-04-28T19:37:02.601700+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:05 smithi059 bash[23786]: cluster 2022-04-28T19:37:03.919246+0000 mgr.y (mgr.14140) 78 : cluster [DBG] pgmap v51: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:05.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:05 smithi059 bash[29114]: cluster 2022-04-28T19:37:03.919246+0000 mgr.y (mgr.14140) 78 : cluster [DBG] pgmap v51: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:05.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:05 smithi139 bash[23439]: cluster 2022-04-28T19:37:03.919246+0000 mgr.y (mgr.14140) 78 : cluster [DBG] pgmap v51: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[23786]: cluster 2022-04-28T19:37:05.919864+0000 mgr.y (mgr.14140) 79 : cluster [DBG] pgmap v52: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[23786]: audit 2022-04-28T19:37:06.245769+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-04-28T19:37:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[23786]: audit 2022-04-28T19:37:06.246863+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[23786]: cephadm 2022-04-28T19:37:06.247750+0000 mgr.y (mgr.14140) 80 : cephadm [INF] Deploying daemon osd.0 on smithi059 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[23786]: audit 2022-04-28T19:37:06.248126+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.0", "key": "container_image"}]: dispatch 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[29114]: cluster 2022-04-28T19:37:05.919864+0000 mgr.y (mgr.14140) 79 : cluster [DBG] pgmap v52: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[29114]: audit 2022-04-28T19:37:06.245769+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[29114]: audit 2022-04-28T19:37:06.246863+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[29114]: cephadm 2022-04-28T19:37:06.247750+0000 mgr.y (mgr.14140) 80 : cephadm [INF] Deploying daemon osd.0 on smithi059 2022-04-28T19:37:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:07 smithi059 bash[29114]: audit 2022-04-28T19:37:06.248126+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.0", "key": "container_image"}]: dispatch 2022-04-28T19:37:07.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:07 smithi139 bash[23439]: cluster 2022-04-28T19:37:05.919864+0000 mgr.y (mgr.14140) 79 : cluster [DBG] pgmap v52: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:07.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:07 smithi139 bash[23439]: audit 2022-04-28T19:37:06.245769+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2022-04-28T19:37:07.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:07 smithi139 bash[23439]: audit 2022-04-28T19:37:06.246863+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:07.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:07 smithi139 bash[23439]: cephadm 2022-04-28T19:37:06.247750+0000 mgr.y (mgr.14140) 80 : cephadm [INF] Deploying daemon osd.0 on smithi059 2022-04-28T19:37:07.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:07 smithi139 bash[23439]: audit 2022-04-28T19:37:06.248126+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.0", "key": "container_image"}]: dispatch 2022-04-28T19:37:08.405 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 0 on host 'smithi059' 2022-04-28T19:37:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:09 smithi059 bash[23786]: cluster 2022-04-28T19:37:07.920393+0000 mgr.y (mgr.14140) 81 : cluster [DBG] pgmap v53: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:09 smithi059 bash[23786]: audit 2022-04-28T19:37:08.401927+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:09.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:09 smithi059 bash[23786]: audit 2022-04-28T19:37:08.426886+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:09 smithi059 bash[29114]: cluster 2022-04-28T19:37:07.920393+0000 mgr.y (mgr.14140) 81 : cluster [DBG] pgmap v53: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:09 smithi059 bash[29114]: audit 2022-04-28T19:37:08.401927+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:09 smithi059 bash[29114]: audit 2022-04-28T19:37:08.426886+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:09 smithi139 bash[23439]: cluster 2022-04-28T19:37:07.920393+0000 mgr.y (mgr.14140) 81 : cluster [DBG] pgmap v53: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:09 smithi139 bash[23439]: audit 2022-04-28T19:37:08.401927+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:09 smithi139 bash[23439]: audit 2022-04-28T19:37:08.426886+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:09.456 DEBUG:teuthology.orchestra.run.smithi059:osd.0> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.0.service 2022-04-28T19:37:09.458 INFO:tasks.cephadm:Deploying osd.1 on smithi059 with /dev/vg_nvme/lv_3... 2022-04-28T19:37:09.458 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2022-04-28T19:37:09.492 INFO:journalctl@ceph.osd.0.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:37:10.527 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[23786]: cephadm 2022-04-28T19:37:08.426668+0000 mgr.y (mgr.14140) 82 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:37:10.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[23786]: audit 2022-04-28T19:37:09.201329+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:10.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[23786]: audit 2022-04-28T19:37:09.202606+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:10.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[29114]: cephadm 2022-04-28T19:37:08.426668+0000 mgr.y (mgr.14140) 82 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:37:10.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[29114]: audit 2022-04-28T19:37:09.201329+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:10.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[29114]: audit 2022-04-28T19:37:09.202606+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2022-04-28T19:37:10.529 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:10 smithi059 bash[31905]: --> ceph-volume lvm activate successful for osd ID: 0 2022-04-28T19:37:10.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:10 smithi139 bash[23439]: cephadm 2022-04-28T19:37:08.426668+0000 mgr.y (mgr.14140) 82 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:37:10.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:10 smithi139 bash[23439]: audit 2022-04-28T19:37:09.201329+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:10.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:10 smithi139 bash[23439]: audit 2022-04-28T19:37:09.202606+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:11.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:11 smithi139 bash[23439]: cluster 2022-04-28T19:37:09.920879+0000 mgr.y (mgr.14140) 83 : cluster [DBG] pgmap v54: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:11.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:11 smithi059 bash[23786]: cluster 2022-04-28T19:37:09.920879+0000 mgr.y (mgr.14140) 83 : cluster [DBG] pgmap v54: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:11.647 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:11 smithi059 bash[29114]: cluster 2022-04-28T19:37:09.920879+0000 mgr.y (mgr.14140) 83 : cluster [DBG] pgmap v54: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:12 smithi059 bash[23786]: cluster 2022-04-28T19:37:11.921448+0000 mgr.y (mgr.14140) 84 : cluster [DBG] pgmap v55: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:12 smithi059 bash[23786]: cluster 2022-04-28T19:37:12.212769+0000 mon.a (mon.0) 252 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2022-04-28T19:37:13.343 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:13 smithi059 bash[31905]: debug 2022-04-28T19:37:13.189+0000 7ff8357dd700 1 -- 172.21.15.59:0/1444134004 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555ec17d04e0 con 0x555ec184b400 2022-04-28T19:37:13.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:12 smithi059 bash[29114]: cluster 2022-04-28T19:37:11.921448+0000 mgr.y (mgr.14140) 84 : cluster [DBG] pgmap v55: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:13.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:12 smithi059 bash[29114]: cluster 2022-04-28T19:37:12.212769+0000 mon.a (mon.0) 252 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2022-04-28T19:37:13.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:12 smithi139 bash[23439]: cluster 2022-04-28T19:37:11.921448+0000 mgr.y (mgr.14140) 84 : cluster [DBG] pgmap v55: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:13.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:12 smithi139 bash[23439]: cluster 2022-04-28T19:37:12.212769+0000 mon.a (mon.0) 252 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2022-04-28T19:37:14.092 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:13 smithi059 bash[31905]: debug 2022-04-28T19:37:13.737+0000 7ff83ca7bf00 -1 Falling back to public interface 2022-04-28T19:37:14.842 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:14 smithi059 bash[31905]: debug 2022-04-28T19:37:14.341+0000 7ff83ca7bf00 -1 osd.0 0 log_to_monitors {default=true} 2022-04-28T19:37:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:15 smithi059 bash[23786]: cluster 2022-04-28T19:37:13.922028+0000 mgr.y (mgr.14140) 85 : cluster [DBG] pgmap v56: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:15 smithi059 bash[23786]: audit 2022-04-28T19:37:14.349836+0000 mon.a (mon.0) 253 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:37:15.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:15 smithi059 bash[29114]: cluster 2022-04-28T19:37:13.922028+0000 mgr.y (mgr.14140) 85 : cluster [DBG] pgmap v56: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:15.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:15 smithi059 bash[29114]: audit 2022-04-28T19:37:14.349836+0000 mon.a (mon.0) 253 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:37:15.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:15 smithi139 bash[23439]: cluster 2022-04-28T19:37:13.922028+0000 mgr.y (mgr.14140) 85 : cluster [DBG] pgmap v56: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:15.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:15 smithi139 bash[23439]: audit 2022-04-28T19:37:14.349836+0000 mon.a (mon.0) 253 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:37:15.401 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_3 2022-04-28T19:37:15.401 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2022-04-28T19:37:15.401 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2022-04-28T19:37:15.401 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2022-04-28T19:37:15.402 INFO:teuthology.orchestra.run.smithi059.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0130896 s, 801 MB/s 2022-04-28T19:37:15.402 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2022-04-28T19:37:16.308 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[31905]: debug 2022-04-28T19:37:16.005+0000 7ff823f49700 -1 osd.0 0 waiting for initial osdmap 2022-04-28T19:37:16.308 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[31905]: debug 2022-04-28T19:37:16.009+0000 7ff830762700 -1 osd.0 9 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:37:16.309 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[23786]: audit 2022-04-28T19:37:14.998302+0000 mon.a (mon.0) 254 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-04-28T19:37:16.309 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[23786]: cluster 2022-04-28T19:37:14.998741+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e8: 1 total, 0 up, 0 in 2022-04-28T19:37:16.309 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[23786]: audit 2022-04-28T19:37:14.999438+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:16.309 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[23786]: audit 2022-04-28T19:37:15.000374+0000 mon.a (mon.0) 257 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:16.309 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[29114]: audit 2022-04-28T19:37:14.998302+0000 mon.a (mon.0) 254 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-04-28T19:37:16.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[29114]: cluster 2022-04-28T19:37:14.998741+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e8: 1 total, 0 up, 0 in 2022-04-28T19:37:16.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[29114]: audit 2022-04-28T19:37:14.999438+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:16.310 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:16 smithi059 bash[29114]: audit 2022-04-28T19:37:15.000374+0000 mon.a (mon.0) 257 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:16.311 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi059:vg_nvme/lv_3 2022-04-28T19:37:16.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:16 smithi139 bash[23439]: audit 2022-04-28T19:37:14.998302+0000 mon.a (mon.0) 254 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-04-28T19:37:16.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:16 smithi139 bash[23439]: cluster 2022-04-28T19:37:14.998741+0000 mon.a (mon.0) 255 : cluster [DBG] osdmap e8: 1 total, 0 up, 0 in 2022-04-28T19:37:16.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:16 smithi139 bash[23439]: audit 2022-04-28T19:37:14.999438+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:16.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:16 smithi139 bash[23439]: audit 2022-04-28T19:37:15.000374+0000 mon.a (mon.0) 257 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: cluster 2022-04-28T19:37:15.922590+0000 mgr.y (mgr.14140) 86 : cluster [DBG] pgmap v58: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.002630+0000 mon.a (mon.0) 258 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: cluster 2022-04-28T19:37:16.002838+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 0 up, 0 in 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.003127+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.011241+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.105384+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.107924+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.480966+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.785193+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:17.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[29114]: audit 2022-04-28T19:37:16.793548+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: cluster 2022-04-28T19:37:15.922590+0000 mgr.y (mgr.14140) 86 : cluster [DBG] pgmap v58: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.002630+0000 mon.a (mon.0) 258 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: cluster 2022-04-28T19:37:16.002838+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 0 up, 0 in 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.003127+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.011241+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.105384+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.107924+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.480966+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:17.294 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.785193+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:17.295 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:17 smithi059 bash[23786]: audit 2022-04-28T19:37:16.793548+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: cluster 2022-04-28T19:37:15.922590+0000 mgr.y (mgr.14140) 86 : cluster [DBG] pgmap v58: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.002630+0000 mon.a (mon.0) 258 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: cluster 2022-04-28T19:37:16.002838+0000 mon.a (mon.0) 259 : cluster [DBG] osdmap e9: 1 total, 0 up, 0 in 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.003127+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.011241+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.105384+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:37:17.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.107924+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:37:17.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.480966+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:17.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.785193+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:17.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:17 smithi139 bash[23439]: audit 2022-04-28T19:37:16.793548+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[23786]: cluster 2022-04-28T19:37:15.357761+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:37:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[23786]: cluster 2022-04-28T19:37:15.357820+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:37:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[23786]: audit 2022-04-28T19:37:17.009603+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[23786]: cluster 2022-04-28T19:37:17.023295+0000 mon.a (mon.0) 268 : cluster [INF] osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759] boot 2022-04-28T19:37:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[23786]: cluster 2022-04-28T19:37:17.023377+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2022-04-28T19:37:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[23786]: audit 2022-04-28T19:37:17.023750+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[29114]: cluster 2022-04-28T19:37:15.357761+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:37:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[29114]: cluster 2022-04-28T19:37:15.357820+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:37:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[29114]: audit 2022-04-28T19:37:17.009603+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:18.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[29114]: cluster 2022-04-28T19:37:17.023295+0000 mon.a (mon.0) 268 : cluster [INF] osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759] boot 2022-04-28T19:37:18.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[29114]: cluster 2022-04-28T19:37:17.023377+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2022-04-28T19:37:18.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:18 smithi059 bash[29114]: audit 2022-04-28T19:37:17.023750+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:18.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:18 smithi139 bash[23439]: cluster 2022-04-28T19:37:15.357761+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:37:18.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:18 smithi139 bash[23439]: cluster 2022-04-28T19:37:15.357820+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:37:18.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:18 smithi139 bash[23439]: audit 2022-04-28T19:37:17.009603+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:18.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:18 smithi139 bash[23439]: cluster 2022-04-28T19:37:17.023295+0000 mon.a (mon.0) 268 : cluster [INF] osd.0 [v2:172.21.15.59:6802/1828778759,v1:172.21.15.59:6803/1828778759] boot 2022-04-28T19:37:18.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:18 smithi139 bash[23439]: cluster 2022-04-28T19:37:17.023377+0000 mon.a (mon.0) 269 : cluster [DBG] osdmap e10: 1 total, 1 up, 1 in 2022-04-28T19:37:18.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:18 smithi139 bash[23439]: audit 2022-04-28T19:37:17.023750+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:37:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:19 smithi059 bash[23786]: cluster 2022-04-28T19:37:17.923144+0000 mgr.y (mgr.14140) 87 : cluster [DBG] pgmap v61: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:19 smithi059 bash[23786]: cluster 2022-04-28T19:37:18.025945+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 1 total, 1 up, 1 in 2022-04-28T19:37:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:19 smithi059 bash[29114]: cluster 2022-04-28T19:37:17.923144+0000 mgr.y (mgr.14140) 87 : cluster [DBG] pgmap v61: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:19 smithi059 bash[29114]: cluster 2022-04-28T19:37:18.025945+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 1 total, 1 up, 1 in 2022-04-28T19:37:19.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:19 smithi139 bash[23439]: cluster 2022-04-28T19:37:17.923144+0000 mgr.y (mgr.14140) 87 : cluster [DBG] pgmap v61: 1 pgs: 1 unknown; 0 B data, 0 B used, 0 B / 0 B avail 2022-04-28T19:37:19.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:19 smithi139 bash[23439]: cluster 2022-04-28T19:37:18.025945+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e11: 1 total, 1 up, 1 in 2022-04-28T19:37:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[23786]: cluster 2022-04-28T19:37:19.026575+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e12: 1 total, 1 up, 1 in 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[23786]: audit 2022-04-28T19:37:19.294373+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[23786]: audit 2022-04-28T19:37:19.296957+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[23786]: audit 2022-04-28T19:37:19.298495+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[23786]: audit 2022-04-28T19:37:19.300273+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[29114]: cluster 2022-04-28T19:37:19.026575+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e12: 1 total, 1 up, 1 in 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[29114]: audit 2022-04-28T19:37:19.294373+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:37:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[29114]: audit 2022-04-28T19:37:19.296957+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:37:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[29114]: audit 2022-04-28T19:37:19.298495+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:20 smithi059 bash[29114]: audit 2022-04-28T19:37:19.300273+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:20.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:20 smithi139 bash[23439]: cluster 2022-04-28T19:37:19.026575+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e12: 1 total, 1 up, 1 in 2022-04-28T19:37:20.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:20 smithi139 bash[23439]: audit 2022-04-28T19:37:19.294373+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:37:20.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:20 smithi139 bash[23439]: audit 2022-04-28T19:37:19.296957+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:37:20.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:20 smithi139 bash[23439]: audit 2022-04-28T19:37:19.298495+0000 mon.a (mon.0) 275 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:20.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:20 smithi139 bash[23439]: audit 2022-04-28T19:37:19.300273+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[23786]: audit 2022-04-28T19:37:19.293076+0000 mgr.y (mgr.14140) 88 : audit [DBG] from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[23786]: cephadm 2022-04-28T19:37:19.296477+0000 mgr.y (mgr.14140) 89 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[23786]: cluster 2022-04-28T19:37:19.923829+0000 mgr.y (mgr.14140) 90 : cluster [DBG] pgmap v64: 1 pgs: 1 creating+peering; 0 B data, 96 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[23786]: cluster 2022-04-28T19:37:20.030279+0000 mon.a (mon.0) 277 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[29114]: audit 2022-04-28T19:37:19.293076+0000 mgr.y (mgr.14140) 88 : audit [DBG] from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[29114]: cephadm 2022-04-28T19:37:19.296477+0000 mgr.y (mgr.14140) 89 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[29114]: cluster 2022-04-28T19:37:19.923829+0000 mgr.y (mgr.14140) 90 : cluster [DBG] pgmap v64: 1 pgs: 1 creating+peering; 0 B data, 96 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:21.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:21 smithi059 bash[29114]: cluster 2022-04-28T19:37:20.030279+0000 mon.a (mon.0) 277 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2022-04-28T19:37:21.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:21 smithi139 bash[23439]: audit 2022-04-28T19:37:19.293076+0000 mgr.y (mgr.14140) 88 : audit [DBG] from='client.24148 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:37:21.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:21 smithi139 bash[23439]: cephadm 2022-04-28T19:37:19.296477+0000 mgr.y (mgr.14140) 89 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:37:21.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:21 smithi139 bash[23439]: cluster 2022-04-28T19:37:19.923829+0000 mgr.y (mgr.14140) 90 : cluster [DBG] pgmap v64: 1 pgs: 1 creating+peering; 0 B data, 96 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:21.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:21 smithi139 bash[23439]: cluster 2022-04-28T19:37:20.030279+0000 mon.a (mon.0) 277 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2022-04-28T19:37:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[23786]: cluster 2022-04-28T19:37:21.924411+0000 mgr.y (mgr.14140) 91 : cluster [DBG] pgmap v65: 1 pgs: 1 creating+peering; 0 B data, 96 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[23786]: audit 2022-04-28T19:37:22.836624+0000 mon.a (mon.0) 278 : audit [INF] from='client.? 172.21.15.59:0/517221413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5173783f-e72e-45f9-943b-bfa78f399cf4"}]: dispatch 2022-04-28T19:37:23.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[23786]: audit 2022-04-28T19:37:22.843606+0000 mon.a (mon.0) 279 : audit [INF] from='client.? 172.21.15.59:0/517221413' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5173783f-e72e-45f9-943b-bfa78f399cf4"}]': finished 2022-04-28T19:37:23.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[23786]: cluster 2022-04-28T19:37:22.843781+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e13: 2 total, 1 up, 1 in 2022-04-28T19:37:23.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[23786]: audit 2022-04-28T19:37:22.844023+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:23.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[29114]: cluster 2022-04-28T19:37:21.924411+0000 mgr.y (mgr.14140) 91 : cluster [DBG] pgmap v65: 1 pgs: 1 creating+peering; 0 B data, 96 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:23.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[29114]: audit 2022-04-28T19:37:22.836624+0000 mon.a (mon.0) 278 : audit [INF] from='client.? 172.21.15.59:0/517221413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5173783f-e72e-45f9-943b-bfa78f399cf4"}]: dispatch 2022-04-28T19:37:23.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[29114]: audit 2022-04-28T19:37:22.843606+0000 mon.a (mon.0) 279 : audit [INF] from='client.? 172.21.15.59:0/517221413' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5173783f-e72e-45f9-943b-bfa78f399cf4"}]': finished 2022-04-28T19:37:23.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[29114]: cluster 2022-04-28T19:37:22.843781+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e13: 2 total, 1 up, 1 in 2022-04-28T19:37:23.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:23 smithi059 bash[29114]: audit 2022-04-28T19:37:22.844023+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:23.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:23 smithi139 bash[23439]: cluster 2022-04-28T19:37:21.924411+0000 mgr.y (mgr.14140) 91 : cluster [DBG] pgmap v65: 1 pgs: 1 creating+peering; 0 B data, 96 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:23.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:23 smithi139 bash[23439]: audit 2022-04-28T19:37:22.836624+0000 mon.a (mon.0) 278 : audit [INF] from='client.? 172.21.15.59:0/517221413' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5173783f-e72e-45f9-943b-bfa78f399cf4"}]: dispatch 2022-04-28T19:37:23.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:23 smithi139 bash[23439]: audit 2022-04-28T19:37:22.843606+0000 mon.a (mon.0) 279 : audit [INF] from='client.? 172.21.15.59:0/517221413' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5173783f-e72e-45f9-943b-bfa78f399cf4"}]': finished 2022-04-28T19:37:23.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:23 smithi139 bash[23439]: cluster 2022-04-28T19:37:22.843781+0000 mon.a (mon.0) 280 : cluster [DBG] osdmap e13: 2 total, 1 up, 1 in 2022-04-28T19:37:23.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:23 smithi139 bash[23439]: audit 2022-04-28T19:37:22.844023+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:24 smithi059 bash[23786]: audit 2022-04-28T19:37:23.388084+0000 mon.c (mon.1) 4 : audit [DBG] from='client.? 172.21.15.59:0/2274913479' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:24.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:24 smithi059 bash[29114]: audit 2022-04-28T19:37:23.388084+0000 mon.c (mon.1) 4 : audit [DBG] from='client.? 172.21.15.59:0/2274913479' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:24.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:24 smithi139 bash[23439]: audit 2022-04-28T19:37:23.388084+0000 mon.c (mon.1) 4 : audit [DBG] from='client.? 172.21.15.59:0/2274913479' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:25 smithi059 bash[23786]: cluster 2022-04-28T19:37:23.925027+0000 mgr.y (mgr.14140) 92 : cluster [DBG] pgmap v67: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:25 smithi059 bash[29114]: cluster 2022-04-28T19:37:23.925027+0000 mgr.y (mgr.14140) 92 : cluster [DBG] pgmap v67: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:25.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:25 smithi139 bash[23439]: cluster 2022-04-28T19:37:23.925027+0000 mgr.y (mgr.14140) 92 : cluster [DBG] pgmap v67: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:27 smithi059 bash[23786]: cluster 2022-04-28T19:37:25.925659+0000 mgr.y (mgr.14140) 93 : cluster [DBG] pgmap v68: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:27 smithi059 bash[23786]: audit 2022-04-28T19:37:26.178869+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:27 smithi059 bash[23786]: cluster 2022-04-28T19:37:26.989503+0000 mon.a (mon.0) 283 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2022-04-28T19:37:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:27 smithi059 bash[29114]: cluster 2022-04-28T19:37:25.925659+0000 mgr.y (mgr.14140) 93 : cluster [DBG] pgmap v68: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:27 smithi059 bash[29114]: audit 2022-04-28T19:37:26.178869+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:27 smithi059 bash[29114]: cluster 2022-04-28T19:37:26.989503+0000 mon.a (mon.0) 283 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2022-04-28T19:37:27.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:27 smithi139 bash[23439]: cluster 2022-04-28T19:37:25.925659+0000 mgr.y (mgr.14140) 93 : cluster [DBG] pgmap v68: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:27.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:27 smithi139 bash[23439]: audit 2022-04-28T19:37:26.178869+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:27.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:27 smithi139 bash[23439]: cluster 2022-04-28T19:37:26.989503+0000 mon.a (mon.0) 283 : cluster [WRN] Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2022-04-28T19:37:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:29 smithi059 bash[29114]: cluster 2022-04-28T19:37:27.926263+0000 mgr.y (mgr.14140) 94 : cluster [DBG] pgmap v69: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:29 smithi059 bash[23786]: cluster 2022-04-28T19:37:27.926263+0000 mgr.y (mgr.14140) 94 : cluster [DBG] pgmap v69: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:29.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:29 smithi139 bash[23439]: cluster 2022-04-28T19:37:27.926263+0000 mgr.y (mgr.14140) 94 : cluster [DBG] pgmap v69: 1 pgs: 1 undersized+peered; 0 B data, 112 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:30 smithi059 bash[23786]: audit 2022-04-28T19:37:29.824708+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-04-28T19:37:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:30 smithi059 bash[23786]: audit 2022-04-28T19:37:29.826274+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:30 smithi059 bash[23786]: audit 2022-04-28T19:37:29.827893+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.1", "key": "container_image"}]: dispatch 2022-04-28T19:37:30.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:30 smithi059 bash[29114]: audit 2022-04-28T19:37:29.824708+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-04-28T19:37:30.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:30 smithi059 bash[29114]: audit 2022-04-28T19:37:29.826274+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:30.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:30 smithi059 bash[29114]: audit 2022-04-28T19:37:29.827893+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.1", "key": "container_image"}]: dispatch 2022-04-28T19:37:30.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:30 smithi139 bash[23439]: audit 2022-04-28T19:37:29.824708+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2022-04-28T19:37:30.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:30 smithi139 bash[23439]: audit 2022-04-28T19:37:29.826274+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:30.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:30 smithi139 bash[23439]: audit 2022-04-28T19:37:29.827893+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.1", "key": "container_image"}]: dispatch 2022-04-28T19:37:31.268 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:31 smithi059 bash[29114]: cephadm 2022-04-28T19:37:29.827335+0000 mgr.y (mgr.14140) 95 : cephadm [INF] Deploying daemon osd.1 on smithi059 2022-04-28T19:37:31.269 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:31 smithi059 bash[29114]: cluster 2022-04-28T19:37:29.926851+0000 mgr.y (mgr.14140) 96 : cluster [DBG] pgmap v70: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:31.269 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:31 smithi059 bash[23786]: cephadm 2022-04-28T19:37:29.827335+0000 mgr.y (mgr.14140) 95 : cephadm [INF] Deploying daemon osd.1 on smithi059 2022-04-28T19:37:31.269 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:31 smithi059 bash[23786]: cluster 2022-04-28T19:37:29.926851+0000 mgr.y (mgr.14140) 96 : cluster [DBG] pgmap v70: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:31.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:31 smithi139 bash[23439]: cephadm 2022-04-28T19:37:29.827335+0000 mgr.y (mgr.14140) 95 : cephadm [INF] Deploying daemon osd.1 on smithi059 2022-04-28T19:37:31.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:31 smithi139 bash[23439]: cluster 2022-04-28T19:37:29.926851+0000 mgr.y (mgr.14140) 96 : cluster [DBG] pgmap v70: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:32.095 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 1 on host 'smithi059' 2022-04-28T19:37:33.236 DEBUG:teuthology.orchestra.run.smithi059:osd.1> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.1.service 2022-04-28T19:37:33.238 INFO:tasks.cephadm:Deploying osd.2 on smithi059 with /dev/vg_nvme/lv_2... 2022-04-28T19:37:33.238 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2022-04-28T19:37:33.274 INFO:journalctl@ceph.osd.1.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:37:33.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:33 smithi139 bash[23439]: cluster 2022-04-28T19:37:31.927403+0000 mgr.y (mgr.14140) 97 : cluster [DBG] pgmap v71: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:33.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:33 smithi139 bash[23439]: audit 2022-04-28T19:37:32.092337+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:33.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:33 smithi139 bash[23439]: audit 2022-04-28T19:37:32.099731+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:33.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:33 smithi139 bash[23439]: audit 2022-04-28T19:37:32.912472+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:33.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:33 smithi139 bash[23439]: audit 2022-04-28T19:37:32.913681+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[23786]: cluster 2022-04-28T19:37:31.927403+0000 mgr.y (mgr.14140) 97 : cluster [DBG] pgmap v71: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[23786]: audit 2022-04-28T19:37:32.092337+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[23786]: audit 2022-04-28T19:37:32.099731+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[23786]: audit 2022-04-28T19:37:32.912472+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:33.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[23786]: audit 2022-04-28T19:37:32.913681+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[29114]: cluster 2022-04-28T19:37:31.927403+0000 mgr.y (mgr.14140) 97 : cluster [DBG] pgmap v71: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[29114]: audit 2022-04-28T19:37:32.092337+0000 mon.a (mon.0) 287 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[29114]: audit 2022-04-28T19:37:32.099731+0000 mon.a (mon.0) 288 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[29114]: audit 2022-04-28T19:37:32.912472+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:33 smithi059 bash[29114]: audit 2022-04-28T19:37:32.913681+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:34.842 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2022-04-28T19:37:34.842 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2022-04-28T19:37:34.842 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2022-04-28T19:37:34.842 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2022-04-28T19:37:34.843 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2022-04-28T19:37:34.843 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2022-04-28T19:37:34.843 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:34 smithi059 bash[33921]: --> ceph-volume lvm activate successful for osd ID: 1 2022-04-28T19:37:35.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:35 smithi139 bash[23439]: cluster 2022-04-28T19:37:33.927943+0000 mgr.y (mgr.14140) 98 : cluster [DBG] pgmap v72: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:35.487 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:35 smithi059 bash[23786]: cluster 2022-04-28T19:37:33.927943+0000 mgr.y (mgr.14140) 98 : cluster [DBG] pgmap v72: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:35.487 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:35 smithi059 bash[29114]: cluster 2022-04-28T19:37:33.927943+0000 mgr.y (mgr.14140) 98 : cluster [DBG] pgmap v72: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:37.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:37 smithi139 bash[23439]: cluster 2022-04-28T19:37:35.928547+0000 mgr.y (mgr.14140) 99 : cluster [DBG] pgmap v73: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:37 smithi059 bash[23786]: cluster 2022-04-28T19:37:35.928547+0000 mgr.y (mgr.14140) 99 : cluster [DBG] pgmap v73: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:37 smithi059 bash[29114]: cluster 2022-04-28T19:37:35.928547+0000 mgr.y (mgr.14140) 99 : cluster [DBG] pgmap v73: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:38.092 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:37 smithi059 bash[33921]: debug 2022-04-28T19:37:37.893+0000 7fa474bbe700 1 -- 172.21.15.59:0/3591868750 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cebab6a4e0 con 0x55cebabe5000 2022-04-28T19:37:38.749 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:38 smithi059 bash[33921]: debug 2022-04-28T19:37:38.433+0000 7fa47be5cf00 -1 Falling back to public interface 2022-04-28T19:37:39.342 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[33921]: debug 2022-04-28T19:37:39.057+0000 7fa47be5cf00 -1 osd.1 0 log_to_monitors {default=true} 2022-04-28T19:37:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[23786]: cluster 2022-04-28T19:37:37.929131+0000 mgr.y (mgr.14140) 100 : cluster [DBG] pgmap v74: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[23786]: audit 2022-04-28T19:37:39.063611+0000 mon.c (mon.1) 5 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:37:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[23786]: audit 2022-04-28T19:37:39.063972+0000 mon.a (mon.0) 291 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:37:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[29114]: cluster 2022-04-28T19:37:37.929131+0000 mgr.y (mgr.14140) 100 : cluster [DBG] pgmap v74: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[29114]: audit 2022-04-28T19:37:39.063611+0000 mon.c (mon.1) 5 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:37:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:39 smithi059 bash[29114]: audit 2022-04-28T19:37:39.063972+0000 mon.a (mon.0) 291 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:37:39.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:39 smithi139 bash[23439]: cluster 2022-04-28T19:37:37.929131+0000 mgr.y (mgr.14140) 100 : cluster [DBG] pgmap v74: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:39.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:39 smithi139 bash[23439]: audit 2022-04-28T19:37:39.063611+0000 mon.c (mon.1) 5 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:37:39.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:39 smithi139 bash[23439]: audit 2022-04-28T19:37:39.063972+0000 mon.a (mon.0) 291 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:37:39.399 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_2 2022-04-28T19:37:39.399 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2022-04-28T19:37:39.400 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2022-04-28T19:37:39.400 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2022-04-28T19:37:39.400 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0186218 s, 563 MB/s 2022-04-28T19:37:39.400 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2022-04-28T19:37:40.351 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi059:vg_nvme/lv_2 2022-04-28T19:37:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[23786]: audit 2022-04-28T19:37:39.108140+0000 mon.a (mon.0) 292 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-04-28T19:37:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[23786]: cluster 2022-04-28T19:37:39.108206+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e14: 2 total, 1 up, 1 in 2022-04-28T19:37:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[23786]: audit 2022-04-28T19:37:39.108578+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[23786]: audit 2022-04-28T19:37:39.109496+0000 mon.c (mon.1) 6 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[23786]: audit 2022-04-28T19:37:39.115138+0000 mon.a (mon.0) 295 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:40.593 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[33921]: debug 2022-04-28T19:37:40.125+0000 7fa45f322700 -1 osd.1 0 waiting for initial osdmap 2022-04-28T19:37:40.593 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[33921]: debug 2022-04-28T19:37:40.129+0000 7fa46bb3b700 -1 osd.1 15 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:37:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[29114]: audit 2022-04-28T19:37:39.108140+0000 mon.a (mon.0) 292 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-04-28T19:37:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[29114]: cluster 2022-04-28T19:37:39.108206+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e14: 2 total, 1 up, 1 in 2022-04-28T19:37:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[29114]: audit 2022-04-28T19:37:39.108578+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[29114]: audit 2022-04-28T19:37:39.109496+0000 mon.c (mon.1) 6 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:40.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:40 smithi059 bash[29114]: audit 2022-04-28T19:37:39.115138+0000 mon.a (mon.0) 295 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:40.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:40 smithi139 bash[23439]: audit 2022-04-28T19:37:39.108140+0000 mon.a (mon.0) 292 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-04-28T19:37:40.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:40 smithi139 bash[23439]: cluster 2022-04-28T19:37:39.108206+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e14: 2 total, 1 up, 1 in 2022-04-28T19:37:40.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:40 smithi139 bash[23439]: audit 2022-04-28T19:37:39.108578+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:40.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:40 smithi139 bash[23439]: audit 2022-04-28T19:37:39.109496+0000 mon.c (mon.1) 6 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:40.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:40 smithi139 bash[23439]: audit 2022-04-28T19:37:39.115138+0000 mon.a (mon.0) 295 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:37:41.456 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: cluster 2022-04-28T19:37:39.929723+0000 mgr.y (mgr.14140) 101 : cluster [DBG] pgmap v76: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: audit 2022-04-28T19:37:40.120498+0000 mon.a (mon.0) 296 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: cluster 2022-04-28T19:37:40.120723+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 2 total, 1 up, 1 in 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: audit 2022-04-28T19:37:40.121467+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: audit 2022-04-28T19:37:40.129148+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: audit 2022-04-28T19:37:40.251911+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: audit 2022-04-28T19:37:40.570186+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[29114]: audit 2022-04-28T19:37:40.923637+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:41.457 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: cluster 2022-04-28T19:37:39.929723+0000 mgr.y (mgr.14140) 101 : cluster [DBG] pgmap v76: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: audit 2022-04-28T19:37:40.120498+0000 mon.a (mon.0) 296 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: cluster 2022-04-28T19:37:40.120723+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 2 total, 1 up, 1 in 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: audit 2022-04-28T19:37:40.121467+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: audit 2022-04-28T19:37:40.129148+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: audit 2022-04-28T19:37:40.251911+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: audit 2022-04-28T19:37:40.570186+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:41.458 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:41 smithi059 bash[23786]: audit 2022-04-28T19:37:40.923637+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:41.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: cluster 2022-04-28T19:37:39.929723+0000 mgr.y (mgr.14140) 101 : cluster [DBG] pgmap v76: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:41.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: audit 2022-04-28T19:37:40.120498+0000 mon.a (mon.0) 296 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:37:41.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: cluster 2022-04-28T19:37:40.120723+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 2 total, 1 up, 1 in 2022-04-28T19:37:41.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: audit 2022-04-28T19:37:40.121467+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:41.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: audit 2022-04-28T19:37:40.129148+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:41.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: audit 2022-04-28T19:37:40.251911+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:41.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: audit 2022-04-28T19:37:40.570186+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:41.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:41 smithi139 bash[23439]: audit 2022-04-28T19:37:40.923637+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: cluster 2022-04-28T19:37:40.048514+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:37:42.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: cluster 2022-04-28T19:37:40.048595+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:37:42.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: cephadm 2022-04-28T19:37:40.570649+0000 mgr.y (mgr.14140) 102 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:37:42.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: audit 2022-04-28T19:37:41.127054+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:42.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: cluster 2022-04-28T19:37:41.136317+0000 mon.a (mon.0) 304 : cluster [INF] osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240] boot 2022-04-28T19:37:42.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: cluster 2022-04-28T19:37:41.136401+0000 mon.a (mon.0) 305 : cluster [DBG] osdmap e16: 2 total, 2 up, 2 in 2022-04-28T19:37:42.710 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[23786]: audit 2022-04-28T19:37:41.137036+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:42.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: cluster 2022-04-28T19:37:40.048514+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:37:42.710 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: cluster 2022-04-28T19:37:40.048595+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:37:42.711 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: cephadm 2022-04-28T19:37:40.570649+0000 mgr.y (mgr.14140) 102 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:37:42.711 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: audit 2022-04-28T19:37:41.127054+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:42.711 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: cluster 2022-04-28T19:37:41.136317+0000 mon.a (mon.0) 304 : cluster [INF] osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240] boot 2022-04-28T19:37:42.711 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: cluster 2022-04-28T19:37:41.136401+0000 mon.a (mon.0) 305 : cluster [DBG] osdmap e16: 2 total, 2 up, 2 in 2022-04-28T19:37:42.712 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:42 smithi059 bash[29114]: audit 2022-04-28T19:37:41.137036+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:42.712 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: cluster 2022-04-28T19:37:40.048514+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:37:42.713 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: cluster 2022-04-28T19:37:40.048595+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:37:42.713 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: cephadm 2022-04-28T19:37:40.570649+0000 mgr.y (mgr.14140) 102 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:37:42.713 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: audit 2022-04-28T19:37:41.127054+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:42.713 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: cluster 2022-04-28T19:37:41.136317+0000 mon.a (mon.0) 304 : cluster [INF] osd.1 [v2:172.21.15.59:6810/222479240,v1:172.21.15.59:6811/222479240] boot 2022-04-28T19:37:42.713 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: cluster 2022-04-28T19:37:41.136401+0000 mon.a (mon.0) 305 : cluster [DBG] osdmap e16: 2 total, 2 up, 2 in 2022-04-28T19:37:42.714 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:42 smithi139 bash[23439]: audit 2022-04-28T19:37:41.137036+0000 mon.a (mon.0) 306 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:37:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:43 smithi059 bash[29114]: cluster 2022-04-28T19:37:41.930363+0000 mgr.y (mgr.14140) 103 : cluster [DBG] pgmap v79: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:43 smithi059 bash[29114]: cluster 2022-04-28T19:37:42.155307+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e17: 2 total, 2 up, 2 in 2022-04-28T19:37:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:43 smithi059 bash[23786]: cluster 2022-04-28T19:37:41.930363+0000 mgr.y (mgr.14140) 103 : cluster [DBG] pgmap v79: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:43 smithi059 bash[23786]: cluster 2022-04-28T19:37:42.155307+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e17: 2 total, 2 up, 2 in 2022-04-28T19:37:43.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:43 smithi139 bash[23439]: cluster 2022-04-28T19:37:41.930363+0000 mgr.y (mgr.14140) 103 : cluster [DBG] pgmap v79: 1 pgs: 1 undersized+peered; 0 B data, 120 KiB used, 88 GiB / 89 GiB avail 2022-04-28T19:37:43.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:43 smithi139 bash[23439]: cluster 2022-04-28T19:37:42.155307+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e17: 2 total, 2 up, 2 in 2022-04-28T19:37:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[23786]: audit 2022-04-28T19:37:43.576575+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:37:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[23786]: audit 2022-04-28T19:37:43.579700+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:37:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[23786]: audit 2022-04-28T19:37:43.581049+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[23786]: audit 2022-04-28T19:37:43.582637+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:44.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[29114]: audit 2022-04-28T19:37:43.576575+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:37:44.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[29114]: audit 2022-04-28T19:37:43.579700+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:37:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[29114]: audit 2022-04-28T19:37:43.581049+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:44 smithi059 bash[29114]: audit 2022-04-28T19:37:43.582637+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:44.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:44 smithi139 bash[23439]: audit 2022-04-28T19:37:43.576575+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:37:44.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:44 smithi139 bash[23439]: audit 2022-04-28T19:37:43.579700+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:37:44.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:44 smithi139 bash[23439]: audit 2022-04-28T19:37:43.581049+0000 mon.a (mon.0) 310 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:44.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:44 smithi139 bash[23439]: audit 2022-04-28T19:37:43.582637+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:45 smithi059 bash[23786]: audit 2022-04-28T19:37:43.574813+0000 mgr.y (mgr.14140) 104 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:37:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:45 smithi059 bash[23786]: cephadm 2022-04-28T19:37:43.579235+0000 mgr.y (mgr.14140) 105 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:37:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:45 smithi059 bash[23786]: cluster 2022-04-28T19:37:43.930966+0000 mgr.y (mgr.14140) 106 : cluster [DBG] pgmap v81: 1 pgs: 1 undersized+peered; 0 B data, 296 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:45 smithi059 bash[29114]: audit 2022-04-28T19:37:43.574813+0000 mgr.y (mgr.14140) 104 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:37:45.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:45 smithi059 bash[29114]: cephadm 2022-04-28T19:37:43.579235+0000 mgr.y (mgr.14140) 105 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:37:45.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:45 smithi059 bash[29114]: cluster 2022-04-28T19:37:43.930966+0000 mgr.y (mgr.14140) 106 : cluster [DBG] pgmap v81: 1 pgs: 1 undersized+peered; 0 B data, 296 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:45.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:45 smithi139 bash[23439]: audit 2022-04-28T19:37:43.574813+0000 mgr.y (mgr.14140) 104 : audit [DBG] from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:37:45.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:45 smithi139 bash[23439]: cephadm 2022-04-28T19:37:43.579235+0000 mgr.y (mgr.14140) 105 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:37:45.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:45 smithi139 bash[23439]: cluster 2022-04-28T19:37:43.930966+0000 mgr.y (mgr.14140) 106 : cluster [DBG] pgmap v81: 1 pgs: 1 undersized+peered; 0 B data, 296 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:47 smithi059 bash[23786]: cluster 2022-04-28T19:37:45.931593+0000 mgr.y (mgr.14140) 107 : cluster [DBG] pgmap v82: 1 pgs: 1 undersized+peered; 0 B data, 296 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:47 smithi059 bash[29114]: cluster 2022-04-28T19:37:45.931593+0000 mgr.y (mgr.14140) 107 : cluster [DBG] pgmap v82: 1 pgs: 1 undersized+peered; 0 B data, 296 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:47.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:47 smithi139 bash[23439]: cluster 2022-04-28T19:37:45.931593+0000 mgr.y (mgr.14140) 107 : cluster [DBG] pgmap v82: 1 pgs: 1 undersized+peered; 0 B data, 296 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[23786]: audit 2022-04-28T19:37:47.181551+0000 mon.c (mon.1) 7 : audit [INF] from='client.? 172.21.15.59:0/881350217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]: dispatch 2022-04-28T19:37:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[23786]: audit 2022-04-28T19:37:47.182283+0000 mon.a (mon.0) 312 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]: dispatch 2022-04-28T19:37:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[23786]: audit 2022-04-28T19:37:47.188699+0000 mon.a (mon.0) 313 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]': finished 2022-04-28T19:37:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[23786]: cluster 2022-04-28T19:37:47.188796+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e18: 3 total, 2 up, 2 in 2022-04-28T19:37:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[23786]: audit 2022-04-28T19:37:47.189071+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:37:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[23786]: audit 2022-04-28T19:37:47.727434+0000 mon.a (mon.0) 316 : audit [DBG] from='client.? 172.21.15.59:0/1951058229' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[29114]: audit 2022-04-28T19:37:47.181551+0000 mon.c (mon.1) 7 : audit [INF] from='client.? 172.21.15.59:0/881350217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]: dispatch 2022-04-28T19:37:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[29114]: audit 2022-04-28T19:37:47.182283+0000 mon.a (mon.0) 312 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]: dispatch 2022-04-28T19:37:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[29114]: audit 2022-04-28T19:37:47.188699+0000 mon.a (mon.0) 313 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]': finished 2022-04-28T19:37:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[29114]: cluster 2022-04-28T19:37:47.188796+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e18: 3 total, 2 up, 2 in 2022-04-28T19:37:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[29114]: audit 2022-04-28T19:37:47.189071+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:37:48.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:48 smithi059 bash[29114]: audit 2022-04-28T19:37:47.727434+0000 mon.a (mon.0) 316 : audit [DBG] from='client.? 172.21.15.59:0/1951058229' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:48.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:48 smithi139 bash[23439]: audit 2022-04-28T19:37:47.181551+0000 mon.c (mon.1) 7 : audit [INF] from='client.? 172.21.15.59:0/881350217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]: dispatch 2022-04-28T19:37:48.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:48 smithi139 bash[23439]: audit 2022-04-28T19:37:47.182283+0000 mon.a (mon.0) 312 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]: dispatch 2022-04-28T19:37:48.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:48 smithi139 bash[23439]: audit 2022-04-28T19:37:47.188699+0000 mon.a (mon.0) 313 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3832226-ed91-463c-917e-209d876b2656"}]': finished 2022-04-28T19:37:48.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:48 smithi139 bash[23439]: cluster 2022-04-28T19:37:47.188796+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e18: 3 total, 2 up, 2 in 2022-04-28T19:37:48.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:48 smithi139 bash[23439]: audit 2022-04-28T19:37:47.189071+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:37:48.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:48 smithi139 bash[23439]: audit 2022-04-28T19:37:47.727434+0000 mon.a (mon.0) 316 : audit [DBG] from='client.? 172.21.15.59:0/1951058229' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:37:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:49 smithi059 bash[23786]: cluster 2022-04-28T19:37:47.932163+0000 mgr.y (mgr.14140) 108 : cluster [DBG] pgmap v84: 1 pgs: 1 active+undersized; 0 B data, 304 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:49 smithi059 bash[23786]: cluster 2022-04-28T19:37:48.186149+0000 mon.a (mon.0) 317 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2022-04-28T19:37:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:49 smithi059 bash[23786]: cluster 2022-04-28T19:37:48.186217+0000 mon.a (mon.0) 318 : cluster [INF] Cluster is now healthy 2022-04-28T19:37:49.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:49 smithi059 bash[29114]: cluster 2022-04-28T19:37:47.932163+0000 mgr.y (mgr.14140) 108 : cluster [DBG] pgmap v84: 1 pgs: 1 active+undersized; 0 B data, 304 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:49.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:49 smithi059 bash[29114]: cluster 2022-04-28T19:37:48.186149+0000 mon.a (mon.0) 317 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2022-04-28T19:37:49.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:49 smithi059 bash[29114]: cluster 2022-04-28T19:37:48.186217+0000 mon.a (mon.0) 318 : cluster [INF] Cluster is now healthy 2022-04-28T19:37:49.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:49 smithi139 bash[23439]: cluster 2022-04-28T19:37:47.932163+0000 mgr.y (mgr.14140) 108 : cluster [DBG] pgmap v84: 1 pgs: 1 active+undersized; 0 B data, 304 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:49.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:49 smithi139 bash[23439]: cluster 2022-04-28T19:37:48.186149+0000 mon.a (mon.0) 317 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2022-04-28T19:37:49.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:49 smithi139 bash[23439]: cluster 2022-04-28T19:37:48.186217+0000 mon.a (mon.0) 318 : cluster [INF] Cluster is now healthy 2022-04-28T19:37:51.496 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:51 smithi059 bash[23786]: cluster 2022-04-28T19:37:49.932716+0000 mgr.y (mgr.14140) 109 : cluster [DBG] pgmap v85: 1 pgs: 1 active+undersized; 0 B data, 312 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:51.496 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:51 smithi059 bash[23786]: audit 2022-04-28T19:37:50.436862+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:51.496 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:51 smithi059 bash[29114]: cluster 2022-04-28T19:37:49.932716+0000 mgr.y (mgr.14140) 109 : cluster [DBG] pgmap v85: 1 pgs: 1 active+undersized; 0 B data, 312 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:51.497 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:51 smithi059 bash[29114]: audit 2022-04-28T19:37:50.436862+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:51.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:51 smithi139 bash[23439]: cluster 2022-04-28T19:37:49.932716+0000 mgr.y (mgr.14140) 109 : cluster [DBG] pgmap v85: 1 pgs: 1 active+undersized; 0 B data, 312 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:51.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:51 smithi139 bash[23439]: audit 2022-04-28T19:37:50.436862+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:53 smithi059 bash[29114]: cluster 2022-04-28T19:37:51.933251+0000 mgr.y (mgr.14140) 110 : cluster [DBG] pgmap v86: 1 pgs: 1 active+undersized; 0 B data, 312 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:53.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:53 smithi059 bash[23786]: cluster 2022-04-28T19:37:51.933251+0000 mgr.y (mgr.14140) 110 : cluster [DBG] pgmap v86: 1 pgs: 1 active+undersized; 0 B data, 312 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:53.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:53 smithi139 bash[23439]: cluster 2022-04-28T19:37:51.933251+0000 mgr.y (mgr.14140) 110 : cluster [DBG] pgmap v86: 1 pgs: 1 active+undersized; 0 B data, 312 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:55.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[23786]: cluster 2022-04-28T19:37:53.933931+0000 mgr.y (mgr.14140) 111 : cluster [DBG] pgmap v87: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:55.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[23786]: audit 2022-04-28T19:37:54.254750+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-04-28T19:37:55.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[23786]: audit 2022-04-28T19:37:54.256449+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:55.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[23786]: cephadm 2022-04-28T19:37:54.257740+0000 mgr.y (mgr.14140) 112 : cephadm [INF] Deploying daemon osd.2 on smithi059 2022-04-28T19:37:55.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[23786]: audit 2022-04-28T19:37:54.258403+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.2", "key": "container_image"}]: dispatch 2022-04-28T19:37:55.449 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[29114]: cluster 2022-04-28T19:37:53.933931+0000 mgr.y (mgr.14140) 111 : cluster [DBG] pgmap v87: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:55.450 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[29114]: audit 2022-04-28T19:37:54.254750+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-04-28T19:37:55.450 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[29114]: audit 2022-04-28T19:37:54.256449+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:55.450 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[29114]: cephadm 2022-04-28T19:37:54.257740+0000 mgr.y (mgr.14140) 112 : cephadm [INF] Deploying daemon osd.2 on smithi059 2022-04-28T19:37:55.450 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:55 smithi059 bash[29114]: audit 2022-04-28T19:37:54.258403+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.2", "key": "container_image"}]: dispatch 2022-04-28T19:37:55.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:55 smithi139 bash[23439]: cluster 2022-04-28T19:37:53.933931+0000 mgr.y (mgr.14140) 111 : cluster [DBG] pgmap v87: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:55.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:55 smithi139 bash[23439]: audit 2022-04-28T19:37:54.254750+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2022-04-28T19:37:55.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:55 smithi139 bash[23439]: audit 2022-04-28T19:37:54.256449+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:37:55.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:55 smithi139 bash[23439]: cephadm 2022-04-28T19:37:54.257740+0000 mgr.y (mgr.14140) 112 : cephadm [INF] Deploying daemon osd.2 on smithi059 2022-04-28T19:37:55.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:55 smithi139 bash[23439]: audit 2022-04-28T19:37:54.258403+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.2", "key": "container_image"}]: dispatch 2022-04-28T19:37:56.423 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 2 on host 'smithi059' 2022-04-28T19:37:57.435 DEBUG:teuthology.orchestra.run.smithi059:osd.2> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.2.service 2022-04-28T19:37:57.437 INFO:tasks.cephadm:Deploying osd.3 on smithi059 with /dev/vg_nvme/lv_1... 2022-04-28T19:37:57.437 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2022-04-28T19:37:57.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:57 smithi059 bash[29114]: cluster 2022-04-28T19:37:55.934562+0000 mgr.y (mgr.14140) 113 : cluster [DBG] pgmap v88: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:57.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:57 smithi059 bash[29114]: audit 2022-04-28T19:37:56.420480+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:57.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:57 smithi059 bash[29114]: audit 2022-04-28T19:37:56.429298+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:57.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:57 smithi059 bash[23786]: cluster 2022-04-28T19:37:55.934562+0000 mgr.y (mgr.14140) 113 : cluster [DBG] pgmap v88: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:57.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:57 smithi059 bash[23786]: audit 2022-04-28T19:37:56.420480+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:57.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:57 smithi059 bash[23786]: audit 2022-04-28T19:37:56.429298+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:57.497 INFO:journalctl@ceph.osd.2.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:37:57.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:57 smithi139 bash[23439]: cluster 2022-04-28T19:37:55.934562+0000 mgr.y (mgr.14140) 113 : cluster [DBG] pgmap v88: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:57.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:57 smithi139 bash[23439]: audit 2022-04-28T19:37:56.420480+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:57.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:57 smithi139 bash[23439]: audit 2022-04-28T19:37:56.429298+0000 mon.a (mon.0) 324 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:37:58.493 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2022-04-28T19:37:58.494 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2022-04-28T19:37:58.494 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2022-04-28T19:37:58.494 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2022-04-28T19:37:58.494 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2022-04-28T19:37:58.494 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2022-04-28T19:37:58.495 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[35970]: --> ceph-volume lvm activate successful for osd ID: 2 2022-04-28T19:37:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[23786]: audit 2022-04-28T19:37:57.484923+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[23786]: audit 2022-04-28T19:37:57.485928+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[23786]: cluster 2022-04-28T19:37:57.935248+0000 mgr.y (mgr.14140) 114 : cluster [DBG] pgmap v89: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[29114]: audit 2022-04-28T19:37:57.484923+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[29114]: audit 2022-04-28T19:37:57.485928+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:58.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:37:58 smithi059 bash[29114]: cluster 2022-04-28T19:37:57.935248+0000 mgr.y (mgr.14140) 114 : cluster [DBG] pgmap v89: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:37:58.863 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:58 smithi139 bash[23439]: audit 2022-04-28T19:37:57.484923+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:37:58.864 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:58 smithi139 bash[23439]: audit 2022-04-28T19:37:57.485928+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:37:58.864 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:37:58 smithi139 bash[23439]: cluster 2022-04-28T19:37:57.935248+0000 mgr.y (mgr.14140) 114 : cluster [DBG] pgmap v89: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:00.842 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:38:00 smithi059 bash[35970]: debug 2022-04-28T19:38:00.565+0000 7ff3ee55a700 1 -- 172.21.15.59:0/252353739 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a28f8364e0 con 0x55a28f8b1400 2022-04-28T19:38:01.342 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:38:01 smithi059 bash[35970]: debug 2022-04-28T19:38:01.101+0000 7ff3f57f8f00 -1 Falling back to public interface 2022-04-28T19:38:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:00 smithi059 bash[23786]: cluster 2022-04-28T19:37:59.935944+0000 mgr.y (mgr.14140) 115 : cluster [DBG] pgmap v90: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:00 smithi059 bash[29114]: cluster 2022-04-28T19:37:59.935944+0000 mgr.y (mgr.14140) 115 : cluster [DBG] pgmap v90: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:01.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:00 smithi139 bash[23439]: cluster 2022-04-28T19:37:59.935944+0000 mgr.y (mgr.14140) 115 : cluster [DBG] pgmap v90: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:01.996 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:38:01 smithi059 bash[35970]: debug 2022-04-28T19:38:01.701+0000 7ff3f57f8f00 -1 osd.2 0 log_to_monitors {default=true} 2022-04-28T19:38:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:01 smithi059 bash[23786]: audit 2022-04-28T19:38:01.711212+0000 mon.c (mon.1) 8 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:38:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:01 smithi059 bash[23786]: audit 2022-04-28T19:38:01.711651+0000 mon.a (mon.0) 327 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:38:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:01 smithi059 bash[29114]: audit 2022-04-28T19:38:01.711212+0000 mon.c (mon.1) 8 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:38:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:01 smithi059 bash[29114]: audit 2022-04-28T19:38:01.711651+0000 mon.a (mon.0) 327 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:38:02.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:01 smithi139 bash[23439]: audit 2022-04-28T19:38:01.711212+0000 mon.c (mon.1) 8 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:38:02.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:01 smithi139 bash[23439]: audit 2022-04-28T19:38:01.711651+0000 mon.a (mon.0) 327 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:38:03.021 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[29114]: cluster 2022-04-28T19:38:01.936507+0000 mgr.y (mgr.14140) 116 : cluster [DBG] pgmap v91: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[29114]: audit 2022-04-28T19:38:02.009851+0000 mon.a (mon.0) 328 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[29114]: cluster 2022-04-28T19:38:02.010013+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e19: 3 total, 2 up, 2 in 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[29114]: audit 2022-04-28T19:38:02.010585+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[29114]: audit 2022-04-28T19:38:02.012361+0000 mon.c (mon.1) 9 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[29114]: audit 2022-04-28T19:38:02.016124+0000 mon.a (mon.0) 331 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[23786]: cluster 2022-04-28T19:38:01.936507+0000 mgr.y (mgr.14140) 116 : cluster [DBG] pgmap v91: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:03.022 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[23786]: audit 2022-04-28T19:38:02.009851+0000 mon.a (mon.0) 328 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-04-28T19:38:03.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[23786]: cluster 2022-04-28T19:38:02.010013+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e19: 3 total, 2 up, 2 in 2022-04-28T19:38:03.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[23786]: audit 2022-04-28T19:38:02.010585+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:03.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[23786]: audit 2022-04-28T19:38:02.012361+0000 mon.c (mon.1) 9 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:03.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[23786]: audit 2022-04-28T19:38:02.016124+0000 mon.a (mon.0) 331 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:03.282 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping: /dev/vg_nvme/lv_1 2022-04-28T19:38:03.282 INFO:teuthology.orchestra.run.smithi059.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2022-04-28T19:38:03.282 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10+0 records in 2022-04-28T19:38:03.282 INFO:teuthology.orchestra.run.smithi059.stderr:10+0 records out 2022-04-28T19:38:03.282 INFO:teuthology.orchestra.run.smithi059.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0214142 s, 490 MB/s 2022-04-28T19:38:03.282 INFO:teuthology.orchestra.run.smithi059.stderr:--> Zapping successful for: 2022-04-28T19:38:03.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[35970]: debug 2022-04-28T19:38:03.017+0000 7ff3dccc6700 -1 osd.2 0 waiting for initial osdmap 2022-04-28T19:38:03.342 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:38:03 smithi059 bash[35970]: debug 2022-04-28T19:38:03.021+0000 7ff3e94df700 -1 osd.2 20 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:38:03.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:03 smithi139 bash[23439]: cluster 2022-04-28T19:38:01.936507+0000 mgr.y (mgr.14140) 116 : cluster [DBG] pgmap v91: 1 pgs: 1 active+undersized; 0 B data, 320 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:03.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:03 smithi139 bash[23439]: audit 2022-04-28T19:38:02.009851+0000 mon.a (mon.0) 328 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-04-28T19:38:03.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:03 smithi139 bash[23439]: cluster 2022-04-28T19:38:02.010013+0000 mon.a (mon.0) 329 : cluster [DBG] osdmap e19: 3 total, 2 up, 2 in 2022-04-28T19:38:03.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:03 smithi139 bash[23439]: audit 2022-04-28T19:38:02.010585+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:03.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:03 smithi139 bash[23439]: audit 2022-04-28T19:38:02.012361+0000 mon.c (mon.1) 9 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:03.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:03 smithi139 bash[23439]: audit 2022-04-28T19:38:02.016124+0000 mon.a (mon.0) 331 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:04.153 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi059:vg_nvme/lv_1 2022-04-28T19:38:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[23786]: audit 2022-04-28T19:38:03.016923+0000 mon.a (mon.0) 332 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:38:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[23786]: cluster 2022-04-28T19:38:03.017019+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e20: 3 total, 2 up, 2 in 2022-04-28T19:38:04.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[23786]: audit 2022-04-28T19:38:03.017156+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:04.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[23786]: audit 2022-04-28T19:38:03.020982+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[29114]: audit 2022-04-28T19:38:03.016923+0000 mon.a (mon.0) 332 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:38:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[29114]: cluster 2022-04-28T19:38:03.017019+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e20: 3 total, 2 up, 2 in 2022-04-28T19:38:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[29114]: audit 2022-04-28T19:38:03.017156+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:04.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:04 smithi059 bash[29114]: audit 2022-04-28T19:38:03.020982+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:04.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:04 smithi139 bash[23439]: audit 2022-04-28T19:38:03.016923+0000 mon.a (mon.0) 332 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:38:04.443 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:04 smithi139 bash[23439]: cluster 2022-04-28T19:38:03.017019+0000 mon.a (mon.0) 333 : cluster [DBG] osdmap e20: 3 total, 2 up, 2 in 2022-04-28T19:38:04.444 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:04 smithi139 bash[23439]: audit 2022-04-28T19:38:03.017156+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:04.444 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:04 smithi139 bash[23439]: audit 2022-04-28T19:38:03.020982+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: cluster 2022-04-28T19:38:02.747459+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: cluster 2022-04-28T19:38:02.747591+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:05.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: cluster 2022-04-28T19:38:03.937058+0000 mgr.y (mgr.14140) 117 : cluster [DBG] pgmap v94: 1 pgs: 1 active+undersized; 0 B data, 328 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:05.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: cluster 2022-04-28T19:38:04.023567+0000 mon.a (mon.0) 336 : cluster [INF] osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329] boot 2022-04-28T19:38:05.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: cluster 2022-04-28T19:38:04.023668+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2022-04-28T19:38:05.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: audit 2022-04-28T19:38:04.024371+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:05.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: audit 2022-04-28T19:38:04.382142+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:05.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: audit 2022-04-28T19:38:04.703017+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:05.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[23786]: audit 2022-04-28T19:38:04.708643+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:05.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: cluster 2022-04-28T19:38:02.747459+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:05.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: cluster 2022-04-28T19:38:02.747591+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:05.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: cluster 2022-04-28T19:38:03.937058+0000 mgr.y (mgr.14140) 117 : cluster [DBG] pgmap v94: 1 pgs: 1 active+undersized; 0 B data, 328 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:05.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: cluster 2022-04-28T19:38:04.023567+0000 mon.a (mon.0) 336 : cluster [INF] osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329] boot 2022-04-28T19:38:05.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: cluster 2022-04-28T19:38:04.023668+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2022-04-28T19:38:05.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: audit 2022-04-28T19:38:04.024371+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:05.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: audit 2022-04-28T19:38:04.382142+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:05.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: audit 2022-04-28T19:38:04.703017+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:05.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:05 smithi059 bash[29114]: audit 2022-04-28T19:38:04.708643+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:05.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: cluster 2022-04-28T19:38:02.747459+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:05.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: cluster 2022-04-28T19:38:02.747591+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:05.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: cluster 2022-04-28T19:38:03.937058+0000 mgr.y (mgr.14140) 117 : cluster [DBG] pgmap v94: 1 pgs: 1 active+undersized; 0 B data, 328 KiB used, 177 GiB / 179 GiB avail 2022-04-28T19:38:05.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: cluster 2022-04-28T19:38:04.023567+0000 mon.a (mon.0) 336 : cluster [INF] osd.2 [v2:172.21.15.59:6818/1472619329,v1:172.21.15.59:6819/1472619329] boot 2022-04-28T19:38:05.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: cluster 2022-04-28T19:38:04.023668+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2022-04-28T19:38:05.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: audit 2022-04-28T19:38:04.024371+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:38:05.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: audit 2022-04-28T19:38:04.382142+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:05.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: audit 2022-04-28T19:38:04.703017+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:05.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:05 smithi139 bash[23439]: audit 2022-04-28T19:38:04.708643+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:06.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:06 smithi059 bash[23786]: cluster 2022-04-28T19:38:05.033918+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2022-04-28T19:38:06.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:06 smithi059 bash[29114]: cluster 2022-04-28T19:38:05.033918+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2022-04-28T19:38:06.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:06 smithi139 bash[23439]: cluster 2022-04-28T19:38:05.033918+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 3 total, 3 up, 3 in 2022-04-28T19:38:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:07 smithi059 bash[29114]: cluster 2022-04-28T19:38:05.937722+0000 mgr.y (mgr.14140) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+undersized; 0 B data, 512 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:07 smithi059 bash[23786]: cluster 2022-04-28T19:38:05.937722+0000 mgr.y (mgr.14140) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+undersized; 0 B data, 512 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:07.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:07 smithi139 bash[23439]: cluster 2022-04-28T19:38:05.937722+0000 mgr.y (mgr.14140) 118 : cluster [DBG] pgmap v97: 1 pgs: 1 active+undersized; 0 B data, 512 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[23786]: audit 2022-04-28T19:38:07.300187+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[23786]: audit 2022-04-28T19:38:07.303157+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[23786]: audit 2022-04-28T19:38:07.304618+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[23786]: audit 2022-04-28T19:38:07.306165+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[29114]: audit 2022-04-28T19:38:07.300187+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[29114]: audit 2022-04-28T19:38:07.303157+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[29114]: audit 2022-04-28T19:38:07.304618+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:08 smithi059 bash[29114]: audit 2022-04-28T19:38:07.306165+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:08.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:08 smithi139 bash[23439]: audit 2022-04-28T19:38:07.300187+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:08.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:08 smithi139 bash[23439]: audit 2022-04-28T19:38:07.303157+0000 mon.a (mon.0) 344 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:08.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:08 smithi139 bash[23439]: audit 2022-04-28T19:38:07.304618+0000 mon.a (mon.0) 345 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:08.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:08 smithi139 bash[23439]: audit 2022-04-28T19:38:07.306165+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:09 smithi059 bash[23786]: audit 2022-04-28T19:38:07.298281+0000 mgr.y (mgr.14140) 119 : audit [DBG] from='client.14283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:09 smithi059 bash[23786]: cephadm 2022-04-28T19:38:07.302722+0000 mgr.y (mgr.14140) 120 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:38:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:09 smithi059 bash[23786]: cluster 2022-04-28T19:38:07.938354+0000 mgr.y (mgr.14140) 121 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 544 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:09 smithi059 bash[29114]: audit 2022-04-28T19:38:07.298281+0000 mgr.y (mgr.14140) 119 : audit [DBG] from='client.14283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:09 smithi059 bash[29114]: cephadm 2022-04-28T19:38:07.302722+0000 mgr.y (mgr.14140) 120 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:38:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:09 smithi059 bash[29114]: cluster 2022-04-28T19:38:07.938354+0000 mgr.y (mgr.14140) 121 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 544 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:09.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:09 smithi139 bash[23439]: audit 2022-04-28T19:38:07.298281+0000 mgr.y (mgr.14140) 119 : audit [DBG] from='client.14283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi059:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:09.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:09 smithi139 bash[23439]: cephadm 2022-04-28T19:38:07.302722+0000 mgr.y (mgr.14140) 120 : cephadm [INF] Applying drive group None on host smithi059... 2022-04-28T19:38:09.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:09 smithi139 bash[23439]: cluster 2022-04-28T19:38:07.938354+0000 mgr.y (mgr.14140) 121 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 544 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:11 smithi059 bash[23786]: cluster 2022-04-28T19:38:09.938911+0000 mgr.y (mgr.14140) 122 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 568 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:11.399 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:11 smithi059 bash[29114]: cluster 2022-04-28T19:38:09.938911+0000 mgr.y (mgr.14140) 122 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 568 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:11.400 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:11 smithi139 bash[23439]: cluster 2022-04-28T19:38:09.938911+0000 mgr.y (mgr.14140) 122 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 568 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[23786]: audit 2022-04-28T19:38:11.100595+0000 mon.c (mon.1) 10 : audit [INF] from='client.? 172.21.15.59:0/675498700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]: dispatch 2022-04-28T19:38:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[23786]: audit 2022-04-28T19:38:11.101340+0000 mon.a (mon.0) 347 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]: dispatch 2022-04-28T19:38:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[23786]: audit 2022-04-28T19:38:11.108584+0000 mon.a (mon.0) 348 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]': finished 2022-04-28T19:38:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[23786]: cluster 2022-04-28T19:38:11.108696+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e23: 4 total, 3 up, 3 in 2022-04-28T19:38:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[23786]: audit 2022-04-28T19:38:11.108850+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[23786]: audit 2022-04-28T19:38:11.656694+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.59:0/3697214837' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[29114]: audit 2022-04-28T19:38:11.100595+0000 mon.c (mon.1) 10 : audit [INF] from='client.? 172.21.15.59:0/675498700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]: dispatch 2022-04-28T19:38:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[29114]: audit 2022-04-28T19:38:11.101340+0000 mon.a (mon.0) 347 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]: dispatch 2022-04-28T19:38:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[29114]: audit 2022-04-28T19:38:11.108584+0000 mon.a (mon.0) 348 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]': finished 2022-04-28T19:38:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[29114]: cluster 2022-04-28T19:38:11.108696+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e23: 4 total, 3 up, 3 in 2022-04-28T19:38:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[29114]: audit 2022-04-28T19:38:11.108850+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:12 smithi059 bash[29114]: audit 2022-04-28T19:38:11.656694+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.59:0/3697214837' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:12.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:12 smithi139 bash[23439]: audit 2022-04-28T19:38:11.100595+0000 mon.c (mon.1) 10 : audit [INF] from='client.? 172.21.15.59:0/675498700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]: dispatch 2022-04-28T19:38:12.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:12 smithi139 bash[23439]: audit 2022-04-28T19:38:11.101340+0000 mon.a (mon.0) 347 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]: dispatch 2022-04-28T19:38:12.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:12 smithi139 bash[23439]: audit 2022-04-28T19:38:11.108584+0000 mon.a (mon.0) 348 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2e34b16e-61a7-4684-a305-7d3c3a64f393"}]': finished 2022-04-28T19:38:12.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:12 smithi139 bash[23439]: cluster 2022-04-28T19:38:11.108696+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e23: 4 total, 3 up, 3 in 2022-04-28T19:38:12.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:12 smithi139 bash[23439]: audit 2022-04-28T19:38:11.108850+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:12.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:12 smithi139 bash[23439]: audit 2022-04-28T19:38:11.656694+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.59:0/3697214837' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:13.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:13 smithi139 bash[23439]: cluster 2022-04-28T19:38:11.939451+0000 mgr.y (mgr.14140) 123 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 576 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:13 smithi059 bash[23786]: cluster 2022-04-28T19:38:11.939451+0000 mgr.y (mgr.14140) 123 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 576 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:13 smithi059 bash[29114]: cluster 2022-04-28T19:38:11.939451+0000 mgr.y (mgr.14140) 123 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 576 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:15.490 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:15 smithi059 bash[29114]: cluster 2022-04-28T19:38:13.940027+0000 mgr.y (mgr.14140) 124 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 584 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:15.491 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:15 smithi059 bash[29114]: audit 2022-04-28T19:38:14.431442+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:15.492 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:15 smithi059 bash[23786]: cluster 2022-04-28T19:38:13.940027+0000 mgr.y (mgr.14140) 124 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 584 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:15.492 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:15 smithi059 bash[23786]: audit 2022-04-28T19:38:14.431442+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:15.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:15 smithi139 bash[23439]: cluster 2022-04-28T19:38:13.940027+0000 mgr.y (mgr.14140) 124 : cluster [DBG] pgmap v102: 1 pgs: 1 active+clean; 0 B data, 584 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:15.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:15 smithi139 bash[23439]: audit 2022-04-28T19:38:14.431442+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:16 smithi059 bash[23786]: audit 2022-04-28T19:38:16.107301+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:38:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:16 smithi059 bash[23786]: audit 2022-04-28T19:38:16.109910+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:38:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:16 smithi059 bash[29114]: audit 2022-04-28T19:38:16.107301+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:38:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:16 smithi059 bash[29114]: audit 2022-04-28T19:38:16.109910+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:38:16.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:16 smithi139 bash[23439]: audit 2022-04-28T19:38:16.107301+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:38:16.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:16 smithi139 bash[23439]: audit 2022-04-28T19:38:16.109910+0000 mon.a (mon.0) 353 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:38:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:17 smithi059 bash[29114]: cluster 2022-04-28T19:38:15.940714+0000 mgr.y (mgr.14140) 125 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 592 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:17 smithi059 bash[23786]: cluster 2022-04-28T19:38:15.940714+0000 mgr.y (mgr.14140) 125 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 592 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:17.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:17 smithi139 bash[23439]: cluster 2022-04-28T19:38:15.940714+0000 mgr.y (mgr.14140) 125 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 592 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:19.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[29114]: cluster 2022-04-28T19:38:17.941353+0000 mgr.y (mgr.14140) 126 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:19.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[29114]: audit 2022-04-28T19:38:18.324598+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-04-28T19:38:19.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[29114]: audit 2022-04-28T19:38:18.326216+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:19.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[29114]: audit 2022-04-28T19:38:18.327842+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.3", "key": "container_image"}]: dispatch 2022-04-28T19:38:19.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[23786]: cluster 2022-04-28T19:38:17.941353+0000 mgr.y (mgr.14140) 126 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:19.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[23786]: audit 2022-04-28T19:38:18.324598+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-04-28T19:38:19.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[23786]: audit 2022-04-28T19:38:18.326216+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:19.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:19 smithi059 bash[23786]: audit 2022-04-28T19:38:18.327842+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.3", "key": "container_image"}]: dispatch 2022-04-28T19:38:19.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:19 smithi139 bash[23439]: cluster 2022-04-28T19:38:17.941353+0000 mgr.y (mgr.14140) 126 : cluster [DBG] pgmap v104: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:19.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:19 smithi139 bash[23439]: audit 2022-04-28T19:38:18.324598+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2022-04-28T19:38:19.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:19 smithi139 bash[23439]: audit 2022-04-28T19:38:18.326216+0000 mon.a (mon.0) 355 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:19.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:19 smithi139 bash[23439]: audit 2022-04-28T19:38:18.327842+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.3", "key": "container_image"}]: dispatch 2022-04-28T19:38:20.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:20 smithi059 bash[29114]: cephadm 2022-04-28T19:38:18.327309+0000 mgr.y (mgr.14140) 127 : cephadm [INF] Deploying daemon osd.3 on smithi059 2022-04-28T19:38:20.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:20 smithi059 bash[23786]: cephadm 2022-04-28T19:38:18.327309+0000 mgr.y (mgr.14140) 127 : cephadm [INF] Deploying daemon osd.3 on smithi059 2022-04-28T19:38:20.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:20 smithi139 bash[23439]: cephadm 2022-04-28T19:38:18.327309+0000 mgr.y (mgr.14140) 127 : cephadm [INF] Deploying daemon osd.3 on smithi059 2022-04-28T19:38:20.622 INFO:teuthology.orchestra.run.smithi059.stdout:Created osd(s) 3 on host 'smithi059' 2022-04-28T19:38:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:21 smithi059 bash[29114]: cluster 2022-04-28T19:38:19.941987+0000 mgr.y (mgr.14140) 128 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:21 smithi059 bash[29114]: audit 2022-04-28T19:38:20.616211+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:21 smithi059 bash[29114]: audit 2022-04-28T19:38:20.619980+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:21 smithi059 bash[23786]: cluster 2022-04-28T19:38:19.941987+0000 mgr.y (mgr.14140) 128 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:21 smithi059 bash[23786]: audit 2022-04-28T19:38:20.616211+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:21 smithi059 bash[23786]: audit 2022-04-28T19:38:20.619980+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:21.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:21 smithi139 bash[23439]: cluster 2022-04-28T19:38:19.941987+0000 mgr.y (mgr.14140) 128 : cluster [DBG] pgmap v105: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:21.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:21 smithi139 bash[23439]: audit 2022-04-28T19:38:20.616211+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:21.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:21 smithi139 bash[23439]: audit 2022-04-28T19:38:20.619980+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:21.839 DEBUG:teuthology.orchestra.run.smithi059:osd.3> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.3.service 2022-04-28T19:38:21.842 INFO:tasks.cephadm:Deploying osd.4 on smithi139 with /dev/vg_nvme/lv_4... 2022-04-28T19:38:21.842 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2022-04-28T19:38:21.854 INFO:journalctl@ceph.osd.3.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:38:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[29114]: cephadm 2022-04-28T19:38:20.615673+0000 mgr.y (mgr.14140) 129 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:38:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[29114]: audit 2022-04-28T19:38:21.545705+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[29114]: audit 2022-04-28T19:38:21.546878+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[23786]: cephadm 2022-04-28T19:38:20.615673+0000 mgr.y (mgr.14140) 129 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:38:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[23786]: audit 2022-04-28T19:38:21.545705+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[23786]: audit 2022-04-28T19:38:21.546878+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:22.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:22 smithi139 bash[23439]: cephadm 2022-04-28T19:38:20.615673+0000 mgr.y (mgr.14140) 129 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:38:22.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:22 smithi139 bash[23439]: audit 2022-04-28T19:38:21.545705+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:22.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:22 smithi139 bash[23439]: audit 2022-04-28T19:38:21.546878+0000 mon.a (mon.0) 360 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:23.091 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2022-04-28T19:38:23.092 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2022-04-28T19:38:23.092 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2022-04-28T19:38:23.092 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2022-04-28T19:38:23.092 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2022-04-28T19:38:23.092 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2022-04-28T19:38:23.092 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:22 smithi059 bash[38013]: --> ceph-volume lvm activate successful for osd ID: 3 2022-04-28T19:38:23.209 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:23 smithi139 bash[23439]: cluster 2022-04-28T19:38:21.942554+0000 mgr.y (mgr.14140) 130 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:23.448 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:23 smithi059 bash[29114]: cluster 2022-04-28T19:38:21.942554+0000 mgr.y (mgr.14140) 130 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:23.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:23 smithi059 bash[23786]: cluster 2022-04-28T19:38:21.942554+0000 mgr.y (mgr.14140) 130 : cluster [DBG] pgmap v106: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:25.116 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_4 2022-04-28T19:38:25.116 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2022-04-28T19:38:25.116 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2022-04-28T19:38:25.116 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2022-04-28T19:38:25.116 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0145987 s, 718 MB/s 2022-04-28T19:38:25.116 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2022-04-28T19:38:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:25 smithi059 bash[23786]: cluster 2022-04-28T19:38:23.943171+0000 mgr.y (mgr.14140) 131 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:25 smithi059 bash[29114]: cluster 2022-04-28T19:38:23.943171+0000 mgr.y (mgr.14140) 131 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:25.342 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:25 smithi059 bash[38013]: debug 2022-04-28T19:38:25.069+0000 7fde2fa84700 1 -- 172.21.15.59:0/4119460953 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ea177684e0 con 0x55ea177e2c00 2022-04-28T19:38:25.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:25 smithi139 bash[23439]: cluster 2022-04-28T19:38:23.943171+0000 mgr.y (mgr.14140) 131 : cluster [DBG] pgmap v107: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:25.734 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi139:vg_nvme/lv_4 2022-04-28T19:38:25.841 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:25 smithi059 bash[38013]: debug 2022-04-28T19:38:25.633+0000 7fde36d22f00 -1 Falling back to public interface 2022-04-28T19:38:26.591 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:26 smithi059 bash[38013]: debug 2022-04-28T19:38:26.253+0000 7fde36d22f00 -1 osd.3 0 log_to_monitors {default=true} 2022-04-28T19:38:27.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:27 smithi139 bash[23439]: cluster 2022-04-28T19:38:25.943851+0000 mgr.y (mgr.14140) 132 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:27.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:27 smithi139 bash[23439]: audit 2022-04-28T19:38:26.260207+0000 mon.c (mon.1) 12 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:38:27.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:27 smithi139 bash[23439]: audit 2022-04-28T19:38:26.260656+0000 mon.a (mon.0) 361 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:38:27.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:27 smithi139 bash[23439]: audit 2022-04-28T19:38:26.891263+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[23786]: cluster 2022-04-28T19:38:25.943851+0000 mgr.y (mgr.14140) 132 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[23786]: audit 2022-04-28T19:38:26.260207+0000 mon.c (mon.1) 12 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[23786]: audit 2022-04-28T19:38:26.260656+0000 mon.a (mon.0) 361 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[23786]: audit 2022-04-28T19:38:26.891263+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[29114]: cluster 2022-04-28T19:38:25.943851+0000 mgr.y (mgr.14140) 132 : cluster [DBG] pgmap v108: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[29114]: audit 2022-04-28T19:38:26.260207+0000 mon.c (mon.1) 12 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[29114]: audit 2022-04-28T19:38:26.260656+0000 mon.a (mon.0) 361 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:38:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:27 smithi059 bash[29114]: audit 2022-04-28T19:38:26.891263+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:28.591 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[38013]: debug 2022-04-28T19:38:28.181+0000 7fde1a1e8700 -1 osd.3 0 waiting for initial osdmap 2022-04-28T19:38:28.591 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[38013]: debug 2022-04-28T19:38:28.185+0000 7fde26a01700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.172088+0000 mon.a (mon.0) 363 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: cluster 2022-04-28T19:38:27.172209+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e24: 4 total, 3 up, 3 in 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.172451+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.172980+0000 mon.c (mon.1) 13 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.173407+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.206743+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.213460+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[29114]: audit 2022-04-28T19:38:27.223771+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.172088+0000 mon.a (mon.0) 363 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: cluster 2022-04-28T19:38:27.172209+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e24: 4 total, 3 up, 3 in 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.172451+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.172980+0000 mon.c (mon.1) 13 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.173407+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.206743+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.213460+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:28 smithi059 bash[23786]: audit 2022-04-28T19:38:27.223771+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.172088+0000 mon.a (mon.0) 363 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-04-28T19:38:28.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: cluster 2022-04-28T19:38:27.172209+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e24: 4 total, 3 up, 3 in 2022-04-28T19:38:28.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.172451+0000 mon.a (mon.0) 365 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:28.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.172980+0000 mon.c (mon.1) 13 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:28.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.173407+0000 mon.a (mon.0) 366 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:38:28.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.206743+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.213460+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:28.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:28 smithi139 bash[23439]: audit 2022-04-28T19:38:27.223771+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: cluster 2022-04-28T19:38:27.944427+0000 mgr.y (mgr.14140) 133 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.175107+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:38:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: cluster 2022-04-28T19:38:28.175187+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e25: 4 total, 3 up, 3 in 2022-04-28T19:38:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.175835+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.185454+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.735833+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.739780+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.741493+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[23786]: audit 2022-04-28T19:38:28.743417+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: cluster 2022-04-28T19:38:27.944427+0000 mgr.y (mgr.14140) 133 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.175107+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:38:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: cluster 2022-04-28T19:38:28.175187+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e25: 4 total, 3 up, 3 in 2022-04-28T19:38:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.175835+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.185454+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.735833+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.739780+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.741493+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:29 smithi059 bash[29114]: audit 2022-04-28T19:38:28.743417+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:29.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: cluster 2022-04-28T19:38:27.944427+0000 mgr.y (mgr.14140) 133 : cluster [DBG] pgmap v110: 1 pgs: 1 active+clean; 0 B data, 600 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:29.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.175107+0000 mon.a (mon.0) 370 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]': finished 2022-04-28T19:38:29.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: cluster 2022-04-28T19:38:28.175187+0000 mon.a (mon.0) 371 : cluster [DBG] osdmap e25: 4 total, 3 up, 3 in 2022-04-28T19:38:29.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.175835+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:29.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.185454+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:29.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.735833+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:29.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.739780+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:29.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.741493+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:29.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:29 smithi139 bash[23439]: audit 2022-04-28T19:38:28.743417+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:30.452 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: cluster 2022-04-28T19:38:27.304773+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:30.452 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: cluster 2022-04-28T19:38:27.304839+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:30.452 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: audit 2022-04-28T19:38:28.733781+0000 mgr.y (mgr.14140) 134 : audit [DBG] from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:30.452 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: cephadm 2022-04-28T19:38:28.739260+0000 mgr.y (mgr.14140) 135 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:38:30.452 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: cluster 2022-04-28T19:38:29.188576+0000 mon.a (mon.0) 378 : cluster [INF] osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803] boot 2022-04-28T19:38:30.453 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: cluster 2022-04-28T19:38:29.188699+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2022-04-28T19:38:30.453 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:30 smithi139 bash[23439]: audit 2022-04-28T19:38:29.189332+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: cluster 2022-04-28T19:38:27.304773+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: cluster 2022-04-28T19:38:27.304839+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: audit 2022-04-28T19:38:28.733781+0000 mgr.y (mgr.14140) 134 : audit [DBG] from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: cephadm 2022-04-28T19:38:28.739260+0000 mgr.y (mgr.14140) 135 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: cluster 2022-04-28T19:38:29.188576+0000 mon.a (mon.0) 378 : cluster [INF] osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803] boot 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: cluster 2022-04-28T19:38:29.188699+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[23786]: audit 2022-04-28T19:38:29.189332+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: cluster 2022-04-28T19:38:27.304773+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: cluster 2022-04-28T19:38:27.304839+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: audit 2022-04-28T19:38:28.733781+0000 mgr.y (mgr.14140) 134 : audit [DBG] from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: cephadm 2022-04-28T19:38:28.739260+0000 mgr.y (mgr.14140) 135 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:38:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: cluster 2022-04-28T19:38:29.188576+0000 mon.a (mon.0) 378 : cluster [INF] osd.3 [v2:172.21.15.59:6826/2946747803,v1:172.21.15.59:6827/2946747803] boot 2022-04-28T19:38:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: cluster 2022-04-28T19:38:29.188699+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2022-04-28T19:38:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:30 smithi059 bash[29114]: audit 2022-04-28T19:38:29.189332+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:38:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:31 smithi059 bash[23786]: cluster 2022-04-28T19:38:29.944987+0000 mgr.y (mgr.14140) 136 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 608 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:31 smithi059 bash[23786]: cluster 2022-04-28T19:38:30.205672+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2022-04-28T19:38:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:31 smithi059 bash[29114]: cluster 2022-04-28T19:38:29.944987+0000 mgr.y (mgr.14140) 136 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 608 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:31 smithi059 bash[29114]: cluster 2022-04-28T19:38:30.205672+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2022-04-28T19:38:31.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:31 smithi139 bash[23439]: cluster 2022-04-28T19:38:29.944987+0000 mgr.y (mgr.14140) 136 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 608 KiB used, 265 GiB / 268 GiB avail 2022-04-28T19:38:31.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:31 smithi139 bash[23439]: cluster 2022-04-28T19:38:30.205672+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e27: 4 total, 4 up, 4 in 2022-04-28T19:38:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: cluster 2022-04-28T19:38:31.945525+0000 mgr.y (mgr.14140) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 848 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: audit 2022-04-28T19:38:32.603034+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]: dispatch 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: audit 2022-04-28T19:38:32.606416+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.139:0/1793638472' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]: dispatch 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: audit 2022-04-28T19:38:32.613453+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]': finished 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: cluster 2022-04-28T19:38:32.613528+0000 mon.a (mon.0) 384 : cluster [DBG] osdmap e28: 5 total, 4 up, 4 in 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: audit 2022-04-28T19:38:32.624920+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[23786]: audit 2022-04-28T19:38:33.121407+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.139:0/3316023391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: cluster 2022-04-28T19:38:31.945525+0000 mgr.y (mgr.14140) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 848 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: audit 2022-04-28T19:38:32.603034+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]: dispatch 2022-04-28T19:38:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: audit 2022-04-28T19:38:32.606416+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.139:0/1793638472' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]: dispatch 2022-04-28T19:38:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: audit 2022-04-28T19:38:32.613453+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]': finished 2022-04-28T19:38:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: cluster 2022-04-28T19:38:32.613528+0000 mon.a (mon.0) 384 : cluster [DBG] osdmap e28: 5 total, 4 up, 4 in 2022-04-28T19:38:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: audit 2022-04-28T19:38:32.624920+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:33.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:33 smithi059 bash[29114]: audit 2022-04-28T19:38:33.121407+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.139:0/3316023391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:33.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: cluster 2022-04-28T19:38:31.945525+0000 mgr.y (mgr.14140) 137 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 848 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:33.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: audit 2022-04-28T19:38:32.603034+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]: dispatch 2022-04-28T19:38:33.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: audit 2022-04-28T19:38:32.606416+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.139:0/1793638472' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]: dispatch 2022-04-28T19:38:33.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: audit 2022-04-28T19:38:32.613453+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fb8003d3-2e2b-49dc-9c09-7951da79c977"}]': finished 2022-04-28T19:38:33.614 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: cluster 2022-04-28T19:38:32.613528+0000 mon.a (mon.0) 384 : cluster [DBG] osdmap e28: 5 total, 4 up, 4 in 2022-04-28T19:38:33.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: audit 2022-04-28T19:38:32.624920+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:33.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:33 smithi139 bash[23439]: audit 2022-04-28T19:38:33.121407+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.139:0/3316023391' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:35 smithi059 bash[23786]: cluster 2022-04-28T19:38:33.946064+0000 mgr.y (mgr.14140) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 904 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:35 smithi059 bash[29114]: cluster 2022-04-28T19:38:33.946064+0000 mgr.y (mgr.14140) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 904 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:35.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:35 smithi139 bash[23439]: cluster 2022-04-28T19:38:33.946064+0000 mgr.y (mgr.14140) 138 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 904 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:36 smithi059 bash[23786]: audit 2022-04-28T19:38:35.868816+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:36.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:36 smithi059 bash[29114]: audit 2022-04-28T19:38:35.868816+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:36.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:36 smithi139 bash[23439]: audit 2022-04-28T19:38:35.868816+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:37.263 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:37 smithi139 bash[23439]: cluster 2022-04-28T19:38:35.946746+0000 mgr.y (mgr.14140) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 936 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:37 smithi059 bash[23786]: cluster 2022-04-28T19:38:35.946746+0000 mgr.y (mgr.14140) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 936 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:37 smithi059 bash[29114]: cluster 2022-04-28T19:38:35.946746+0000 mgr.y (mgr.14140) 139 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 936 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:39.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:39 smithi059 bash[23786]: cluster 2022-04-28T19:38:37.947365+0000 mgr.y (mgr.14140) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 944 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:39 smithi059 bash[29114]: cluster 2022-04-28T19:38:37.947365+0000 mgr.y (mgr.14140) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 944 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:39.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:39 smithi139 bash[23439]: cluster 2022-04-28T19:38:37.947365+0000 mgr.y (mgr.14140) 140 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 944 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:40 smithi059 bash[23786]: audit 2022-04-28T19:38:39.515385+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-04-28T19:38:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:40 smithi059 bash[23786]: audit 2022-04-28T19:38:39.517261+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:40 smithi059 bash[23786]: audit 2022-04-28T19:38:39.519436+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.4", "key": "container_image"}]: dispatch 2022-04-28T19:38:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:40 smithi059 bash[29114]: audit 2022-04-28T19:38:39.515385+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-04-28T19:38:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:40 smithi059 bash[29114]: audit 2022-04-28T19:38:39.517261+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:40 smithi059 bash[29114]: audit 2022-04-28T19:38:39.519436+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.4", "key": "container_image"}]: dispatch 2022-04-28T19:38:40.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:40 smithi139 bash[23439]: audit 2022-04-28T19:38:39.515385+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2022-04-28T19:38:40.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:40 smithi139 bash[23439]: audit 2022-04-28T19:38:39.517261+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:40.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:40 smithi139 bash[23439]: audit 2022-04-28T19:38:39.519436+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.4", "key": "container_image"}]: dispatch 2022-04-28T19:38:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:41 smithi059 bash[29114]: cephadm 2022-04-28T19:38:39.518714+0000 mgr.y (mgr.14140) 141 : cephadm [INF] Deploying daemon osd.4 on smithi139 2022-04-28T19:38:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:41 smithi059 bash[29114]: cluster 2022-04-28T19:38:39.947915+0000 mgr.y (mgr.14140) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 952 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:41 smithi059 bash[23786]: cephadm 2022-04-28T19:38:39.518714+0000 mgr.y (mgr.14140) 141 : cephadm [INF] Deploying daemon osd.4 on smithi139 2022-04-28T19:38:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:41 smithi059 bash[23786]: cluster 2022-04-28T19:38:39.947915+0000 mgr.y (mgr.14140) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 952 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:41.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:41 smithi139 bash[23439]: cephadm 2022-04-28T19:38:39.518714+0000 mgr.y (mgr.14140) 141 : cephadm [INF] Deploying daemon osd.4 on smithi139 2022-04-28T19:38:41.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:41 smithi139 bash[23439]: cluster 2022-04-28T19:38:39.947915+0000 mgr.y (mgr.14140) 142 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 952 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:42.038 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 4 on host 'smithi139' 2022-04-28T19:38:43.105 DEBUG:teuthology.orchestra.run.smithi139:osd.4> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.4.service 2022-04-28T19:38:43.107 INFO:tasks.cephadm:Deploying osd.5 on smithi139 with /dev/vg_nvme/lv_3... 2022-04-28T19:38:43.107 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2022-04-28T19:38:43.130 INFO:journalctl@ceph.osd.4.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:38:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[29114]: cluster 2022-04-28T19:38:41.948471+0000 mgr.y (mgr.14140) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[29114]: audit 2022-04-28T19:38:42.036380+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[29114]: cephadm 2022-04-28T19:38:42.079668+0000 mgr.y (mgr.14140) 144 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:38:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[29114]: audit 2022-04-28T19:38:42.080379+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[29114]: audit 2022-04-28T19:38:42.774607+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[29114]: audit 2022-04-28T19:38:42.776359+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[23786]: cluster 2022-04-28T19:38:41.948471+0000 mgr.y (mgr.14140) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[23786]: audit 2022-04-28T19:38:42.036380+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[23786]: cephadm 2022-04-28T19:38:42.079668+0000 mgr.y (mgr.14140) 144 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[23786]: audit 2022-04-28T19:38:42.080379+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[23786]: audit 2022-04-28T19:38:42.774607+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:43 smithi059 bash[23786]: audit 2022-04-28T19:38:42.776359+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:43.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:43 smithi139 bash[23439]: cluster 2022-04-28T19:38:41.948471+0000 mgr.y (mgr.14140) 143 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:43.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:43 smithi139 bash[23439]: audit 2022-04-28T19:38:42.036380+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:43.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:43 smithi139 bash[23439]: cephadm 2022-04-28T19:38:42.079668+0000 mgr.y (mgr.14140) 144 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:38:43.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:43 smithi139 bash[23439]: audit 2022-04-28T19:38:42.080379+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:43.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:43 smithi139 bash[23439]: audit 2022-04-28T19:38:42.774607+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:43.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:43 smithi139 bash[23439]: audit 2022-04-28T19:38:42.776359+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2022-04-28T19:38:44.588 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:44 smithi139 bash[25976]: --> ceph-volume lvm activate successful for osd ID: 4 2022-04-28T19:38:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:45 smithi059 bash[23786]: cluster 2022-04-28T19:38:43.949051+0000 mgr.y (mgr.14140) 145 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:45 smithi059 bash[29114]: cluster 2022-04-28T19:38:43.949051+0000 mgr.y (mgr.14140) 145 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:45.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:45 smithi139 bash[23439]: cluster 2022-04-28T19:38:43.949051+0000 mgr.y (mgr.14140) 145 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:47 smithi059 bash[23786]: cluster 2022-04-28T19:38:45.949596+0000 mgr.y (mgr.14140) 146 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:47 smithi059 bash[29114]: cluster 2022-04-28T19:38:45.949596+0000 mgr.y (mgr.14140) 146 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:47.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:47 smithi139 bash[23439]: cluster 2022-04-28T19:38:45.949596+0000 mgr.y (mgr.14140) 146 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:47.810 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:47 smithi139 bash[25976]: debug 2022-04-28T19:38:47.521+0000 7f4c1c11d700 1 -- 172.21.15.139:0/3416975673 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fe751024e0 con 0x55fe7517d000 2022-04-28T19:38:48.362 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:48 smithi139 bash[25976]: debug 2022-04-28T19:38:48.073+0000 7f4c233bbf00 -1 Falling back to public interface 2022-04-28T19:38:48.428 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_3 2022-04-28T19:38:48.428 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2022-04-28T19:38:48.428 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2022-04-28T19:38:48.428 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2022-04-28T19:38:48.428 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122658 s, 855 MB/s 2022-04-28T19:38:48.429 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2022-04-28T19:38:48.863 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:48 smithi139 bash[25976]: debug 2022-04-28T19:38:48.693+0000 7f4c233bbf00 -1 osd.4 0 log_to_monitors {default=true} 2022-04-28T19:38:49.268 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi139:vg_nvme/lv_3 2022-04-28T19:38:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:49 smithi059 bash[23786]: cluster 2022-04-28T19:38:47.950172+0000 mgr.y (mgr.14140) 147 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:49 smithi059 bash[23786]: audit 2022-04-28T19:38:48.701200+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:38:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:49 smithi059 bash[23786]: audit 2022-04-28T19:38:48.704690+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:38:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:49 smithi059 bash[29114]: cluster 2022-04-28T19:38:47.950172+0000 mgr.y (mgr.14140) 147 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:49 smithi059 bash[29114]: audit 2022-04-28T19:38:48.701200+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:38:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:49 smithi059 bash[29114]: audit 2022-04-28T19:38:48.704690+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:38:49.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:49 smithi139 bash[23439]: cluster 2022-04-28T19:38:47.950172+0000 mgr.y (mgr.14140) 147 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:49.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:49 smithi139 bash[23439]: audit 2022-04-28T19:38:48.701200+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:38:49.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:49 smithi139 bash[23439]: audit 2022-04-28T19:38:48.704690+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:38:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.054120+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-04-28T19:38:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: cluster 2022-04-28T19:38:49.054188+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e29: 5 total, 4 up, 4 in 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.054634+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.060055+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.061162+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.371122+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.687088+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[23786]: audit 2022-04-28T19:38:49.694112+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.054120+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-04-28T19:38:50.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: cluster 2022-04-28T19:38:49.054188+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e29: 5 total, 4 up, 4 in 2022-04-28T19:38:50.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.054634+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:50.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.060055+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:38:50.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.061162+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:38:50.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.371122+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:50.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.687088+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:50.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:50 smithi059 bash[29114]: audit 2022-04-28T19:38:49.694112+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:50.349 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[25976]: debug 2022-04-28T19:38:50.073+0000 7f4c0a889700 -1 osd.4 0 waiting for initial osdmap 2022-04-28T19:38:50.349 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[25976]: debug 2022-04-28T19:38:50.085+0000 7f4c170a2700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:38:50.349 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.054120+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: cluster 2022-04-28T19:38:49.054188+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e29: 5 total, 4 up, 4 in 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.054634+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.060055+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.061162+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.371122+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.687088+0000 mon.a (mon.0) 400 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:50.350 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:50 smithi139 bash[23439]: audit 2022-04-28T19:38:49.694112+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:38:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[23786]: cluster 2022-04-28T19:38:49.950751+0000 mgr.y (mgr.14140) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:51.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[23786]: audit 2022-04-28T19:38:50.065766+0000 mon.a (mon.0) 402 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:38:51.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[23786]: cluster 2022-04-28T19:38:50.065991+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 4 up, 4 in 2022-04-28T19:38:51.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[23786]: audit 2022-04-28T19:38:50.066848+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:51.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[23786]: audit 2022-04-28T19:38:50.076152+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:51.364 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[29114]: cluster 2022-04-28T19:38:49.950751+0000 mgr.y (mgr.14140) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:51.364 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[29114]: audit 2022-04-28T19:38:50.065766+0000 mon.a (mon.0) 402 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:38:51.364 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[29114]: cluster 2022-04-28T19:38:50.065991+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 4 up, 4 in 2022-04-28T19:38:51.364 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[29114]: audit 2022-04-28T19:38:50.066848+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:51.364 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:51 smithi059 bash[29114]: audit 2022-04-28T19:38:50.076152+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:51.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:51 smithi139 bash[23439]: cluster 2022-04-28T19:38:49.950751+0000 mgr.y (mgr.14140) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 960 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:51.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:51 smithi139 bash[23439]: audit 2022-04-28T19:38:50.065766+0000 mon.a (mon.0) 402 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:38:51.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:51 smithi139 bash[23439]: cluster 2022-04-28T19:38:50.065991+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e30: 5 total, 4 up, 4 in 2022-04-28T19:38:51.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:51 smithi139 bash[23439]: audit 2022-04-28T19:38:50.066848+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:51.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:51 smithi139 bash[23439]: audit 2022-04-28T19:38:50.076152+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[23786]: cluster 2022-04-28T19:38:49.708030+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[23786]: cluster 2022-04-28T19:38:49.708136+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[23786]: audit 2022-04-28T19:38:51.078051+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[23786]: cluster 2022-04-28T19:38:51.083238+0000 mon.a (mon.0) 407 : cluster [INF] osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841] boot 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[23786]: cluster 2022-04-28T19:38:51.083310+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[23786]: audit 2022-04-28T19:38:51.083496+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[29114]: cluster 2022-04-28T19:38:49.708030+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[29114]: cluster 2022-04-28T19:38:49.708136+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[29114]: audit 2022-04-28T19:38:51.078051+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[29114]: cluster 2022-04-28T19:38:51.083238+0000 mon.a (mon.0) 407 : cluster [INF] osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841] boot 2022-04-28T19:38:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[29114]: cluster 2022-04-28T19:38:51.083310+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2022-04-28T19:38:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:52 smithi059 bash[29114]: audit 2022-04-28T19:38:51.083496+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:52.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:52 smithi139 bash[23439]: cluster 2022-04-28T19:38:49.708030+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:38:52.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:52 smithi139 bash[23439]: cluster 2022-04-28T19:38:49.708136+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:38:52.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:52 smithi139 bash[23439]: audit 2022-04-28T19:38:51.078051+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:52.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:52 smithi139 bash[23439]: cluster 2022-04-28T19:38:51.083238+0000 mon.a (mon.0) 407 : cluster [INF] osd.4 [v2:172.21.15.139:6800/611845841,v1:172.21.15.139:6801/611845841] boot 2022-04-28T19:38:52.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:52 smithi139 bash[23439]: cluster 2022-04-28T19:38:51.083310+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2022-04-28T19:38:52.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:52 smithi139 bash[23439]: audit 2022-04-28T19:38:51.083496+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:38:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[23786]: cluster 2022-04-28T19:38:51.951385+0000 mgr.y (mgr.14140) 149 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 968 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[23786]: cluster 2022-04-28T19:38:52.086172+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[23786]: audit 2022-04-28T19:38:52.355074+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[23786]: audit 2022-04-28T19:38:52.358817+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[23786]: audit 2022-04-28T19:38:52.360718+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[23786]: audit 2022-04-28T19:38:52.362775+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[29114]: cluster 2022-04-28T19:38:51.951385+0000 mgr.y (mgr.14140) 149 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 968 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[29114]: cluster 2022-04-28T19:38:52.086172+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[29114]: audit 2022-04-28T19:38:52.355074+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[29114]: audit 2022-04-28T19:38:52.358817+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:53.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[29114]: audit 2022-04-28T19:38:52.360718+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:53.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:53 smithi059 bash[29114]: audit 2022-04-28T19:38:52.362775+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:53.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:53 smithi139 bash[23439]: cluster 2022-04-28T19:38:51.951385+0000 mgr.y (mgr.14140) 149 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 968 KiB used, 354 GiB / 358 GiB avail 2022-04-28T19:38:53.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:53 smithi139 bash[23439]: cluster 2022-04-28T19:38:52.086172+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2022-04-28T19:38:53.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:53 smithi139 bash[23439]: audit 2022-04-28T19:38:52.355074+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:38:53.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:53 smithi139 bash[23439]: audit 2022-04-28T19:38:52.358817+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:38:53.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:53 smithi139 bash[23439]: audit 2022-04-28T19:38:52.360718+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:38:53.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:53 smithi139 bash[23439]: audit 2022-04-28T19:38:52.362775+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:38:54.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:54 smithi139 bash[23439]: audit 2022-04-28T19:38:52.353159+0000 mgr.y (mgr.14140) 150 : audit [DBG] from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:54.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:54 smithi139 bash[23439]: cephadm 2022-04-28T19:38:52.358252+0000 mgr.y (mgr.14140) 151 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:38:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:54 smithi059 bash[23786]: audit 2022-04-28T19:38:52.353159+0000 mgr.y (mgr.14140) 150 : audit [DBG] from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:54 smithi059 bash[23786]: cephadm 2022-04-28T19:38:52.358252+0000 mgr.y (mgr.14140) 151 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:38:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:54 smithi059 bash[29114]: audit 2022-04-28T19:38:52.353159+0000 mgr.y (mgr.14140) 150 : audit [DBG] from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:38:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:54 smithi059 bash[29114]: cephadm 2022-04-28T19:38:52.358252+0000 mgr.y (mgr.14140) 151 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:38:55.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:55 smithi139 bash[23439]: cluster 2022-04-28T19:38:53.951997+0000 mgr.y (mgr.14140) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:55 smithi059 bash[29114]: cluster 2022-04-28T19:38:53.951997+0000 mgr.y (mgr.14140) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:55.798 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:55 smithi059 bash[23786]: cluster 2022-04-28T19:38:53.951997+0000 mgr.y (mgr.14140) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:56.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:56 smithi139 bash[23439]: audit 2022-04-28T19:38:56.000085+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]: dispatch 2022-04-28T19:38:56.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:56 smithi139 bash[23439]: audit 2022-04-28T19:38:56.003728+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.139:0/1063963026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]: dispatch 2022-04-28T19:38:56.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:56 smithi139 bash[23439]: audit 2022-04-28T19:38:56.007089+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]': finished 2022-04-28T19:38:56.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:56 smithi139 bash[23439]: cluster 2022-04-28T19:38:56.007168+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e33: 6 total, 5 up, 5 in 2022-04-28T19:38:56.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:56 smithi139 bash[23439]: audit 2022-04-28T19:38:56.007308+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:38:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[23786]: audit 2022-04-28T19:38:56.000085+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]: dispatch 2022-04-28T19:38:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[23786]: audit 2022-04-28T19:38:56.003728+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.139:0/1063963026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]: dispatch 2022-04-28T19:38:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[23786]: audit 2022-04-28T19:38:56.007089+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]': finished 2022-04-28T19:38:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[23786]: cluster 2022-04-28T19:38:56.007168+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e33: 6 total, 5 up, 5 in 2022-04-28T19:38:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[23786]: audit 2022-04-28T19:38:56.007308+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:38:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[29114]: audit 2022-04-28T19:38:56.000085+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]: dispatch 2022-04-28T19:38:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[29114]: audit 2022-04-28T19:38:56.003728+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.139:0/1063963026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]: dispatch 2022-04-28T19:38:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[29114]: audit 2022-04-28T19:38:56.007089+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c6583c6c-c41c-45b4-8035-9c5158a4fc9d"}]': finished 2022-04-28T19:38:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[29114]: cluster 2022-04-28T19:38:56.007168+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e33: 6 total, 5 up, 5 in 2022-04-28T19:38:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:56 smithi059 bash[29114]: audit 2022-04-28T19:38:56.007308+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:38:57.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:57 smithi139 bash[23439]: cluster 2022-04-28T19:38:55.952548+0000 mgr.y (mgr.14140) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:57.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:57 smithi139 bash[23439]: audit 2022-04-28T19:38:56.536699+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.139:0/2320572455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:57 smithi059 bash[23786]: cluster 2022-04-28T19:38:55.952548+0000 mgr.y (mgr.14140) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:57 smithi059 bash[23786]: audit 2022-04-28T19:38:56.536699+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.139:0/2320572455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:57 smithi059 bash[29114]: cluster 2022-04-28T19:38:55.952548+0000 mgr.y (mgr.14140) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:57 smithi059 bash[29114]: audit 2022-04-28T19:38:56.536699+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.139:0/2320572455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:38:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:38:59 smithi059 bash[23786]: cluster 2022-04-28T19:38:57.953238+0000 mgr.y (mgr.14140) 154 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:38:59 smithi059 bash[29114]: cluster 2022-04-28T19:38:57.953238+0000 mgr.y (mgr.14140) 154 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:38:59.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:38:59 smithi139 bash[23439]: cluster 2022-04-28T19:38:57.953238+0000 mgr.y (mgr.14140) 154 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 1.3 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:00.398 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:00 smithi139 bash[23439]: audit 2022-04-28T19:38:59.323462+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:00 smithi059 bash[29114]: audit 2022-04-28T19:38:59.323462+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:00 smithi059 bash[23786]: audit 2022-04-28T19:38:59.323462+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:01 smithi059 bash[23786]: cluster 2022-04-28T19:38:59.953931+0000 mgr.y (mgr.14140) 155 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:01 smithi059 bash[29114]: cluster 2022-04-28T19:38:59.953931+0000 mgr.y (mgr.14140) 155 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:01.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:01 smithi139 bash[23439]: cluster 2022-04-28T19:38:59.953931+0000 mgr.y (mgr.14140) 155 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[23786]: cluster 2022-04-28T19:39:01.954567+0000 mgr.y (mgr.14140) 156 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[23786]: audit 2022-04-28T19:39:03.038529+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-04-28T19:39:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[23786]: audit 2022-04-28T19:39:03.040477+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[23786]: audit 2022-04-28T19:39:03.042642+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.5", "key": "container_image"}]: dispatch 2022-04-28T19:39:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[29114]: cluster 2022-04-28T19:39:01.954567+0000 mgr.y (mgr.14140) 156 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:03.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[29114]: audit 2022-04-28T19:39:03.038529+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-04-28T19:39:03.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[29114]: audit 2022-04-28T19:39:03.040477+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:03.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:03 smithi059 bash[29114]: audit 2022-04-28T19:39:03.042642+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.5", "key": "container_image"}]: dispatch 2022-04-28T19:39:03.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:03 smithi139 bash[23439]: cluster 2022-04-28T19:39:01.954567+0000 mgr.y (mgr.14140) 156 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:03.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:03 smithi139 bash[23439]: audit 2022-04-28T19:39:03.038529+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2022-04-28T19:39:03.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:03 smithi139 bash[23439]: audit 2022-04-28T19:39:03.040477+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:03.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:03 smithi139 bash[23439]: audit 2022-04-28T19:39:03.042642+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.5", "key": "container_image"}]: dispatch 2022-04-28T19:39:04.490 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:04 smithi139 bash[23439]: cephadm 2022-04-28T19:39:03.041944+0000 mgr.y (mgr.14140) 157 : cephadm [INF] Deploying daemon osd.5 on smithi139 2022-04-28T19:39:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:04 smithi059 bash[23786]: cephadm 2022-04-28T19:39:03.041944+0000 mgr.y (mgr.14140) 157 : cephadm [INF] Deploying daemon osd.5 on smithi139 2022-04-28T19:39:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:04 smithi059 bash[29114]: cephadm 2022-04-28T19:39:03.041944+0000 mgr.y (mgr.14140) 157 : cephadm [INF] Deploying daemon osd.5 on smithi139 2022-04-28T19:39:05.278 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:05 smithi139 bash[23439]: cluster 2022-04-28T19:39:03.955146+0000 mgr.y (mgr.14140) 158 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:05.319 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 5 on host 'smithi139' 2022-04-28T19:39:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:05 smithi059 bash[23786]: cluster 2022-04-28T19:39:03.955146+0000 mgr.y (mgr.14140) 158 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:05 smithi059 bash[29114]: cluster 2022-04-28T19:39:03.955146+0000 mgr.y (mgr.14140) 158 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:06.295 DEBUG:teuthology.orchestra.run.smithi139:osd.5> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.5.service 2022-04-28T19:39:06.296 INFO:tasks.cephadm:Deploying osd.6 on smithi139 with /dev/vg_nvme/lv_2... 2022-04-28T19:39:06.297 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2022-04-28T19:39:06.343 INFO:journalctl@ceph.osd.5.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:39:06.453 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:06 smithi139 bash[23439]: audit 2022-04-28T19:39:05.308758+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:06.453 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:06 smithi139 bash[23439]: audit 2022-04-28T19:39:05.313699+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.453 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:06 smithi139 bash[23439]: audit 2022-04-28T19:39:06.075250+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.454 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:06 smithi139 bash[23439]: audit 2022-04-28T19:39:06.076934+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:06.454 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:06 smithi139 bash[23439]: audit 2022-04-28T19:39:06.101851+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[23786]: audit 2022-04-28T19:39:05.308758+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[23786]: audit 2022-04-28T19:39:05.313699+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[23786]: audit 2022-04-28T19:39:06.075250+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[23786]: audit 2022-04-28T19:39:06.076934+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[23786]: audit 2022-04-28T19:39:06.101851+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[29114]: audit 2022-04-28T19:39:05.308758+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[29114]: audit 2022-04-28T19:39:05.313699+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[29114]: audit 2022-04-28T19:39:06.075250+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[29114]: audit 2022-04-28T19:39:06.076934+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:06 smithi059 bash[29114]: audit 2022-04-28T19:39:06.101851+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:07 smithi059 bash[29114]: cluster 2022-04-28T19:39:05.955735+0000 mgr.y (mgr.14140) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:07 smithi059 bash[23786]: cluster 2022-04-28T19:39:05.955735+0000 mgr.y (mgr.14140) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:07.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[23439]: cluster 2022-04-28T19:39:05.955735+0000 mgr.y (mgr.14140) 159 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:07.612 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2022-04-28T19:39:07.612 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2022-04-28T19:39:07.612 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2022-04-28T19:39:07.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2022-04-28T19:39:07.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2022-04-28T19:39:07.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2022-04-28T19:39:07.613 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:07 smithi139 bash[27981]: --> ceph-volume lvm activate successful for osd ID: 5 2022-04-28T19:39:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:09 smithi059 bash[23786]: cluster 2022-04-28T19:39:07.956360+0000 mgr.y (mgr.14140) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:09 smithi059 bash[29114]: cluster 2022-04-28T19:39:07.956360+0000 mgr.y (mgr.14140) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:09.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:09 smithi139 bash[23439]: cluster 2022-04-28T19:39:07.956360+0000 mgr.y (mgr.14140) 160 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:11.112 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:10 smithi139 bash[27981]: debug 2022-04-28T19:39:10.697+0000 7f1bd49cb700 1 -- 172.21.15.139:0/1445319215 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bcf33544e0 con 0x55bcf33cf000 2022-04-28T19:39:11.580 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:11 smithi139 bash[23439]: cluster 2022-04-28T19:39:09.956943+0000 mgr.y (mgr.14140) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:11.580 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:11 smithi139 bash[27981]: debug 2022-04-28T19:39:11.253+0000 7f1bdbc69f00 -1 Falling back to public interface 2022-04-28T19:39:11.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:11 smithi059 bash[23786]: cluster 2022-04-28T19:39:09.956943+0000 mgr.y (mgr.14140) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:11 smithi059 bash[29114]: cluster 2022-04-28T19:39:09.956943+0000 mgr.y (mgr.14140) 161 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:12.112 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:11 smithi139 bash[27981]: debug 2022-04-28T19:39:11.881+0000 7f1bdbc69f00 -1 osd.5 0 log_to_monitors {default=true} 2022-04-28T19:39:12.287 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_2 2022-04-28T19:39:12.287 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2022-04-28T19:39:12.287 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2022-04-28T19:39:12.287 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2022-04-28T19:39:12.287 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0181942 s, 576 MB/s 2022-04-28T19:39:12.287 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2022-04-28T19:39:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:12 smithi059 bash[23786]: audit 2022-04-28T19:39:11.885315+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:39:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:12 smithi059 bash[23786]: audit 2022-04-28T19:39:11.889112+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:39:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:12 smithi059 bash[29114]: audit 2022-04-28T19:39:11.885315+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:39:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:12 smithi059 bash[29114]: audit 2022-04-28T19:39:11.889112+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:39:12.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:12 smithi139 bash[23439]: audit 2022-04-28T19:39:11.885315+0000 mon.a (mon.0) 428 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:39:12.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:12 smithi139 bash[23439]: audit 2022-04-28T19:39:11.889112+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:39:13.198 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi139:vg_nvme/lv_2 2022-04-28T19:39:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[23786]: cluster 2022-04-28T19:39:11.957566+0000 mgr.y (mgr.14140) 162 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[23786]: audit 2022-04-28T19:39:12.172543+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-04-28T19:39:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[23786]: cluster 2022-04-28T19:39:12.172686+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 5 in 2022-04-28T19:39:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[23786]: audit 2022-04-28T19:39:12.172943+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[23786]: audit 2022-04-28T19:39:12.173827+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[23786]: audit 2022-04-28T19:39:12.177868+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[29114]: cluster 2022-04-28T19:39:11.957566+0000 mgr.y (mgr.14140) 162 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[29114]: audit 2022-04-28T19:39:12.172543+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-04-28T19:39:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[29114]: cluster 2022-04-28T19:39:12.172686+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 5 in 2022-04-28T19:39:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[29114]: audit 2022-04-28T19:39:12.172943+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[29114]: audit 2022-04-28T19:39:12.173827+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:13 smithi059 bash[29114]: audit 2022-04-28T19:39:12.177868+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:13.612 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[27981]: debug 2022-04-28T19:39:13.177+0000 7f1bbf12f700 -1 osd.5 0 waiting for initial osdmap 2022-04-28T19:39:13.612 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[27981]: debug 2022-04-28T19:39:13.185+0000 7f1bcb948700 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:39:13.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[23439]: cluster 2022-04-28T19:39:11.957566+0000 mgr.y (mgr.14140) 162 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:13.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[23439]: audit 2022-04-28T19:39:12.172543+0000 mon.a (mon.0) 429 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-04-28T19:39:13.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[23439]: cluster 2022-04-28T19:39:12.172686+0000 mon.a (mon.0) 430 : cluster [DBG] osdmap e34: 6 total, 5 up, 5 in 2022-04-28T19:39:13.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[23439]: audit 2022-04-28T19:39:12.172943+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:13.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[23439]: audit 2022-04-28T19:39:12.173827+0000 mon.a (mon.0) 432 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:13.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:13 smithi139 bash[23439]: audit 2022-04-28T19:39:12.177868+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:14.446 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.176015+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:39:14.446 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: cluster 2022-04-28T19:39:13.176277+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 5 in 2022-04-28T19:39:14.446 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.178001+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.446 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.180071+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.446 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.300515+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:14.446 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.623264+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.447 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.630518+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.447 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:13.637361+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.447 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: cluster 2022-04-28T19:39:14.174427+0000 mon.a (mon.0) 441 : cluster [INF] osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393] boot 2022-04-28T19:39:14.447 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: cluster 2022-04-28T19:39:14.174492+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2022-04-28T19:39:14.447 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:14 smithi139 bash[23439]: audit 2022-04-28T19:39:14.174641+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.176015+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:39:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: cluster 2022-04-28T19:39:13.176277+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 5 in 2022-04-28T19:39:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.178001+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.180071+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.300515+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.623264+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.630518+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:13.637361+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: cluster 2022-04-28T19:39:14.174427+0000 mon.a (mon.0) 441 : cluster [INF] osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393] boot 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: cluster 2022-04-28T19:39:14.174492+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[23786]: audit 2022-04-28T19:39:14.174641+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.176015+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:39:14.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: cluster 2022-04-28T19:39:13.176277+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e35: 6 total, 5 up, 5 in 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.178001+0000 mon.a (mon.0) 435 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.180071+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.300515+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.623264+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.630518+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:13.637361+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: cluster 2022-04-28T19:39:14.174427+0000 mon.a (mon.0) 441 : cluster [INF] osd.5 [v2:172.21.15.139:6808/2795762393,v1:172.21.15.139:6809/2795762393] boot 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: cluster 2022-04-28T19:39:14.174492+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2022-04-28T19:39:14.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:14 smithi059 bash[29114]: audit 2022-04-28T19:39:14.174641+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:39:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:15 smithi059 bash[23786]: cluster 2022-04-28T19:39:12.852688+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:39:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:15 smithi059 bash[23786]: cluster 2022-04-28T19:39:12.852743+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:39:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:15 smithi059 bash[23786]: cluster 2022-04-28T19:39:13.958148+0000 mgr.y (mgr.14140) 163 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:15 smithi059 bash[29114]: cluster 2022-04-28T19:39:12.852688+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:39:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:15 smithi059 bash[29114]: cluster 2022-04-28T19:39:12.852743+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:39:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:15 smithi059 bash[29114]: cluster 2022-04-28T19:39:13.958148+0000 mgr.y (mgr.14140) 163 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:15.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:15 smithi139 bash[23439]: cluster 2022-04-28T19:39:12.852688+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:39:15.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:15 smithi139 bash[23439]: cluster 2022-04-28T19:39:12.852743+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:39:15.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:15 smithi139 bash[23439]: cluster 2022-04-28T19:39:13.958148+0000 mgr.y (mgr.14140) 163 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.4 MiB used, 442 GiB / 447 GiB avail 2022-04-28T19:39:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:16 smithi059 bash[23786]: cluster 2022-04-28T19:39:15.191276+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2022-04-28T19:39:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:16 smithi059 bash[23786]: audit 2022-04-28T19:39:16.109628+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:39:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:16 smithi059 bash[23786]: audit 2022-04-28T19:39:16.112472+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:39:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:16 smithi059 bash[29114]: cluster 2022-04-28T19:39:15.191276+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2022-04-28T19:39:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:16 smithi059 bash[29114]: audit 2022-04-28T19:39:16.109628+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:39:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:16 smithi059 bash[29114]: audit 2022-04-28T19:39:16.112472+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:39:16.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:16 smithi139 bash[23439]: cluster 2022-04-28T19:39:15.191276+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2022-04-28T19:39:16.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:16 smithi139 bash[23439]: audit 2022-04-28T19:39:16.109628+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:39:16.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:16 smithi139 bash[23439]: audit 2022-04-28T19:39:16.112472+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:39:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[23786]: cluster 2022-04-28T19:39:15.958691+0000 mgr.y (mgr.14140) 164 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.7 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[23786]: audit 2022-04-28T19:39:16.549590+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:39:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[23786]: audit 2022-04-28T19:39:16.553103+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:39:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[23786]: audit 2022-04-28T19:39:16.554829+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[23786]: audit 2022-04-28T19:39:16.556762+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[29114]: cluster 2022-04-28T19:39:15.958691+0000 mgr.y (mgr.14140) 164 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.7 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[29114]: audit 2022-04-28T19:39:16.549590+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:39:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[29114]: audit 2022-04-28T19:39:16.553103+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:39:17.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[29114]: audit 2022-04-28T19:39:16.554829+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:17.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:17 smithi059 bash[29114]: audit 2022-04-28T19:39:16.556762+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:17.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:17 smithi139 bash[23439]: cluster 2022-04-28T19:39:15.958691+0000 mgr.y (mgr.14140) 164 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.7 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:17.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:17 smithi139 bash[23439]: audit 2022-04-28T19:39:16.549590+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:39:17.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:17 smithi139 bash[23439]: audit 2022-04-28T19:39:16.553103+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:39:17.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:17 smithi139 bash[23439]: audit 2022-04-28T19:39:16.554829+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:17.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:17 smithi139 bash[23439]: audit 2022-04-28T19:39:16.556762+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:18.243 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:18 smithi139 bash[23439]: audit 2022-04-28T19:39:16.547674+0000 mgr.y (mgr.14140) 165 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:39:18.244 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:18 smithi139 bash[23439]: cephadm 2022-04-28T19:39:16.552613+0000 mgr.y (mgr.14140) 166 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:39:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:18 smithi059 bash[23786]: audit 2022-04-28T19:39:16.547674+0000 mgr.y (mgr.14140) 165 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:39:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:18 smithi059 bash[23786]: cephadm 2022-04-28T19:39:16.552613+0000 mgr.y (mgr.14140) 166 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:39:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:18 smithi059 bash[29114]: audit 2022-04-28T19:39:16.547674+0000 mgr.y (mgr.14140) 165 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:39:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:18 smithi059 bash[29114]: cephadm 2022-04-28T19:39:16.552613+0000 mgr.y (mgr.14140) 166 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:39:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:19 smithi059 bash[23786]: cluster 2022-04-28T19:39:17.959374+0000 mgr.y (mgr.14140) 167 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.8 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:19 smithi059 bash[29114]: cluster 2022-04-28T19:39:17.959374+0000 mgr.y (mgr.14140) 167 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.8 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:19.606 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:19 smithi139 bash[23439]: cluster 2022-04-28T19:39:17.959374+0000 mgr.y (mgr.14140) 167 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.8 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: cluster 2022-04-28T19:39:19.960010+0000 mgr.y (mgr.14140) 168 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.8 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: audit 2022-04-28T19:39:20.397849+0000 mon.a (mon.0) 451 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]: dispatch 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: audit 2022-04-28T19:39:20.401545+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.139:0/2815089705' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]: dispatch 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: audit 2022-04-28T19:39:20.406199+0000 mon.a (mon.0) 452 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]': finished 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: cluster 2022-04-28T19:39:20.406379+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e38: 7 total, 6 up, 6 in 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: audit 2022-04-28T19:39:20.406592+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[23786]: audit 2022-04-28T19:39:20.925473+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.139:0/3908196237' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: cluster 2022-04-28T19:39:19.960010+0000 mgr.y (mgr.14140) 168 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.8 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: audit 2022-04-28T19:39:20.397849+0000 mon.a (mon.0) 451 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]: dispatch 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: audit 2022-04-28T19:39:20.401545+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.139:0/2815089705' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]: dispatch 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: audit 2022-04-28T19:39:20.406199+0000 mon.a (mon.0) 452 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]': finished 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: cluster 2022-04-28T19:39:20.406379+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e38: 7 total, 6 up, 6 in 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: audit 2022-04-28T19:39:20.406592+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:21 smithi059 bash[29114]: audit 2022-04-28T19:39:20.925473+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.139:0/3908196237' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: cluster 2022-04-28T19:39:19.960010+0000 mgr.y (mgr.14140) 168 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.8 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: audit 2022-04-28T19:39:20.397849+0000 mon.a (mon.0) 451 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]: dispatch 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: audit 2022-04-28T19:39:20.401545+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.139:0/2815089705' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]: dispatch 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: audit 2022-04-28T19:39:20.406199+0000 mon.a (mon.0) 452 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab0065ec-bbb4-403d-97b7-d4756f91e824"}]': finished 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: cluster 2022-04-28T19:39:20.406379+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e38: 7 total, 6 up, 6 in 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: audit 2022-04-28T19:39:20.406592+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:21.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:21 smithi139 bash[23439]: audit 2022-04-28T19:39:20.925473+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.139:0/3908196237' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:39:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:23 smithi059 bash[23786]: cluster 2022-04-28T19:39:21.960638+0000 mgr.y (mgr.14140) 169 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:23 smithi059 bash[29114]: cluster 2022-04-28T19:39:21.960638+0000 mgr.y (mgr.14140) 169 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:23.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:23 smithi139 bash[23439]: cluster 2022-04-28T19:39:21.960638+0000 mgr.y (mgr.14140) 169 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:24.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:24 smithi059 bash[23786]: audit 2022-04-28T19:39:23.727273+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:24.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:24 smithi059 bash[29114]: audit 2022-04-28T19:39:23.727273+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:24.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:24 smithi139 bash[23439]: audit 2022-04-28T19:39:23.727273+0000 mon.a (mon.0) 455 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:25 smithi059 bash[29114]: cluster 2022-04-28T19:39:23.961245+0000 mgr.y (mgr.14140) 170 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:25 smithi059 bash[23786]: cluster 2022-04-28T19:39:23.961245+0000 mgr.y (mgr.14140) 170 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:25.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:25 smithi139 bash[23439]: cluster 2022-04-28T19:39:23.961245+0000 mgr.y (mgr.14140) 170 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:27.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:27 smithi139 bash[23439]: cluster 2022-04-28T19:39:25.961865+0000 mgr.y (mgr.14140) 171 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:27.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:27 smithi059 bash[23786]: cluster 2022-04-28T19:39:25.961865+0000 mgr.y (mgr.14140) 171 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:27 smithi059 bash[29114]: cluster 2022-04-28T19:39:25.961865+0000 mgr.y (mgr.14140) 171 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:28.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:28 smithi059 bash[23786]: audit 2022-04-28T19:39:27.552718+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-04-28T19:39:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:28 smithi059 bash[23786]: audit 2022-04-28T19:39:27.554182+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:28 smithi059 bash[23786]: audit 2022-04-28T19:39:27.556040+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.6", "key": "container_image"}]: dispatch 2022-04-28T19:39:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:28 smithi059 bash[29114]: audit 2022-04-28T19:39:27.552718+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-04-28T19:39:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:28 smithi059 bash[29114]: audit 2022-04-28T19:39:27.554182+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:28 smithi059 bash[29114]: audit 2022-04-28T19:39:27.556040+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.6", "key": "container_image"}]: dispatch 2022-04-28T19:39:28.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:28 smithi139 bash[23439]: audit 2022-04-28T19:39:27.552718+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2022-04-28T19:39:28.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:28 smithi139 bash[23439]: audit 2022-04-28T19:39:27.554182+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:28.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:28 smithi139 bash[23439]: audit 2022-04-28T19:39:27.556040+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.6", "key": "container_image"}]: dispatch 2022-04-28T19:39:29.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:29 smithi139 bash[23439]: cephadm 2022-04-28T19:39:27.555304+0000 mgr.y (mgr.14140) 172 : cephadm [INF] Deploying daemon osd.6 on smithi139 2022-04-28T19:39:29.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:29 smithi139 bash[23439]: cluster 2022-04-28T19:39:27.962422+0000 mgr.y (mgr.14140) 173 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:29.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:29 smithi059 bash[23786]: cephadm 2022-04-28T19:39:27.555304+0000 mgr.y (mgr.14140) 172 : cephadm [INF] Deploying daemon osd.6 on smithi139 2022-04-28T19:39:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:29 smithi059 bash[23786]: cluster 2022-04-28T19:39:27.962422+0000 mgr.y (mgr.14140) 173 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:29 smithi059 bash[29114]: cephadm 2022-04-28T19:39:27.555304+0000 mgr.y (mgr.14140) 172 : cephadm [INF] Deploying daemon osd.6 on smithi139 2022-04-28T19:39:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:29 smithi059 bash[29114]: cluster 2022-04-28T19:39:27.962422+0000 mgr.y (mgr.14140) 173 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:29.893 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 6 on host 'smithi139' 2022-04-28T19:39:30.956 DEBUG:teuthology.orchestra.run.smithi139:osd.6> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.6.service 2022-04-28T19:39:30.958 INFO:tasks.cephadm:Deploying osd.7 on smithi139 with /dev/vg_nvme/lv_1... 2022-04-28T19:39:30.958 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2022-04-28T19:39:31.018 INFO:journalctl@ceph.osd.6.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:39:31.330 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:30 smithi139 bash[23439]: audit 2022-04-28T19:39:29.891027+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:31.331 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:30 smithi139 bash[23439]: audit 2022-04-28T19:39:29.898329+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:31.331 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:30 smithi139 bash[23439]: cluster 2022-04-28T19:39:29.963000+0000 mgr.y (mgr.14140) 174 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:31.331 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:30 smithi139 bash[23439]: audit 2022-04-28T19:39:30.717993+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:31.331 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:30 smithi139 bash[23439]: audit 2022-04-28T19:39:30.719736+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[23786]: audit 2022-04-28T19:39:29.891027+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[23786]: audit 2022-04-28T19:39:29.898329+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[23786]: cluster 2022-04-28T19:39:29.963000+0000 mgr.y (mgr.14140) 174 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[23786]: audit 2022-04-28T19:39:30.717993+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[23786]: audit 2022-04-28T19:39:30.719736+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[29114]: audit 2022-04-28T19:39:29.891027+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[29114]: audit 2022-04-28T19:39:29.898329+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[29114]: cluster 2022-04-28T19:39:29.963000+0000 mgr.y (mgr.14140) 174 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[29114]: audit 2022-04-28T19:39:30.717993+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:30 smithi059 bash[29114]: audit 2022-04-28T19:39:30.719736+0000 mon.a (mon.0) 462 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:32.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[23439]: audit 2022-04-28T19:39:31.110153+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:32.362 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2022-04-28T19:39:32.362 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2022-04-28T19:39:32.362 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2022-04-28T19:39:32.362 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2022-04-28T19:39:32.363 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2022-04-28T19:39:32.363 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2022-04-28T19:39:32.363 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:32 smithi139 bash[29993]: --> ceph-volume lvm activate successful for osd ID: 6 2022-04-28T19:39:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:32 smithi059 bash[23786]: audit 2022-04-28T19:39:31.110153+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:32 smithi059 bash[29114]: audit 2022-04-28T19:39:31.110153+0000 mon.a (mon.0) 463 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:33.506 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:33 smithi139 bash[23439]: cluster 2022-04-28T19:39:31.963727+0000 mgr.y (mgr.14140) 175 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:33.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:33 smithi059 bash[23786]: cluster 2022-04-28T19:39:31.963727+0000 mgr.y (mgr.14140) 175 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:33.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:33 smithi059 bash[29114]: cluster 2022-04-28T19:39:31.963727+0000 mgr.y (mgr.14140) 175 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:35.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:35 smithi139 bash[23439]: cluster 2022-04-28T19:39:33.964320+0000 mgr.y (mgr.14140) 176 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:35.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:35 smithi059 bash[23786]: cluster 2022-04-28T19:39:33.964320+0000 mgr.y (mgr.14140) 176 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:35 smithi059 bash[29114]: cluster 2022-04-28T19:39:33.964320+0000 mgr.y (mgr.14140) 176 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:35.761 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:35 smithi139 bash[29993]: debug 2022-04-28T19:39:35.393+0000 7fa540958700 1 -- 172.21.15.139:0/649001431 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560ab5bf24e0 con 0x560ab5c6cc00 2022-04-28T19:39:36.111 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:35 smithi139 bash[29993]: debug 2022-04-28T19:39:35.945+0000 7fa547bf6f00 -1 Falling back to public interface 2022-04-28T19:39:36.290 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping: /dev/vg_nvme/lv_1 2022-04-28T19:39:36.290 INFO:teuthology.orchestra.run.smithi139.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2022-04-28T19:39:36.290 INFO:teuthology.orchestra.run.smithi139.stderr: stderr: 10+0 records in 2022-04-28T19:39:36.290 INFO:teuthology.orchestra.run.smithi139.stderr:10+0 records out 2022-04-28T19:39:36.290 INFO:teuthology.orchestra.run.smithi139.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0126329 s, 830 MB/s 2022-04-28T19:39:36.290 INFO:teuthology.orchestra.run.smithi139.stderr:--> Zapping successful for: 2022-04-28T19:39:36.611 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:36 smithi139 bash[29993]: debug 2022-04-28T19:39:36.573+0000 7fa547bf6f00 -1 osd.6 0 log_to_monitors {default=true} 2022-04-28T19:39:37.148 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch daemon add osd smithi139:vg_nvme/lv_1 2022-04-28T19:39:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:37 smithi059 bash[29114]: cluster 2022-04-28T19:39:35.964945+0000 mgr.y (mgr.14140) 177 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:37 smithi059 bash[29114]: audit 2022-04-28T19:39:36.580439+0000 mon.a (mon.0) 464 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:39:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:37 smithi059 bash[29114]: audit 2022-04-28T19:39:36.584413+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:39:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:37 smithi059 bash[23786]: cluster 2022-04-28T19:39:35.964945+0000 mgr.y (mgr.14140) 177 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:37 smithi059 bash[23786]: audit 2022-04-28T19:39:36.580439+0000 mon.a (mon.0) 464 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:39:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:37 smithi059 bash[23786]: audit 2022-04-28T19:39:36.584413+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:39:37.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:37 smithi139 bash[23439]: cluster 2022-04-28T19:39:35.964945+0000 mgr.y (mgr.14140) 177 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:37.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:37 smithi139 bash[23439]: audit 2022-04-28T19:39:36.580439+0000 mon.a (mon.0) 464 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:39:37.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:37 smithi139 bash[23439]: audit 2022-04-28T19:39:36.584413+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:39:38.491 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[29993]: debug 2022-04-28T19:39:38.137+0000 7fa52f0c4700 -1 osd.6 0 waiting for initial osdmap 2022-04-28T19:39:38.491 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[29993]: debug 2022-04-28T19:39:38.149+0000 7fa53b8dd700 -1 osd.6 40 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:39:38.491 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.132120+0000 mon.a (mon.0) 465 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: cluster 2022-04-28T19:39:37.132388+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e39: 7 total, 6 up, 6 in 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.133040+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.133820+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.138122+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.236107+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.553184+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:37.948604+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:38.133319+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:39:38.492 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: cluster 2022-04-28T19:39:38.133449+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e40: 7 total, 6 up, 6 in 2022-04-28T19:39:38.493 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:38 smithi139 bash[23439]: audit 2022-04-28T19:39:38.134014+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.132120+0000 mon.a (mon.0) 465 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: cluster 2022-04-28T19:39:37.132388+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e39: 7 total, 6 up, 6 in 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.133040+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.133820+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.138122+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.236107+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.553184+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:37.948604+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:38.133319+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: cluster 2022-04-28T19:39:38.133449+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e40: 7 total, 6 up, 6 in 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[23786]: audit 2022-04-28T19:39:38.134014+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.132120+0000 mon.a (mon.0) 465 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: cluster 2022-04-28T19:39:37.132388+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e39: 7 total, 6 up, 6 in 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.133040+0000 mon.a (mon.0) 467 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.133820+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.138122+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:39:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.236107+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.553184+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:37.948604+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:38.133319+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:39:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: cluster 2022-04-28T19:39:38.133449+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e40: 7 total, 6 up, 6 in 2022-04-28T19:39:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:38 smithi059 bash[29114]: audit 2022-04-28T19:39:38.134014+0000 mon.a (mon.0) 474 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:39.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[23786]: cephadm 2022-04-28T19:39:37.554324+0000 mgr.y (mgr.14140) 178 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:39:39.611 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[23786]: cluster 2022-04-28T19:39:37.965478+0000 mgr.y (mgr.14140) 179 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:39.611 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[23786]: audit 2022-04-28T19:39:38.143162+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:39.611 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[23786]: cluster 2022-04-28T19:39:39.136201+0000 mon.a (mon.0) 476 : cluster [INF] osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322] boot 2022-04-28T19:39:39.611 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[23786]: cluster 2022-04-28T19:39:39.136249+0000 mon.a (mon.0) 477 : cluster [DBG] osdmap e41: 7 total, 7 up, 7 in 2022-04-28T19:39:39.611 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[23786]: audit 2022-04-28T19:39:39.136420+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:39.611 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[29114]: cephadm 2022-04-28T19:39:37.554324+0000 mgr.y (mgr.14140) 178 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:39:39.612 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[29114]: cluster 2022-04-28T19:39:37.965478+0000 mgr.y (mgr.14140) 179 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:39.612 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[29114]: audit 2022-04-28T19:39:38.143162+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:39.612 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[29114]: cluster 2022-04-28T19:39:39.136201+0000 mon.a (mon.0) 476 : cluster [INF] osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322] boot 2022-04-28T19:39:39.612 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[29114]: cluster 2022-04-28T19:39:39.136249+0000 mon.a (mon.0) 477 : cluster [DBG] osdmap e41: 7 total, 7 up, 7 in 2022-04-28T19:39:39.612 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:39 smithi059 bash[29114]: audit 2022-04-28T19:39:39.136420+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:39.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:39 smithi139 bash[23439]: cephadm 2022-04-28T19:39:37.554324+0000 mgr.y (mgr.14140) 178 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:39:39.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:39 smithi139 bash[23439]: cluster 2022-04-28T19:39:37.965478+0000 mgr.y (mgr.14140) 179 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 1.9 MiB used, 530 GiB / 536 GiB avail 2022-04-28T19:39:39.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:39 smithi139 bash[23439]: audit 2022-04-28T19:39:38.143162+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:39.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:39 smithi139 bash[23439]: cluster 2022-04-28T19:39:39.136201+0000 mon.a (mon.0) 476 : cluster [INF] osd.6 [v2:172.21.15.139:6816/1808526322,v1:172.21.15.139:6817/1808526322] boot 2022-04-28T19:39:39.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:39 smithi139 bash[23439]: cluster 2022-04-28T19:39:39.136249+0000 mon.a (mon.0) 477 : cluster [DBG] osdmap e41: 7 total, 7 up, 7 in 2022-04-28T19:39:39.613 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:39 smithi139 bash[23439]: audit 2022-04-28T19:39:39.136420+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:39:40.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:40 smithi139 bash[23439]: cluster 2022-04-28T19:39:37.626633+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:39:40.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:40 smithi139 bash[23439]: cluster 2022-04-28T19:39:37.626698+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:39:40.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:40 smithi059 bash[23786]: cluster 2022-04-28T19:39:37.626633+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:39:40.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:40 smithi059 bash[23786]: cluster 2022-04-28T19:39:37.626698+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:39:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:40 smithi059 bash[29114]: cluster 2022-04-28T19:39:37.626633+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:39:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:40 smithi059 bash[29114]: cluster 2022-04-28T19:39:37.626698+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:39:41.506 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:41 smithi139 bash[23439]: cluster 2022-04-28T19:39:39.966089+0000 mgr.y (mgr.14140) 180 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 2.3 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:41.506 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:41 smithi139 bash[23439]: cluster 2022-04-28T19:39:40.152944+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2022-04-28T19:39:41.506 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:41 smithi139 bash[23439]: audit 2022-04-28T19:39:40.336688+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:39:41.507 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:41 smithi139 bash[23439]: audit 2022-04-28T19:39:40.340116+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:39:41.507 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:41 smithi139 bash[23439]: audit 2022-04-28T19:39:40.341737+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:41.507 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:41 smithi139 bash[23439]: audit 2022-04-28T19:39:40.343642+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[23786]: cluster 2022-04-28T19:39:39.966089+0000 mgr.y (mgr.14140) 180 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 2.3 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[23786]: cluster 2022-04-28T19:39:40.152944+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2022-04-28T19:39:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[23786]: audit 2022-04-28T19:39:40.336688+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:39:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[23786]: audit 2022-04-28T19:39:40.340116+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:39:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[23786]: audit 2022-04-28T19:39:40.341737+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[23786]: audit 2022-04-28T19:39:40.343642+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[29114]: cluster 2022-04-28T19:39:39.966089+0000 mgr.y (mgr.14140) 180 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 2.3 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[29114]: cluster 2022-04-28T19:39:40.152944+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2022-04-28T19:39:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[29114]: audit 2022-04-28T19:39:40.336688+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2022-04-28T19:39:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[29114]: audit 2022-04-28T19:39:40.340116+0000 mon.a (mon.0) 481 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2022-04-28T19:39:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[29114]: audit 2022-04-28T19:39:40.341737+0000 mon.a (mon.0) 482 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:41 smithi059 bash[29114]: audit 2022-04-28T19:39:40.343642+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:42.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:42 smithi059 bash[23786]: audit 2022-04-28T19:39:40.334694+0000 mgr.y (mgr.14140) 181 : audit [DBG] from='client.34301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:39:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:42 smithi059 bash[23786]: cephadm 2022-04-28T19:39:40.339585+0000 mgr.y (mgr.14140) 182 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:39:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:42 smithi059 bash[29114]: audit 2022-04-28T19:39:40.334694+0000 mgr.y (mgr.14140) 181 : audit [DBG] from='client.34301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:39:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:42 smithi059 bash[29114]: cephadm 2022-04-28T19:39:40.339585+0000 mgr.y (mgr.14140) 182 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:39:42.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:42 smithi139 bash[23439]: audit 2022-04-28T19:39:40.334694+0000 mgr.y (mgr.14140) 181 : audit [DBG] from='client.34301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi139:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:39:42.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:42 smithi139 bash[23439]: cephadm 2022-04-28T19:39:40.339585+0000 mgr.y (mgr.14140) 182 : cephadm [INF] Applying drive group None on host smithi139... 2022-04-28T19:39:43.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:43 smithi059 bash[29114]: cluster 2022-04-28T19:39:41.966728+0000 mgr.y (mgr.14140) 183 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 2.4 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:43 smithi059 bash[23786]: cluster 2022-04-28T19:39:41.966728+0000 mgr.y (mgr.14140) 183 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 2.4 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:43.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:43 smithi139 bash[23439]: cluster 2022-04-28T19:39:41.966728+0000 mgr.y (mgr.14140) 183 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 2.4 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:44.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[23786]: audit 2022-04-28T19:39:43.912269+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]: dispatch 2022-04-28T19:39:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[23786]: audit 2022-04-28T19:39:43.916302+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.139:0/3390277568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]: dispatch 2022-04-28T19:39:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[23786]: audit 2022-04-28T19:39:43.922254+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]': finished 2022-04-28T19:39:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[23786]: cluster 2022-04-28T19:39:43.922422+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e43: 8 total, 7 up, 7 in 2022-04-28T19:39:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[23786]: audit 2022-04-28T19:39:43.922665+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:39:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[29114]: audit 2022-04-28T19:39:43.912269+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]: dispatch 2022-04-28T19:39:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[29114]: audit 2022-04-28T19:39:43.916302+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.139:0/3390277568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]: dispatch 2022-04-28T19:39:44.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[29114]: audit 2022-04-28T19:39:43.922254+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]': finished 2022-04-28T19:39:44.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[29114]: cluster 2022-04-28T19:39:43.922422+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e43: 8 total, 7 up, 7 in 2022-04-28T19:39:44.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:44 smithi059 bash[29114]: audit 2022-04-28T19:39:43.922665+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:39:44.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:44 smithi139 bash[23439]: audit 2022-04-28T19:39:43.912269+0000 mon.a (mon.0) 484 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]: dispatch 2022-04-28T19:39:44.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:44 smithi139 bash[23439]: audit 2022-04-28T19:39:43.916302+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.139:0/3390277568' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]: dispatch 2022-04-28T19:39:44.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:44 smithi139 bash[23439]: audit 2022-04-28T19:39:43.922254+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04593407-3ff3-42d0-bc54-fe0e3e96c835"}]': finished 2022-04-28T19:39:44.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:44 smithi139 bash[23439]: cluster 2022-04-28T19:39:43.922422+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e43: 8 total, 7 up, 7 in 2022-04-28T19:39:44.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:44 smithi139 bash[23439]: audit 2022-04-28T19:39:43.922665+0000 mon.a (mon.0) 487 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:39:45.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:45 smithi059 bash[23786]: cluster 2022-04-28T19:39:43.967328+0000 mgr.y (mgr.14140) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 2.5 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:45 smithi059 bash[23786]: audit 2022-04-28T19:39:44.472753+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.139:0/3648605051' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:39:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:45 smithi059 bash[29114]: cluster 2022-04-28T19:39:43.967328+0000 mgr.y (mgr.14140) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 2.5 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:45 smithi059 bash[29114]: audit 2022-04-28T19:39:44.472753+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.139:0/3648605051' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:39:45.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:45 smithi139 bash[23439]: cluster 2022-04-28T19:39:43.967328+0000 mgr.y (mgr.14140) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 2.5 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:45.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:45 smithi139 bash[23439]: audit 2022-04-28T19:39:44.472753+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.139:0/3648605051' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2022-04-28T19:39:47.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:47 smithi059 bash[29114]: cluster 2022-04-28T19:39:45.967949+0000 mgr.y (mgr.14140) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 2.6 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:47 smithi059 bash[23786]: cluster 2022-04-28T19:39:45.967949+0000 mgr.y (mgr.14140) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 2.6 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:47.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:47 smithi139 bash[23439]: cluster 2022-04-28T19:39:45.967949+0000 mgr.y (mgr.14140) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 2.6 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:48.464 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:48 smithi139 bash[23439]: audit 2022-04-28T19:39:47.255869+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:48 smithi059 bash[29114]: audit 2022-04-28T19:39:47.255869+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:48.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:48 smithi059 bash[23786]: audit 2022-04-28T19:39:47.255869+0000 mon.a (mon.0) 488 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:49.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:49 smithi059 bash[29114]: cluster 2022-04-28T19:39:47.968550+0000 mgr.y (mgr.14140) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 2.6 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:49.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:49 smithi059 bash[23786]: cluster 2022-04-28T19:39:47.968550+0000 mgr.y (mgr.14140) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 2.6 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:49.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:49 smithi139 bash[23439]: cluster 2022-04-28T19:39:47.968550+0000 mgr.y (mgr.14140) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 2.6 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:51.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:51 smithi059 bash[29114]: cluster 2022-04-28T19:39:49.969163+0000 mgr.y (mgr.14140) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:51 smithi059 bash[23786]: cluster 2022-04-28T19:39:49.969163+0000 mgr.y (mgr.14140) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:51.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:51 smithi139 bash[23439]: cluster 2022-04-28T19:39:49.969163+0000 mgr.y (mgr.14140) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:52.530 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:52 smithi139 bash[23439]: audit 2022-04-28T19:39:51.257697+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-04-28T19:39:52.531 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:52 smithi139 bash[23439]: audit 2022-04-28T19:39:51.259806+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:52.531 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:52 smithi139 bash[23439]: cephadm 2022-04-28T19:39:51.261421+0000 mgr.y (mgr.14140) 188 : cephadm [INF] Deploying daemon osd.7 on smithi139 2022-04-28T19:39:52.531 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:52 smithi139 bash[23439]: audit 2022-04-28T19:39:51.262242+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.7", "key": "container_image"}]: dispatch 2022-04-28T19:39:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[23786]: audit 2022-04-28T19:39:51.257697+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-04-28T19:39:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[23786]: audit 2022-04-28T19:39:51.259806+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[23786]: cephadm 2022-04-28T19:39:51.261421+0000 mgr.y (mgr.14140) 188 : cephadm [INF] Deploying daemon osd.7 on smithi139 2022-04-28T19:39:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[23786]: audit 2022-04-28T19:39:51.262242+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.7", "key": "container_image"}]: dispatch 2022-04-28T19:39:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[29114]: audit 2022-04-28T19:39:51.257697+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2022-04-28T19:39:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[29114]: audit 2022-04-28T19:39:51.259806+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:39:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[29114]: cephadm 2022-04-28T19:39:51.261421+0000 mgr.y (mgr.14140) 188 : cephadm [INF] Deploying daemon osd.7 on smithi139 2022-04-28T19:39:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:52 smithi059 bash[29114]: audit 2022-04-28T19:39:51.262242+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd.7", "key": "container_image"}]: dispatch 2022-04-28T19:39:53.463 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:53 smithi139 bash[23439]: cluster 2022-04-28T19:39:51.969820+0000 mgr.y (mgr.14140) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:53.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:53 smithi059 bash[23786]: cluster 2022-04-28T19:39:51.969820+0000 mgr.y (mgr.14140) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:53 smithi059 bash[29114]: cluster 2022-04-28T19:39:51.969820+0000 mgr.y (mgr.14140) 189 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:53.642 INFO:teuthology.orchestra.run.smithi139.stdout:Created osd(s) 7 on host 'smithi139' 2022-04-28T19:39:54.720 DEBUG:teuthology.orchestra.run.smithi139:osd.7> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@osd.7.service 2022-04-28T19:39:54.722 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2022-04-28T19:39:54.722 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd stat -f json 2022-04-28T19:39:54.729 INFO:journalctl@ceph.osd.7.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:39:54.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:54 smithi139 bash[23439]: audit 2022-04-28T19:39:53.640930+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:54.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:54 smithi139 bash[23439]: cephadm 2022-04-28T19:39:53.657863+0000 mgr.y (mgr.14140) 190 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:39:54.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:54 smithi139 bash[23439]: audit 2022-04-28T19:39:53.658635+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:54.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:54 smithi139 bash[23439]: cluster 2022-04-28T19:39:53.970271+0000 mgr.y (mgr.14140) 191 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[23786]: audit 2022-04-28T19:39:53.640930+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[23786]: cephadm 2022-04-28T19:39:53.657863+0000 mgr.y (mgr.14140) 190 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:39:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[23786]: audit 2022-04-28T19:39:53.658635+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[23786]: cluster 2022-04-28T19:39:53.970271+0000 mgr.y (mgr.14140) 191 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[29114]: audit 2022-04-28T19:39:53.640930+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[29114]: cephadm 2022-04-28T19:39:53.657863+0000 mgr.y (mgr.14140) 190 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:39:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[29114]: audit 2022-04-28T19:39:53.658635+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:39:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:54 smithi059 bash[29114]: cluster 2022-04-28T19:39:53.970271+0000 mgr.y (mgr.14140) 191 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:55.611 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2022-04-28T19:39:55.611 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2022-04-28T19:39:55.612 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2022-04-28T19:39:55.612 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2022-04-28T19:39:55.612 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2022-04-28T19:39:55.612 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2022-04-28T19:39:55.612 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[32027]: --> ceph-volume lvm activate successful for osd ID: 7 2022-04-28T19:39:56.075 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:55 smithi059 bash[29114]: audit 2022-04-28T19:39:54.664200+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:56.076 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:55 smithi059 bash[29114]: audit 2022-04-28T19:39:54.665391+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:56.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:55 smithi059 bash[23786]: audit 2022-04-28T19:39:54.664200+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:56.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:55 smithi059 bash[23786]: audit 2022-04-28T19:39:54.665391+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:56.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[23439]: audit 2022-04-28T19:39:54.664200+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:56.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:55 smithi139 bash[23439]: audit 2022-04-28T19:39:54.665391+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "osd", "key": "container_image"}]: dispatch 2022-04-28T19:39:57.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:57 smithi059 bash[23786]: cluster 2022-04-28T19:39:55.970863+0000 mgr.y (mgr.14140) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:57.439 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:57 smithi059 bash[23786]: audit 2022-04-28T19:39:56.117156+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:57.439 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:57 smithi059 bash[29114]: cluster 2022-04-28T19:39:55.970863+0000 mgr.y (mgr.14140) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:57.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:57 smithi059 bash[29114]: audit 2022-04-28T19:39:56.117156+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:57.631 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:57 smithi139 bash[23439]: cluster 2022-04-28T19:39:55.970863+0000 mgr.y (mgr.14140) 192 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:57.631 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:57 smithi139 bash[23439]: audit 2022-04-28T19:39:56.117156+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:39:57.813 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:39:58.111 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:57 smithi139 bash[32027]: debug 2022-04-28T19:39:57.965+0000 7ff64333a700 1 -- 172.21.15.139:0/3699500638 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b9b743c4e0 con 0x55b9b74b7400 2022-04-28T19:39:58.420 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:58 smithi059 bash[23786]: audit 2022-04-28T19:39:57.813460+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.59:0/591983925' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-04-28T19:39:58.420 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:58 smithi059 bash[29114]: audit 2022-04-28T19:39:57.813460+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.59:0/591983925' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-04-28T19:39:58.421 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":43,"num_osds":8,"num_up_osds":7,"osd_up_since":1651174779,"num_in_osds":7,"osd_in_since":1651174779,"num_remapped_pgs":0} 2022-04-28T19:39:58.498 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:58 smithi139 bash[23439]: audit 2022-04-28T19:39:57.813460+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.59:0/591983925' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-04-28T19:39:58.861 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:58 smithi139 bash[32027]: debug 2022-04-28T19:39:58.497+0000 7ff64a5d8f00 -1 Falling back to public interface 2022-04-28T19:39:59.361 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:39:59 smithi139 bash[32027]: debug 2022-04-28T19:39:59.113+0000 7ff64a5d8f00 -1 osd.7 0 log_to_monitors {default=true} 2022-04-28T19:39:59.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:59 smithi139 bash[23439]: cluster 2022-04-28T19:39:57.971470+0000 mgr.y (mgr.14140) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:59.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:59 smithi139 bash[23439]: audit 2022-04-28T19:39:59.119232+0000 mon.a (mon.0) 497 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:39:59.362 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:39:59 smithi139 bash[23439]: audit 2022-04-28T19:39:59.123445+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:39:59.422 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd stat -f json 2022-04-28T19:39:59.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:59 smithi059 bash[29114]: cluster 2022-04-28T19:39:57.971470+0000 mgr.y (mgr.14140) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:59.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:59 smithi059 bash[29114]: audit 2022-04-28T19:39:59.119232+0000 mon.a (mon.0) 497 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:39:59.435 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:39:59 smithi059 bash[29114]: audit 2022-04-28T19:39:59.123445+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:39:59.436 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:59 smithi059 bash[23786]: cluster 2022-04-28T19:39:57.971470+0000 mgr.y (mgr.14140) 193 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:39:59.436 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:59 smithi059 bash[23786]: audit 2022-04-28T19:39:59.119232+0000 mon.a (mon.0) 497 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:39:59.436 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:39:59 smithi059 bash[23786]: audit 2022-04-28T19:39:59.123445+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:40:00.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[23786]: audit 2022-04-28T19:39:59.138910+0000 mon.a (mon.0) 498 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-04-28T19:40:00.529 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[23786]: cluster 2022-04-28T19:39:59.139101+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e44: 8 total, 7 up, 7 in 2022-04-28T19:40:00.530 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[23786]: audit 2022-04-28T19:39:59.139403+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:00.530 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[23786]: audit 2022-04-28T19:39:59.140603+0000 mon.a (mon.0) 501 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:40:00.530 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[23786]: audit 2022-04-28T19:39:59.144777+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:40:00.530 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[23786]: cluster 2022-04-28T19:40:00.000129+0000 mon.a (mon.0) 502 : cluster [INF] overall HEALTH_OK 2022-04-28T19:40:00.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[29114]: audit 2022-04-28T19:39:59.138910+0000 mon.a (mon.0) 498 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-04-28T19:40:00.530 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[29114]: cluster 2022-04-28T19:39:59.139101+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e44: 8 total, 7 up, 7 in 2022-04-28T19:40:00.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[29114]: audit 2022-04-28T19:39:59.139403+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:00.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[29114]: audit 2022-04-28T19:39:59.140603+0000 mon.a (mon.0) 501 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:40:00.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[29114]: audit 2022-04-28T19:39:59.144777+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:40:00.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:00 smithi059 bash[29114]: cluster 2022-04-28T19:40:00.000129+0000 mon.a (mon.0) 502 : cluster [INF] overall HEALTH_OK 2022-04-28T19:40:00.611 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[32027]: debug 2022-04-28T19:40:00.153+0000 7ff631aa6700 -1 osd.7 0 waiting for initial osdmap 2022-04-28T19:40:00.611 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[32027]: debug 2022-04-28T19:40:00.165+0000 7ff63e2bf700 -1 osd.7 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:40:00.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[23439]: audit 2022-04-28T19:39:59.138910+0000 mon.a (mon.0) 498 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-04-28T19:40:00.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[23439]: cluster 2022-04-28T19:39:59.139101+0000 mon.a (mon.0) 499 : cluster [DBG] osdmap e44: 8 total, 7 up, 7 in 2022-04-28T19:40:00.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[23439]: audit 2022-04-28T19:39:59.139403+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:00.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[23439]: audit 2022-04-28T19:39:59.140603+0000 mon.a (mon.0) 501 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:40:00.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[23439]: audit 2022-04-28T19:39:59.144777+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:40:00.612 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:00 smithi139 bash[23439]: cluster 2022-04-28T19:40:00.000129+0000 mon.a (mon.0) 502 : cluster [INF] overall HEALTH_OK 2022-04-28T19:40:01.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: cluster 2022-04-28T19:39:59.972155+0000 mgr.y (mgr.14140) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:00.145232+0000 mon.a (mon.0) 503 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: cluster 2022-04-28T19:40:00.145467+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e45: 8 total, 7 up, 7 in 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:00.146176+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:00.153734+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:00.311132+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:00.630057+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:00.636364+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: cluster 2022-04-28T19:40:01.145479+0000 mon.a (mon.0) 510 : cluster [INF] osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349] boot 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: cluster 2022-04-28T19:40:01.145546+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2022-04-28T19:40:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[23786]: audit 2022-04-28T19:40:01.145732+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: cluster 2022-04-28T19:39:59.972155+0000 mgr.y (mgr.14140) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:00.145232+0000 mon.a (mon.0) 503 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: cluster 2022-04-28T19:40:00.145467+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e45: 8 total, 7 up, 7 in 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:00.146176+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:00.153734+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:00.311132+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:00.630057+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:00.636364+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: cluster 2022-04-28T19:40:01.145479+0000 mon.a (mon.0) 510 : cluster [INF] osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349] boot 2022-04-28T19:40:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: cluster 2022-04-28T19:40:01.145546+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2022-04-28T19:40:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:01 smithi059 bash[29114]: audit 2022-04-28T19:40:01.145732+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: cluster 2022-04-28T19:39:59.972155+0000 mgr.y (mgr.14140) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 2.7 MiB used, 619 GiB / 626 GiB avail 2022-04-28T19:40:01.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:00.145232+0000 mon.a (mon.0) 503 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]': finished 2022-04-28T19:40:01.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: cluster 2022-04-28T19:40:00.145467+0000 mon.a (mon.0) 504 : cluster [DBG] osdmap e45: 8 total, 7 up, 7 in 2022-04-28T19:40:01.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:00.146176+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:00.153734+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:00.311132+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:00.630057+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:00.636364+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: cluster 2022-04-28T19:40:01.145479+0000 mon.a (mon.0) 510 : cluster [INF] osd.7 [v2:172.21.15.139:6824/2154117349,v1:172.21.15.139:6825/2154117349] boot 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: cluster 2022-04-28T19:40:01.145546+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e46: 8 total, 8 up, 8 in 2022-04-28T19:40:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:01 smithi139 bash[23439]: audit 2022-04-28T19:40:01.145732+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:40:02.653 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:40:03.257 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":47,"num_osds":8,"num_up_osds":8,"osd_up_since":1651174801,"num_in_osds":8,"osd_in_since":1651174801,"num_remapped_pgs":0} 2022-04-28T19:40:03.258 INFO:tasks.cephadm:Adding prometheus.a on smithi139 2022-04-28T19:40:03.258 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch apply prometheus '1;smithi139=a' 2022-04-28T19:40:03.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[23786]: cluster 2022-04-28T19:40:00.076914+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:40:03.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[23786]: cluster 2022-04-28T19:40:00.076990+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[23786]: cluster 2022-04-28T19:40:01.972734+0000 mgr.y (mgr.14140) 195 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 3.1 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[23786]: cluster 2022-04-28T19:40:02.163932+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[23786]: audit 2022-04-28T19:40:02.653584+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.59:0/3055235742' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[29114]: cluster 2022-04-28T19:40:00.076914+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[29114]: cluster 2022-04-28T19:40:00.076990+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[29114]: cluster 2022-04-28T19:40:01.972734+0000 mgr.y (mgr.14140) 195 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 3.1 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[29114]: cluster 2022-04-28T19:40:02.163932+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2022-04-28T19:40:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:03 smithi059 bash[29114]: audit 2022-04-28T19:40:02.653584+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.59:0/3055235742' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-04-28T19:40:03.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:03 smithi139 bash[23439]: cluster 2022-04-28T19:40:00.076914+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2022-04-28T19:40:03.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:03 smithi139 bash[23439]: cluster 2022-04-28T19:40:00.076990+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2022-04-28T19:40:03.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:03 smithi139 bash[23439]: cluster 2022-04-28T19:40:01.972734+0000 mgr.y (mgr.14140) 195 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 3.1 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:03.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:03 smithi139 bash[23439]: cluster 2022-04-28T19:40:02.163932+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 8 up, 8 in 2022-04-28T19:40:03.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:03 smithi139 bash[23439]: audit 2022-04-28T19:40:02.653584+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.59:0/3055235742' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2022-04-28T19:40:05.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:05 smithi059 bash[23786]: cluster 2022-04-28T19:40:03.973316+0000 mgr.y (mgr.14140) 196 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 3.2 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:05.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:05 smithi059 bash[29114]: cluster 2022-04-28T19:40:03.973316+0000 mgr.y (mgr.14140) 196 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 3.2 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:05.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:05 smithi139 bash[23439]: cluster 2022-04-28T19:40:03.973316+0000 mgr.y (mgr.14140) 196 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 3.2 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:06.609 INFO:teuthology.orchestra.run.smithi139.stdout:Scheduled prometheus update... 2022-04-28T19:40:07.278 DEBUG:teuthology.orchestra.run.smithi139:prometheus.a> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@prometheus.a.service 2022-04-28T19:40:07.281 INFO:tasks.cephadm:Adding node-exporter.a on smithi059 2022-04-28T19:40:07.281 INFO:tasks.cephadm:Adding node-exporter.b on smithi139 2022-04-28T19:40:07.281 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch apply node-exporter '2;smithi059=a;smithi139=b' 2022-04-28T19:40:07.308 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:40:07.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:07 smithi139 bash[23439]: cluster 2022-04-28T19:40:05.973898+0000 mgr.y (mgr.14140) 197 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:07.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:07 smithi139 bash[23439]: audit 2022-04-28T19:40:06.607741+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:07.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:07 smithi139 bash[23439]: audit 2022-04-28T19:40:06.627390+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:07 smithi059 bash[23786]: cluster 2022-04-28T19:40:05.973898+0000 mgr.y (mgr.14140) 197 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:07 smithi059 bash[23786]: audit 2022-04-28T19:40:06.607741+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:07 smithi059 bash[23786]: audit 2022-04-28T19:40:06.627390+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:07 smithi059 bash[29114]: cluster 2022-04-28T19:40:05.973898+0000 mgr.y (mgr.14140) 197 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:07 smithi059 bash[29114]: audit 2022-04-28T19:40:06.607741+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:07 smithi059 bash[29114]: audit 2022-04-28T19:40:06.627390+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:08.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:08 smithi059 bash[23786]: audit 2022-04-28T19:40:06.599826+0000 mgr.y (mgr.14140) 198 : audit [DBG] from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi139=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:08.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:08 smithi059 bash[23786]: cephadm 2022-04-28T19:40:06.601488+0000 mgr.y (mgr.14140) 199 : cephadm [INF] Saving service prometheus spec with placement smithi139=a;count:1 2022-04-28T19:40:08.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:08 smithi059 bash[23786]: cephadm 2022-04-28T19:40:06.646319+0000 mgr.y (mgr.14140) 200 : cephadm [INF] Deploying daemon prometheus.a on smithi139 2022-04-28T19:40:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:08 smithi059 bash[29114]: audit 2022-04-28T19:40:06.599826+0000 mgr.y (mgr.14140) 198 : audit [DBG] from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi139=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:08 smithi059 bash[29114]: cephadm 2022-04-28T19:40:06.601488+0000 mgr.y (mgr.14140) 199 : cephadm [INF] Saving service prometheus spec with placement smithi139=a;count:1 2022-04-28T19:40:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:08 smithi059 bash[29114]: cephadm 2022-04-28T19:40:06.646319+0000 mgr.y (mgr.14140) 200 : cephadm [INF] Deploying daemon prometheus.a on smithi139 2022-04-28T19:40:08.428 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:08 smithi139 bash[23439]: audit 2022-04-28T19:40:06.599826+0000 mgr.y (mgr.14140) 198 : audit [DBG] from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi139=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:08.428 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:08 smithi139 bash[23439]: cephadm 2022-04-28T19:40:06.601488+0000 mgr.y (mgr.14140) 199 : cephadm [INF] Saving service prometheus spec with placement smithi139=a;count:1 2022-04-28T19:40:08.429 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:08 smithi139 bash[23439]: cephadm 2022-04-28T19:40:06.646319+0000 mgr.y (mgr.14140) 200 : cephadm [INF] Deploying daemon prometheus.a on smithi139 2022-04-28T19:40:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:09 smithi059 bash[29114]: cluster 2022-04-28T19:40:07.974481+0000 mgr.y (mgr.14140) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:09.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:09 smithi059 bash[23786]: cluster 2022-04-28T19:40:07.974481+0000 mgr.y (mgr.14140) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:09.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:09 smithi139 bash[23439]: cluster 2022-04-28T19:40:07.974481+0000 mgr.y (mgr.14140) 201 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:11.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:11 smithi059 bash[23786]: cluster 2022-04-28T19:40:09.975054+0000 mgr.y (mgr.14140) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:11 smithi059 bash[29114]: cluster 2022-04-28T19:40:09.975054+0000 mgr.y (mgr.14140) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:11.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:11 smithi139 bash[23439]: cluster 2022-04-28T19:40:09.975054+0000 mgr.y (mgr.14140) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:12.059 INFO:teuthology.orchestra.run.smithi139.stdout:Scheduled node-exporter update... 2022-04-28T19:40:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:12 smithi059 bash[29114]: audit 2022-04-28T19:40:11.124764+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:12 smithi059 bash[29114]: audit 2022-04-28T19:40:12.058673+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:12 smithi059 bash[23786]: audit 2022-04-28T19:40:11.124764+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:12 smithi059 bash[23786]: audit 2022-04-28T19:40:12.058673+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:12.721 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:12 smithi139 bash[23439]: audit 2022-04-28T19:40:11.124764+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:12.721 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:12 smithi139 bash[23439]: audit 2022-04-28T19:40:12.058673+0000 mon.a (mon.0) 517 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:13.433 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:13 smithi139 bash[23439]: cluster 2022-04-28T19:40:11.975656+0000 mgr.y (mgr.14140) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:13.433 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:13 smithi139 bash[23439]: audit 2022-04-28T19:40:12.050904+0000 mgr.y (mgr.14140) 204 : audit [DBG] from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi059=a;smithi139=b", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:13.433 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:13 smithi139 bash[23439]: cephadm 2022-04-28T19:40:12.052626+0000 mgr.y (mgr.14140) 205 : cephadm [INF] Saving service node-exporter spec with placement smithi059=a;smithi139=b;count:2 2022-04-28T19:40:13.434 DEBUG:teuthology.orchestra.run.smithi059:node-exporter.a> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@node-exporter.a.service 2022-04-28T19:40:13.437 DEBUG:teuthology.orchestra.run.smithi139:node-exporter.b> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@node-exporter.b.service 2022-04-28T19:40:13.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:13 smithi059 bash[29114]: cluster 2022-04-28T19:40:11.975656+0000 mgr.y (mgr.14140) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:13.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:13 smithi059 bash[29114]: audit 2022-04-28T19:40:12.050904+0000 mgr.y (mgr.14140) 204 : audit [DBG] from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi059=a;smithi139=b", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:13.447 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:13 smithi059 bash[29114]: cephadm 2022-04-28T19:40:12.052626+0000 mgr.y (mgr.14140) 205 : cephadm [INF] Saving service node-exporter spec with placement smithi059=a;smithi139=b;count:2 2022-04-28T19:40:13.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:13 smithi059 bash[23786]: cluster 2022-04-28T19:40:11.975656+0000 mgr.y (mgr.14140) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:13.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:13 smithi059 bash[23786]: audit 2022-04-28T19:40:12.050904+0000 mgr.y (mgr.14140) 204 : audit [DBG] from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi059=a;smithi139=b", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:13.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:13 smithi059 bash[23786]: cephadm 2022-04-28T19:40:12.052626+0000 mgr.y (mgr.14140) 205 : cephadm [INF] Saving service node-exporter spec with placement smithi059=a;smithi139=b;count:2 2022-04-28T19:40:13.469 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:40:13.477 INFO:tasks.cephadm:Adding alertmanager.a on smithi059 2022-04-28T19:40:13.477 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch apply alertmanager '1;smithi059=a' 2022-04-28T19:40:13.522 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:40:15.322 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:15 smithi139 bash[23439]: cluster 2022-04-28T19:40:13.976220+0000 mgr.y (mgr.14140) 206 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:15 smithi059 bash[23786]: cluster 2022-04-28T19:40:13.976220+0000 mgr.y (mgr.14140) 206 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:15.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:15 smithi059 bash[29114]: cluster 2022-04-28T19:40:13.976220+0000 mgr.y (mgr.14140) 206 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:16 smithi059 bash[29114]: audit 2022-04-28T19:40:16.112263+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:40:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:16 smithi059 bash[29114]: audit 2022-04-28T19:40:16.115851+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:40:16.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:16 smithi059 bash[23786]: audit 2022-04-28T19:40:16.112263+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:40:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:16 smithi059 bash[23786]: audit 2022-04-28T19:40:16.115851+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:40:16.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:16 smithi139 bash[23439]: audit 2022-04-28T19:40:16.112263+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:40:16.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:16 smithi139 bash[23439]: audit 2022-04-28T19:40:16.115851+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:40:17.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:17 smithi059 bash[29114]: cluster 2022-04-28T19:40:15.976835+0000 mgr.y (mgr.14140) 207 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:17.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:17 smithi059 bash[23786]: cluster 2022-04-28T19:40:15.976835+0000 mgr.y (mgr.14140) 207 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:17.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:17 smithi139 bash[23439]: cluster 2022-04-28T19:40:15.976835+0000 mgr.y (mgr.14140) 207 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:18.115 INFO:teuthology.orchestra.run.smithi139.stdout:Scheduled alertmanager update... 2022-04-28T19:40:18.962 DEBUG:teuthology.orchestra.run.smithi059:alertmanager.a> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@alertmanager.a.service 2022-04-28T19:40:18.964 INFO:tasks.cephadm:Adding grafana.a on smithi139 2022-04-28T19:40:18.965 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph orch apply grafana '1;smithi139=a' 2022-04-28T19:40:18.985 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:40:19.138 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:19 smithi139 bash[23439]: cluster 2022-04-28T19:40:17.977440+0000 mgr.y (mgr.14140) 208 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:19.138 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:19 smithi139 bash[23439]: audit 2022-04-28T19:40:18.106370+0000 mgr.y (mgr.14140) 209 : audit [DBG] from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi059=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:19.138 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:19 smithi139 bash[23439]: cephadm 2022-04-28T19:40:18.107992+0000 mgr.y (mgr.14140) 210 : cephadm [INF] Saving service alertmanager spec with placement smithi059=a;count:1 2022-04-28T19:40:19.138 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:19 smithi139 bash[23439]: audit 2022-04-28T19:40:18.114562+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[29114]: cluster 2022-04-28T19:40:17.977440+0000 mgr.y (mgr.14140) 208 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[29114]: audit 2022-04-28T19:40:18.106370+0000 mgr.y (mgr.14140) 209 : audit [DBG] from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi059=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[29114]: cephadm 2022-04-28T19:40:18.107992+0000 mgr.y (mgr.14140) 210 : cephadm [INF] Saving service alertmanager spec with placement smithi059=a;count:1 2022-04-28T19:40:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[29114]: audit 2022-04-28T19:40:18.114562+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[23786]: cluster 2022-04-28T19:40:17.977440+0000 mgr.y (mgr.14140) 208 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[23786]: audit 2022-04-28T19:40:18.106370+0000 mgr.y (mgr.14140) 209 : audit [DBG] from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi059=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[23786]: cephadm 2022-04-28T19:40:18.107992+0000 mgr.y (mgr.14140) 210 : cephadm [INF] Saving service alertmanager spec with placement smithi059=a;count:1 2022-04-28T19:40:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:19 smithi059 bash[23786]: audit 2022-04-28T19:40:18.114562+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:21.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:21 smithi139 bash[23439]: cluster 2022-04-28T19:40:19.978016+0000 mgr.y (mgr.14140) 211 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:21 smithi059 bash[29114]: cluster 2022-04-28T19:40:19.978016+0000 mgr.y (mgr.14140) 211 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:21.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:21 smithi059 bash[23786]: cluster 2022-04-28T19:40:19.978016+0000 mgr.y (mgr.14140) 211 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:23.229 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:23 smithi139 bash[23439]: cluster 2022-04-28T19:40:21.978600+0000 mgr.y (mgr.14140) 212 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:23.428 INFO:teuthology.orchestra.run.smithi139.stdout:Scheduled grafana update... 2022-04-28T19:40:23.483 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:23 smithi139 systemd[1]: Started Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:40:23.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:23 smithi059 bash[23786]: cluster 2022-04-28T19:40:21.978600+0000 mgr.y (mgr.14140) 212 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:23.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:23 smithi059 bash[29114]: cluster 2022-04-28T19:40:21.978600+0000 mgr.y (mgr.14140) 212 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:24.384 DEBUG:teuthology.orchestra.run.smithi139:grafana.a> sudo journalctl -f -n 0 -u ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@grafana.a.service 2022-04-28T19:40:24.386 INFO:tasks.cephadm:Setting up client nodes... 2022-04-28T19:40:24.386 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-04-28T19:40:24.409 INFO:journalctl@ceph.grafana.a.smithi139.stdout:-- Logs begin at Sun 2022-04-24 12:19:44 UTC. -- 2022-04-28T19:40:24.515 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[23439]: audit 2022-04-28T19:40:23.261971+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:24.515 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[23439]: audit 2022-04-28T19:40:23.264694+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:24.516 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[23439]: audit 2022-04-28T19:40:23.428325+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:24.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:24 smithi059 bash[23786]: audit 2022-04-28T19:40:23.261971+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:24.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:24 smithi059 bash[23786]: audit 2022-04-28T19:40:23.264694+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:24.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:24 smithi059 bash[23786]: audit 2022-04-28T19:40:23.428325+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:24 smithi059 bash[29114]: audit 2022-04-28T19:40:23.261971+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:24 smithi059 bash[29114]: audit 2022-04-28T19:40:23.264694+0000 mon.a (mon.0) 522 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:24 smithi059 bash[29114]: audit 2022-04-28T19:40:23.428325+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:24.861 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.751Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2022-04-28T19:40:24.861 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.751Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2022-04-28T19:40:24.861 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.751Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2022-04-28T19:40:24.861 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.751Z caller=main.go:339 host_details="(Linux 4.15.0-176-generic #185-Ubuntu SMP Tue Mar 29 17:40:04 UTC 2022 x86_64 smithi139 (none))" 2022-04-28T19:40:24.861 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.751Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2022-04-28T19:40:24.861 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.752Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.754Z caller=main.go:678 msg="Starting TSDB ..." 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.754Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.761Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.762Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.762Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=372.722µs 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.763Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.763Z caller=main.go:695 msg="TSDB started" 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.763Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:40:24.862 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.769Z caller=main.go:827 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:40:24.863 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:40:24 smithi139 bash[34185]: level=info ts=2022-04-28T19:40:24.769Z caller=main.go:646 msg="Server is ready to receive web requests." 2022-04-28T19:40:25.568 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:25 smithi139 bash[23439]: audit 2022-04-28T19:40:23.420926+0000 mgr.y (mgr.14140) 213 : audit [DBG] from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi139=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:25.569 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:25 smithi139 bash[23439]: cephadm 2022-04-28T19:40:23.422335+0000 mgr.y (mgr.14140) 214 : cephadm [INF] Saving service grafana spec with placement smithi139=a;count:1 2022-04-28T19:40:25.569 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:25 smithi139 bash[23439]: cluster 2022-04-28T19:40:23.979248+0000 mgr.y (mgr.14140) 215 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:25.569 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:25 smithi139 bash[23439]: audit 2022-04-28T19:40:24.914809+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:25.569 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:25 smithi139 bash[23439]: audit 2022-04-28T19:40:24.923656+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[23786]: audit 2022-04-28T19:40:23.420926+0000 mgr.y (mgr.14140) 213 : audit [DBG] from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi139=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[23786]: cephadm 2022-04-28T19:40:23.422335+0000 mgr.y (mgr.14140) 214 : cephadm [INF] Saving service grafana spec with placement smithi139=a;count:1 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[23786]: cluster 2022-04-28T19:40:23.979248+0000 mgr.y (mgr.14140) 215 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[23786]: audit 2022-04-28T19:40:24.914809+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[23786]: audit 2022-04-28T19:40:24.923656+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[29114]: audit 2022-04-28T19:40:23.420926+0000 mgr.y (mgr.14140) 213 : audit [DBG] from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi139=a", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:40:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[29114]: cephadm 2022-04-28T19:40:23.422335+0000 mgr.y (mgr.14140) 214 : cephadm [INF] Saving service grafana spec with placement smithi139=a;count:1 2022-04-28T19:40:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[29114]: cluster 2022-04-28T19:40:23.979248+0000 mgr.y (mgr.14140) 215 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[29114]: audit 2022-04-28T19:40:24.914809+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:25 smithi059 bash[29114]: audit 2022-04-28T19:40:24.923656+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:25.861 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:25 smithi139 systemd[1]: Started Ceph node-exporter.b for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:40:25.861 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:25 smithi139 bash[34564]: Unable to find image 'quay.io/prometheus/node-exporter:v0.18.1' locally 2022-04-28T19:40:26.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[23439]: cephadm 2022-04-28T19:40:24.927317+0000 mgr.y (mgr.14140) 216 : cephadm [INF] Deploying daemon node-exporter.b on smithi139 2022-04-28T19:40:26.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[23439]: audit 2022-04-28T19:40:25.601312+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:26.361 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: v0.18.1: Pulling from prometheus/node-exporter 2022-04-28T19:40:26.361 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 49a2d53aa1af: Pulling fs layer 2022-04-28T19:40:26.361 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 3589a6efd9ce: Pulling fs layer 2022-04-28T19:40:26.361 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 190160031744: Pulling fs layer 2022-04-28T19:40:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:26 smithi059 bash[29114]: cephadm 2022-04-28T19:40:24.927317+0000 mgr.y (mgr.14140) 216 : cephadm [INF] Deploying daemon node-exporter.b on smithi139 2022-04-28T19:40:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:26 smithi059 bash[29114]: audit 2022-04-28T19:40:25.601312+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:26.590 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:26 smithi059 systemd[1]: Started Ceph node-exporter.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:40:26.590 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:26 smithi059 bash[39484]: Unable to find image 'quay.io/prometheus/node-exporter:v0.18.1' locally 2022-04-28T19:40:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:26 smithi059 bash[23786]: cephadm 2022-04-28T19:40:24.927317+0000 mgr.y (mgr.14140) 216 : cephadm [INF] Deploying daemon node-exporter.b on smithi139 2022-04-28T19:40:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:26 smithi059 bash[23786]: audit 2022-04-28T19:40:25.601312+0000 mon.a (mon.0) 526 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:26.758 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 3589a6efd9ce: Verifying Checksum 2022-04-28T19:40:26.758 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 3589a6efd9ce: Download complete 2022-04-28T19:40:26.758 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 49a2d53aa1af: Verifying Checksum 2022-04-28T19:40:26.758 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 49a2d53aa1af: Download complete 2022-04-28T19:40:26.758 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 190160031744: Verifying Checksum 2022-04-28T19:40:26.758 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 190160031744: Download complete 2022-04-28T19:40:27.110 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:26 smithi139 bash[34564]: 49a2d53aa1af: Pull complete 2022-04-28T19:40:27.605 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: v0.18.1: Pulling from prometheus/node-exporter 2022-04-28T19:40:27.605 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[23786]: cephadm 2022-04-28T19:40:25.602640+0000 mgr.y (mgr.14140) 217 : cephadm [INF] Deploying daemon node-exporter.a on smithi059 2022-04-28T19:40:27.605 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[23786]: cluster 2022-04-28T19:40:25.979790+0000 mgr.y (mgr.14140) 218 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:27.606 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[23786]: audit 2022-04-28T19:40:26.308965+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[29114]: cephadm 2022-04-28T19:40:25.602640+0000 mgr.y (mgr.14140) 217 : cephadm [INF] Deploying daemon node-exporter.a on smithi059 2022-04-28T19:40:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[29114]: cluster 2022-04-28T19:40:25.979790+0000 mgr.y (mgr.14140) 218 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[29114]: audit 2022-04-28T19:40:26.308965+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:27.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[23439]: cephadm 2022-04-28T19:40:25.602640+0000 mgr.y (mgr.14140) 217 : cephadm [INF] Deploying daemon node-exporter.a on smithi059 2022-04-28T19:40:27.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[23439]: cluster 2022-04-28T19:40:25.979790+0000 mgr.y (mgr.14140) 218 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:27.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[23439]: audit 2022-04-28T19:40:26.308965+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:27.611 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[34564]: 3589a6efd9ce: Pull complete 2022-04-28T19:40:27.887 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 49a2d53aa1af: Pulling fs layer 2022-04-28T19:40:27.887 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 3589a6efd9ce: Pulling fs layer 2022-04-28T19:40:27.887 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 190160031744: Pulling fs layer 2022-04-28T19:40:27.887 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 3589a6efd9ce: Verifying Checksum 2022-04-28T19:40:27.887 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 3589a6efd9ce: Download complete 2022-04-28T19:40:27.888 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 49a2d53aa1af: Download complete 2022-04-28T19:40:28.110 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[34564]: 190160031744: Pull complete 2022-04-28T19:40:28.111 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[34564]: Digest: sha256:a2f29256e53cc3e0b64d7a472512600b2e9410347d53cdc85b49f659c17e02ee 2022-04-28T19:40:28.111 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:27 smithi139 bash[34564]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v0.18.1 2022-04-28T19:40:28.280 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 190160031744: Verifying Checksum 2022-04-28T19:40:28.280 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:27 smithi059 bash[39484]: 190160031744: Download complete 2022-04-28T19:40:28.280 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:28 smithi059 bash[39484]: 49a2d53aa1af: Pull complete 2022-04-28T19:40:28.315 INFO:teuthology.orchestra.run.smithi059.stdout:[client.0] 2022-04-28T19:40:28.315 INFO:teuthology.orchestra.run.smithi059.stdout: key = AQCs7WpiNraKEhAAU171gkCs4tLwvXN+YWRuTQ== 2022-04-28T19:40:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:28 smithi059 bash[29114]: cephadm 2022-04-28T19:40:26.315838+0000 mgr.y (mgr.14140) 219 : cephadm [INF] Deploying daemon alertmanager.a on smithi059 2022-04-28T19:40:28.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:28 smithi059 bash[23786]: cephadm 2022-04-28T19:40:26.315838+0000 mgr.y (mgr.14140) 219 : cephadm [INF] Deploying daemon alertmanager.a on smithi059 2022-04-28T19:40:28.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:28 smithi139 bash[23439]: cephadm 2022-04-28T19:40:26.315838+0000 mgr.y (mgr.14140) 219 : cephadm [INF] Deploying daemon alertmanager.a on smithi059 2022-04-28T19:40:29.090 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:28 smithi059 bash[39484]: 3589a6efd9ce: Pull complete 2022-04-28T19:40:29.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[23439]: cluster 2022-04-28T19:40:27.980242+0000 mgr.y (mgr.14140) 220 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:29.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[23439]: audit 2022-04-28T19:40:28.310367+0000 mon.c (mon.1) 16 : audit [INF] from='client.? 172.21.15.59:0/331578028' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:29.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[23439]: audit 2022-04-28T19:40:28.310849+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:29.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[23439]: audit 2022-04-28T19:40:28.314946+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-04-28T19:40:29.397 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg="Starting node_exporter (version=0.18.1, branch=HEAD, revision=3db77732e925c08f675d7404a8c46466b2ece83e)" source="node_exporter.go:156" 2022-04-28T19:40:29.397 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg="Build context (go=go1.12.5, user=root@b50852a1acba, date=20190604-16:41:18)" source="node_exporter.go:157" 2022-04-28T19:40:29.397 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg="Enabled collectors:" source="node_exporter.go:97" 2022-04-28T19:40:29.398 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - arp" source="node_exporter.go:104" 2022-04-28T19:40:29.398 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - bcache" source="node_exporter.go:104" 2022-04-28T19:40:29.398 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - bonding" source="node_exporter.go:104" 2022-04-28T19:40:29.398 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - conntrack" source="node_exporter.go:104" 2022-04-28T19:40:29.398 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - cpu" source="node_exporter.go:104" 2022-04-28T19:40:29.398 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - cpufreq" source="node_exporter.go:104" 2022-04-28T19:40:29.399 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - diskstats" source="node_exporter.go:104" 2022-04-28T19:40:29.399 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - edac" source="node_exporter.go:104" 2022-04-28T19:40:29.399 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - entropy" source="node_exporter.go:104" 2022-04-28T19:40:29.399 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - filefd" source="node_exporter.go:104" 2022-04-28T19:40:29.399 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - filesystem" source="node_exporter.go:104" 2022-04-28T19:40:29.400 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - hwmon" source="node_exporter.go:104" 2022-04-28T19:40:29.400 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - infiniband" source="node_exporter.go:104" 2022-04-28T19:40:29.400 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - ipvs" source="node_exporter.go:104" 2022-04-28T19:40:29.400 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - loadavg" source="node_exporter.go:104" 2022-04-28T19:40:29.400 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - mdadm" source="node_exporter.go:104" 2022-04-28T19:40:29.400 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - meminfo" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - netclass" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - netdev" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - netstat" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - nfs" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - nfsd" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - pressure" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - sockstat" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - stat" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - textfile" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - time" source="node_exporter.go:104" 2022-04-28T19:40:29.401 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - uname" source="node_exporter.go:104" 2022-04-28T19:40:29.402 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - vmstat" source="node_exporter.go:104" 2022-04-28T19:40:29.402 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - xfs" source="node_exporter.go:104" 2022-04-28T19:40:29.402 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg=" - zfs" source="node_exporter.go:104" 2022-04-28T19:40:29.402 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:40:29 smithi139 bash[34564]: time="2022-04-28T19:40:29Z" level=info msg="Listening on :9100" source="node_exporter.go:170" 2022-04-28T19:40:29.570 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[23786]: cluster 2022-04-28T19:40:27.980242+0000 mgr.y (mgr.14140) 220 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:29.570 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[23786]: audit 2022-04-28T19:40:28.310367+0000 mon.c (mon.1) 16 : audit [INF] from='client.? 172.21.15.59:0/331578028' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:29.570 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[23786]: audit 2022-04-28T19:40:28.310849+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:29.570 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[23786]: audit 2022-04-28T19:40:28.314946+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-04-28T19:40:29.570 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[29114]: cluster 2022-04-28T19:40:27.980242+0000 mgr.y (mgr.14140) 220 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:29.570 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[29114]: audit 2022-04-28T19:40:28.310367+0000 mon.c (mon.1) 16 : audit [INF] from='client.? 172.21.15.59:0/331578028' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:29.571 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[29114]: audit 2022-04-28T19:40:28.310849+0000 mon.a (mon.0) 528 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:29.571 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[29114]: audit 2022-04-28T19:40:28.314946+0000 mon.a (mon.0) 529 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-04-28T19:40:29.572 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:40:29.572 DEBUG:teuthology.orchestra.run.smithi059:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2022-04-28T19:40:29.572 DEBUG:teuthology.orchestra.run.smithi059:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2022-04-28T19:40:29.585 DEBUG:teuthology.orchestra.run.smithi139:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2022-04-28T19:40:29.840 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[39484]: 190160031744: Pull complete 2022-04-28T19:40:30.339 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:29 smithi059 bash[39484]: Digest: sha256:a2f29256e53cc3e0b64d7a472512600b2e9410347d53cdc85b49f659c17e02ee 2022-04-28T19:40:30.340 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:30 smithi059 bash[39484]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v0.18.1 2022-04-28T19:40:31.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:31 smithi139 bash[23439]: cluster 2022-04-28T19:40:29.980779+0000 mgr.y (mgr.14140) 221 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:31.589 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:31 smithi059 bash[23786]: cluster 2022-04-28T19:40:29.980779+0000 mgr.y (mgr.14140) 221 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:31 smithi059 bash[29114]: cluster 2022-04-28T19:40:29.980779+0000 mgr.y (mgr.14140) 221 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:32.340 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg="Starting node_exporter (version=0.18.1, branch=HEAD, revision=3db77732e925c08f675d7404a8c46466b2ece83e)" source="node_exporter.go:156" 2022-04-28T19:40:32.340 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg="Build context (go=go1.12.5, user=root@b50852a1acba, date=20190604-16:41:18)" source="node_exporter.go:157" 2022-04-28T19:40:32.340 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg="Enabled collectors:" source="node_exporter.go:97" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - arp" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - bcache" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - bonding" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - conntrack" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - cpu" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - cpufreq" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - diskstats" source="node_exporter.go:104" 2022-04-28T19:40:32.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - edac" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - entropy" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - filefd" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - filesystem" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - hwmon" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - infiniband" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - ipvs" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - loadavg" source="node_exporter.go:104" 2022-04-28T19:40:32.342 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - mdadm" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - meminfo" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - netclass" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - netdev" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - netstat" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - nfs" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - nfsd" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - pressure" source="node_exporter.go:104" 2022-04-28T19:40:32.343 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - sockstat" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - stat" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - textfile" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - time" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - uname" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - vmstat" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - xfs" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg=" - zfs" source="node_exporter.go:104" 2022-04-28T19:40:32.344 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:40:32 smithi059 bash[39484]: time="2022-04-28T19:40:32Z" level=info msg="Listening on :9100" source="node_exporter.go:170" 2022-04-28T19:40:32.753 INFO:teuthology.orchestra.run.smithi139.stdout:[client.1] 2022-04-28T19:40:32.753 INFO:teuthology.orchestra.run.smithi139.stdout: key = AQCw7WpidKKELBAAC0UoIYbMspj3ylxK3aOGMQ== 2022-04-28T19:40:33.308 DEBUG:teuthology.orchestra.run.smithi139:> set -ex 2022-04-28T19:40:33.308 DEBUG:teuthology.orchestra.run.smithi139:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2022-04-28T19:40:33.308 DEBUG:teuthology.orchestra.run.smithi139:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2022-04-28T19:40:33.333 INFO:tasks.cephadm:Waiting for OSDs to come up 2022-04-28T19:40:33.333 DEBUG:teuthology.orchestra.run.smithi059:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2022-04-28T19:40:33.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[29114]: cluster 2022-04-28T19:40:31.981172+0000 mgr.y (mgr.14140) 222 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:33.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[29114]: audit 2022-04-28T19:40:32.746431+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:33.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[29114]: audit 2022-04-28T19:40:32.750822+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.139:0/1435966233' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:33.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[29114]: audit 2022-04-28T19:40:32.753705+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-04-28T19:40:33.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[23786]: cluster 2022-04-28T19:40:31.981172+0000 mgr.y (mgr.14140) 222 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[23786]: audit 2022-04-28T19:40:32.746431+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[23786]: audit 2022-04-28T19:40:32.750822+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.139:0/1435966233' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:33 smithi059 bash[23786]: audit 2022-04-28T19:40:32.753705+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-04-28T19:40:33.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:33 smithi139 bash[23439]: cluster 2022-04-28T19:40:31.981172+0000 mgr.y (mgr.14140) 222 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:33.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:33 smithi139 bash[23439]: audit 2022-04-28T19:40:32.746431+0000 mon.a (mon.0) 530 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:33.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:33 smithi139 bash[23439]: audit 2022-04-28T19:40:32.750822+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.139:0/1435966233' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2022-04-28T19:40:33.611 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:33 smithi139 bash[23439]: audit 2022-04-28T19:40:32.753705+0000 mon.a (mon.0) 531 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2022-04-28T19:40:33.642 INFO:teuthology.misc.health.smithi059.stdout: 2022-04-28T19:40:33.642 INFO:teuthology.misc.health.smithi059.stdout:{"epoch":47,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:40:02.152775+0000","last_up_change":"2022-04-28T19:40:01.137034+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"octopus","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6803","nonce":1828778759}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6805","nonce":1828778759}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6809","nonce":1828778759}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6807","nonce":1828778759}]},"public_addr":"172.21.15.59:6803/1828778759","cluster_addr":"172.21.15.59:6805/1828778759","heartbeat_back_addr":"172.21.15.59:6809/1828778759","heartbeat_front_addr":"172.21.15.59:6807/1828778759","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":16,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6811","nonce":222479240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6813","nonce":222479240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6817","nonce":222479240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6815","nonce":222479240}]},"public_addr":"172.21.15.59:6811/222479240","cluster_addr":"172.21.15.59:6813/222479240","heartbeat_back_addr":"172.21.15.59:6817/222479240","heartbeat_front_addr":"172.21.15.59:6815/222479240","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6819","nonce":1472619329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6821","nonce":1472619329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6825","nonce":1472619329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6823","nonce":1472619329}]},"public_addr":"172.21.15.59:6819/1472619329","cluster_addr":"172.21.15.59:6821/1472619329","heartbeat_back_addr":"172.21.15.59:6825/1472619329","heartbeat_front_addr":"172.21.15.59:6823/1472619329","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6827","nonce":2946747803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6829","nonce":2946747803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6833","nonce":2946747803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6831","nonce":2946747803}]},"public_addr":"172.21.15.59:6827/2946747803","cluster_addr":"172.21.15.59:6829/2946747803","heartbeat_back_addr":"172.21.15.59:6833/2946747803","heartbeat_front_addr":"172.21.15.59:6831/2946747803","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6801","nonce":611845841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6803","nonce":611845841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6807","nonce":611845841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6805","nonce":611845841}]},"public_addr":"172.21.15.139:6801/611845841","cluster_addr":"172.21.15.139:6803/611845841","heartbeat_back_addr":"172.21.15.139:6807/611845841","heartbeat_front_addr":"172.21.15.139:6805/611845841","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":36,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6809","nonce":2795762393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6811","nonce":2795762393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6815","nonce":2795762393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6813","nonce":2795762393}]},"public_addr":"172.21.15.139:6809/2795762393","cluster_addr":"172.21.15.139:6811/2795762393","heartbeat_back_addr":"172.21.15.139:6815/2795762393","heartbeat_front_addr":"172.21.15.139:6813/2795762393","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6817","nonce":1808526322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6819","nonce":1808526322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6823","nonce":1808526322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6821","nonce":1808526322}]},"public_addr":"172.21.15.139:6817/1808526322","cluster_addr":"172.21.15.139:6819/1808526322","heartbeat_back_addr":"172.21.15.139:6823/1808526322","heartbeat_front_addr":"172.21.15.139:6821/1808526322","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":46,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6825","nonce":2154117349}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6827","nonce":2154117349}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6831","nonce":2154117349}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6829","nonce":2154117349}]},"public_addr":"172.21.15.139:6825/2154117349","cluster_addr":"172.21.15.139:6827/2154117349","heartbeat_back_addr":"172.21.15.139:6831/2154117349","heartbeat_front_addr":"172.21.15.139:6829/2154117349","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{"172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} 2022-04-28T19:40:33.662 DEBUG:teuthology.misc:8 of 8 OSDs are up 2022-04-28T19:40:33.759 INFO:tasks.cephadm:Creating RBD pool 2022-04-28T19:40:33.760 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- sudo ceph --cluster ceph osd pool create rbd 8 2022-04-28T19:40:34.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:34 smithi059 bash[29114]: audit 2022-04-28T19:40:33.642319+0000 mon.a (mon.0) 532 : audit [DBG] from='client.? 172.21.15.59:0/221103669' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:40:34.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:34 smithi059 bash[23786]: audit 2022-04-28T19:40:33.642319+0000 mon.a (mon.0) 532 : audit [DBG] from='client.? 172.21.15.59:0/221103669' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:40:34.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:34 smithi139 bash[23439]: audit 2022-04-28T19:40:33.642319+0000 mon.a (mon.0) 532 : audit [DBG] from='client.? 172.21.15.59:0/221103669' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:40:35.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:35 smithi059 bash[29114]: cluster 2022-04-28T19:40:33.981687+0000 mgr.y (mgr.14140) 223 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:35.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:35 smithi059 bash[23786]: cluster 2022-04-28T19:40:33.981687+0000 mgr.y (mgr.14140) 223 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:35.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:35 smithi139 bash[23439]: cluster 2022-04-28T19:40:33.981687+0000 mgr.y (mgr.14140) 223 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:36.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:36 smithi059 bash[23786]: cluster 2022-04-28T19:40:35.982116+0000 mgr.y (mgr.14140) 224 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:36.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:36 smithi059 bash[29114]: cluster 2022-04-28T19:40:35.982116+0000 mgr.y (mgr.14140) 224 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:36.610 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:36 smithi139 bash[23439]: cluster 2022-04-28T19:40:35.982116+0000 mgr.y (mgr.14140) 224 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:37.839 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:37 smithi059 systemd[1]: Started Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:40:38.835 INFO:teuthology.orchestra.run.smithi059.stderr:pool 'rbd' created 2022-04-28T19:40:38.893 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.621068+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.893 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.803387+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.813919+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.814632+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.815132+0000 mgr.y (mgr.14140) 225 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.820154+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: cephadm 2022-04-28T19:40:37.825612+0000 mgr.y (mgr.14140) 226 : cephadm [INF] Deploying daemon grafana.a on smithi139 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: audit 2022-04-28T19:40:37.965397+0000 mon.a (mon.0) 538 : audit [INF] from='client.? 172.21.15.59:0/4204184736' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-04-28T19:40:38.894 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[29114]: cluster 2022-04-28T19:40:37.982460+0000 mgr.y (mgr.14140) 227 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.621068+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.803387+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.813919+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.814632+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.815132+0000 mgr.y (mgr.14140) 225 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.820154+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: cephadm 2022-04-28T19:40:37.825612+0000 mgr.y (mgr.14140) 226 : cephadm [INF] Deploying daemon grafana.a on smithi139 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: audit 2022-04-28T19:40:37.965397+0000 mon.a (mon.0) 538 : audit [INF] from='client.? 172.21.15.59:0/4204184736' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-04-28T19:40:38.895 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[23786]: cluster 2022-04-28T19:40:37.982460+0000 mgr.y (mgr.14140) 227 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:39.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.621068+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:39.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.803387+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.813919+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.814632+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.815132+0000 mgr.y (mgr.14140) 225 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.820154+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: cephadm 2022-04-28T19:40:37.825612+0000 mgr.y (mgr.14140) 226 : cephadm [INF] Deploying daemon grafana.a on smithi139 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: audit 2022-04-28T19:40:37.965397+0000 mon.a (mon.0) 538 : audit [INF] from='client.? 172.21.15.59:0/4204184736' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2022-04-28T19:40:39.111 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:38 smithi139 bash[23439]: cluster 2022-04-28T19:40:37.982460+0000 mgr.y (mgr.14140) 227 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:38.968Z caller=main.go:231 msg="Starting Alertmanager" version="(version=0.20.0, branch=HEAD, revision=f74be0400a6243d10bb53812d6fa408ad71ff32d)" 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:38.968Z caller=main.go:232 build_context="(go=go1.13.5, user=root@00c3106655f8, date=20191211-14:13:14)" 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:38.970Z caller=cluster.go:161 component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:38 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:38.971Z caller=cluster.go:623 component=cluster msg="Waiting for gossip to settle..." interval=2s 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:39.008Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:39.008Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2022-04-28T19:40:39.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:39.013Z caller=main.go:497 msg=Listening address=:9093 2022-04-28T19:40:39.535 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2022-04-28T19:40:40.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:39 smithi139 bash[23439]: audit 2022-04-28T19:40:38.831067+0000 mon.a (mon.0) 539 : audit [INF] from='client.? 172.21.15.59:0/4204184736' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2022-04-28T19:40:40.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:39 smithi139 bash[23439]: cluster 2022-04-28T19:40:38.831146+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2022-04-28T19:40:40.249 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[29114]: audit 2022-04-28T19:40:38.831067+0000 mon.a (mon.0) 539 : audit [INF] from='client.? 172.21.15.59:0/4204184736' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2022-04-28T19:40:40.249 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[29114]: cluster 2022-04-28T19:40:38.831146+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2022-04-28T19:40:40.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[23786]: audit 2022-04-28T19:40:38.831067+0000 mon.a (mon.0) 539 : audit [INF] from='client.? 172.21.15.59:0/4204184736' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2022-04-28T19:40:40.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:39 smithi059 bash[23786]: cluster 2022-04-28T19:40:38.831146+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2022-04-28T19:40:40.948 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:40 smithi059 bash[23786]: cluster 2022-04-28T19:40:39.834600+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2022-04-28T19:40:40.948 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:40 smithi059 bash[23786]: cluster 2022-04-28T19:40:39.983058+0000 mgr.y (mgr.14140) 228 : cluster [DBG] pgmap v201: 9 pgs: 1 creating+peering, 1 active+clean, 7 unknown; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:40.948 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:40 smithi059 bash[29114]: cluster 2022-04-28T19:40:39.834600+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2022-04-28T19:40:40.948 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:40 smithi059 bash[29114]: cluster 2022-04-28T19:40:39.983058+0000 mgr.y (mgr.14140) 228 : cluster [DBG] pgmap v201: 9 pgs: 1 creating+peering, 1 active+clean, 7 unknown; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:41.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:40 smithi139 bash[23439]: cluster 2022-04-28T19:40:39.834600+0000 mon.a (mon.0) 541 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2022-04-28T19:40:41.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:40 smithi139 bash[23439]: cluster 2022-04-28T19:40:39.983058+0000 mgr.y (mgr.14140) 228 : cluster [DBG] pgmap v201: 9 pgs: 1 creating+peering, 1 active+clean, 7 unknown; 0 B data, 3.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:41.339 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:40 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:40.971Z caller=cluster.go:648 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000136819s 2022-04-28T19:40:42.110 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:41 smithi139 bash[23439]: cluster 2022-04-28T19:40:40.849235+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2022-04-28T19:40:42.291 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:41 smithi059 bash[29114]: cluster 2022-04-28T19:40:40.849235+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2022-04-28T19:40:42.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:41 smithi059 bash[23786]: cluster 2022-04-28T19:40:40.849235+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2022-04-28T19:40:43.063 INFO:teuthology.orchestra.run.smithi059.stderr:enabled application 'rbd' on pool 'rbd' 2022-04-28T19:40:43.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:43 smithi059 bash[23786]: cluster 2022-04-28T19:40:41.983733+0000 mgr.y (mgr.14140) 229 : cluster [DBG] pgmap v203: 9 pgs: 3 active+clean, 1 creating+activating, 1 creating+peering, 4 unknown; 0 B data, 3.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:43.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:43 smithi059 bash[23786]: audit 2022-04-28T19:40:42.664550+0000 mon.a (mon.0) 543 : audit [INF] from='client.? 172.21.15.59:0/187541120' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:40:43.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:43 smithi059 bash[29114]: cluster 2022-04-28T19:40:41.983733+0000 mgr.y (mgr.14140) 229 : cluster [DBG] pgmap v203: 9 pgs: 3 active+clean, 1 creating+activating, 1 creating+peering, 4 unknown; 0 B data, 3.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:43.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:43 smithi059 bash[29114]: audit 2022-04-28T19:40:42.664550+0000 mon.a (mon.0) 543 : audit [INF] from='client.? 172.21.15.59:0/187541120' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:40:43.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:43 smithi139 bash[23439]: cluster 2022-04-28T19:40:41.983733+0000 mgr.y (mgr.14140) 229 : cluster [DBG] pgmap v203: 9 pgs: 3 active+clean, 1 creating+activating, 1 creating+peering, 4 unknown; 0 B data, 3.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:43.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:43 smithi139 bash[23439]: audit 2022-04-28T19:40:42.664550+0000 mon.a (mon.0) 543 : audit [INF] from='client.? 172.21.15.59:0/187541120' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:40:43.684 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd dump --format=json 2022-04-28T19:40:43.822 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:40:44.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:44 smithi059 bash[23786]: audit 2022-04-28T19:40:43.057392+0000 mon.a (mon.0) 544 : audit [INF] from='client.? 172.21.15.59:0/187541120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2022-04-28T19:40:44.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:44 smithi059 bash[23786]: cluster 2022-04-28T19:40:43.057612+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2022-04-28T19:40:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:44 smithi059 bash[29114]: audit 2022-04-28T19:40:43.057392+0000 mon.a (mon.0) 544 : audit [INF] from='client.? 172.21.15.59:0/187541120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2022-04-28T19:40:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:44 smithi059 bash[29114]: cluster 2022-04-28T19:40:43.057612+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2022-04-28T19:40:44.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:44 smithi139 bash[23439]: audit 2022-04-28T19:40:43.057392+0000 mon.a (mon.0) 544 : audit [INF] from='client.? 172.21.15.59:0/187541120' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2022-04-28T19:40:44.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:44 smithi139 bash[23439]: cluster 2022-04-28T19:40:43.057612+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2022-04-28T19:40:45.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:45 smithi059 bash[23786]: cluster 2022-04-28T19:40:43.984399+0000 mgr.y (mgr.14140) 230 : cluster [DBG] pgmap v205: 9 pgs: 5 active+clean, 1 creating+activating, 1 creating+peering, 2 unknown; 0 B data, 3.6 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:45.102 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:45 smithi059 bash[29114]: cluster 2022-04-28T19:40:43.984399+0000 mgr.y (mgr.14140) 230 : cluster [DBG] pgmap v205: 9 pgs: 5 active+clean, 1 creating+activating, 1 creating+peering, 2 unknown; 0 B data, 3.6 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:45.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:45 smithi139 bash[23439]: cluster 2022-04-28T19:40:43.984399+0000 mgr.y (mgr.14140) 230 : cluster [DBG] pgmap v205: 9 pgs: 5 active+clean, 1 creating+activating, 1 creating+peering, 2 unknown; 0 B data, 3.6 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:46.913 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:40:46.913 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":51,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:40:43.049867+0000","last_up_change":"2022-04-28T19:40:01.137034+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"octopus","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-04-28T19:40:38.422278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6803","nonce":1828778759}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6805","nonce":1828778759}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6809","nonce":1828778759}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6807","nonce":1828778759}]},"public_addr":"172.21.15.59:6803/1828778759","cluster_addr":"172.21.15.59:6805/1828778759","heartbeat_back_addr":"172.21.15.59:6809/1828778759","heartbeat_front_addr":"172.21.15.59:6807/1828778759","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":16,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6811","nonce":222479240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6813","nonce":222479240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6817","nonce":222479240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6815","nonce":222479240}]},"public_addr":"172.21.15.59:6811/222479240","cluster_addr":"172.21.15.59:6813/222479240","heartbeat_back_addr":"172.21.15.59:6817/222479240","heartbeat_front_addr":"172.21.15.59:6815/222479240","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6819","nonce":1472619329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6821","nonce":1472619329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6825","nonce":1472619329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6823","nonce":1472619329}]},"public_addr":"172.21.15.59:6819/1472619329","cluster_addr":"172.21.15.59:6821/1472619329","heartbeat_back_addr":"172.21.15.59:6825/1472619329","heartbeat_front_addr":"172.21.15.59:6823/1472619329","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6827","nonce":2946747803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6829","nonce":2946747803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6833","nonce":2946747803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6831","nonce":2946747803}]},"public_addr":"172.21.15.59:6827/2946747803","cluster_addr":"172.21.15.59:6829/2946747803","heartbeat_back_addr":"172.21.15.59:6833/2946747803","heartbeat_front_addr":"172.21.15.59:6831/2946747803","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6801","nonce":611845841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6803","nonce":611845841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6807","nonce":611845841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6805","nonce":611845841}]},"public_addr":"172.21.15.139:6801/611845841","cluster_addr":"172.21.15.139:6803/611845841","heartbeat_back_addr":"172.21.15.139:6807/611845841","heartbeat_front_addr":"172.21.15.139:6805/611845841","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6809","nonce":2795762393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6811","nonce":2795762393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6815","nonce":2795762393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6813","nonce":2795762393}]},"public_addr":"172.21.15.139:6809/2795762393","cluster_addr":"172.21.15.139:6811/2795762393","heartbeat_back_addr":"172.21.15.139:6815/2795762393","heartbeat_front_addr":"172.21.15.139:6813/2795762393","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6817","nonce":1808526322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6819","nonce":1808526322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6823","nonce":1808526322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6821","nonce":1808526322}]},"public_addr":"172.21.15.139:6817/1808526322","cluster_addr":"172.21.15.139:6819/1808526322","heartbeat_back_addr":"172.21.15.139:6823/1808526322","heartbeat_front_addr":"172.21.15.139:6821/1808526322","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6825","nonce":2154117349}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6827","nonce":2154117349}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6831","nonce":2154117349}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6829","nonce":2154117349}]},"public_addr":"172.21.15.139:6825/2154117349","cluster_addr":"172.21.15.139:6827/2154117349","heartbeat_back_addr":"172.21.15.139:6831/2154117349","heartbeat_front_addr":"172.21.15.139:6829/2154117349","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{"172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} 2022-04-28T19:40:47.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:47 smithi059 bash[29114]: cluster 2022-04-28T19:40:45.985091+0000 mgr.y (mgr.14140) 231 : cluster [DBG] pgmap v206: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:47.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:47 smithi059 bash[29114]: audit 2022-04-28T19:40:46.912087+0000 mon.a (mon.0) 546 : audit [DBG] from='client.? 172.21.15.59:0/3923722165' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:40:47.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:47 smithi059 bash[23786]: cluster 2022-04-28T19:40:45.985091+0000 mgr.y (mgr.14140) 231 : cluster [DBG] pgmap v206: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:47.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:47 smithi059 bash[23786]: audit 2022-04-28T19:40:46.912087+0000 mon.a (mon.0) 546 : audit [DBG] from='client.? 172.21.15.59:0/3923722165' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:40:47.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:47 smithi139 bash[23439]: cluster 2022-04-28T19:40:45.985091+0000 mgr.y (mgr.14140) 231 : cluster [DBG] pgmap v206: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:47.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:47 smithi139 bash[23439]: audit 2022-04-28T19:40:46.912087+0000 mon.a (mon.0) 546 : audit [DBG] from='client.? 172.21.15.59:0/3923722165' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:40:47.564 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2022-04-28T19:36:09.679269+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '12', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}, {'pool': 2, 'pool_name': 'rbd', 'create_time': '2022-04-28T19:40:38.422278+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '51', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}}] 2022-04-28T19:40:47.565 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd pool get device_health_metrics pg_num 2022-04-28T19:40:47.713 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:40:49.339 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:49 smithi059 bash[23786]: cluster 2022-04-28T19:40:47.985773+0000 mgr.y (mgr.14140) 232 : cluster [DBG] pgmap v207: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:49.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:49 smithi059 bash[29114]: cluster 2022-04-28T19:40:47.985773+0000 mgr.y (mgr.14140) 232 : cluster [DBG] pgmap v207: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:49.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:40:48 smithi059 bash[40315]: level=info ts=2022-04-28T19:40:48.972Z caller=cluster.go:640 component=cluster msg="gossip settled; proceeding" elapsed=10.000957389s 2022-04-28T19:40:49.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:49 smithi139 bash[23439]: cluster 2022-04-28T19:40:47.985773+0000 mgr.y (mgr.14140) 232 : cluster [DBG] pgmap v207: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:50.678 INFO:teuthology.orchestra.run.smithi059.stdout:pg_num: 1 2022-04-28T19:40:51.294 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd pool get rbd pg_num 2022-04-28T19:40:51.339 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:51 smithi059 bash[29114]: cluster 2022-04-28T19:40:49.986370+0000 mgr.y (mgr.14140) 233 : cluster [DBG] pgmap v208: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:51.339 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:51 smithi059 bash[29114]: audit 2022-04-28T19:40:50.678510+0000 mon.a (mon.0) 547 : audit [DBG] from='client.? 172.21.15.59:0/2838822015' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-04-28T19:40:51.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:51 smithi059 bash[23786]: cluster 2022-04-28T19:40:49.986370+0000 mgr.y (mgr.14140) 233 : cluster [DBG] pgmap v208: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:51.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:51 smithi059 bash[23786]: audit 2022-04-28T19:40:50.678510+0000 mon.a (mon.0) 547 : audit [DBG] from='client.? 172.21.15.59:0/2838822015' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-04-28T19:40:51.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:51 smithi139 bash[23439]: cluster 2022-04-28T19:40:49.986370+0000 mgr.y (mgr.14140) 233 : cluster [DBG] pgmap v208: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:51.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:51 smithi139 bash[23439]: audit 2022-04-28T19:40:50.678510+0000 mon.a (mon.0) 547 : audit [DBG] from='client.? 172.21.15.59:0/2838822015' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2022-04-28T19:40:51.431 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:40:53.032 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:53 smithi139 systemd[1]: Started Ceph grafana.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:40:53.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:53 smithi139 bash[23439]: cluster 2022-04-28T19:40:51.986974+0000 mgr.y (mgr.14140) 234 : cluster [DBG] pgmap v209: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:53.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:53 smithi139 bash[23439]: audit 2022-04-28T19:40:53.066923+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:53.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:53 smithi139 bash[23439]: audit 2022-04-28T19:40:53.069425+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:53.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:53 smithi139 bash[23439]: audit 2022-04-28T19:40:53.069906+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:53.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[29114]: cluster 2022-04-28T19:40:51.986974+0000 mgr.y (mgr.14140) 234 : cluster [DBG] pgmap v209: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[29114]: audit 2022-04-28T19:40:53.066923+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[29114]: audit 2022-04-28T19:40:53.069425+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[29114]: audit 2022-04-28T19:40:53.069906+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[23786]: cluster 2022-04-28T19:40:51.986974+0000 mgr.y (mgr.14140) 234 : cluster [DBG] pgmap v209: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[23786]: audit 2022-04-28T19:40:53.066923+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[23786]: audit 2022-04-28T19:40:53.069425+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:53.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:53 smithi059 bash[23786]: audit 2022-04-28T19:40:53.069906+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:40:54.309 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Starting Grafana" logger=server version=6.7.4 commit=8e44bbc5f5 branch=HEAD compiled=2020-05-26T17:35:38+0000 2022-04-28T19:40:54.309 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2022-04-28T19:40:54.309 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2022-04-28T19:40:54.309 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2022-04-28T19:40:54.309 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2022-04-28T19:40:54.309 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="App mode production" logger=settings 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Initializing SqlStore" logger=server 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2022-04-28T19:40:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Starting DB migration" logger=migrator 2022-04-28T19:40:54.311 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2022-04-28T19:40:54.567 INFO:teuthology.orchestra.run.smithi059.stdout:pg_num: 8 2022-04-28T19:40:54.602 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2022-04-28T19:40:54.602 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2022-04-28T19:40:54.603 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2022-04-28T19:40:54.603 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2022-04-28T19:40:54.860 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2022-04-28T19:40:54.860 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2022-04-28T19:40:54.860 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2022-04-28T19:40:54.860 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2022-04-28T19:40:54.861 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2022-04-28T19:40:55.148 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2022-04-28T19:40:55.148 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2022-04-28T19:40:55.149 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:54 smithi139 bash[35227]: t=2022-04-28T19:40:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2022-04-28T19:40:55.149 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2022-04-28T19:40:55.149 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[23439]: cluster 2022-04-28T19:40:53.987416+0000 mgr.y (mgr.14140) 235 : cluster [DBG] pgmap v210: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:55.149 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[23439]: audit 2022-04-28T19:40:54.404254+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.150 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[23439]: audit 2022-04-28T19:40:54.526924+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.150 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[23439]: audit 2022-04-28T19:40:54.568890+0000 mon.a (mon.0) 553 : audit [DBG] from='client.? 172.21.15.59:0/3679920323' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2022-04-28T19:40:55.150 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[23439]: audit 2022-04-28T19:40:54.904523+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.220 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2022-04-28T19:40:55.220 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2022-04-28T19:40:55.220 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph mgr dump --format=json 2022-04-28T19:40:55.354 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:40:55.425 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2022-04-28T19:40:55.425 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2022-04-28T19:40:55.425 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2022-04-28T19:40:55.425 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2022-04-28T19:40:55.426 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2022-04-28T19:40:55.426 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2022-04-28T19:40:55.426 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2022-04-28T19:40:55.589 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[29114]: cluster 2022-04-28T19:40:53.987416+0000 mgr.y (mgr.14140) 235 : cluster [DBG] pgmap v210: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[29114]: audit 2022-04-28T19:40:54.404254+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[29114]: audit 2022-04-28T19:40:54.526924+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[29114]: audit 2022-04-28T19:40:54.568890+0000 mon.a (mon.0) 553 : audit [DBG] from='client.? 172.21.15.59:0/3679920323' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2022-04-28T19:40:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[29114]: audit 2022-04-28T19:40:54.904523+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[23786]: cluster 2022-04-28T19:40:53.987416+0000 mgr.y (mgr.14140) 235 : cluster [DBG] pgmap v210: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:55.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[23786]: audit 2022-04-28T19:40:54.404254+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[23786]: audit 2022-04-28T19:40:54.526924+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[23786]: audit 2022-04-28T19:40:54.568890+0000 mon.a (mon.0) 553 : audit [DBG] from='client.? 172.21.15.59:0/3679920323' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "rbd", "var": "pg_num"}]: dispatch 2022-04-28T19:40:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:55 smithi059 bash[23786]: audit 2022-04-28T19:40:54.904523+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:40:55.710 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2022-04-28T19:40:55.802 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2022-04-28T19:40:55.802 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2022-04-28T19:40:55.802 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2022-04-28T19:40:55.978 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2022-04-28T19:40:55.978 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2022-04-28T19:40:55.978 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2022-04-28T19:40:55.979 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2022-04-28T19:40:56.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[23439]: cephadm 2022-04-28T19:40:54.405025+0000 mgr.y (mgr.14140) 236 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:40:56.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[23439]: cephadm 2022-04-28T19:40:54.528118+0000 mgr.y (mgr.14140) 237 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:40:56.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[23439]: cephadm 2022-04-28T19:40:54.910935+0000 mgr.y (mgr.14140) 238 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2022-04-28T19:40:56.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[23439]: cephadm 2022-04-28T19:40:54.917461+0000 mgr.y (mgr.14140) 239 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi059 2022-04-28T19:40:56.255 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:55 smithi139 bash[35227]: t=2022-04-28T19:40:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2022-04-28T19:40:56.256 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2022-04-28T19:40:56.256 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2022-04-28T19:40:56.256 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2022-04-28T19:40:56.553 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[29114]: cephadm 2022-04-28T19:40:54.405025+0000 mgr.y (mgr.14140) 236 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[29114]: cephadm 2022-04-28T19:40:54.528118+0000 mgr.y (mgr.14140) 237 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[29114]: cephadm 2022-04-28T19:40:54.910935+0000 mgr.y (mgr.14140) 238 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[29114]: cephadm 2022-04-28T19:40:54.917461+0000 mgr.y (mgr.14140) 239 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi059 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[23786]: cephadm 2022-04-28T19:40:54.405025+0000 mgr.y (mgr.14140) 236 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[23786]: cephadm 2022-04-28T19:40:54.528118+0000 mgr.y (mgr.14140) 237 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[23786]: cephadm 2022-04-28T19:40:54.910935+0000 mgr.y (mgr.14140) 238 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2022-04-28T19:40:56.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:56 smithi059 bash[23786]: cephadm 2022-04-28T19:40:54.917461+0000 mgr.y (mgr.14140) 239 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi059 2022-04-28T19:40:56.581 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2022-04-28T19:40:56.581 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2022-04-28T19:40:56.581 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2022-04-28T19:40:56.582 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2022-04-28T19:40:56.858 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2022-04-28T19:40:56.858 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2022-04-28T19:40:56.858 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2022-04-28T19:40:56.858 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2022-04-28T19:40:57.108 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2022-04-28T19:40:57.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:56 smithi139 bash[35227]: t=2022-04-28T19:40:56+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2022-04-28T19:40:57.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2022-04-28T19:40:57.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2022-04-28T19:40:57.129 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:57 smithi059 bash[23786]: cluster 2022-04-28T19:40:55.988104+0000 mgr.y (mgr.14140) 240 : cluster [DBG] pgmap v211: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:57.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[23439]: cluster 2022-04-28T19:40:55.988104+0000 mgr.y (mgr.14140) 240 : cluster [DBG] pgmap v211: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:57.361 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2022-04-28T19:40:57.361 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2022-04-28T19:40:57.361 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2022-04-28T19:40:57.361 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2022-04-28T19:40:57.361 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2022-04-28T19:40:57.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:57 smithi059 bash[29114]: cluster 2022-04-28T19:40:55.988104+0000 mgr.y (mgr.14140) 240 : cluster [DBG] pgmap v211: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:57.645 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2022-04-28T19:40:57.646 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2022-04-28T19:40:57.646 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2022-04-28T19:40:57.646 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2022-04-28T19:40:57.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2022-04-28T19:40:57.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2022-04-28T19:40:57.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2022-04-28T19:40:57.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2022-04-28T19:40:58.265 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:57 smithi139 bash[35227]: t=2022-04-28T19:40:57+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2022-04-28T19:40:58.266 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2022-04-28T19:40:58.266 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2022-04-28T19:40:58.266 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2022-04-28T19:40:58.266 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2022-04-28T19:40:58.525 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2022-04-28T19:40:58.525 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2022-04-28T19:40:58.525 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2022-04-28T19:40:58.525 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2022-04-28T19:40:58.785 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2022-04-28T19:40:58.891 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2022-04-28T19:40:58.891 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2022-04-28T19:40:59.095 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2022-04-28T19:40:59.108 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2022-04-28T19:40:59.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2022-04-28T19:40:59.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:58 smithi139 bash[35227]: t=2022-04-28T19:40:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2022-04-28T19:40:59.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2022-04-28T19:40:59.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:40:59 smithi059 bash[23786]: cluster 2022-04-28T19:40:57.988788+0000 mgr.y (mgr.14140) 241 : cluster [DBG] pgmap v212: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:59.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:40:59 smithi059 bash[29114]: cluster 2022-04-28T19:40:57.988788+0000 mgr.y (mgr.14140) 241 : cluster [DBG] pgmap v212: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[23439]: cluster 2022-04-28T19:40:57.988788+0000 mgr.y (mgr.14140) 241 : cluster [DBG] pgmap v212: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:40:59.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2022-04-28T19:40:59.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2022-04-28T19:40:59.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2022-04-28T19:40:59.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2022-04-28T19:40:59.503 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:40:59.683 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2022-04-28T19:40:59.684 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2022-04-28T19:40:59.684 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2022-04-28T19:40:59.684 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2022-04-28T19:40:59.684 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2022-04-28T19:40:59.960 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2022-04-28T19:40:59.960 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2022-04-28T19:40:59.960 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2022-04-28T19:40:59.960 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2022-04-28T19:40:59.960 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:40:59 smithi139 bash[35227]: t=2022-04-28T19:40:59+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2022-04-28T19:41:00.245 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[23439]: audit 2022-04-28T19:40:59.494676+0000 mon.a (mon.0) 555 : audit [DBG] from='client.? 172.21.15.59:0/1114922293' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-04-28T19:41:00.245 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2022-04-28T19:41:00.246 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2022-04-28T19:41:00.246 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2022-04-28T19:41:00.246 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2022-04-28T19:41:00.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:00 smithi059 bash[29114]: audit 2022-04-28T19:40:59.494676+0000 mon.a (mon.0) 555 : audit [DBG] from='client.? 172.21.15.59:0/1114922293' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-04-28T19:41:00.446 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:00 smithi059 systemd[1]: Stopping Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:41:00.446 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:00 smithi059 bash[23786]: audit 2022-04-28T19:40:59.494676+0000 mon.a (mon.0) 555 : audit [DBG] from='client.? 172.21.15.59:0/1114922293' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-04-28T19:41:00.447 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":14,"active_gid":14140,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6800","nonce":103934933},{"type":"v1","addr":"172.21.15.59:6801","nonce":103934933}]},"active_addr":"172.21.15.59:6801/103934933","active_change":"2022-04-28T19:35:15.837768+0000","active_mgr_features":4540138292840890367,"available":true,"standbys":[{"gid":34112,"name":"x","mgr_features":4540138292840890367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"none","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"int","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"None","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"none","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"int","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"None","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://smithi059:8443/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":3,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":397086884}]},{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":3381593068}]}]}} 2022-04-28T19:41:00.469 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2022-04-28T19:41:00.469 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2022-04-28T19:41:00.469 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd dump --format=json 2022-04-28T19:41:00.496 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2022-04-28T19:41:00.496 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2022-04-28T19:41:00.496 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2022-04-28T19:41:00.496 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2022-04-28T19:41:00.618 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:00.819 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2022-04-28T19:41:00.819 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2022-04-28T19:41:00.819 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2022-04-28T19:41:00.819 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2022-04-28T19:41:00.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:00 smithi059 bash[40315]: level=info ts=2022-04-28T19:41:00.449Z caller=main.go:536 msg="Received SIGTERM, exiting gracefully..." 2022-04-28T19:41:01.111 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2022-04-28T19:41:01.112 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2022-04-28T19:41:01.112 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:00 smithi139 bash[35227]: t=2022-04-28T19:41:00+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2022-04-28T19:41:01.112 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2022-04-28T19:41:01.112 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2022-04-28T19:41:01.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[23439]: cluster 2022-04-28T19:40:59.989442+0000 mgr.y (mgr.14140) 242 : cluster [DBG] pgmap v213: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:01.410 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2022-04-28T19:41:01.410 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2022-04-28T19:41:01.410 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2022-04-28T19:41:01.410 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2022-04-28T19:41:01.430 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:01 smithi059 bash[29114]: cluster 2022-04-28T19:40:59.989442+0000 mgr.y (mgr.14140) 242 : cluster [DBG] pgmap v213: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:01.430 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:01 smithi059 bash[23786]: cluster 2022-04-28T19:40:59.989442+0000 mgr.y (mgr.14140) 242 : cluster [DBG] pgmap v213: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:01.661 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2022-04-28T19:41:01.661 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2022-04-28T19:41:01.661 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2022-04-28T19:41:01.661 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2022-04-28T19:41:01.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:01 smithi059 docker[42170]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-alertmanager.a 2022-04-28T19:41:01.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:01 smithi059 systemd[1]: Stopped Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:41:01.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:01 smithi059 systemd[1]: Started Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:41:01.912 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2022-04-28T19:41:01.912 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2022-04-28T19:41:01.912 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2022-04-28T19:41:01.912 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2022-04-28T19:41:02.304 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:01 smithi139 bash[35227]: t=2022-04-28T19:41:01+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2022-04-28T19:41:02.576 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[35227]: t=2022-04-28T19:41:02+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2022-04-28T19:41:02.576 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[35227]: t=2022-04-28T19:41:02+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2022-04-28T19:41:02.807 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[29114]: audit 2022-04-28T19:41:01.564682+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:02.807 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[29114]: cephadm 2022-04-28T19:41:01.565826+0000 mgr.y (mgr.14140) 243 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-04-28T19:41:02.808 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[29114]: cephadm 2022-04-28T19:41:01.569318+0000 mgr.y (mgr.14140) 244 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi139 2022-04-28T19:41:02.808 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[29114]: cluster 2022-04-28T19:41:01.990130+0000 mgr.y (mgr.14140) 245 : cluster [DBG] pgmap v214: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:02.808 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[23786]: audit 2022-04-28T19:41:01.564682+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:02.808 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[23786]: cephadm 2022-04-28T19:41:01.565826+0000 mgr.y (mgr.14140) 243 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-04-28T19:41:02.808 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[23786]: cephadm 2022-04-28T19:41:01.569318+0000 mgr.y (mgr.14140) 244 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi139 2022-04-28T19:41:02.808 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:02 smithi059 bash[23786]: cluster 2022-04-28T19:41:01.990130+0000 mgr.y (mgr.14140) 245 : cluster [DBG] pgmap v214: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:02.862 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[35227]: t=2022-04-28T19:41:02+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2022-04-28T19:41:02.862 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[23439]: audit 2022-04-28T19:41:01.564682+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:02.862 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[23439]: cephadm 2022-04-28T19:41:01.565826+0000 mgr.y (mgr.14140) 243 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-04-28T19:41:02.862 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[23439]: cephadm 2022-04-28T19:41:01.569318+0000 mgr.y (mgr.14140) 244 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi139 2022-04-28T19:41:02.862 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[23439]: cluster 2022-04-28T19:41:01.990130+0000 mgr.y (mgr.14140) 245 : cluster [DBG] pgmap v214: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:03.078 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.023Z caller=main.go:231 msg="Starting Alertmanager" version="(version=0.20.0, branch=HEAD, revision=f74be0400a6243d10bb53812d6fa408ad71ff32d)" 2022-04-28T19:41:03.078 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.023Z caller=main.go:232 build_context="(go=go1.13.5, user=root@00c3106655f8, date=20191211-14:13:14)" 2022-04-28T19:41:03.078 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.026Z caller=cluster.go:161 component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2022-04-28T19:41:03.078 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.028Z caller=cluster.go:623 component=cluster msg="Waiting for gossip to settle..." interval=2s 2022-04-28T19:41:03.152 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:02 smithi139 bash[35227]: t=2022-04-28T19:41:02+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2022-04-28T19:41:03.152 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2022-04-28T19:41:03.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.078Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2022-04-28T19:41:03.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.078Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2022-04-28T19:41:03.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:03 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:03.083Z caller=main.go:497 msg=Listening address=:9093 2022-04-28T19:41:03.437 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2022-04-28T19:41:03.437 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2022-04-28T19:41:03.437 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2022-04-28T19:41:03.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2022-04-28T19:41:03.734 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2022-04-28T19:41:03.734 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2022-04-28T19:41:03.734 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2022-04-28T19:41:03.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2022-04-28T19:41:03.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2022-04-28T19:41:04.266 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:03 smithi139 bash[35227]: t=2022-04-28T19:41:03+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2022-04-28T19:41:04.590 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:04 smithi139 bash[35227]: t=2022-04-28T19:41:04+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2022-04-28T19:41:04.602 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:04 smithi139 bash[35227]: t=2022-04-28T19:41:04+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2022-04-28T19:41:04.602 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:04 smithi139 bash[35227]: t=2022-04-28T19:41:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2022-04-28T19:41:04.727 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:41:04.727 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":51,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:40:43.049867+0000","last_up_change":"2022-04-28T19:40:01.137034+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"octopus","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-04-28T19:40:38.422278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6803","nonce":1828778759}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6805","nonce":1828778759}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6809","nonce":1828778759}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6807","nonce":1828778759}]},"public_addr":"172.21.15.59:6803/1828778759","cluster_addr":"172.21.15.59:6805/1828778759","heartbeat_back_addr":"172.21.15.59:6809/1828778759","heartbeat_front_addr":"172.21.15.59:6807/1828778759","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":16,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6811","nonce":222479240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6813","nonce":222479240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6817","nonce":222479240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6815","nonce":222479240}]},"public_addr":"172.21.15.59:6811/222479240","cluster_addr":"172.21.15.59:6813/222479240","heartbeat_back_addr":"172.21.15.59:6817/222479240","heartbeat_front_addr":"172.21.15.59:6815/222479240","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6819","nonce":1472619329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6821","nonce":1472619329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6825","nonce":1472619329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6823","nonce":1472619329}]},"public_addr":"172.21.15.59:6819/1472619329","cluster_addr":"172.21.15.59:6821/1472619329","heartbeat_back_addr":"172.21.15.59:6825/1472619329","heartbeat_front_addr":"172.21.15.59:6823/1472619329","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6827","nonce":2946747803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6829","nonce":2946747803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6833","nonce":2946747803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6831","nonce":2946747803}]},"public_addr":"172.21.15.59:6827/2946747803","cluster_addr":"172.21.15.59:6829/2946747803","heartbeat_back_addr":"172.21.15.59:6833/2946747803","heartbeat_front_addr":"172.21.15.59:6831/2946747803","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6801","nonce":611845841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6803","nonce":611845841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6807","nonce":611845841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6805","nonce":611845841}]},"public_addr":"172.21.15.139:6801/611845841","cluster_addr":"172.21.15.139:6803/611845841","heartbeat_back_addr":"172.21.15.139:6807/611845841","heartbeat_front_addr":"172.21.15.139:6805/611845841","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6809","nonce":2795762393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6811","nonce":2795762393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6815","nonce":2795762393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6813","nonce":2795762393}]},"public_addr":"172.21.15.139:6809/2795762393","cluster_addr":"172.21.15.139:6811/2795762393","heartbeat_back_addr":"172.21.15.139:6815/2795762393","heartbeat_front_addr":"172.21.15.139:6813/2795762393","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6817","nonce":1808526322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6819","nonce":1808526322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6823","nonce":1808526322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6821","nonce":1808526322}]},"public_addr":"172.21.15.139:6817/1808526322","cluster_addr":"172.21.15.139:6819/1808526322","heartbeat_back_addr":"172.21.15.139:6823/1808526322","heartbeat_front_addr":"172.21.15.139:6821/1808526322","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6825","nonce":2154117349}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6827","nonce":2154117349}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6831","nonce":2154117349}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6829","nonce":2154117349}]},"public_addr":"172.21.15.139:6825/2154117349","cluster_addr":"172.21.15.139:6827/2154117349","heartbeat_back_addr":"172.21.15.139:6831/2154117349","heartbeat_front_addr":"172.21.15.139:6829/2154117349","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{"172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} 2022-04-28T19:41:04.862 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:04 smithi139 bash[35227]: t=2022-04-28T19:41:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2022-04-28T19:41:05.047 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:05 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:05.028Z caller=cluster.go:648 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000153345s 2022-04-28T19:41:05.330 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:05 smithi059 bash[29114]: cluster 2022-04-28T19:41:03.990754+0000 mgr.y (mgr.14140) 246 : cluster [DBG] pgmap v215: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:05.330 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:05 smithi059 bash[29114]: audit 2022-04-28T19:41:04.725577+0000 mon.a (mon.0) 557 : audit [DBG] from='client.? 172.21.15.59:0/4163735976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:41:05.330 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:05 smithi059 bash[23786]: cluster 2022-04-28T19:41:03.990754+0000 mgr.y (mgr.14140) 246 : cluster [DBG] pgmap v215: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:05.331 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:05 smithi059 bash[23786]: audit 2022-04-28T19:41:04.725577+0000 mon.a (mon.0) 557 : audit [DBG] from='client.? 172.21.15.59:0/4163735976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:41:05.332 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2022-04-28T19:41:05.332 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd dump --format=json 2022-04-28T19:41:05.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[23439]: cluster 2022-04-28T19:41:03.990754+0000 mgr.y (mgr.14140) 246 : cluster [DBG] pgmap v215: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:05.352 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[23439]: audit 2022-04-28T19:41:04.725577+0000 mon.a (mon.0) 557 : audit [DBG] from='client.? 172.21.15.59:0/4163735976' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:41:05.352 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:04 smithi139 bash[35227]: t=2022-04-28T19:41:04+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2022-04-28T19:41:05.352 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[35227]: t=2022-04-28T19:41:05+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2022-04-28T19:41:05.473 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:05.612 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[35227]: t=2022-04-28T19:41:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2022-04-28T19:41:05.612 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[35227]: t=2022-04-28T19:41:05+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2022-04-28T19:41:05.915 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[35227]: t=2022-04-28T19:41:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2022-04-28T19:41:05.915 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[35227]: t=2022-04-28T19:41:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2022-04-28T19:41:06.234 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:05 smithi139 bash[35227]: t=2022-04-28T19:41:05+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2022-04-28T19:41:06.235 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2022-04-28T19:41:06.235 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2022-04-28T19:41:06.235 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2022-04-28T19:41:06.612 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2022-04-28T19:41:06.612 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2022-04-28T19:41:06.964 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2022-04-28T19:41:06.965 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2022-04-28T19:41:07.230 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[23439]: cluster 2022-04-28T19:41:05.991389+0000 mgr.y (mgr.14140) 247 : cluster [DBG] pgmap v216: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:07.231 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 systemd[1]: Stopping Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:41:07.231 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:06 smithi139 bash[35227]: t=2022-04-28T19:41:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2022-04-28T19:41:07.231 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[35227]: t=2022-04-28T19:41:07+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2022-04-28T19:41:07.231 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[35227]: t=2022-04-28T19:41:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2022-04-28T19:41:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:07 smithi059 bash[29114]: cluster 2022-04-28T19:41:05.991389+0000 mgr.y (mgr.14140) 247 : cluster [DBG] pgmap v216: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:07.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:07 smithi059 bash[23786]: cluster 2022-04-28T19:41:05.991389+0000 mgr.y (mgr.14140) 247 : cluster [DBG] pgmap v216: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:07.613 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[35227]: t=2022-04-28T19:41:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2022-04-28T19:41:07.613 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[35227]: t=2022-04-28T19:41:07+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2022-04-28T19:41:07.613 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=warn ts=2022-04-28T19:41:07.234Z caller=main.go:524 msg="Received SIGTERM, exiting gracefully..." 2022-04-28T19:41:07.613 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:561 msg="Stopping notify discovery manager..." 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:583 msg="Stopping scrape manager..." 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:543 msg="Scrape discovery manager stopped" 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:557 msg="Notify discovery manager stopped" 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:577 msg="Scrape manager stopped" 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2022-04-28T19:41:07.614 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.234Z caller=main.go:749 msg="Notifier manager stopped" 2022-04-28T19:41:07.615 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[34185]: level=info ts=2022-04-28T19:41:07.235Z caller=main.go:761 msg="See you next time!" 2022-04-28T19:41:08.096 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[35227]: t=2022-04-28T19:41:07+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2022-04-28T19:41:08.096 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:07 smithi139 bash[35227]: t=2022-04-28T19:41:07+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2022-04-28T19:41:08.363 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:08 smithi139 docker[35823]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-prometheus.a 2022-04-28T19:41:08.363 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:08 smithi139 systemd[1]: Stopped Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:41:08.363 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:08 smithi139 systemd[1]: Started Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:41:08.363 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:08 smithi139 bash[35227]: t=2022-04-28T19:41:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2022-04-28T19:41:08.363 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:08 smithi139 bash[35227]: t=2022-04-28T19:41:08+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2022-04-28T19:41:08.456 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:41:08.456 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":51,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:40:43.049867+0000","last_up_change":"2022-04-28T19:40:01.137034+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"octopus","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-04-28T19:40:38.422278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":45,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6803","nonce":1828778759}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6805","nonce":1828778759}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6809","nonce":1828778759}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6807","nonce":1828778759}]},"public_addr":"172.21.15.59:6803/1828778759","cluster_addr":"172.21.15.59:6805/1828778759","heartbeat_back_addr":"172.21.15.59:6809/1828778759","heartbeat_front_addr":"172.21.15.59:6807/1828778759","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":16,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6811","nonce":222479240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6813","nonce":222479240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6817","nonce":222479240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6815","nonce":222479240}]},"public_addr":"172.21.15.59:6811/222479240","cluster_addr":"172.21.15.59:6813/222479240","heartbeat_back_addr":"172.21.15.59:6817/222479240","heartbeat_front_addr":"172.21.15.59:6815/222479240","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6819","nonce":1472619329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6821","nonce":1472619329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6825","nonce":1472619329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6823","nonce":1472619329}]},"public_addr":"172.21.15.59:6819/1472619329","cluster_addr":"172.21.15.59:6821/1472619329","heartbeat_back_addr":"172.21.15.59:6825/1472619329","heartbeat_front_addr":"172.21.15.59:6823/1472619329","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6827","nonce":2946747803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6829","nonce":2946747803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6833","nonce":2946747803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6831","nonce":2946747803}]},"public_addr":"172.21.15.59:6827/2946747803","cluster_addr":"172.21.15.59:6829/2946747803","heartbeat_back_addr":"172.21.15.59:6833/2946747803","heartbeat_front_addr":"172.21.15.59:6831/2946747803","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6801","nonce":611845841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6803","nonce":611845841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6807","nonce":611845841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6805","nonce":611845841}]},"public_addr":"172.21.15.139:6801/611845841","cluster_addr":"172.21.15.139:6803/611845841","heartbeat_back_addr":"172.21.15.139:6807/611845841","heartbeat_front_addr":"172.21.15.139:6805/611845841","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6809","nonce":2795762393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6811","nonce":2795762393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6815","nonce":2795762393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6813","nonce":2795762393}]},"public_addr":"172.21.15.139:6809/2795762393","cluster_addr":"172.21.15.139:6811/2795762393","heartbeat_back_addr":"172.21.15.139:6815/2795762393","heartbeat_front_addr":"172.21.15.139:6813/2795762393","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6817","nonce":1808526322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6819","nonce":1808526322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6823","nonce":1808526322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6821","nonce":1808526322}]},"public_addr":"172.21.15.139:6817/1808526322","cluster_addr":"172.21.15.139:6819/1808526322","heartbeat_back_addr":"172.21.15.139:6823/1808526322","heartbeat_front_addr":"172.21.15.139:6821/1808526322","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6825","nonce":2154117349}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6827","nonce":2154117349}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6831","nonce":2154117349}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6829","nonce":2154117349}]},"public_addr":"172.21.15.139:6825/2154117349","cluster_addr":"172.21.15.139:6827/2154117349","heartbeat_back_addr":"172.21.15.139:6831/2154117349","heartbeat_front_addr":"172.21.15.139:6829/2154117349","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{"172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} 2022-04-28T19:41:08.863 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:08 smithi139 bash[35227]: t=2022-04-28T19:41:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2022-04-28T19:41:08.863 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:08 smithi139 bash[35227]: t=2022-04-28T19:41:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2022-04-28T19:41:09.093 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2022-04-28T19:41:09.276 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:09.363 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2022-04-28T19:41:09.363 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: cluster 2022-04-28T19:41:07.991978+0000 mgr.y (mgr.14140) 248 : cluster [DBG] pgmap v217: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.173801+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.181650+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.182169+0000 mgr.y (mgr.14140) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.183847+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.184283+0000 mgr.y (mgr.14140) 250 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.189783+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.197091+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:41:09.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.197641+0000 mgr.y (mgr.14140) 251 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.207876+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139:9095"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.208501+0000 mgr.y (mgr.14140) 252 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139:9095"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.213748+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.221980+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.222432+0000 mgr.y (mgr.14140) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.229241+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139:3000"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.229723+0000 mgr.y (mgr.14140) 254 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139:3000"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.235647+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.240002+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:09.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.240338+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:09.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[23439]: audit 2022-04-28T19:41:08.455013+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.59:0/1171644485' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:41:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: cluster 2022-04-28T19:41:07.991978+0000 mgr.y (mgr.14140) 248 : cluster [DBG] pgmap v217: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.173801+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.181650+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:41:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.182169+0000 mgr.y (mgr.14140) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:41:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.183847+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2022-04-28T19:41:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.184283+0000 mgr.y (mgr.14140) 250 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.189783+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.197091+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.197641+0000 mgr.y (mgr.14140) 251 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.207876+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139:9095"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.208501+0000 mgr.y (mgr.14140) 252 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139:9095"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.213748+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.221980+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.222432+0000 mgr.y (mgr.14140) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:41:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.229241+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139:3000"}]: dispatch 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.229723+0000 mgr.y (mgr.14140) 254 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139:3000"}]: dispatch 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.235647+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.240002+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: cluster 2022-04-28T19:41:07.991978+0000 mgr.y (mgr.14140) 248 : cluster [DBG] pgmap v217: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.173801+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.181650+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.182169+0000 mgr.y (mgr.14140) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:41:09.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.183847+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.184283+0000 mgr.y (mgr.14140) 250 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi059:9093"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.189783+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.197091+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.197641+0000 mgr.y (mgr.14140) 251 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.207876+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139:9095"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.208501+0000 mgr.y (mgr.14140) 252 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139:9095"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.213748+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.221980+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.222432+0000 mgr.y (mgr.14140) 253 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:41:09.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.229241+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139:3000"}]: dispatch 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.229723+0000 mgr.y (mgr.14140) 254 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139:3000"}]: dispatch 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.235647+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.240002+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.240338+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[29114]: audit 2022-04-28T19:41:08.455013+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.59:0/1171644485' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.240338+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:09.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:09 smithi059 bash[23786]: audit 2022-04-28T19:41:08.455013+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.59:0/1171644485' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:41:09.622 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2022-04-28T19:41:09.622 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2022-04-28T19:41:09.875 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2022-04-28T19:41:09.875 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2022-04-28T19:41:09.875 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2022-04-28T19:41:09.875 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2022-04-28T19:41:10.141 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2022-04-28T19:41:10.141 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35227]: t=2022-04-28T19:41:09+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2022-04-28T19:41:10.142 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2022-04-28T19:41:10.142 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.878Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2022-04-28T19:41:10.142 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.878Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2022-04-28T19:41:10.142 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.878Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2022-04-28T19:41:10.142 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.878Z caller=main.go:339 host_details="(Linux 4.15.0-176-generic #185-Ubuntu SMP Tue Mar 29 17:40:04 UTC 2022 x86_64 smithi139 (none))" 2022-04-28T19:41:10.142 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.878Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2022-04-28T19:41:10.142 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.878Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.880Z caller=main.go:678 msg="Starting TSDB ..." 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.880Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.886Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.887Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.888Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.888Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=1.60262ms 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.890Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.890Z caller=main.go:695 msg="TSDB started" 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.890Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:41:10.143 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.898Z caller=main.go:827 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:41:10.144 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:41:09 smithi139 bash[35873]: level=info ts=2022-04-28T19:41:09.898Z caller=main.go:646 msg="Server is ready to receive web requests." 2022-04-28T19:41:10.435 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2022-04-28T19:41:10.435 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2022-04-28T19:41:10.436 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2022-04-28T19:41:10.436 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2022-04-28T19:41:10.436 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2022-04-28T19:41:10.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2022-04-28T19:41:10.698 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2022-04-28T19:41:10.698 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2022-04-28T19:41:10.698 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2022-04-28T19:41:10.698 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2022-04-28T19:41:10.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[29114]: audit 2022-04-28T19:41:09.692634+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[29114]: cluster 2022-04-28T19:41:09.992541+0000 mgr.y (mgr.14140) 255 : cluster [DBG] pgmap v218: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[29114]: audit 2022-04-28T19:41:10.432190+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[29114]: audit 2022-04-28T19:41:10.439169+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[29114]: audit 2022-04-28T19:41:10.447232+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[29114]: audit 2022-04-28T19:41:10.447684+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[23786]: audit 2022-04-28T19:41:09.692634+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[23786]: cluster 2022-04-28T19:41:09.992541+0000 mgr.y (mgr.14140) 255 : cluster [DBG] pgmap v218: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[23786]: audit 2022-04-28T19:41:10.432190+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[23786]: audit 2022-04-28T19:41:10.439169+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[23786]: audit 2022-04-28T19:41:10.447232+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:10 smithi059 bash[23786]: audit 2022-04-28T19:41:10.447684+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:10.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2022-04-28T19:41:10.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2022-04-28T19:41:10.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2022-04-28T19:41:10.989 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2022-04-28T19:41:10.989 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[23439]: audit 2022-04-28T19:41:09.692634+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.989 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[23439]: cluster 2022-04-28T19:41:09.992541+0000 mgr.y (mgr.14140) 255 : cluster [DBG] pgmap v218: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:10.990 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[23439]: audit 2022-04-28T19:41:10.432190+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.990 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[23439]: audit 2022-04-28T19:41:10.439169+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:10.990 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[23439]: audit 2022-04-28T19:41:10.447232+0000 mon.a (mon.0) 573 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:10.990 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[23439]: audit 2022-04-28T19:41:10.447684+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:41:11.257 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:10 smithi139 bash[35227]: t=2022-04-28T19:41:10+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2022-04-28T19:41:11.258 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2022-04-28T19:41:11.258 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2022-04-28T19:41:11.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2022-04-28T19:41:11.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2022-04-28T19:41:11.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2022-04-28T19:41:11.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2022-04-28T19:41:11.786 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2022-04-28T19:41:11.786 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2022-04-28T19:41:11.786 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2022-04-28T19:41:11.787 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2022-04-28T19:41:12.038 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2022-04-28T19:41:12.038 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2022-04-28T19:41:12.039 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2022-04-28T19:41:12.039 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:11 smithi139 bash[35227]: t=2022-04-28T19:41:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2022-04-28T19:41:12.299 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2022-04-28T19:41:12.299 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2022-04-28T19:41:12.299 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2022-04-28T19:41:12.299 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2022-04-28T19:41:12.609 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2022-04-28T19:41:12.609 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2022-04-28T19:41:12.609 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2022-04-28T19:41:12.609 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2022-04-28T19:41:12.609 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2022-04-28T19:41:12.863 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[23439]: audit 2022-04-28T19:41:11.803613+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:12.864 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[23439]: audit 2022-04-28T19:41:11.885229+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:12.864 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[23439]: audit 2022-04-28T19:41:11.891538+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:12.864 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[23439]: cluster 2022-04-28T19:41:11.993208+0000 mgr.y (mgr.14140) 256 : cluster [DBG] pgmap v219: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:12.864 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2022-04-28T19:41:12.864 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2022-04-28T19:41:12.864 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2022-04-28T19:41:13.001 INFO:teuthology.orchestra.run.smithi059.stdout:42949673009 2022-04-28T19:41:13.002 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2022-04-28T19:41:13.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[23786]: audit 2022-04-28T19:41:11.803613+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[23786]: audit 2022-04-28T19:41:11.885229+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[23786]: audit 2022-04-28T19:41:11.891538+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[23786]: cluster 2022-04-28T19:41:11.993208+0000 mgr.y (mgr.14140) 256 : cluster [DBG] pgmap v219: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[29114]: audit 2022-04-28T19:41:11.803613+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[29114]: audit 2022-04-28T19:41:11.885229+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[29114]: audit 2022-04-28T19:41:11.891538+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:41:13.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:12 smithi059 bash[29114]: cluster 2022-04-28T19:41:11.993208+0000 mgr.y (mgr.14140) 256 : cluster [DBG] pgmap v219: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:13.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:41:13 smithi059 bash[42244]: level=info ts=2022-04-28T19:41:13.029Z caller=cluster.go:640 component=cluster msg="gossip settled; proceeding" elapsed=10.000913924s 2022-04-28T19:41:13.143 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:13.179 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2022-04-28T19:41:13.179 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:12 smithi139 bash[35227]: t=2022-04-28T19:41:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2022-04-28T19:41:13.179 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2022-04-28T19:41:13.179 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2022-04-28T19:41:13.430 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2022-04-28T19:41:13.486 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2022-04-28T19:41:13.486 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2022-04-28T19:41:13.487 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2022-04-28T19:41:13.699 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2022-04-28T19:41:13.699 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2022-04-28T19:41:13.699 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2022-04-28T19:41:13.699 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2022-04-28T19:41:13.699 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2022-04-28T19:41:13.699 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2022-04-28T19:41:13.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2022-04-28T19:41:13.976 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2022-04-28T19:41:13.976 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:13 smithi139 bash[35227]: t=2022-04-28T19:41:13+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Created default admin" logger=sqlstore user=admin 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing HTTPServer" logger=server 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing BackendPluginManager" logger=server 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing PluginManager" logger=server 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Starting plugin search" logger=plugins 2022-04-28T19:41:14.289 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Registering plugin" logger=plugins name="Pie Chart" 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Registering plugin" logger=plugins name="Status Panel" 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing HooksService" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing OSSLicensingService" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing InternalMetricsService" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing RemoteCache" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing RenderingService" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing AlertEngine" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing QuotaService" logger=server 2022-04-28T19:41:14.290 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing ServerLockService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing UserAuthTokenService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing DatasourceCacheService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing LoginService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing SearchService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing TracingService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing UsageStatsService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing CleanUpService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing NotificationService" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing provisioningServiceImpl" logger=server 2022-04-28T19:41:14.291 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2022-04-28T19:41:14.292 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 2022-04-28T19:41:14.292 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Backend rendering via phantomJS" logger=rendering renderer=phantomJS 2022-04-28T19:41:14.613 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=warn msg="phantomJS is deprecated and will be removed in a future release. You should consider migrating from phantomJS to grafana-image-renderer plugin. Read more at https://grafana.com/docs/grafana/latest/administration/image_rendering/" logger=rendering renderer=phantomJS 2022-04-28T19:41:14.614 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="Initializing Stream Manager" 2022-04-28T19:41:14.614 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:41:14 smithi139 bash[35227]: t=2022-04-28T19:41:14+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2022-04-28T19:41:15.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:15 smithi059 bash[23786]: cluster 2022-04-28T19:41:13.993856+0000 mgr.y (mgr.14140) 257 : cluster [DBG] pgmap v220: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:15.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:15 smithi059 bash[29114]: cluster 2022-04-28T19:41:13.993856+0000 mgr.y (mgr.14140) 257 : cluster [DBG] pgmap v220: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:15.363 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:15 smithi139 bash[23439]: cluster 2022-04-28T19:41:13.993856+0000 mgr.y (mgr.14140) 257 : cluster [DBG] pgmap v220: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:16.755 INFO:teuthology.orchestra.run.smithi059.stdout:68719476782 2022-04-28T19:41:16.755 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2022-04-28T19:41:16.896 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:17 smithi059 bash[29114]: cluster 2022-04-28T19:41:15.994425+0000 mgr.y (mgr.14140) 258 : cluster [DBG] pgmap v221: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:17 smithi059 bash[29114]: audit 2022-04-28T19:41:16.114710+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:41:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:17 smithi059 bash[29114]: audit 2022-04-28T19:41:16.119131+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:41:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:17 smithi059 bash[23786]: cluster 2022-04-28T19:41:15.994425+0000 mgr.y (mgr.14140) 258 : cluster [DBG] pgmap v221: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:17 smithi059 bash[23786]: audit 2022-04-28T19:41:16.114710+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:41:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:17 smithi059 bash[23786]: audit 2022-04-28T19:41:16.119131+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:41:17.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:17 smithi139 bash[23439]: cluster 2022-04-28T19:41:15.994425+0000 mgr.y (mgr.14140) 258 : cluster [DBG] pgmap v221: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:17.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:17 smithi139 bash[23439]: audit 2022-04-28T19:41:16.114710+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:41:17.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:17 smithi139 bash[23439]: audit 2022-04-28T19:41:16.119131+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:41:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:19 smithi059 bash[29114]: cluster 2022-04-28T19:41:17.995030+0000 mgr.y (mgr.14140) 259 : cluster [DBG] pgmap v222: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:19 smithi059 bash[23786]: cluster 2022-04-28T19:41:17.995030+0000 mgr.y (mgr.14140) 259 : cluster [DBG] pgmap v222: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:19.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:19 smithi139 bash[23439]: cluster 2022-04-28T19:41:17.995030+0000 mgr.y (mgr.14140) 259 : cluster [DBG] pgmap v222: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:20.506 INFO:teuthology.orchestra.run.smithi059.stdout:90194313258 2022-04-28T19:41:20.507 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2022-04-28T19:41:20.652 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:21 smithi059 bash[23786]: cluster 2022-04-28T19:41:19.995628+0000 mgr.y (mgr.14140) 260 : cluster [DBG] pgmap v223: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:21 smithi059 bash[29114]: cluster 2022-04-28T19:41:19.995628+0000 mgr.y (mgr.14140) 260 : cluster [DBG] pgmap v223: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:21.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:21 smithi139 bash[23439]: cluster 2022-04-28T19:41:19.995628+0000 mgr.y (mgr.14140) 260 : cluster [DBG] pgmap v223: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:23 smithi059 bash[23786]: cluster 2022-04-28T19:41:21.996256+0000 mgr.y (mgr.14140) 261 : cluster [DBG] pgmap v224: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:23 smithi059 bash[29114]: cluster 2022-04-28T19:41:21.996256+0000 mgr.y (mgr.14140) 261 : cluster [DBG] pgmap v224: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:23.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:23 smithi139 bash[23439]: cluster 2022-04-28T19:41:21.996256+0000 mgr.y (mgr.14140) 261 : cluster [DBG] pgmap v224: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:24.332 INFO:teuthology.orchestra.run.smithi059.stdout:111669149733 2022-04-28T19:41:24.515 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2022-04-28T19:41:24.661 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:25 smithi059 bash[23786]: cluster 2022-04-28T19:41:23.996798+0000 mgr.y (mgr.14140) 262 : cluster [DBG] pgmap v225: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:25 smithi059 bash[29114]: cluster 2022-04-28T19:41:23.996798+0000 mgr.y (mgr.14140) 262 : cluster [DBG] pgmap v225: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:25.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:25 smithi139 bash[23439]: cluster 2022-04-28T19:41:23.996798+0000 mgr.y (mgr.14140) 262 : cluster [DBG] pgmap v225: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:27.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:27 smithi139 bash[23439]: cluster 2022-04-28T19:41:25.997240+0000 mgr.y (mgr.14140) 263 : cluster [DBG] pgmap v226: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:27.443 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:27 smithi059 bash[23786]: cluster 2022-04-28T19:41:25.997240+0000 mgr.y (mgr.14140) 263 : cluster [DBG] pgmap v226: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:27.444 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:27 smithi059 bash[29114]: cluster 2022-04-28T19:41:25.997240+0000 mgr.y (mgr.14140) 263 : cluster [DBG] pgmap v226: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:28.340 INFO:teuthology.orchestra.run.smithi059.stdout:133143986211 2022-04-28T19:41:28.341 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2022-04-28T19:41:28.486 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:29.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:29 smithi139 bash[23439]: cluster 2022-04-28T19:41:27.997795+0000 mgr.y (mgr.14140) 264 : cluster [DBG] pgmap v227: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:29.381 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:29 smithi059 bash[29114]: cluster 2022-04-28T19:41:27.997795+0000 mgr.y (mgr.14140) 264 : cluster [DBG] pgmap v227: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:29.381 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:29 smithi059 bash[23786]: cluster 2022-04-28T19:41:27.997795+0000 mgr.y (mgr.14140) 264 : cluster [DBG] pgmap v227: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:31.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:31 smithi059 bash[29114]: cluster 2022-04-28T19:41:29.998344+0000 mgr.y (mgr.14140) 265 : cluster [DBG] pgmap v228: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:31 smithi059 bash[23786]: cluster 2022-04-28T19:41:29.998344+0000 mgr.y (mgr.14140) 265 : cluster [DBG] pgmap v228: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:31.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:31 smithi139 bash[23439]: cluster 2022-04-28T19:41:29.998344+0000 mgr.y (mgr.14140) 265 : cluster [DBG] pgmap v228: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:32.002 INFO:teuthology.orchestra.run.smithi059.stdout:154618822685 2022-04-28T19:41:32.003 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2022-04-28T19:41:32.152 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:33.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:33 smithi139 bash[23439]: cluster 2022-04-28T19:41:31.998913+0000 mgr.y (mgr.14140) 266 : cluster [DBG] pgmap v229: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:33.418 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:33 smithi059 bash[29114]: cluster 2022-04-28T19:41:31.998913+0000 mgr.y (mgr.14140) 266 : cluster [DBG] pgmap v229: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:33.419 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:33 smithi059 bash[23786]: cluster 2022-04-28T19:41:31.998913+0000 mgr.y (mgr.14140) 266 : cluster [DBG] pgmap v229: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:35 smithi059 bash[23786]: cluster 2022-04-28T19:41:33.999348+0000 mgr.y (mgr.14140) 267 : cluster [DBG] pgmap v230: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:35.442 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:35 smithi059 bash[29114]: cluster 2022-04-28T19:41:33.999348+0000 mgr.y (mgr.14140) 267 : cluster [DBG] pgmap v230: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:35.443 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:35 smithi139 bash[23439]: cluster 2022-04-28T19:41:33.999348+0000 mgr.y (mgr.14140) 267 : cluster [DBG] pgmap v230: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:35.640 INFO:teuthology.orchestra.run.smithi059.stdout:176093659162 2022-04-28T19:41:35.734 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2022-04-28T19:41:35.881 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:37 smithi059 bash[29114]: cluster 2022-04-28T19:41:35.999963+0000 mgr.y (mgr.14140) 268 : cluster [DBG] pgmap v231: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:37 smithi059 bash[23786]: cluster 2022-04-28T19:41:35.999963+0000 mgr.y (mgr.14140) 268 : cluster [DBG] pgmap v231: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:37.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:37 smithi139 bash[23439]: cluster 2022-04-28T19:41:35.999963+0000 mgr.y (mgr.14140) 268 : cluster [DBG] pgmap v231: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:39 smithi059 bash[29114]: cluster 2022-04-28T19:41:38.000559+0000 mgr.y (mgr.14140) 269 : cluster [DBG] pgmap v232: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:39 smithi059 bash[23786]: cluster 2022-04-28T19:41:38.000559+0000 mgr.y (mgr.14140) 269 : cluster [DBG] pgmap v232: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:39.443 INFO:teuthology.orchestra.run.smithi059.stdout:197568495637 2022-04-28T19:41:39.443 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.0 2022-04-28T19:41:39.588 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:39.615 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:39 smithi139 bash[23439]: cluster 2022-04-28T19:41:38.000559+0000 mgr.y (mgr.14140) 269 : cluster [DBG] pgmap v232: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:41 smithi059 bash[29114]: cluster 2022-04-28T19:41:40.001269+0000 mgr.y (mgr.14140) 270 : cluster [DBG] pgmap v233: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:41 smithi059 bash[23786]: cluster 2022-04-28T19:41:40.001269+0000 mgr.y (mgr.14140) 270 : cluster [DBG] pgmap v233: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:41.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:41 smithi139 bash[23439]: cluster 2022-04-28T19:41:40.001269+0000 mgr.y (mgr.14140) 270 : cluster [DBG] pgmap v233: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:42.582 INFO:teuthology.orchestra.run.smithi059.stdout:42949673015 2022-04-28T19:41:43.266 INFO:tasks.cephadm.ceph_manager.ceph:need seq 42949673009 got 42949673015 for osd.0 2022-04-28T19:41:43.266 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.1 2022-04-28T19:41:43.413 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:43 smithi059 bash[23786]: cluster 2022-04-28T19:41:42.001936+0000 mgr.y (mgr.14140) 271 : cluster [DBG] pgmap v234: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:43 smithi059 bash[23786]: audit 2022-04-28T19:41:42.581764+0000 mon.a (mon.0) 580 : audit [DBG] from='client.? 172.21.15.59:0/997550776' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-04-28T19:41:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:43 smithi059 bash[29114]: cluster 2022-04-28T19:41:42.001936+0000 mgr.y (mgr.14140) 271 : cluster [DBG] pgmap v234: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:43 smithi059 bash[29114]: audit 2022-04-28T19:41:42.581764+0000 mon.a (mon.0) 580 : audit [DBG] from='client.? 172.21.15.59:0/997550776' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-04-28T19:41:43.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:43 smithi139 bash[23439]: cluster 2022-04-28T19:41:42.001936+0000 mgr.y (mgr.14140) 271 : cluster [DBG] pgmap v234: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:43.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:43 smithi139 bash[23439]: audit 2022-04-28T19:41:42.581764+0000 mon.a (mon.0) 580 : audit [DBG] from='client.? 172.21.15.59:0/997550776' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-04-28T19:41:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:45 smithi059 bash[29114]: cluster 2022-04-28T19:41:44.002569+0000 mgr.y (mgr.14140) 272 : cluster [DBG] pgmap v235: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:45 smithi059 bash[23786]: cluster 2022-04-28T19:41:44.002569+0000 mgr.y (mgr.14140) 272 : cluster [DBG] pgmap v235: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:45.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:45 smithi139 bash[23439]: cluster 2022-04-28T19:41:44.002569+0000 mgr.y (mgr.14140) 272 : cluster [DBG] pgmap v235: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:46.449 INFO:teuthology.orchestra.run.smithi059.stdout:68719476788 2022-04-28T19:41:47.095 INFO:tasks.cephadm.ceph_manager.ceph:need seq 68719476782 got 68719476788 for osd.1 2022-04-28T19:41:47.096 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.2 2022-04-28T19:41:47.238 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:47 smithi059 bash[23786]: cluster 2022-04-28T19:41:46.003220+0000 mgr.y (mgr.14140) 273 : cluster [DBG] pgmap v236: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:47 smithi059 bash[23786]: audit 2022-04-28T19:41:46.447776+0000 mon.c (mon.1) 18 : audit [DBG] from='client.? 172.21.15.59:0/3242060102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-04-28T19:41:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:47 smithi059 bash[29114]: cluster 2022-04-28T19:41:46.003220+0000 mgr.y (mgr.14140) 273 : cluster [DBG] pgmap v236: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:47 smithi059 bash[29114]: audit 2022-04-28T19:41:46.447776+0000 mon.c (mon.1) 18 : audit [DBG] from='client.? 172.21.15.59:0/3242060102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-04-28T19:41:47.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:47 smithi139 bash[23439]: cluster 2022-04-28T19:41:46.003220+0000 mgr.y (mgr.14140) 273 : cluster [DBG] pgmap v236: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:47.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:47 smithi139 bash[23439]: audit 2022-04-28T19:41:46.447776+0000 mon.c (mon.1) 18 : audit [DBG] from='client.? 172.21.15.59:0/3242060102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-04-28T19:41:49.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:49 smithi059 bash[23786]: cluster 2022-04-28T19:41:48.004011+0000 mgr.y (mgr.14140) 274 : cluster [DBG] pgmap v237: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:49 smithi059 bash[29114]: cluster 2022-04-28T19:41:48.004011+0000 mgr.y (mgr.14140) 274 : cluster [DBG] pgmap v237: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:49.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:49 smithi139 bash[23439]: cluster 2022-04-28T19:41:48.004011+0000 mgr.y (mgr.14140) 274 : cluster [DBG] pgmap v237: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:50.340 INFO:teuthology.orchestra.run.smithi059.stdout:90194313264 2022-04-28T19:41:50.965 INFO:tasks.cephadm.ceph_manager.ceph:need seq 90194313258 got 90194313264 for osd.2 2022-04-28T19:41:50.965 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.3 2022-04-28T19:41:51.112 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:51 smithi059 bash[29114]: cluster 2022-04-28T19:41:50.004884+0000 mgr.y (mgr.14140) 275 : cluster [DBG] pgmap v238: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:51 smithi059 bash[29114]: audit 2022-04-28T19:41:50.339336+0000 mon.c (mon.1) 19 : audit [DBG] from='client.? 172.21.15.59:0/1382579719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-04-28T19:41:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:51 smithi059 bash[23786]: cluster 2022-04-28T19:41:50.004884+0000 mgr.y (mgr.14140) 275 : cluster [DBG] pgmap v238: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:51 smithi059 bash[23786]: audit 2022-04-28T19:41:50.339336+0000 mon.c (mon.1) 19 : audit [DBG] from='client.? 172.21.15.59:0/1382579719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-04-28T19:41:51.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:51 smithi139 bash[23439]: cluster 2022-04-28T19:41:50.004884+0000 mgr.y (mgr.14140) 275 : cluster [DBG] pgmap v238: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:51.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:51 smithi139 bash[23439]: audit 2022-04-28T19:41:50.339336+0000 mon.c (mon.1) 19 : audit [DBG] from='client.? 172.21.15.59:0/1382579719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-04-28T19:41:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:53 smithi059 bash[29114]: cluster 2022-04-28T19:41:52.005577+0000 mgr.y (mgr.14140) 276 : cluster [DBG] pgmap v239: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:53.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:53 smithi059 bash[23786]: cluster 2022-04-28T19:41:52.005577+0000 mgr.y (mgr.14140) 276 : cluster [DBG] pgmap v239: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:53.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:53 smithi139 bash[23439]: cluster 2022-04-28T19:41:52.005577+0000 mgr.y (mgr.14140) 276 : cluster [DBG] pgmap v239: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:54.262 INFO:teuthology.orchestra.run.smithi059.stdout:111669149739 2022-04-28T19:41:54.894 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149733 got 111669149739 for osd.3 2022-04-28T19:41:54.894 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.4 2022-04-28T19:41:55.043 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:55.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:55 smithi059 bash[29114]: cluster 2022-04-28T19:41:54.006366+0000 mgr.y (mgr.14140) 277 : cluster [DBG] pgmap v240: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:55.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:55 smithi059 bash[29114]: audit 2022-04-28T19:41:54.261572+0000 mon.a (mon.0) 581 : audit [DBG] from='client.? 172.21.15.59:0/2880867683' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-04-28T19:41:55.605 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:55 smithi059 bash[23786]: cluster 2022-04-28T19:41:54.006366+0000 mgr.y (mgr.14140) 277 : cluster [DBG] pgmap v240: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:55.605 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:55 smithi059 bash[23786]: audit 2022-04-28T19:41:54.261572+0000 mon.a (mon.0) 581 : audit [DBG] from='client.? 172.21.15.59:0/2880867683' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-04-28T19:41:55.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:55 smithi139 bash[23439]: cluster 2022-04-28T19:41:54.006366+0000 mgr.y (mgr.14140) 277 : cluster [DBG] pgmap v240: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:55.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:55 smithi139 bash[23439]: audit 2022-04-28T19:41:54.261572+0000 mon.a (mon.0) 581 : audit [DBG] from='client.? 172.21.15.59:0/2880867683' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-04-28T19:41:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:57 smithi059 bash[29114]: cluster 2022-04-28T19:41:56.007057+0000 mgr.y (mgr.14140) 278 : cluster [DBG] pgmap v241: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:57 smithi059 bash[23786]: cluster 2022-04-28T19:41:56.007057+0000 mgr.y (mgr.14140) 278 : cluster [DBG] pgmap v241: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:57.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:57 smithi139 bash[23439]: cluster 2022-04-28T19:41:56.007057+0000 mgr.y (mgr.14140) 278 : cluster [DBG] pgmap v241: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:58.211 INFO:teuthology.orchestra.run.smithi059.stdout:133143986217 2022-04-28T19:41:58.830 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986211 got 133143986217 for osd.4 2022-04-28T19:41:58.830 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.5 2022-04-28T19:41:58.978 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:41:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:59 smithi059 bash[29114]: cluster 2022-04-28T19:41:58.007725+0000 mgr.y (mgr.14140) 279 : cluster [DBG] pgmap v242: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:41:59 smithi059 bash[29114]: audit 2022-04-28T19:41:58.210174+0000 mon.c (mon.1) 20 : audit [DBG] from='client.? 172.21.15.59:0/2268615670' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-04-28T19:41:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:59 smithi059 bash[23786]: cluster 2022-04-28T19:41:58.007725+0000 mgr.y (mgr.14140) 279 : cluster [DBG] pgmap v242: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:41:59 smithi059 bash[23786]: audit 2022-04-28T19:41:58.210174+0000 mon.c (mon.1) 20 : audit [DBG] from='client.? 172.21.15.59:0/2268615670' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-04-28T19:41:59.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:59 smithi139 bash[23439]: cluster 2022-04-28T19:41:58.007725+0000 mgr.y (mgr.14140) 279 : cluster [DBG] pgmap v242: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:41:59.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:41:59 smithi139 bash[23439]: audit 2022-04-28T19:41:58.210174+0000 mon.c (mon.1) 20 : audit [DBG] from='client.? 172.21.15.59:0/2268615670' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-04-28T19:42:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:01 smithi059 bash[29114]: cluster 2022-04-28T19:42:00.008382+0000 mgr.y (mgr.14140) 280 : cluster [DBG] pgmap v243: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:01 smithi059 bash[23786]: cluster 2022-04-28T19:42:00.008382+0000 mgr.y (mgr.14140) 280 : cluster [DBG] pgmap v243: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:01.616 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:01 smithi139 bash[23439]: cluster 2022-04-28T19:42:00.008382+0000 mgr.y (mgr.14140) 280 : cluster [DBG] pgmap v243: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:02.162 INFO:teuthology.orchestra.run.smithi059.stdout:154618822691 2022-04-28T19:42:02.812 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822685 got 154618822691 for osd.5 2022-04-28T19:42:02.812 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.6 2022-04-28T19:42:02.954 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:03 smithi059 bash[29114]: cluster 2022-04-28T19:42:02.009001+0000 mgr.y (mgr.14140) 281 : cluster [DBG] pgmap v244: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:03 smithi059 bash[29114]: audit 2022-04-28T19:42:02.161505+0000 mon.a (mon.0) 582 : audit [DBG] from='client.? 172.21.15.59:0/752913344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-04-28T19:42:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:03 smithi059 bash[23786]: cluster 2022-04-28T19:42:02.009001+0000 mgr.y (mgr.14140) 281 : cluster [DBG] pgmap v244: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:03 smithi059 bash[23786]: audit 2022-04-28T19:42:02.161505+0000 mon.a (mon.0) 582 : audit [DBG] from='client.? 172.21.15.59:0/752913344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-04-28T19:42:03.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:03 smithi139 bash[23439]: cluster 2022-04-28T19:42:02.009001+0000 mgr.y (mgr.14140) 281 : cluster [DBG] pgmap v244: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:03.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:03 smithi139 bash[23439]: audit 2022-04-28T19:42:02.161505+0000 mon.a (mon.0) 582 : audit [DBG] from='client.? 172.21.15.59:0/752913344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-04-28T19:42:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:05 smithi059 bash[23786]: cluster 2022-04-28T19:42:04.009534+0000 mgr.y (mgr.14140) 282 : cluster [DBG] pgmap v245: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:05.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:05 smithi059 bash[29114]: cluster 2022-04-28T19:42:04.009534+0000 mgr.y (mgr.14140) 282 : cluster [DBG] pgmap v245: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:05.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:05 smithi139 bash[23439]: cluster 2022-04-28T19:42:04.009534+0000 mgr.y (mgr.14140) 282 : cluster [DBG] pgmap v245: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:06.080 INFO:teuthology.orchestra.run.smithi059.stdout:176093659167 2022-04-28T19:42:06.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:06 smithi139 bash[23439]: audit 2022-04-28T19:42:06.079420+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.59:0/2141569774' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-04-28T19:42:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:06 smithi059 bash[23786]: audit 2022-04-28T19:42:06.079420+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.59:0/2141569774' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-04-28T19:42:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:06 smithi059 bash[29114]: audit 2022-04-28T19:42:06.079420+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.59:0/2141569774' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-04-28T19:42:06.773 INFO:tasks.cephadm.ceph_manager.ceph:need seq 176093659162 got 176093659167 for osd.6 2022-04-28T19:42:06.773 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.7 2022-04-28T19:42:06.915 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:07.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:07 smithi139 bash[23439]: cluster 2022-04-28T19:42:06.010137+0000 mgr.y (mgr.14140) 283 : cluster [DBG] pgmap v246: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:07 smithi059 bash[29114]: cluster 2022-04-28T19:42:06.010137+0000 mgr.y (mgr.14140) 283 : cluster [DBG] pgmap v246: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:07 smithi059 bash[23786]: cluster 2022-04-28T19:42:06.010137+0000 mgr.y (mgr.14140) 283 : cluster [DBG] pgmap v246: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:09.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:09 smithi139 bash[23439]: cluster 2022-04-28T19:42:08.010870+0000 mgr.y (mgr.14140) 284 : cluster [DBG] pgmap v247: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:09 smithi059 bash[29114]: cluster 2022-04-28T19:42:08.010870+0000 mgr.y (mgr.14140) 284 : cluster [DBG] pgmap v247: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:09 smithi059 bash[23786]: cluster 2022-04-28T19:42:08.010870+0000 mgr.y (mgr.14140) 284 : cluster [DBG] pgmap v247: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:10.117 INFO:teuthology.orchestra.run.smithi059.stdout:197568495643 2022-04-28T19:42:10.736 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495637 got 197568495643 for osd.7 2022-04-28T19:42:10.736 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2022-04-28T19:42:10.737 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph pg dump --format=json 2022-04-28T19:42:10.879 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:11.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:11 smithi139 bash[23439]: cluster 2022-04-28T19:42:10.011456+0000 mgr.y (mgr.14140) 285 : cluster [DBG] pgmap v248: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:11.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:11 smithi139 bash[23439]: audit 2022-04-28T19:42:10.116202+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.59:0/3333982933' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-04-28T19:42:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:11 smithi059 bash[23786]: cluster 2022-04-28T19:42:10.011456+0000 mgr.y (mgr.14140) 285 : cluster [DBG] pgmap v248: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:11 smithi059 bash[23786]: audit 2022-04-28T19:42:10.116202+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.59:0/3333982933' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-04-28T19:42:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:11 smithi059 bash[29114]: cluster 2022-04-28T19:42:10.011456+0000 mgr.y (mgr.14140) 285 : cluster [DBG] pgmap v248: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:11 smithi059 bash[29114]: audit 2022-04-28T19:42:10.116202+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.59:0/3333982933' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-04-28T19:42:13.510 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:13 smithi059 bash[29114]: cluster 2022-04-28T19:42:12.012096+0000 mgr.y (mgr.14140) 286 : cluster [DBG] pgmap v249: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:13.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:13 smithi059 bash[23786]: cluster 2022-04-28T19:42:12.012096+0000 mgr.y (mgr.14140) 286 : cluster [DBG] pgmap v249: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:13.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:13 smithi139 bash[23439]: cluster 2022-04-28T19:42:12.012096+0000 mgr.y (mgr.14140) 286 : cluster [DBG] pgmap v249: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:13.508Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:42:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:13.508Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:42:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:13.509Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:42:13.980 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:42:13.986 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2022-04-28T19:42:14.673 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":249,"stamp":"2022-04-28T19:42:12.011748+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8392512,"kb_used_data":3840,"kb_used_omap":0,"kb_used_meta":8388608,"kb_avail":741568704,"statfs":{"total":767960285184,"available":759366352896,"internally_reserved":8589934592,"allocated":3932160,"data_stored":1602744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8589934592},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.099290+0000","last_change":"2022-04-28T19:40:39.845995+0000","last_active":"2022-04-28T19:40:41.099290+0000","last_peered":"2022-04-28T19:40:41.099290+0000","last_clean":"2022-04-28T19:40:41.099290+0000","last_became_active":"2022-04-28T19:40:39.845574+0000","last_became_peered":"2022-04-28T19:40:39.845574+0000","last_unstale":"2022-04-28T19:40:41.099290+0000","last_undegraded":"2022-04-28T19:40:41.099290+0000","last_fullsized":"2022-04-28T19:40:41.099290+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.845534+0000","last_change":"2022-04-28T19:40:39.837533+0000","last_active":"2022-04-28T19:40:40.845534+0000","last_peered":"2022-04-28T19:40:40.845534+0000","last_clean":"2022-04-28T19:40:40.845534+0000","last_became_active":"2022-04-28T19:40:39.837213+0000","last_became_peered":"2022-04-28T19:40:39.837213+0000","last_unstale":"2022-04-28T19:40:40.845534+0000","last_undegraded":"2022-04-28T19:40:40.845534+0000","last_fullsized":"2022-04-28T19:40:40.845534+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.847880+0000","last_change":"2022-04-28T19:40:39.846944+0000","last_active":"2022-04-28T19:40:40.847880+0000","last_peered":"2022-04-28T19:40:40.847880+0000","last_clean":"2022-04-28T19:40:40.847880+0000","last_became_active":"2022-04-28T19:40:39.846597+0000","last_became_peered":"2022-04-28T19:40:39.846597+0000","last_unstale":"2022-04-28T19:40:40.847880+0000","last_undegraded":"2022-04-28T19:40:40.847880+0000","last_fullsized":"2022-04-28T19:40:40.847880+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.847504+0000","last_change":"2022-04-28T19:40:39.848985+0000","last_active":"2022-04-28T19:40:40.847504+0000","last_peered":"2022-04-28T19:40:40.847504+0000","last_clean":"2022-04-28T19:40:40.847504+0000","last_became_active":"2022-04-28T19:40:39.848622+0000","last_became_peered":"2022-04-28T19:40:39.848622+0000","last_unstale":"2022-04-28T19:40:40.847504+0000","last_undegraded":"2022-04-28T19:40:40.847504+0000","last_fullsized":"2022-04-28T19:40:40.847504+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":14,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.847501+0000","last_change":"2022-04-28T19:40:02.912608+0000","last_active":"2022-04-28T19:40:40.847501+0000","last_peered":"2022-04-28T19:40:40.847501+0000","last_clean":"2022-04-28T19:40:40.847501+0000","last_became_active":"2022-04-28T19:40:02.911686+0000","last_became_peered":"2022-04-28T19:40:02.911686+0000","last_unstale":"2022-04-28T19:40:40.847501+0000","last_undegraded":"2022-04-28T19:40:40.847501+0000","last_fullsized":"2022-04-28T19:40:40.847501+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":5,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_clean_scrub_stamp":"2022-04-28T19:36:10.237993+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.098636+0000","last_change":"2022-04-28T19:40:39.847019+0000","last_active":"2022-04-28T19:40:41.098636+0000","last_peered":"2022-04-28T19:40:41.098636+0000","last_clean":"2022-04-28T19:40:41.098636+0000","last_became_active":"2022-04-28T19:40:39.846557+0000","last_became_peered":"2022-04-28T19:40:39.846557+0000","last_unstale":"2022-04-28T19:40:41.098636+0000","last_undegraded":"2022-04-28T19:40:41.098636+0000","last_fullsized":"2022-04-28T19:40:41.098636+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.093515+0000","last_change":"2022-04-28T19:40:40.101205+0000","last_active":"2022-04-28T19:40:41.093515+0000","last_peered":"2022-04-28T19:40:41.093515+0000","last_clean":"2022-04-28T19:40:41.093515+0000","last_became_active":"2022-04-28T19:40:40.100696+0000","last_became_peered":"2022-04-28T19:40:40.100696+0000","last_unstale":"2022-04-28T19:40:41.093515+0000","last_undegraded":"2022-04-28T19:40:41.093515+0000","last_fullsized":"2022-04-28T19:40:41.093515+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.098634+0000","last_change":"2022-04-28T19:40:39.848860+0000","last_active":"2022-04-28T19:40:41.098634+0000","last_peered":"2022-04-28T19:40:41.098634+0000","last_clean":"2022-04-28T19:40:41.098634+0000","last_became_active":"2022-04-28T19:40:39.848587+0000","last_became_peered":"2022-04-28T19:40:39.848587+0000","last_unstale":"2022-04-28T19:40:41.098634+0000","last_undegraded":"2022-04-28T19:40:41.098634+0000","last_fullsized":"2022-04-28T19:40:41.098634+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.845456+0000","last_change":"2022-04-28T19:40:39.838356+0000","last_active":"2022-04-28T19:40:40.845456+0000","last_peered":"2022-04-28T19:40:40.845456+0000","last_clean":"2022-04-28T19:40:40.845456+0000","last_became_active":"2022-04-28T19:40:39.837847+0000","last_became_peered":"2022-04-28T19:40:39.837847+0000","last_unstale":"2022-04-28T19:40:40.845456+0000","last_undegraded":"2022-04-28T19:40:40.845456+0000","last_fullsized":"2022-04-28T19:40:40.845456+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":46,"seq":197568495644,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.81399999999999995,"15min":0.81799999999999995},"min":{"1min":0.52200000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.046,"5min":1.048,"15min":1.048},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.82599999999999996,"15min":0.83699999999999997},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.98999999999999999,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.78100000000000003}]},{"osd":1,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.84399999999999997,"15min":0.84599999999999997},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.1519999999999999,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":1.004},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.81999999999999995,"15min":0.81399999999999995},"min":{"1min":0.61699999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.0409999999999999,"5min":1.151,"15min":1.151},"last":0.93799999999999994}]},{"osd":2,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.82799999999999996,"15min":0.83099999999999996},"min":{"1min":0.623,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.94799999999999995,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.85099999999999998,"15min":0.84499999999999997},"min":{"1min":0.71699999999999997,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.107,"5min":1.145,"15min":1.145},"last":0.71199999999999997}]},{"osd":3,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.68799999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.03,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.85799999999999998,"15min":0.85399999999999998},"min":{"1min":0.66300000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.0900000000000001,"5min":1.21,"15min":1.21},"last":0.64600000000000002}]},{"osd":4,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.753,"15min":0.754},"min":{"1min":0.50600000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.101,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.80500000000000005,"15min":0.80900000000000005},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.0700000000000001,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.85799999999999998}]},{"osd":5,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.79700000000000004,"15min":0.79500000000000004},"min":{"1min":0.59899999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.1830000000000001,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.85899999999999999,"15min":0.86499999999999999},"min":{"1min":0.64900000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0209999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.88}]},{"osd":6,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.93200000000000005,"15min":0.93700000000000006},"min":{"1min":0.61299999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.131,"5min":1.409,"15min":1.409},"last":1.026},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.92000000000000004,"15min":0.92400000000000004},"min":{"1min":0.63700000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0700000000000001,"5min":1.252,"15min":1.252},"last":0.67600000000000005}]}]},{"osd":6,"up_from":41,"seq":176093659169,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.875,"15min":0.88200000000000001},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":1.22},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.88300000000000001,"15min":0.88900000000000001},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":1.165}]},{"osd":1,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.85899999999999999,"15min":0.85799999999999998},"min":{"1min":0.63100000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.872,"5min":0.92000000000000004,"15min":0.92800000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.0880000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":1.1359999999999999}]},{"osd":2,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.877,"15min":0.88500000000000001},"min":{"1min":0.61499999999999999,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.123,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":1.3200000000000001},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.91100000000000003,"15min":0.92500000000000004},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1060000000000001,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.073}]},{"osd":3,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.85699999999999998,"15min":0.84899999999999998},"min":{"1min":0.68300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.218,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.97999999999999998,"15min":0.996},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.123,"5min":1.401,"15min":1.401},"last":1.4059999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.82199999999999995,"15min":0.83599999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0640000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.3720000000000001},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.91300000000000003,"15min":0.92300000000000004},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.1279999999999999,"5min":1.5,"15min":1.5},"last":1.486}]},{"osd":5,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.878,"15min":0.88400000000000001},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.145,"5min":1.663,"15min":1.663},"last":1.2869999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.879,"15min":0.88600000000000001},"min":{"1min":0.54000000000000004,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.365,"5min":1.619,"15min":1.619},"last":1.01}]},{"osd":7,"last update":"Thu Apr 28 19:42:05 2022","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.74299999999999999,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":1.431},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.92100000000000004,"15min":0.90700000000000003},"min":{"1min":0.76200000000000001,"5min":0.495,"15min":0.495},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.2490000000000001}]}]},{"osd":1,"up_from":16,"seq":68719476793,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:53 2022","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59999999999999998,"15min":0.54700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0429999999999999,"5min":1.736,"15min":1.736},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65900000000000003,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.0169999999999999,"5min":1.544,"15min":1.544},"last":0.65800000000000003}]},{"osd":2,"last update":"Thu Apr 28 19:41:12 2022","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72799999999999998,"15min":0.71199999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.153,"5min":1.377,"15min":1.377},"last":0.747},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.67300000000000004,"15min":0.66400000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.59599999999999997}]},{"osd":3,"last update":"Thu Apr 28 19:41:36 2022","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.79000000000000004,"15min":0.78500000000000003},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.76600000000000001,"15min":0.76500000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.874}]},{"osd":4,"last update":"Thu Apr 28 19:41:59 2022","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.86199999999999999,"15min":0.83999999999999997},"min":{"1min":0.74099999999999999,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1299999999999999,"5min":1.446,"15min":1.446},"last":1.101},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.94299999999999995,"15min":0.90900000000000003},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.24,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":1.1539999999999999}]},{"osd":5,"last update":"Thu Apr 28 19:41:16 2022","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.002,"15min":1.002},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.4850000000000001,"5min":1.788,"15min":1.788},"last":1.0660000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.97399999999999998,"15min":0.97199999999999998},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":1.048}]},{"osd":6,"last update":"Thu Apr 28 19:41:42 2022","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.0609999999999999,"15min":1.083},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.3520000000000001,"5min":1.653,"15min":1.653},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.123,"15min":1.1479999999999999},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2909999999999999,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":0.91600000000000004}]},{"osd":7,"last update":"Thu Apr 28 19:42:04 2022","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.1060000000000001,"15min":1.119},"min":{"1min":0.70999999999999996,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.391,"5min":1.907,"15min":1.907},"last":0.94199999999999995},{"interface":"front","average":{"1min":1.093,"5min":1.1220000000000001,"15min":1.127},"min":{"1min":0.92900000000000005,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.268,"5min":1.673,"15min":1.673},"last":1.248}]}]},{"osd":0,"up_from":10,"seq":42949673021,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 28 19:41:51 2022","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.626,"15min":0.57399999999999995},"min":{"1min":0.47199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.349,"5min":1.829,"15min":1.829},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.81299999999999994,"15min":0.90900000000000003},"min":{"1min":0.47799999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.91000000000000003,"5min":9.4420000000000002,"15min":9.4420000000000002},"last":0.57299999999999995}]},{"osd":2,"last update":"Thu Apr 28 19:41:08 2022","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.626,"15min":0.59599999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.70599999999999996,"15min":0.68200000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.1100000000000001,"5min":1.1950000000000001,"15min":1.1950000000000001},"last":0.68799999999999994}]},{"osd":3,"last update":"Thu Apr 28 19:41:33 2022","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.76100000000000001,"15min":0.76300000000000001},"min":{"1min":0.41899999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0860000000000001,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80900000000000005,"15min":0.80600000000000005},"min":{"1min":0.52500000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.125,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.91000000000000003}]},{"osd":4,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.95699999999999996,"15min":0.96199999999999997},"min":{"1min":0.79400000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.367,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.081,"5min":0.98799999999999999,"15min":0.95899999999999996},"min":{"1min":0.82999999999999996,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":1.129}]},{"osd":5,"last update":"Thu Apr 28 19:41:16 2022","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.0269999999999999,"15min":1.0249999999999999},"min":{"1min":0.747,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.252,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.218},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.97499999999999998,"15min":0.98599999999999999},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.014}]},{"osd":6,"last update":"Thu Apr 28 19:41:44 2022","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.0069999999999999,"15min":0.999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.1890000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0029999999999999,"15min":0.99399999999999999},"min":{"1min":0.64900000000000002,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.264}]},{"osd":7,"last update":"Thu Apr 28 19:41:04 2022","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"min":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.284},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":1.292}]}]},{"osd":2,"up_from":21,"seq":90194313269,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:08 2022","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.64800000000000002,"15min":0.60599999999999998},"min":{"1min":0.48199999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.0089999999999999,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.68999999999999995,"15min":0.65800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.54500000000000004}]},{"osd":1,"last update":"Thu Apr 28 19:42:08 2022","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.67800000000000005,"15min":0.65800000000000003},"min":{"1min":0.439,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84899999999999998,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.754,"15min":0.69499999999999995},"min":{"1min":0.53900000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.099,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":1.099}]},{"osd":3,"last update":"Thu Apr 28 19:41:36 2022","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.81000000000000005,"15min":0.78300000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.278,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.74199999999999999,"15min":0.73199999999999998},"min":{"1min":0.44600000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.1599999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.70299999999999996}]},{"osd":4,"last update":"Thu Apr 28 19:41:54 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.876,"15min":0.86599999999999999},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.24,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.008,"15min":0.98899999999999999},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.3580000000000001,"5min":1.615,"15min":1.615},"last":1.0580000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:41:19 2022","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.107,"15min":1.113},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.4019999999999999,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.995},{"interface":"front","average":{"1min":1.056,"5min":1.0249999999999999,"15min":1.02},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.329,"5min":1.637,"15min":1.637},"last":0.75900000000000001}]},{"osd":6,"last update":"Thu Apr 28 19:41:44 2022","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.038,"15min":1.044},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":0.88600000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.2250000000000001,"15min":1.2470000000000001},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.4750000000000001,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":1.1459999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:42:08 2022","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.0760000000000001,"15min":1.0920000000000001},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3839999999999999,"5min":1.536,"15min":1.536},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.216,"15min":1.2310000000000001},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.5469999999999999,"5min":1.655,"15min":1.655},"last":1.1850000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149743,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:32 2022","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66200000000000003,"15min":0.65300000000000002},"min":{"1min":0.44600000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.78700000000000003,"15min":0.78100000000000003},"min":{"1min":0.52800000000000002,"5min":0.443,"15min":0.443},"max":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":0.84799999999999998}]},{"osd":1,"last update":"Thu Apr 28 19:41:32 2022","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.72499999999999998,"15min":0.71299999999999997},"min":{"1min":0.46899999999999997,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.2970000000000002,"5min":2.2970000000000002,"15min":2.2970000000000002},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.71599999999999997,"15min":0.69199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.63500000000000001}]},{"osd":2,"last update":"Thu Apr 28 19:41:32 2022","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.78500000000000003,"15min":0.78300000000000003},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.0800000000000001,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.73299999999999998,"15min":0.71399999999999997},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":1.3049999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:41:59 2022","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92400000000000004,"15min":0.90600000000000003},"min":{"1min":0.69799999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.3300000000000001,"5min":2.415,"15min":2.415},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.91400000000000003,"15min":0.90200000000000002},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.1919999999999999,"5min":1.363,"15min":1.363},"last":1.1080000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.94799999999999995,"15min":0.94099999999999995},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":1.472},{"interface":"front","average":{"1min":1.091,"5min":1.0349999999999999,"15min":1.0249999999999999},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":2.5590000000000002,"5min":2.5590000000000002,"15min":2.5590000000000002},"last":1.292}]},{"osd":6,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.0149999999999999,"15min":1.0109999999999999},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":1.1899999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.0569999999999999,"15min":1.0589999999999999},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.5600000000000001}]},{"osd":7,"last update":"Thu Apr 28 19:42:07 2022","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.101,"15min":1.103},"min":{"1min":0.76800000000000002,"5min":0.747,"15min":0.747},"max":{"1min":1.329,"5min":1.538,"15min":1.538},"last":1.6339999999999999},{"interface":"front","average":{"1min":1.103,"5min":1.1479999999999999,"15min":1.1559999999999999},"min":{"1min":0.88400000000000001,"5min":0.83499999999999996,"15min":0.83499999999999996},"max":{"1min":1.4950000000000001,"5min":2.4900000000000002,"15min":2.4900000000000002},"last":1.347}]}]},{"osd":4,"up_from":31,"seq":133143986219,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.82899999999999996,"15min":0.82299999999999995},"min":{"1min":0.58199999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.0640000000000001,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":1.006},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.82599999999999996,"15min":0.81799999999999995},"min":{"1min":0.56299999999999994,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.196,"5min":1.456,"15min":1.456},"last":0.83799999999999997}]},{"osd":1,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.83299999999999996,"15min":0.81499999999999995},"min":{"1min":0.51000000000000001,"5min":0.44,"15min":0.44},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.79500000000000004,"15min":0.79900000000000004},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.96899999999999997}]},{"osd":2,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.82399999999999995,"15min":0.82199999999999995},"min":{"1min":0.66100000000000003,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.042,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.85899999999999999,"15min":0.871},"min":{"1min":0.69499999999999995,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.0489999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.75800000000000001}]},{"osd":3,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.83599999999999997,"15min":0.83699999999999997},"min":{"1min":0.64800000000000002,"5min":0.495,"15min":0.495},"max":{"1min":1.111,"5min":1.601,"15min":1.601},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.875,"15min":0.873},"min":{"1min":0.67700000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":0.91200000000000003}]},{"osd":5,"last update":"Thu Apr 28 19:41:19 2022","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.75700000000000001,"15min":0.746},"min":{"1min":0.52700000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.087,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.79500000000000004,"15min":0.77500000000000002},"min":{"1min":0.61799999999999999,"5min":0.436,"15min":0.436},"max":{"1min":1.399,"5min":1.399,"15min":1.399},"last":0.94999999999999996}]},{"osd":6,"last update":"Thu Apr 28 19:41:46 2022","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.159,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.872,"5min":0.89900000000000002,"15min":0.90300000000000002},"min":{"1min":0.60299999999999998,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.1100000000000001,"5min":1.427,"15min":1.427},"last":1.032}]},{"osd":7,"last update":"Thu Apr 28 19:42:03 2022","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90200000000000002,"15min":0.89600000000000002},"min":{"1min":0.58299999999999996,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.141,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.93400000000000005,"15min":0.92500000000000004},"min":{"1min":0.66800000000000004,"5min":0.498,"15min":0.498},"max":{"1min":1.155,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.1279999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822693,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.80600000000000005,"15min":0.80900000000000005},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.105,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.83299999999999996,"15min":0.82799999999999996},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.88400000000000001}]},{"osd":1,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.80100000000000005,"15min":0.80400000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0069999999999999,"5min":1.034,"15min":1.034},"last":1.171},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.8300000000000001,"5min":1.8300000000000001,"15min":1.8300000000000001},"last":0.71599999999999997}]},{"osd":2,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.79100000000000004,"15min":0.78800000000000003},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.88100000000000001,"15min":0.89300000000000002},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":1.085}]},{"osd":3,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.85499999999999998,"15min":0.85199999999999998},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.84399999999999997,"15min":0.83999999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":1.0429999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.73799999999999999,"15min":0.73399999999999999},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.103,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73999999999999999,"15min":0.74299999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.84499999999999997}]},{"osd":6,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.85499999999999998,"15min":0.86299999999999999},"min":{"1min":0.51800000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1240000000000001,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":1.6220000000000001},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.83899999999999997,"15min":0.82499999999999996},"min":{"1min":0.59599999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.887,"5min":1.887,"15min":1.887},"last":1.5629999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:42:05 2022","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.92700000000000005,"15min":0.93100000000000005},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.2390000000000001,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":0.96799999999999997},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.82899999999999996,"15min":0.82499999999999996},"min":{"1min":0.63400000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0389999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.90900000000000003}]}]}]}} 2022-04-28T19:42:14.682 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph pg dump --format=json 2022-04-28T19:42:14.826 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:15 smithi059 bash[23786]: audit 2022-04-28T19:42:13.970191+0000 mgr.y (mgr.14140) 287 : audit [DBG] from='client.24505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:42:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:15 smithi059 bash[23786]: cluster 2022-04-28T19:42:14.012673+0000 mgr.y (mgr.14140) 288 : cluster [DBG] pgmap v250: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:15 smithi059 bash[29114]: audit 2022-04-28T19:42:13.970191+0000 mgr.y (mgr.14140) 287 : audit [DBG] from='client.24505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:42:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:15 smithi059 bash[29114]: cluster 2022-04-28T19:42:14.012673+0000 mgr.y (mgr.14140) 288 : cluster [DBG] pgmap v250: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:15.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:15 smithi139 bash[23439]: audit 2022-04-28T19:42:13.970191+0000 mgr.y (mgr.14140) 287 : audit [DBG] from='client.24505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:42:15.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:15 smithi139 bash[23439]: cluster 2022-04-28T19:42:14.012673+0000 mgr.y (mgr.14140) 288 : cluster [DBG] pgmap v250: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:16 smithi059 bash[23786]: audit 2022-04-28T19:42:16.121793+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:42:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:16 smithi059 bash[23786]: audit 2022-04-28T19:42:16.128106+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:42:16.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:16 smithi059 bash[29114]: audit 2022-04-28T19:42:16.121793+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:42:16.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:16 smithi059 bash[29114]: audit 2022-04-28T19:42:16.128106+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:42:16.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:16 smithi139 bash[23439]: audit 2022-04-28T19:42:16.121793+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:42:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:16 smithi139 bash[23439]: audit 2022-04-28T19:42:16.128106+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:42:17.183 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:17 smithi059 bash[23786]: cluster 2022-04-28T19:42:16.013356+0000 mgr.y (mgr.14140) 289 : cluster [DBG] pgmap v251: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:17.184 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:17 smithi059 bash[29114]: cluster 2022-04-28T19:42:16.013356+0000 mgr.y (mgr.14140) 289 : cluster [DBG] pgmap v251: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:17 smithi139 bash[23439]: cluster 2022-04-28T19:42:16.013356+0000 mgr.y (mgr.14140) 289 : cluster [DBG] pgmap v251: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:17.812 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:42:17.814 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2022-04-28T19:42:18.470 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":251,"stamp":"2022-04-28T19:42:16.012991+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8392512,"kb_used_data":3840,"kb_used_omap":0,"kb_used_meta":8388608,"kb_avail":741568704,"statfs":{"total":767960285184,"available":759366352896,"internally_reserved":8589934592,"allocated":3932160,"data_stored":1602744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8589934592},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.099290+0000","last_change":"2022-04-28T19:40:39.845995+0000","last_active":"2022-04-28T19:40:41.099290+0000","last_peered":"2022-04-28T19:40:41.099290+0000","last_clean":"2022-04-28T19:40:41.099290+0000","last_became_active":"2022-04-28T19:40:39.845574+0000","last_became_peered":"2022-04-28T19:40:39.845574+0000","last_unstale":"2022-04-28T19:40:41.099290+0000","last_undegraded":"2022-04-28T19:40:41.099290+0000","last_fullsized":"2022-04-28T19:40:41.099290+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.845534+0000","last_change":"2022-04-28T19:40:39.837533+0000","last_active":"2022-04-28T19:40:40.845534+0000","last_peered":"2022-04-28T19:40:40.845534+0000","last_clean":"2022-04-28T19:40:40.845534+0000","last_became_active":"2022-04-28T19:40:39.837213+0000","last_became_peered":"2022-04-28T19:40:39.837213+0000","last_unstale":"2022-04-28T19:40:40.845534+0000","last_undegraded":"2022-04-28T19:40:40.845534+0000","last_fullsized":"2022-04-28T19:40:40.845534+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.847880+0000","last_change":"2022-04-28T19:40:39.846944+0000","last_active":"2022-04-28T19:40:40.847880+0000","last_peered":"2022-04-28T19:40:40.847880+0000","last_clean":"2022-04-28T19:40:40.847880+0000","last_became_active":"2022-04-28T19:40:39.846597+0000","last_became_peered":"2022-04-28T19:40:39.846597+0000","last_unstale":"2022-04-28T19:40:40.847880+0000","last_undegraded":"2022-04-28T19:40:40.847880+0000","last_fullsized":"2022-04-28T19:40:40.847880+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.847504+0000","last_change":"2022-04-28T19:40:39.848985+0000","last_active":"2022-04-28T19:40:40.847504+0000","last_peered":"2022-04-28T19:40:40.847504+0000","last_clean":"2022-04-28T19:40:40.847504+0000","last_became_active":"2022-04-28T19:40:39.848622+0000","last_became_peered":"2022-04-28T19:40:39.848622+0000","last_unstale":"2022-04-28T19:40:40.847504+0000","last_undegraded":"2022-04-28T19:40:40.847504+0000","last_fullsized":"2022-04-28T19:40:40.847504+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":14,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.847501+0000","last_change":"2022-04-28T19:40:02.912608+0000","last_active":"2022-04-28T19:40:40.847501+0000","last_peered":"2022-04-28T19:40:40.847501+0000","last_clean":"2022-04-28T19:40:40.847501+0000","last_became_active":"2022-04-28T19:40:02.911686+0000","last_became_peered":"2022-04-28T19:40:02.911686+0000","last_unstale":"2022-04-28T19:40:40.847501+0000","last_undegraded":"2022-04-28T19:40:40.847501+0000","last_fullsized":"2022-04-28T19:40:40.847501+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":5,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_clean_scrub_stamp":"2022-04-28T19:36:10.237993+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.098636+0000","last_change":"2022-04-28T19:40:39.847019+0000","last_active":"2022-04-28T19:40:41.098636+0000","last_peered":"2022-04-28T19:40:41.098636+0000","last_clean":"2022-04-28T19:40:41.098636+0000","last_became_active":"2022-04-28T19:40:39.846557+0000","last_became_peered":"2022-04-28T19:40:39.846557+0000","last_unstale":"2022-04-28T19:40:41.098636+0000","last_undegraded":"2022-04-28T19:40:41.098636+0000","last_fullsized":"2022-04-28T19:40:41.098636+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.093515+0000","last_change":"2022-04-28T19:40:40.101205+0000","last_active":"2022-04-28T19:40:41.093515+0000","last_peered":"2022-04-28T19:40:41.093515+0000","last_clean":"2022-04-28T19:40:41.093515+0000","last_became_active":"2022-04-28T19:40:40.100696+0000","last_became_peered":"2022-04-28T19:40:40.100696+0000","last_unstale":"2022-04-28T19:40:41.093515+0000","last_undegraded":"2022-04-28T19:40:41.093515+0000","last_fullsized":"2022-04-28T19:40:41.093515+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:41.098634+0000","last_change":"2022-04-28T19:40:39.848860+0000","last_active":"2022-04-28T19:40:41.098634+0000","last_peered":"2022-04-28T19:40:41.098634+0000","last_clean":"2022-04-28T19:40:41.098634+0000","last_became_active":"2022-04-28T19:40:39.848587+0000","last_became_peered":"2022-04-28T19:40:39.848587+0000","last_unstale":"2022-04-28T19:40:41.098634+0000","last_undegraded":"2022-04-28T19:40:41.098634+0000","last_fullsized":"2022-04-28T19:40:41.098634+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":50,"state":"active+clean","last_fresh":"2022-04-28T19:40:40.845456+0000","last_change":"2022-04-28T19:40:39.838356+0000","last_active":"2022-04-28T19:40:40.845456+0000","last_peered":"2022-04-28T19:40:40.845456+0000","last_clean":"2022-04-28T19:40:40.845456+0000","last_became_active":"2022-04-28T19:40:39.837847+0000","last_became_peered":"2022-04-28T19:40:39.837847+0000","last_unstale":"2022-04-28T19:40:40.845456+0000","last_undegraded":"2022-04-28T19:40:40.845456+0000","last_fullsized":"2022-04-28T19:40:40.845456+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":46,"seq":197568495644,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.81399999999999995,"15min":0.81799999999999995},"min":{"1min":0.52200000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.046,"5min":1.048,"15min":1.048},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.82599999999999996,"15min":0.83699999999999997},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.98999999999999999,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.78100000000000003}]},{"osd":1,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.84399999999999997,"15min":0.84599999999999997},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.1519999999999999,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":1.004},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.81999999999999995,"15min":0.81399999999999995},"min":{"1min":0.61699999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.0409999999999999,"5min":1.151,"15min":1.151},"last":0.93799999999999994}]},{"osd":2,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.82799999999999996,"15min":0.83099999999999996},"min":{"1min":0.623,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.94799999999999995,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.85099999999999998,"15min":0.84499999999999997},"min":{"1min":0.71699999999999997,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.107,"5min":1.145,"15min":1.145},"last":0.71199999999999997}]},{"osd":3,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.68799999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.03,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.85799999999999998,"15min":0.85399999999999998},"min":{"1min":0.66300000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.0900000000000001,"5min":1.21,"15min":1.21},"last":0.64600000000000002}]},{"osd":4,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.753,"15min":0.754},"min":{"1min":0.50600000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.101,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.80500000000000005,"15min":0.80900000000000005},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.0700000000000001,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.85799999999999998}]},{"osd":5,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.79700000000000004,"15min":0.79500000000000004},"min":{"1min":0.59899999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.1830000000000001,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.85899999999999999,"15min":0.86499999999999999},"min":{"1min":0.64900000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.0209999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.88}]},{"osd":6,"last update":"Thu Apr 28 19:42:02 2022","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":0.93200000000000005,"15min":0.93700000000000006},"min":{"1min":0.61299999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.131,"5min":1.409,"15min":1.409},"last":1.026},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.92000000000000004,"15min":0.92400000000000004},"min":{"1min":0.63700000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0700000000000001,"5min":1.252,"15min":1.252},"last":0.67600000000000005}]}]},{"osd":6,"up_from":41,"seq":176093659170,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.875,"15min":0.88200000000000001},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.88300000000000001,"15min":0.88900000000000001},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.2470000000000001,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.95299999999999996}]},{"osd":1,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.85899999999999999,"15min":0.85799999999999998},"min":{"1min":0.63100000000000001,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.872,"5min":0.92000000000000004,"15min":0.92800000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.0880000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.873}]},{"osd":2,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.877,"15min":0.88500000000000001},"min":{"1min":0.61499999999999999,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.123,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.91100000000000003,"15min":0.92500000000000004},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1060000000000001,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.75700000000000001}]},{"osd":3,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.85699999999999998,"15min":0.84899999999999998},"min":{"1min":0.68300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.218,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.97999999999999998,"15min":0.996},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.123,"5min":1.401,"15min":1.401},"last":0.92800000000000005}]},{"osd":4,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.82199999999999995,"15min":0.83599999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0640000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.91300000000000003,"15min":0.92300000000000004},"min":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.1279999999999999,"5min":1.5,"15min":1.5},"last":0.76900000000000002}]},{"osd":5,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.878,"15min":0.88400000000000001},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.145,"5min":1.663,"15min":1.663},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.879,"15min":0.88600000000000001},"min":{"1min":0.54000000000000004,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.365,"5min":1.619,"15min":1.619},"last":1.077}]},{"osd":7,"last update":"Thu Apr 28 19:42:05 2022","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.74299999999999999,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.431,"5min":1.431,"15min":1.431},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.92100000000000004,"15min":0.90700000000000003},"min":{"1min":0.76200000000000001,"5min":0.495,"15min":0.495},"max":{"1min":1.3180000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.135}]}]},{"osd":1,"up_from":16,"seq":68719476794,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:53 2022","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59999999999999998,"15min":0.54700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0429999999999999,"5min":1.736,"15min":1.736},"last":0.749},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65900000000000003,"15min":0.57699999999999996},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.0169999999999999,"5min":1.544,"15min":1.544},"last":0.98299999999999998}]},{"osd":2,"last update":"Thu Apr 28 19:41:12 2022","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72799999999999998,"15min":0.71199999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.153,"5min":1.377,"15min":1.377},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.67300000000000004,"15min":0.66400000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.84399999999999997}]},{"osd":3,"last update":"Thu Apr 28 19:41:36 2022","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.79000000000000004,"15min":0.78500000000000003},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.76600000000000001,"15min":0.76500000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.012,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.58099999999999996}]},{"osd":4,"last update":"Thu Apr 28 19:41:59 2022","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.86199999999999999,"15min":0.83999999999999997},"min":{"1min":0.74099999999999999,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1299999999999999,"5min":1.446,"15min":1.446},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.94299999999999995,"15min":0.90900000000000003},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.24,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.90000000000000002}]},{"osd":5,"last update":"Thu Apr 28 19:41:16 2022","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.002,"15min":1.002},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.4850000000000001,"5min":1.788,"15min":1.788},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.97399999999999998,"15min":0.97199999999999998},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":0.93700000000000006}]},{"osd":6,"last update":"Thu Apr 28 19:41:42 2022","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.0609999999999999,"15min":1.083},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.3520000000000001,"5min":1.653,"15min":1.653},"last":1.2},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.123,"15min":1.1479999999999999},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2909999999999999,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":1.056}]},{"osd":7,"last update":"Thu Apr 28 19:42:04 2022","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.1060000000000001,"15min":1.119},"min":{"1min":0.70999999999999996,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.391,"5min":1.907,"15min":1.907},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.1220000000000001,"15min":1.127},"min":{"1min":0.92900000000000005,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.268,"5min":1.673,"15min":1.673},"last":1.27}]}]},{"osd":0,"up_from":10,"seq":42949673022,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 28 19:41:51 2022","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.626,"15min":0.57399999999999995},"min":{"1min":0.47199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.349,"5min":1.829,"15min":1.829},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.81299999999999994,"15min":0.90900000000000003},"min":{"1min":0.47799999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.91000000000000003,"5min":9.4420000000000002,"15min":9.4420000000000002},"last":0.45800000000000002}]},{"osd":2,"last update":"Thu Apr 28 19:42:09 2022","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.64900000000000002,"15min":0.60399999999999998},"min":{"1min":0.48999999999999999,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.998,"5min":1.2,"15min":1.2},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.73399999999999999,"15min":0.69099999999999995},"min":{"1min":0.52600000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.69499999999999995}]},{"osd":3,"last update":"Thu Apr 28 19:41:33 2022","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.76100000000000001,"15min":0.76300000000000001},"min":{"1min":0.41899999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0860000000000001,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.442},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80900000000000005,"15min":0.80600000000000005},"min":{"1min":0.52500000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.125,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.75900000000000001}]},{"osd":4,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.95699999999999996,"15min":0.96199999999999997},"min":{"1min":0.79400000000000004,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.367,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.88},{"interface":"front","average":{"1min":1.081,"5min":0.98799999999999999,"15min":0.95899999999999996},"min":{"1min":0.82999999999999996,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":1.069}]},{"osd":5,"last update":"Thu Apr 28 19:41:16 2022","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.0269999999999999,"15min":1.0249999999999999},"min":{"1min":0.747,"5min":0.71299999999999997,"15min":0.71299999999999997},"max":{"1min":1.252,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.90700000000000003,"5min":0.97499999999999998,"15min":0.98599999999999999},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":0.77700000000000002}]},{"osd":6,"last update":"Thu Apr 28 19:41:44 2022","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.0069999999999999,"15min":0.999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.157},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.0029999999999999,"15min":0.99399999999999999},"min":{"1min":0.64900000000000002,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.93600000000000005}]},{"osd":7,"last update":"Thu Apr 28 19:42:09 2022","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.1339999999999999,"15min":1.1319999999999999},"min":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"max":{"1min":1.47,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.0469999999999999,"15min":1.0349999999999999},"min":{"1min":0.89000000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.3089999999999999,"5min":1.504,"15min":1.504},"last":0.98399999999999999}]}]},{"osd":2,"up_from":21,"seq":90194313270,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:08 2022","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.64800000000000002,"15min":0.60599999999999998},"min":{"1min":0.48199999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.0089999999999999,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.68999999999999995,"15min":0.65800000000000003},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.65200000000000002}]},{"osd":1,"last update":"Thu Apr 28 19:42:08 2022","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.67800000000000005,"15min":0.65800000000000003},"min":{"1min":0.439,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84899999999999998,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.754,"15min":0.69499999999999995},"min":{"1min":0.53900000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.099,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.70899999999999996}]},{"osd":3,"last update":"Thu Apr 28 19:41:36 2022","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.81000000000000005,"15min":0.78300000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.278,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":1.008},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.74199999999999999,"15min":0.73199999999999998},"min":{"1min":0.44600000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.1599999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.746}]},{"osd":4,"last update":"Thu Apr 28 19:41:54 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.876,"15min":0.86599999999999999},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.24,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.268},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.008,"15min":0.98899999999999999},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.3580000000000001,"5min":1.615,"15min":1.615},"last":1.05}]},{"osd":5,"last update":"Thu Apr 28 19:41:19 2022","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.107,"15min":1.113},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.4019999999999999,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.097},{"interface":"front","average":{"1min":1.056,"5min":1.0249999999999999,"15min":1.02},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.329,"5min":1.637,"15min":1.637},"last":1.29}]},{"osd":6,"last update":"Thu Apr 28 19:41:44 2022","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.038,"15min":1.044},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":1.4159999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.2250000000000001,"15min":1.2470000000000001},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.4750000000000001,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":1.329}]},{"osd":7,"last update":"Thu Apr 28 19:42:08 2022","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.0760000000000001,"15min":1.0920000000000001},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.3839999999999999,"5min":1.536,"15min":1.536},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.216,"15min":1.2310000000000001},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.5469999999999999,"5min":1.655,"15min":1.655},"last":1.3899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149744,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:32 2022","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.66200000000000003,"15min":0.65300000000000002},"min":{"1min":0.44600000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.78700000000000003,"15min":0.78100000000000003},"min":{"1min":0.52800000000000002,"5min":0.443,"15min":0.443},"max":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"last":0.83599999999999997}]},{"osd":1,"last update":"Thu Apr 28 19:41:32 2022","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.72499999999999998,"15min":0.71299999999999997},"min":{"1min":0.46899999999999997,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.2970000000000002,"5min":2.2970000000000002,"15min":2.2970000000000002},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.71599999999999997,"15min":0.69199999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.61499999999999999}]},{"osd":2,"last update":"Thu Apr 28 19:41:32 2022","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.78500000000000003,"15min":0.78300000000000003},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.0800000000000001,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.73299999999999998,"15min":0.71399999999999997},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"last":0.95699999999999996}]},{"osd":4,"last update":"Thu Apr 28 19:41:59 2022","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92400000000000004,"15min":0.90600000000000003},"min":{"1min":0.69799999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.3300000000000001,"5min":2.415,"15min":2.415},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.91400000000000003,"15min":0.90200000000000002},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.1919999999999999,"5min":1.363,"15min":1.363},"last":0.81399999999999995}]},{"osd":5,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.94799999999999995,"15min":0.94099999999999995},"min":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.0349999999999999,"15min":1.0249999999999999},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":2.5590000000000002,"5min":2.5590000000000002,"15min":2.5590000000000002},"last":0.93799999999999994}]},{"osd":6,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.0149999999999999,"15min":1.0109999999999999},"min":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.0569999999999999,"15min":1.0589999999999999},"min":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.1699999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:42:07 2022","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.101,"15min":1.103},"min":{"1min":0.76800000000000002,"5min":0.747,"15min":0.747},"max":{"1min":1.329,"5min":1.538,"15min":1.538},"last":1.111},{"interface":"front","average":{"1min":1.103,"5min":1.1479999999999999,"15min":1.1559999999999999},"min":{"1min":0.88400000000000001,"5min":0.83499999999999996,"15min":0.83499999999999996},"max":{"1min":1.4950000000000001,"5min":2.4900000000000002,"15min":2.4900000000000002},"last":1.1000000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986220,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.82899999999999996,"15min":0.82299999999999995},"min":{"1min":0.58199999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.0640000000000001,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":1.02},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.82599999999999996,"15min":0.81799999999999995},"min":{"1min":0.56299999999999994,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.196,"5min":1.456,"15min":1.456},"last":0.78500000000000003}]},{"osd":1,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.83299999999999996,"15min":0.81499999999999995},"min":{"1min":0.51000000000000001,"5min":0.44,"15min":0.44},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.79500000000000004,"15min":0.79900000000000004},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.1759999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.91000000000000003}]},{"osd":2,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.82399999999999995,"15min":0.82199999999999995},"min":{"1min":0.66100000000000003,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.042,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.85899999999999999,"15min":0.871},"min":{"1min":0.69499999999999995,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.0489999999999999,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.92400000000000004}]},{"osd":3,"last update":"Thu Apr 28 19:41:56 2022","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.83599999999999997,"15min":0.83699999999999997},"min":{"1min":0.64800000000000002,"5min":0.495,"15min":0.495},"max":{"1min":1.111,"5min":1.601,"15min":1.601},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.875,"15min":0.873},"min":{"1min":0.67700000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":1.1020000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:41:19 2022","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.75700000000000001,"15min":0.746},"min":{"1min":0.52700000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.087,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.79500000000000004,"15min":0.77500000000000002},"min":{"1min":0.61799999999999999,"5min":0.436,"15min":0.436},"max":{"1min":1.399,"5min":1.399,"15min":1.399},"last":1.0029999999999999}]},{"osd":6,"last update":"Thu Apr 28 19:41:46 2022","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.159,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.872,"5min":0.89900000000000002,"15min":0.90300000000000002},"min":{"1min":0.60299999999999998,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.1100000000000001,"5min":1.427,"15min":1.427},"last":0.98199999999999998}]},{"osd":7,"last update":"Thu Apr 28 19:42:03 2022","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.90200000000000002,"15min":0.89600000000000002},"min":{"1min":0.58299999999999996,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.141,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.98599999999999999,"5min":0.93400000000000005,"15min":0.92500000000000004},"min":{"1min":0.66800000000000004,"5min":0.498,"15min":0.498},"max":{"1min":1.155,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.0649999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822694,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049064,"kb_used_data":480,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696088,"statfs":{"total":95995035648,"available":94920794112,"internally_reserved":1073741824,"allocated":491520,"data_stored":200343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":0.80600000000000005,"15min":0.80900000000000005},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.105,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.83299999999999996,"15min":0.82799999999999996},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.2649999999999999,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.97399999999999998}]},{"osd":1,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.80100000000000005,"15min":0.80400000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0069999999999999,"5min":1.034,"15min":1.034},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.8300000000000001,"5min":1.8300000000000001,"15min":1.8300000000000001},"last":0.71599999999999997}]},{"osd":2,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.79100000000000004,"15min":0.78800000000000003},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.88100000000000001,"15min":0.89300000000000002},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":0.81699999999999995}]},{"osd":3,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.85499999999999998,"15min":0.85199999999999998},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":0.876},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.84399999999999997,"15min":0.83999999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":1.018}]},{"osd":4,"last update":"Thu Apr 28 19:41:17 2022","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.73799999999999999,"15min":0.73399999999999999},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.103,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.754},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73999999999999999,"15min":0.74299999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.93799999999999994}]},{"osd":6,"last update":"Thu Apr 28 19:41:43 2022","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.85499999999999998,"15min":0.86299999999999999},"min":{"1min":0.51800000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1240000000000001,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.83899999999999997,"15min":0.82499999999999996},"min":{"1min":0.59599999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.887,"5min":1.887,"15min":1.887},"last":0.80100000000000005}]},{"osd":7,"last update":"Thu Apr 28 19:42:05 2022","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.92700000000000005,"15min":0.93100000000000005},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.2390000000000001,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":0.99199999999999999},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.82899999999999996,"15min":0.82499999999999996},"min":{"1min":0.63400000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0389999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.77700000000000002}]}]}]}} 2022-04-28T19:42:18.479 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2022-04-28T19:42:18.479 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2022-04-28T19:42:18.479 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2022-04-28T19:42:18.480 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph health --format=json 2022-04-28T19:42:18.626 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:19.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:19 smithi059 bash[29114]: audit 2022-04-28T19:42:17.806770+0000 mgr.y (mgr.14140) 290 : audit [DBG] from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:42:19.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:19 smithi059 bash[29114]: cluster 2022-04-28T19:42:18.013976+0000 mgr.y (mgr.14140) 291 : cluster [DBG] pgmap v252: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:19.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:19 smithi059 bash[23786]: audit 2022-04-28T19:42:17.806770+0000 mgr.y (mgr.14140) 290 : audit [DBG] from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:42:19.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:19 smithi059 bash[23786]: cluster 2022-04-28T19:42:18.013976+0000 mgr.y (mgr.14140) 291 : cluster [DBG] pgmap v252: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:19.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:19 smithi139 bash[23439]: audit 2022-04-28T19:42:17.806770+0000 mgr.y (mgr.14140) 290 : audit [DBG] from='client.24511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:42:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:19 smithi139 bash[23439]: cluster 2022-04-28T19:42:18.013976+0000 mgr.y (mgr.14140) 291 : cluster [DBG] pgmap v252: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:21 smithi059 bash[29114]: cluster 2022-04-28T19:42:20.014558+0000 mgr.y (mgr.14140) 292 : cluster [DBG] pgmap v253: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:21 smithi059 bash[23786]: cluster 2022-04-28T19:42:20.014558+0000 mgr.y (mgr.14140) 292 : cluster [DBG] pgmap v253: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:21 smithi139 bash[23439]: cluster 2022-04-28T19:42:20.014558+0000 mgr.y (mgr.14140) 292 : cluster [DBG] pgmap v253: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:21.620 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:42:22.317 INFO:teuthology.orchestra.run.smithi059.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2022-04-28T19:42:22.317 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2022-04-28T19:42:22.317 INFO:tasks.cephadm:Setup complete, yielding 2022-04-28T19:42:22.318 INFO:teuthology.run_tasks:Running task print... 2022-04-28T19:42:22.330 INFO:teuthology.task.print:**** done end installing octopus cephadm ... 2022-04-28T19:42:22.330 INFO:teuthology.run_tasks:Running task cephadm.shell... 2022-04-28T19:42:22.388 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi059.front.sepia.ceph.com 2022-04-28T19:42:22.389 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- bash -c 'ceph fs volume create foo' 2022-04-28T19:42:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:22 smithi059 bash[29114]: audit 2022-04-28T19:42:21.618375+0000 mon.c (mon.1) 23 : audit [DBG] from='client.? 172.21.15.59:0/2052347966' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-04-28T19:42:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:22 smithi059 bash[23786]: audit 2022-04-28T19:42:21.618375+0000 mon.c (mon.1) 23 : audit [DBG] from='client.? 172.21.15.59:0/2052347966' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-04-28T19:42:22.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:22 smithi139 bash[23439]: audit 2022-04-28T19:42:21.618375+0000 mon.c (mon.1) 23 : audit [DBG] from='client.? 172.21.15.59:0/2052347966' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-04-28T19:42:23.511 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:23 smithi059 bash[23786]: cluster 2022-04-28T19:42:22.015203+0000 mgr.y (mgr.14140) 293 : cluster [DBG] pgmap v254: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:23.512 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:23 smithi059 bash[29114]: cluster 2022-04-28T19:42:22.015203+0000 mgr.y (mgr.14140) 293 : cluster [DBG] pgmap v254: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:23.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:23 smithi139 bash[23439]: cluster 2022-04-28T19:42:22.015203+0000 mgr.y (mgr.14140) 293 : cluster [DBG] pgmap v254: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:23.509Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:42:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:23.509Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:42:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:23.509Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:42:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:25 smithi059 bash[29114]: cluster 2022-04-28T19:42:24.015871+0000 mgr.y (mgr.14140) 294 : cluster [DBG] pgmap v255: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:25 smithi059 bash[23786]: cluster 2022-04-28T19:42:24.015871+0000 mgr.y (mgr.14140) 294 : cluster [DBG] pgmap v255: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:25.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:25 smithi139 bash[23439]: cluster 2022-04-28T19:42:24.015871+0000 mgr.y (mgr.14140) 294 : cluster [DBG] pgmap v255: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:26 smithi059 bash[29114]: audit 2022-04-28T19:42:25.598014+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2022-04-28T19:42:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:26 smithi059 bash[23786]: audit 2022-04-28T19:42:25.598014+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2022-04-28T19:42:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:26 smithi139 bash[23439]: audit 2022-04-28T19:42:25.598014+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2022-04-28T19:42:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[29114]: audit 2022-04-28T19:42:25.597121+0000 mgr.y (mgr.14140) 295 : audit [DBG] from='client.24523 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:42:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[29114]: cluster 2022-04-28T19:42:26.016456+0000 mgr.y (mgr.14140) 296 : cluster [DBG] pgmap v256: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[29114]: audit 2022-04-28T19:42:26.220325+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2022-04-28T19:42:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[29114]: cluster 2022-04-28T19:42:26.220380+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[29114]: audit 2022-04-28T19:42:26.222689+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[23786]: audit 2022-04-28T19:42:25.597121+0000 mgr.y (mgr.14140) 295 : audit [DBG] from='client.24523 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[23786]: cluster 2022-04-28T19:42:26.016456+0000 mgr.y (mgr.14140) 296 : cluster [DBG] pgmap v256: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[23786]: audit 2022-04-28T19:42:26.220325+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[23786]: cluster 2022-04-28T19:42:26.220380+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[23786]: audit 2022-04-28T19:42:26.222689+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2022-04-28T19:42:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:27 smithi059 bash[23786]: debug 2022-04-28T19:42:27.241+0000 7f64208bd700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2022-04-28T19:42:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:27 smithi139 bash[23439]: audit 2022-04-28T19:42:25.597121+0000 mgr.y (mgr.14140) 295 : audit [DBG] from='client.24523 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:42:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:27 smithi139 bash[23439]: cluster 2022-04-28T19:42:26.016456+0000 mgr.y (mgr.14140) 296 : cluster [DBG] pgmap v256: 9 pgs: 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:27 smithi139 bash[23439]: audit 2022-04-28T19:42:26.220325+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2022-04-28T19:42:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:27 smithi139 bash[23439]: cluster 2022-04-28T19:42:26.220380+0000 mon.a (mon.0) 587 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2022-04-28T19:42:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:27 smithi139 bash[23439]: audit 2022-04-28T19:42:26.222689+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2022-04-28T19:42:27.995 INFO:teuthology.run_tasks:Running task ceph.healthy... 2022-04-28T19:42:28.045 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2022-04-28T19:42:28.045 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2022-04-28T19:42:28.045 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph mgr dump --format=json 2022-04-28T19:42:28.190 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.230996+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cluster 2022-04-28T19:42:27.231295+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.244380+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cluster 2022-04-28T19:42:27.245307+0000 mon.a (mon.0) 592 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cluster 2022-04-28T19:42:27.245346+0000 mon.a (mon.0) 593 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cluster 2022-04-28T19:42:27.251696+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.252695+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2022-04-28T19:42:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cluster 2022-04-28T19:42:27.252809+0000 mon.a (mon.0) 596 : cluster [DBG] fsmap foo:0 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cephadm 2022-04-28T19:42:27.255152+0000 mgr.y (mgr.14140) 297 : cephadm [INF] Saving service mds.foo spec with placement count:2 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.262854+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cephadm 2022-04-28T19:42:27.293410+0000 mgr.y (mgr.14140) 298 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: cephadm 2022-04-28T19:42:27.293861+0000 mgr.y (mgr.14140) 299 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.674541+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.684504+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.685781+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.691586+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.692641+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:42:28.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[23786]: audit 2022-04-28T19:42:27.693900+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mds.foo.smithi139.oefiza", "key": "container_image"}]: dispatch 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.230996+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cluster 2022-04-28T19:42:27.231295+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.244380+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cluster 2022-04-28T19:42:27.245307+0000 mon.a (mon.0) 592 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cluster 2022-04-28T19:42:27.245346+0000 mon.a (mon.0) 593 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cluster 2022-04-28T19:42:27.251696+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.252695+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cluster 2022-04-28T19:42:27.252809+0000 mon.a (mon.0) 596 : cluster [DBG] fsmap foo:0 2022-04-28T19:42:28.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cephadm 2022-04-28T19:42:27.255152+0000 mgr.y (mgr.14140) 297 : cephadm [INF] Saving service mds.foo spec with placement count:2 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.262854+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cephadm 2022-04-28T19:42:27.293410+0000 mgr.y (mgr.14140) 298 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: cephadm 2022-04-28T19:42:27.293861+0000 mgr.y (mgr.14140) 299 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.674541+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.684504+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.685781+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.691586+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.692641+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:42:28.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:28 smithi059 bash[29114]: audit 2022-04-28T19:42:27.693900+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mds.foo.smithi139.oefiza", "key": "container_image"}]: dispatch 2022-04-28T19:42:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.230996+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2022-04-28T19:42:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cluster 2022-04-28T19:42:27.231295+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2022-04-28T19:42:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.244380+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cluster 2022-04-28T19:42:27.245307+0000 mon.a (mon.0) 592 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cluster 2022-04-28T19:42:27.245346+0000 mon.a (mon.0) 593 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cluster 2022-04-28T19:42:27.251696+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.252695+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cluster 2022-04-28T19:42:27.252809+0000 mon.a (mon.0) 596 : cluster [DBG] fsmap foo:0 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cephadm 2022-04-28T19:42:27.255152+0000 mgr.y (mgr.14140) 297 : cephadm [INF] Saving service mds.foo spec with placement count:2 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.262854+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cephadm 2022-04-28T19:42:27.293410+0000 mgr.y (mgr.14140) 298 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:42:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: cephadm 2022-04-28T19:42:27.293861+0000 mgr.y (mgr.14140) 299 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:42:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.674541+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.684504+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.685781+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:42:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.691586+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-04-28T19:42:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.692641+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:42:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:28 smithi139 bash[23439]: audit 2022-04-28T19:42:27.693900+0000 mon.a (mon.0) 603 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mds.foo.smithi139.oefiza", "key": "container_image"}]: dispatch 2022-04-28T19:42:29.352 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:29 smithi139 bash[23439]: cephadm 2022-04-28T19:42:27.693374+0000 mgr.y (mgr.14140) 300 : cephadm [INF] Deploying daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:42:29.352 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:29 smithi139 bash[23439]: cluster 2022-04-28T19:42:28.017117+0000 mgr.y (mgr.14140) 301 : cluster [DBG] pgmap v260: 73 pgs: 7 creating+peering, 57 unknown, 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:29.353 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:29 smithi139 bash[23439]: cluster 2022-04-28T19:42:28.253490+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2022-04-28T19:42:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:29 smithi059 bash[29114]: cephadm 2022-04-28T19:42:27.693374+0000 mgr.y (mgr.14140) 300 : cephadm [INF] Deploying daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:42:29.622 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:29 smithi059 bash[29114]: cluster 2022-04-28T19:42:28.017117+0000 mgr.y (mgr.14140) 301 : cluster [DBG] pgmap v260: 73 pgs: 7 creating+peering, 57 unknown, 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:29.623 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:29 smithi059 bash[29114]: cluster 2022-04-28T19:42:28.253490+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2022-04-28T19:42:29.623 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:29 smithi059 bash[23786]: cephadm 2022-04-28T19:42:27.693374+0000 mgr.y (mgr.14140) 300 : cephadm [INF] Deploying daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:42:29.624 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:29 smithi059 bash[23786]: cluster 2022-04-28T19:42:28.017117+0000 mgr.y (mgr.14140) 301 : cluster [DBG] pgmap v260: 73 pgs: 7 creating+peering, 57 unknown, 9 active+clean; 0 B data, 3.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:29.624 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:29 smithi059 bash[23786]: cluster 2022-04-28T19:42:28.253490+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2022-04-28T19:42:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[23786]: cluster 2022-04-28T19:42:29.264440+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2022-04-28T19:42:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[23786]: audit 2022-04-28T19:42:30.215874+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[23786]: audit 2022-04-28T19:42:30.217630+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:42:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[23786]: audit 2022-04-28T19:42:30.224988+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[23786]: audit 2022-04-28T19:42:30.226669+0000 mon.a (mon.0) 609 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[23786]: audit 2022-04-28T19:42:30.228582+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mds.foo.smithi059.tnizcj", "key": "container_image"}]: dispatch 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[29114]: cluster 2022-04-28T19:42:29.264440+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[29114]: audit 2022-04-28T19:42:30.215874+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[29114]: audit 2022-04-28T19:42:30.217630+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[29114]: audit 2022-04-28T19:42:30.224988+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[29114]: audit 2022-04-28T19:42:30.226669+0000 mon.a (mon.0) 609 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:42:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:30 smithi059 bash[29114]: audit 2022-04-28T19:42:30.228582+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mds.foo.smithi059.tnizcj", "key": "container_image"}]: dispatch 2022-04-28T19:42:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:30 smithi139 bash[23439]: cluster 2022-04-28T19:42:29.264440+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2022-04-28T19:42:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:30 smithi139 bash[23439]: audit 2022-04-28T19:42:30.215874+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:30 smithi139 bash[23439]: audit 2022-04-28T19:42:30.217630+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:42:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:30 smithi139 bash[23439]: audit 2022-04-28T19:42:30.224988+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2022-04-28T19:42:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:30 smithi139 bash[23439]: audit 2022-04-28T19:42:30.226669+0000 mon.a (mon.0) 609 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:42:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:30 smithi139 bash[23439]: audit 2022-04-28T19:42:30.228582+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mds.foo.smithi059.tnizcj", "key": "container_image"}]: dispatch 2022-04-28T19:42:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:31 smithi139 bash[23439]: cluster 2022-04-28T19:42:30.017774+0000 mgr.y (mgr.14140) 302 : cluster [DBG] pgmap v263: 73 pgs: 7 creating+peering, 34 unknown, 32 active+clean; 0 B data, 3.9 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:31 smithi139 bash[23439]: cephadm 2022-04-28T19:42:30.227795+0000 mgr.y (mgr.14140) 303 : cephadm [INF] Deploying daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:42:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:31 smithi059 bash[23786]: cluster 2022-04-28T19:42:30.017774+0000 mgr.y (mgr.14140) 302 : cluster [DBG] pgmap v263: 73 pgs: 7 creating+peering, 34 unknown, 32 active+clean; 0 B data, 3.9 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:31 smithi059 bash[23786]: cephadm 2022-04-28T19:42:30.227795+0000 mgr.y (mgr.14140) 303 : cephadm [INF] Deploying daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:42:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:31 smithi059 bash[29114]: cluster 2022-04-28T19:42:30.017774+0000 mgr.y (mgr.14140) 302 : cluster [DBG] pgmap v263: 73 pgs: 7 creating+peering, 34 unknown, 32 active+clean; 0 B data, 3.9 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:31 smithi059 bash[29114]: cephadm 2022-04-28T19:42:30.227795+0000 mgr.y (mgr.14140) 303 : cephadm [INF] Deploying daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:42:32.148 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:42:33.128 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":14,"active_gid":14140,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6800","nonce":103934933},{"type":"v1","addr":"172.21.15.59:6801","nonce":103934933}]},"active_addr":"172.21.15.59:6801/103934933","active_change":"2022-04-28T19:35:15.837768+0000","active_mgr_features":4540138292840890367,"available":true,"standbys":[{"gid":34112,"name":"x","mgr_features":4540138292840890367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"none","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"int","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"None","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"none","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"int","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"None","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://smithi059:8443/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":3,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":397086884}]},{"addrvec":[{"type":"v2","addr":"172.21.15.59:0","nonce":3381593068}]}]}} 2022-04-28T19:42:33.199 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2022-04-28T19:42:33.200 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2022-04-28T19:42:33.200 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd dump --format=json 2022-04-28T19:42:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[29114]: cluster 2022-04-28T19:42:32.018477+0000 mgr.y (mgr.14140) 304 : cluster [DBG] pgmap v264: 73 pgs: 7 creating+peering, 14 unknown, 52 active+clean; 0 B data, 4.1 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[29114]: audit 2022-04-28T19:42:32.139372+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.59:0/2677899981' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-04-28T19:42:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[23786]: cluster 2022-04-28T19:42:32.018477+0000 mgr.y (mgr.14140) 304 : cluster [DBG] pgmap v264: 73 pgs: 7 creating+peering, 14 unknown, 52 active+clean; 0 B data, 4.1 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[23786]: audit 2022-04-28T19:42:32.139372+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.59:0/2677899981' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-04-28T19:42:33.343 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:33 smithi139 bash[23439]: cluster 2022-04-28T19:42:32.018477+0000 mgr.y (mgr.14140) 304 : cluster [DBG] pgmap v264: 73 pgs: 7 creating+peering, 14 unknown, 52 active+clean; 0 B data, 4.1 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:33 smithi139 bash[23439]: audit 2022-04-28T19:42:32.139372+0000 mon.a (mon.0) 611 : audit [DBG] from='client.? 172.21.15.59:0/2677899981' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2022-04-28T19:42:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:33.509Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:42:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:33.509Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:42:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:33.509Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:42:34.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[29114]: audit 2022-04-28T19:42:33.407958+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:34.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[29114]: audit 2022-04-28T19:42:33.409627+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:42:34.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[29114]: audit 2022-04-28T19:42:33.410012+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:42:34.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[29114]: cluster 2022-04-28T19:42:34.019296+0000 mgr.y (mgr.14140) 305 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 0 B data, 4.2 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:34.668 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[23786]: audit 2022-04-28T19:42:33.407958+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:34.668 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[23786]: audit 2022-04-28T19:42:33.409627+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:42:34.668 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[23786]: audit 2022-04-28T19:42:33.410012+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:42:34.668 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:34 smithi059 bash[23786]: cluster 2022-04-28T19:42:34.019296+0000 mgr.y (mgr.14140) 305 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 0 B data, 4.2 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:34 smithi139 bash[23439]: audit 2022-04-28T19:42:33.407958+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:34 smithi139 bash[23439]: audit 2022-04-28T19:42:33.409627+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:42:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:34 smithi139 bash[23439]: audit 2022-04-28T19:42:33.410012+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:42:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:34 smithi139 bash[23439]: cluster 2022-04-28T19:42:34.019296+0000 mgr.y (mgr.14140) 305 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 0 B data, 4.2 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:36.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[29114]: audit 2022-04-28T19:42:34.933912+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[29114]: audit 2022-04-28T19:42:35.145071+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[29114]: audit 2022-04-28T19:42:35.152853+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[29114]: audit 2022-04-28T19:42:35.167287+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[23786]: audit 2022-04-28T19:42:34.933912+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[23786]: audit 2022-04-28T19:42:35.145071+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[23786]: audit 2022-04-28T19:42:35.152853+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:35 smithi059 bash[23786]: audit 2022-04-28T19:42:35.167287+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:35 smithi139 bash[23439]: audit 2022-04-28T19:42:34.933912+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:35 smithi139 bash[23439]: audit 2022-04-28T19:42:35.145071+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:35 smithi139 bash[23439]: audit 2022-04-28T19:42:35.152853+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:35 smithi139 bash[23439]: audit 2022-04-28T19:42:35.167287+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:42:37.033 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:42:37.033 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":56,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:42:29.250461+0000","last_up_change":"2022-04-28T19:40:01.137034+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"octopus","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-04-28T19:40:38.422278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2022-04-28T19:42:26.047647+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2022-04-28T19:42:26.668967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6803","nonce":1828778759}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6805","nonce":1828778759}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6809","nonce":1828778759}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6807","nonce":1828778759}]},"public_addr":"172.21.15.59:6803/1828778759","cluster_addr":"172.21.15.59:6805/1828778759","heartbeat_back_addr":"172.21.15.59:6809/1828778759","heartbeat_front_addr":"172.21.15.59:6807/1828778759","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":16,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6811","nonce":222479240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6813","nonce":222479240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6817","nonce":222479240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6815","nonce":222479240}]},"public_addr":"172.21.15.59:6811/222479240","cluster_addr":"172.21.15.59:6813/222479240","heartbeat_back_addr":"172.21.15.59:6817/222479240","heartbeat_front_addr":"172.21.15.59:6815/222479240","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6819","nonce":1472619329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6821","nonce":1472619329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6825","nonce":1472619329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6823","nonce":1472619329}]},"public_addr":"172.21.15.59:6819/1472619329","cluster_addr":"172.21.15.59:6821/1472619329","heartbeat_back_addr":"172.21.15.59:6825/1472619329","heartbeat_front_addr":"172.21.15.59:6823/1472619329","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6827","nonce":2946747803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6829","nonce":2946747803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6833","nonce":2946747803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6831","nonce":2946747803}]},"public_addr":"172.21.15.59:6827/2946747803","cluster_addr":"172.21.15.59:6829/2946747803","heartbeat_back_addr":"172.21.15.59:6833/2946747803","heartbeat_front_addr":"172.21.15.59:6831/2946747803","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6801","nonce":611845841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6803","nonce":611845841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6807","nonce":611845841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6805","nonce":611845841}]},"public_addr":"172.21.15.139:6801/611845841","cluster_addr":"172.21.15.139:6803/611845841","heartbeat_back_addr":"172.21.15.139:6807/611845841","heartbeat_front_addr":"172.21.15.139:6805/611845841","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6809","nonce":2795762393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6811","nonce":2795762393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6815","nonce":2795762393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6813","nonce":2795762393}]},"public_addr":"172.21.15.139:6809/2795762393","cluster_addr":"172.21.15.139:6811/2795762393","heartbeat_back_addr":"172.21.15.139:6815/2795762393","heartbeat_front_addr":"172.21.15.139:6813/2795762393","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6817","nonce":1808526322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6819","nonce":1808526322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6823","nonce":1808526322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6821","nonce":1808526322}]},"public_addr":"172.21.15.139:6817/1808526322","cluster_addr":"172.21.15.139:6819/1808526322","heartbeat_back_addr":"172.21.15.139:6823/1808526322","heartbeat_front_addr":"172.21.15.139:6821/1808526322","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6825","nonce":2154117349}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6827","nonce":2154117349}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6831","nonce":2154117349}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6829","nonce":2154117349}]},"public_addr":"172.21.15.139:6825/2154117349","cluster_addr":"172.21.15.139:6827/2154117349","heartbeat_back_addr":"172.21.15.139:6831/2154117349","heartbeat_front_addr":"172.21.15.139:6829/2154117349","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{"172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.020196+0000 mgr.y (mgr.14140) 306 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 0 B data, 4.3 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.171396+0000 mon.a (mon.0) 619 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] up:boot 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.171522+0000 mon.a (mon.0) 620 : cluster [INF] daemon mds.foo.smithi139.oefiza assigned to filesystem foo as rank 0 (now has 1 ranks) 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.171685+0000 mon.a (mon.0) 621 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.171745+0000 mon.a (mon.0) 622 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.171788+0000 mon.a (mon.0) 623 : cluster [INF] Cluster is now healthy 2022-04-28T19:42:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.172217+0000 mon.a (mon.0) 624 : cluster [DBG] fsmap foo:0 1 up:standby 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: audit 2022-04-28T19:42:36.172540+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.178105+0000 mon.a (mon.0) 626 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:creating} 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: cluster 2022-04-28T19:42:36.196480+0000 mon.a (mon.0) 627 : cluster [INF] daemon mds.foo.smithi139.oefiza is now active in filesystem foo as rank 0 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[29114]: audit 2022-04-28T19:42:37.029955+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.59:0/4115087749' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.020196+0000 mgr.y (mgr.14140) 306 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 0 B data, 4.3 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.171396+0000 mon.a (mon.0) 619 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] up:boot 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.171522+0000 mon.a (mon.0) 620 : cluster [INF] daemon mds.foo.smithi139.oefiza assigned to filesystem foo as rank 0 (now has 1 ranks) 2022-04-28T19:42:37.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.171685+0000 mon.a (mon.0) 621 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.171745+0000 mon.a (mon.0) 622 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.171788+0000 mon.a (mon.0) 623 : cluster [INF] Cluster is now healthy 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.172217+0000 mon.a (mon.0) 624 : cluster [DBG] fsmap foo:0 1 up:standby 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: audit 2022-04-28T19:42:36.172540+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.178105+0000 mon.a (mon.0) 626 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:creating} 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: cluster 2022-04-28T19:42:36.196480+0000 mon.a (mon.0) 627 : cluster [INF] daemon mds.foo.smithi139.oefiza is now active in filesystem foo as rank 0 2022-04-28T19:42:37.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:37 smithi059 bash[23786]: audit 2022-04-28T19:42:37.029955+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.59:0/4115087749' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:42:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.020196+0000 mgr.y (mgr.14140) 306 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 0 B data, 4.3 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.171396+0000 mon.a (mon.0) 619 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] up:boot 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.171522+0000 mon.a (mon.0) 620 : cluster [INF] daemon mds.foo.smithi139.oefiza assigned to filesystem foo as rank 0 (now has 1 ranks) 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.171685+0000 mon.a (mon.0) 621 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.171745+0000 mon.a (mon.0) 622 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.171788+0000 mon.a (mon.0) 623 : cluster [INF] Cluster is now healthy 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.172217+0000 mon.a (mon.0) 624 : cluster [DBG] fsmap foo:0 1 up:standby 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: audit 2022-04-28T19:42:36.172540+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.178105+0000 mon.a (mon.0) 626 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:creating} 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: cluster 2022-04-28T19:42:36.196480+0000 mon.a (mon.0) 627 : cluster [INF] daemon mds.foo.smithi139.oefiza is now active in filesystem foo as rank 0 2022-04-28T19:42:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:37 smithi139 bash[23439]: audit 2022-04-28T19:42:37.029955+0000 mon.a (mon.0) 628 : audit [DBG] from='client.? 172.21.15.59:0/4115087749' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:42:37.724 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2022-04-28T19:42:37.724 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd dump --format=json 2022-04-28T19:42:37.868 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:38 smithi059 bash[29114]: cluster 2022-04-28T19:42:37.178979+0000 mon.a (mon.0) 629 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] up:active 2022-04-28T19:42:38.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:38 smithi059 bash[29114]: cluster 2022-04-28T19:42:37.179121+0000 mon.a (mon.0) 630 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 2022-04-28T19:42:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:38 smithi059 bash[23786]: cluster 2022-04-28T19:42:37.178979+0000 mon.a (mon.0) 629 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] up:active 2022-04-28T19:42:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:38 smithi059 bash[23786]: cluster 2022-04-28T19:42:37.179121+0000 mon.a (mon.0) 630 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 2022-04-28T19:42:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:38 smithi139 bash[23439]: cluster 2022-04-28T19:42:37.178979+0000 mon.a (mon.0) 629 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] up:active 2022-04-28T19:42:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:38 smithi139 bash[23439]: cluster 2022-04-28T19:42:37.179121+0000 mon.a (mon.0) 630 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 2022-04-28T19:42:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:39 smithi059 bash[23786]: cluster 2022-04-28T19:42:38.021025+0000 mgr.y (mgr.14140) 307 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 1.5 KiB data, 4.3 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:39 smithi059 bash[29114]: cluster 2022-04-28T19:42:38.021025+0000 mgr.y (mgr.14140) 307 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 1.5 KiB data, 4.3 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:39 smithi139 bash[23439]: cluster 2022-04-28T19:42:38.021025+0000 mgr.y (mgr.14140) 307 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 1.5 KiB data, 4.3 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[23786]: cluster 2022-04-28T19:42:39.200768+0000 mon.a (mon.0) 631 : cluster [DBG] mds.? [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] up:boot 2022-04-28T19:42:40.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[23786]: cluster 2022-04-28T19:42:39.201207+0000 mon.a (mon.0) 632 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:40.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[23786]: audit 2022-04-28T19:42:39.201967+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:42:40.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[23786]: cluster 2022-04-28T19:42:39.207203+0000 mon.a (mon.0) 634 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[29114]: cluster 2022-04-28T19:42:39.200768+0000 mon.a (mon.0) 631 : cluster [DBG] mds.? [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] up:boot 2022-04-28T19:42:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[29114]: cluster 2022-04-28T19:42:39.201207+0000 mon.a (mon.0) 632 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[29114]: audit 2022-04-28T19:42:39.201967+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:42:40.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:40 smithi059 bash[29114]: cluster 2022-04-28T19:42:39.207203+0000 mon.a (mon.0) 634 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:40 smithi139 bash[23439]: cluster 2022-04-28T19:42:39.200768+0000 mon.a (mon.0) 631 : cluster [DBG] mds.? [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] up:boot 2022-04-28T19:42:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:40 smithi139 bash[23439]: cluster 2022-04-28T19:42:39.201207+0000 mon.a (mon.0) 632 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:40 smithi139 bash[23439]: audit 2022-04-28T19:42:39.201967+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:42:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:40 smithi139 bash[23439]: cluster 2022-04-28T19:42:39.207203+0000 mon.a (mon.0) 634 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:41.289 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:42:41.289 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":56,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:42:29.250461+0000","last_up_change":"2022-04-28T19:40:01.137034+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"octopus","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-04-28T19:40:38.422278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2022-04-28T19:42:26.047647+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2022-04-28T19:42:26.668967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":10,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6803","nonce":1828778759}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6805","nonce":1828778759}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6809","nonce":1828778759}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":1828778759},{"type":"v1","addr":"172.21.15.59:6807","nonce":1828778759}]},"public_addr":"172.21.15.59:6803/1828778759","cluster_addr":"172.21.15.59:6805/1828778759","heartbeat_back_addr":"172.21.15.59:6809/1828778759","heartbeat_front_addr":"172.21.15.59:6807/1828778759","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":16,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6811","nonce":222479240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6813","nonce":222479240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6817","nonce":222479240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":222479240},{"type":"v1","addr":"172.21.15.59:6815","nonce":222479240}]},"public_addr":"172.21.15.59:6811/222479240","cluster_addr":"172.21.15.59:6813/222479240","heartbeat_back_addr":"172.21.15.59:6817/222479240","heartbeat_front_addr":"172.21.15.59:6815/222479240","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":21,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6819","nonce":1472619329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6821","nonce":1472619329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6825","nonce":1472619329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":1472619329},{"type":"v1","addr":"172.21.15.59:6823","nonce":1472619329}]},"public_addr":"172.21.15.59:6819/1472619329","cluster_addr":"172.21.15.59:6821/1472619329","heartbeat_back_addr":"172.21.15.59:6825/1472619329","heartbeat_front_addr":"172.21.15.59:6823/1472619329","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6827","nonce":2946747803}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6829","nonce":2946747803}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6833","nonce":2946747803}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":2946747803},{"type":"v1","addr":"172.21.15.59:6831","nonce":2946747803}]},"public_addr":"172.21.15.59:6827/2946747803","cluster_addr":"172.21.15.59:6829/2946747803","heartbeat_back_addr":"172.21.15.59:6833/2946747803","heartbeat_front_addr":"172.21.15.59:6831/2946747803","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6801","nonce":611845841}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6803","nonce":611845841}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6807","nonce":611845841}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":611845841},{"type":"v1","addr":"172.21.15.139:6805","nonce":611845841}]},"public_addr":"172.21.15.139:6801/611845841","cluster_addr":"172.21.15.139:6803/611845841","heartbeat_back_addr":"172.21.15.139:6807/611845841","heartbeat_front_addr":"172.21.15.139:6805/611845841","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6809","nonce":2795762393}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6811","nonce":2795762393}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6815","nonce":2795762393}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2795762393},{"type":"v1","addr":"172.21.15.139:6813","nonce":2795762393}]},"public_addr":"172.21.15.139:6809/2795762393","cluster_addr":"172.21.15.139:6811/2795762393","heartbeat_back_addr":"172.21.15.139:6815/2795762393","heartbeat_front_addr":"172.21.15.139:6813/2795762393","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":41,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6817","nonce":1808526322}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6819","nonce":1808526322}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6823","nonce":1808526322}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":1808526322},{"type":"v1","addr":"172.21.15.139:6821","nonce":1808526322}]},"public_addr":"172.21.15.139:6817/1808526322","cluster_addr":"172.21.15.139:6819/1808526322","heartbeat_back_addr":"172.21.15.139:6823/1808526322","heartbeat_front_addr":"172.21.15.139:6821/1808526322","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6825","nonce":2154117349}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6827","nonce":2154117349}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6831","nonce":2154117349}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":2154117349},{"type":"v1","addr":"172.21.15.139:6829","nonce":2154117349}]},"public_addr":"172.21.15.139:6825/2154117349","cluster_addr":"172.21.15.139:6827/2154117349","heartbeat_back_addr":"172.21.15.139:6831/2154117349","heartbeat_front_addr":"172.21.15.139:6829/2154117349","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138292840890367,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blacklist":{"172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{}} 2022-04-28T19:42:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:41 smithi059 bash[23786]: cluster 2022-04-28T19:42:40.021883+0000 mgr.y (mgr.14140) 308 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 2.0 KiB data, 4.3 MiB used, 707 GiB / 715 GiB avail; 511 B/s wr, 10 op/s 2022-04-28T19:42:41.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:41 smithi059 bash[29114]: cluster 2022-04-28T19:42:40.021883+0000 mgr.y (mgr.14140) 308 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 2.0 KiB data, 4.3 MiB used, 707 GiB / 715 GiB avail; 511 B/s wr, 10 op/s 2022-04-28T19:42:41.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:41 smithi139 bash[23439]: cluster 2022-04-28T19:42:40.021883+0000 mgr.y (mgr.14140) 308 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 2.0 KiB data, 4.3 MiB used, 707 GiB / 715 GiB avail; 511 B/s wr, 10 op/s 2022-04-28T19:42:41.969 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2022-04-28T19:42:42.112 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:42 smithi059 bash[23786]: audit 2022-04-28T19:42:41.285027+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.59:0/3764650904' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:42:42.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:42 smithi059 bash[29114]: audit 2022-04-28T19:42:41.285027+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.59:0/3764650904' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:42:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:42 smithi139 bash[23439]: audit 2022-04-28T19:42:41.285027+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.59:0/3764650904' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:42:43.419 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:43 smithi059 bash[29114]: cluster 2022-04-28T19:42:42.022695+0000 mgr.y (mgr.14140) 309 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s wr, 9 op/s 2022-04-28T19:42:43.419 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:43 smithi059 bash[23786]: cluster 2022-04-28T19:42:42.022695+0000 mgr.y (mgr.14140) 309 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s wr, 9 op/s 2022-04-28T19:42:43.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:43 smithi139 bash[23439]: cluster 2022-04-28T19:42:42.022695+0000 mgr.y (mgr.14140) 309 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s wr, 9 op/s 2022-04-28T19:42:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:43.510Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:42:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:43.510Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:42:43.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:43.510Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:42:44.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:44 smithi059 bash[29114]: cluster 2022-04-28T19:42:43.237185+0000 mon.a (mon.0) 636 : cluster [DBG] mds.? [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] up:standby 2022-04-28T19:42:44.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:44 smithi059 bash[29114]: cluster 2022-04-28T19:42:43.237303+0000 mon.a (mon.0) 637 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:44 smithi059 bash[23786]: cluster 2022-04-28T19:42:43.237185+0000 mon.a (mon.0) 636 : cluster [DBG] mds.? [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] up:standby 2022-04-28T19:42:44.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:44 smithi059 bash[23786]: cluster 2022-04-28T19:42:43.237303+0000 mon.a (mon.0) 637 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:44.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:44 smithi139 bash[23439]: cluster 2022-04-28T19:42:43.237185+0000 mon.a (mon.0) 636 : cluster [DBG] mds.? [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] up:standby 2022-04-28T19:42:44.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:44 smithi139 bash[23439]: cluster 2022-04-28T19:42:43.237303+0000 mon.a (mon.0) 637 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:42:45.406 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:45 smithi059 bash[29114]: cluster 2022-04-28T19:42:44.023632+0000 mgr.y (mgr.14140) 310 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s wr, 6 op/s 2022-04-28T19:42:45.406 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:45 smithi059 bash[23786]: cluster 2022-04-28T19:42:44.023632+0000 mgr.y (mgr.14140) 310 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s wr, 6 op/s 2022-04-28T19:42:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:45 smithi139 bash[23439]: cluster 2022-04-28T19:42:44.023632+0000 mgr.y (mgr.14140) 310 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s wr, 6 op/s 2022-04-28T19:42:46.061 INFO:teuthology.orchestra.run.smithi059.stdout:42949673029 2022-04-28T19:42:46.062 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2022-04-28T19:42:46.209 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:47.432 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:47 smithi059 bash[29114]: cluster 2022-04-28T19:42:46.024546+0000 mgr.y (mgr.14140) 311 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2022-04-28T19:42:47.433 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:47 smithi059 bash[23786]: cluster 2022-04-28T19:42:46.024546+0000 mgr.y (mgr.14140) 311 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2022-04-28T19:42:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:47 smithi139 bash[23439]: cluster 2022-04-28T19:42:46.024546+0000 mgr.y (mgr.14140) 311 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2022-04-28T19:42:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:49 smithi059 bash[23786]: cluster 2022-04-28T19:42:48.025282+0000 mgr.y (mgr.14140) 312 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2022-04-28T19:42:49.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:49 smithi059 bash[29114]: cluster 2022-04-28T19:42:48.025282+0000 mgr.y (mgr.14140) 312 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2022-04-28T19:42:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:49 smithi139 bash[23439]: cluster 2022-04-28T19:42:48.025282+0000 mgr.y (mgr.14140) 312 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2022-04-28T19:42:49.835 INFO:teuthology.orchestra.run.smithi059.stdout:68719476802 2022-04-28T19:42:49.835 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2022-04-28T19:42:49.984 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:51 smithi059 bash[23786]: cluster 2022-04-28T19:42:50.026157+0000 mgr.y (mgr.14140) 313 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 852 B/s wr, 3 op/s 2022-04-28T19:42:51.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:51 smithi059 bash[29114]: cluster 2022-04-28T19:42:50.026157+0000 mgr.y (mgr.14140) 313 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 852 B/s wr, 3 op/s 2022-04-28T19:42:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:51 smithi139 bash[23439]: cluster 2022-04-28T19:42:50.026157+0000 mgr.y (mgr.14140) 313 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 852 B/s wr, 3 op/s 2022-04-28T19:42:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:53 smithi059 bash[29114]: cluster 2022-04-28T19:42:52.027010+0000 mgr.y (mgr.14140) 314 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2022-04-28T19:42:53.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:53.510Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:42:53.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:53.510Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:42:53.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:42:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:42:53.510Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:42:53.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:53 smithi059 bash[23786]: cluster 2022-04-28T19:42:52.027010+0000 mgr.y (mgr.14140) 314 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2022-04-28T19:42:53.612 INFO:teuthology.orchestra.run.smithi059.stdout:90194313278 2022-04-28T19:42:53.651 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2022-04-28T19:42:53.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:53 smithi139 bash[23439]: cluster 2022-04-28T19:42:52.027010+0000 mgr.y (mgr.14140) 314 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2022-04-28T19:42:53.799 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:55 smithi059 bash[23786]: cluster 2022-04-28T19:42:54.028017+0000 mgr.y (mgr.14140) 315 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:42:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:55 smithi059 bash[29114]: cluster 2022-04-28T19:42:54.028017+0000 mgr.y (mgr.14140) 315 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:42:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:55 smithi139 bash[23439]: cluster 2022-04-28T19:42:54.028017+0000 mgr.y (mgr.14140) 315 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:42:57.399 INFO:teuthology.orchestra.run.smithi059.stdout:111669149753 2022-04-28T19:42:57.400 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2022-04-28T19:42:57.546 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:42:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:57 smithi059 bash[29114]: cluster 2022-04-28T19:42:56.028881+0000 mgr.y (mgr.14140) 316 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:57 smithi059 bash[23786]: cluster 2022-04-28T19:42:56.028881+0000 mgr.y (mgr.14140) 316 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:57 smithi139 bash[23439]: cluster 2022-04-28T19:42:56.028881+0000 mgr.y (mgr.14140) 316 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:42:59 smithi059 bash[29114]: cluster 2022-04-28T19:42:58.029965+0000 mgr.y (mgr.14140) 317 : cluster [DBG] pgmap v277: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:59.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:42:59 smithi059 bash[23786]: cluster 2022-04-28T19:42:58.029965+0000 mgr.y (mgr.14140) 317 : cluster [DBG] pgmap v277: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:42:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:42:59 smithi139 bash[23439]: cluster 2022-04-28T19:42:58.029965+0000 mgr.y (mgr.14140) 317 : cluster [DBG] pgmap v277: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:01.387 INFO:teuthology.orchestra.run.smithi059.stdout:133143986230 2022-04-28T19:43:01.388 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2022-04-28T19:43:01.530 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:01 smithi059 bash[23786]: cluster 2022-04-28T19:43:00.030953+0000 mgr.y (mgr.14140) 318 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:01 smithi059 bash[29114]: cluster 2022-04-28T19:43:00.030953+0000 mgr.y (mgr.14140) 318 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:01 smithi139 bash[23439]: cluster 2022-04-28T19:43:00.030953+0000 mgr.y (mgr.14140) 318 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:03 smithi059 bash[23786]: cluster 2022-04-28T19:43:02.031954+0000 mgr.y (mgr.14140) 319 : cluster [DBG] pgmap v279: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:03 smithi059 bash[29114]: cluster 2022-04-28T19:43:02.031954+0000 mgr.y (mgr.14140) 319 : cluster [DBG] pgmap v279: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:03.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:03.510Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:43:03.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:03.510Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:43:03.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:03.511Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:43:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:03 smithi139 bash[23439]: cluster 2022-04-28T19:43:02.031954+0000 mgr.y (mgr.14140) 319 : cluster [DBG] pgmap v279: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:05.415 INFO:teuthology.orchestra.run.smithi059.stdout:154618822705 2022-04-28T19:43:05.416 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2022-04-28T19:43:05.557 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:05 smithi059 bash[23786]: cluster 2022-04-28T19:43:04.032876+0000 mgr.y (mgr.14140) 320 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:05 smithi059 bash[29114]: cluster 2022-04-28T19:43:04.032876+0000 mgr.y (mgr.14140) 320 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:05 smithi139 bash[23439]: cluster 2022-04-28T19:43:04.032876+0000 mgr.y (mgr.14140) 320 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:07 smithi059 bash[29114]: cluster 2022-04-28T19:43:06.033856+0000 mgr.y (mgr.14140) 321 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:07 smithi059 bash[23786]: cluster 2022-04-28T19:43:06.033856+0000 mgr.y (mgr.14140) 321 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:07 smithi139 bash[23439]: cluster 2022-04-28T19:43:06.033856+0000 mgr.y (mgr.14140) 321 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:09.301 INFO:teuthology.orchestra.run.smithi059.stdout:176093659181 2022-04-28T19:43:09.302 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2022-04-28T19:43:09.440 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:09 smithi059 bash[23786]: cluster 2022-04-28T19:43:08.034814+0000 mgr.y (mgr.14140) 322 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:09.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:09 smithi059 bash[29114]: cluster 2022-04-28T19:43:08.034814+0000 mgr.y (mgr.14140) 322 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:09 smithi139 bash[23439]: cluster 2022-04-28T19:43:08.034814+0000 mgr.y (mgr.14140) 322 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:11 smithi059 bash[29114]: cluster 2022-04-28T19:43:10.035689+0000 mgr.y (mgr.14140) 323 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:11 smithi059 bash[23786]: cluster 2022-04-28T19:43:10.035689+0000 mgr.y (mgr.14140) 323 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:11 smithi139 bash[23439]: cluster 2022-04-28T19:43:10.035689+0000 mgr.y (mgr.14140) 323 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:13.129 INFO:teuthology.orchestra.run.smithi059.stdout:197568495657 2022-04-28T19:43:13.129 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.0 2022-04-28T19:43:13.277 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:13.513 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:13 smithi059 bash[29114]: cluster 2022-04-28T19:43:12.036482+0000 mgr.y (mgr.14140) 324 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:13.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:13 smithi059 bash[23786]: cluster 2022-04-28T19:43:12.036482+0000 mgr.y (mgr.14140) 324 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:13 smithi139 bash[23439]: cluster 2022-04-28T19:43:12.036482+0000 mgr.y (mgr.14140) 324 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:13.911 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:13.511Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:43:13.911 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:13.511Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:43:13.911 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:13.511Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:43:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:15 smithi059 bash[29114]: cluster 2022-04-28T19:43:14.037446+0000 mgr.y (mgr.14140) 325 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:15.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:15 smithi059 bash[23786]: cluster 2022-04-28T19:43:14.037446+0000 mgr.y (mgr.14140) 325 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:15 smithi139 bash[23439]: cluster 2022-04-28T19:43:14.037446+0000 mgr.y (mgr.14140) 325 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:16 smithi059 bash[23786]: audit 2022-04-28T19:43:16.126731+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:43:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:16 smithi059 bash[23786]: audit 2022-04-28T19:43:16.132062+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:43:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:16 smithi059 bash[29114]: audit 2022-04-28T19:43:16.126731+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:43:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:16 smithi059 bash[29114]: audit 2022-04-28T19:43:16.132062+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:43:16.618 INFO:teuthology.orchestra.run.smithi059.stdout:42949673035 2022-04-28T19:43:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:16 smithi139 bash[23439]: audit 2022-04-28T19:43:16.126731+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:43:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:16 smithi139 bash[23439]: audit 2022-04-28T19:43:16.132062+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:43:17.267 INFO:tasks.cephadm.ceph_manager.ceph:need seq 42949673029 got 42949673035 for osd.0 2022-04-28T19:43:17.268 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.1 2022-04-28T19:43:17.411 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:17 smithi059 bash[23786]: cluster 2022-04-28T19:43:16.038363+0000 mgr.y (mgr.14140) 326 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:17.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:17 smithi059 bash[23786]: audit 2022-04-28T19:43:16.616929+0000 mon.c (mon.1) 24 : audit [DBG] from='client.? 172.21.15.59:0/1501024764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-04-28T19:43:17.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:17 smithi059 bash[29114]: cluster 2022-04-28T19:43:16.038363+0000 mgr.y (mgr.14140) 326 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:17.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:17 smithi059 bash[29114]: audit 2022-04-28T19:43:16.616929+0000 mon.c (mon.1) 24 : audit [DBG] from='client.? 172.21.15.59:0/1501024764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-04-28T19:43:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:17 smithi139 bash[23439]: cluster 2022-04-28T19:43:16.038363+0000 mgr.y (mgr.14140) 326 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:17 smithi139 bash[23439]: audit 2022-04-28T19:43:16.616929+0000 mon.c (mon.1) 24 : audit [DBG] from='client.? 172.21.15.59:0/1501024764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2022-04-28T19:43:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:18 smithi139 bash[23439]: cluster 2022-04-28T19:43:18.039239+0000 mgr.y (mgr.14140) 327 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:18.657 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:18 smithi059 bash[29114]: cluster 2022-04-28T19:43:18.039239+0000 mgr.y (mgr.14140) 327 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:18.657 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:18 smithi059 bash[23786]: cluster 2022-04-28T19:43:18.039239+0000 mgr.y (mgr.14140) 327 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:20.575 INFO:teuthology.orchestra.run.smithi059.stdout:68719476808 2022-04-28T19:43:21.244 INFO:tasks.cephadm.ceph_manager.ceph:need seq 68719476802 got 68719476808 for osd.1 2022-04-28T19:43:21.244 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.2 2022-04-28T19:43:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:21 smithi139 bash[23439]: cluster 2022-04-28T19:43:20.040064+0000 mgr.y (mgr.14140) 328 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:21 smithi139 bash[23439]: audit 2022-04-28T19:43:20.572998+0000 mon.c (mon.1) 25 : audit [DBG] from='client.? 172.21.15.59:0/2515119175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-04-28T19:43:21.393 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:21 smithi059 bash[23786]: cluster 2022-04-28T19:43:20.040064+0000 mgr.y (mgr.14140) 328 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:21 smithi059 bash[23786]: audit 2022-04-28T19:43:20.572998+0000 mon.c (mon.1) 25 : audit [DBG] from='client.? 172.21.15.59:0/2515119175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-04-28T19:43:21.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:21 smithi059 bash[29114]: cluster 2022-04-28T19:43:20.040064+0000 mgr.y (mgr.14140) 328 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:21.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:21 smithi059 bash[29114]: audit 2022-04-28T19:43:20.572998+0000 mon.c (mon.1) 25 : audit [DBG] from='client.? 172.21.15.59:0/2515119175' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2022-04-28T19:43:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:23 smithi139 bash[23439]: cluster 2022-04-28T19:43:22.040905+0000 mgr.y (mgr.14140) 329 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:23.513 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:23 smithi059 bash[29114]: cluster 2022-04-28T19:43:22.040905+0000 mgr.y (mgr.14140) 329 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:23.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:23 smithi059 bash[23786]: cluster 2022-04-28T19:43:22.040905+0000 mgr.y (mgr.14140) 329 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:23.511Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:43:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:23.511Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:43:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:23.511Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:43:24.634 INFO:teuthology.orchestra.run.smithi059.stdout:90194313284 2022-04-28T19:43:25.273 INFO:tasks.cephadm.ceph_manager.ceph:need seq 90194313278 got 90194313284 for osd.2 2022-04-28T19:43:25.277 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.3 2022-04-28T19:43:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:25 smithi139 bash[23439]: cluster 2022-04-28T19:43:24.041816+0000 mgr.y (mgr.14140) 330 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:25 smithi139 bash[23439]: audit 2022-04-28T19:43:24.632133+0000 mon.a (mon.0) 640 : audit [DBG] from='client.? 172.21.15.59:0/629220432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-04-28T19:43:25.425 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:25 smithi059 bash[29114]: cluster 2022-04-28T19:43:24.041816+0000 mgr.y (mgr.14140) 330 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:25.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:25 smithi059 bash[29114]: audit 2022-04-28T19:43:24.632133+0000 mon.a (mon.0) 640 : audit [DBG] from='client.? 172.21.15.59:0/629220432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-04-28T19:43:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:25 smithi059 bash[23786]: cluster 2022-04-28T19:43:24.041816+0000 mgr.y (mgr.14140) 330 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:25 smithi059 bash[23786]: audit 2022-04-28T19:43:24.632133+0000 mon.a (mon.0) 640 : audit [DBG] from='client.? 172.21.15.59:0/629220432' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2022-04-28T19:43:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:27 smithi059 bash[23786]: cluster 2022-04-28T19:43:26.042726+0000 mgr.y (mgr.14140) 331 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:27.645 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:27 smithi059 bash[29114]: cluster 2022-04-28T19:43:26.042726+0000 mgr.y (mgr.14140) 331 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:27.646 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:27 smithi139 bash[23439]: cluster 2022-04-28T19:43:26.042726+0000 mgr.y (mgr.14140) 331 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:28.590 INFO:teuthology.orchestra.run.smithi059.stdout:111669149759 2022-04-28T19:43:29.249 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149753 got 111669149759 for osd.3 2022-04-28T19:43:29.249 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.4 2022-04-28T19:43:29.395 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:29 smithi059 bash[29114]: cluster 2022-04-28T19:43:28.043673+0000 mgr.y (mgr.14140) 332 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:29 smithi059 bash[29114]: audit 2022-04-28T19:43:28.588624+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.59:0/3965112827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-04-28T19:43:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:29 smithi059 bash[23786]: cluster 2022-04-28T19:43:28.043673+0000 mgr.y (mgr.14140) 332 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:29 smithi059 bash[23786]: audit 2022-04-28T19:43:28.588624+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.59:0/3965112827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-04-28T19:43:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:29 smithi139 bash[23439]: cluster 2022-04-28T19:43:28.043673+0000 mgr.y (mgr.14140) 332 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:29 smithi139 bash[23439]: audit 2022-04-28T19:43:28.588624+0000 mon.a (mon.0) 641 : audit [DBG] from='client.? 172.21.15.59:0/3965112827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2022-04-28T19:43:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:31 smithi059 bash[29114]: cluster 2022-04-28T19:43:30.044461+0000 mgr.y (mgr.14140) 333 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:31 smithi059 bash[23786]: cluster 2022-04-28T19:43:30.044461+0000 mgr.y (mgr.14140) 333 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:31 smithi139 bash[23439]: cluster 2022-04-28T19:43:30.044461+0000 mgr.y (mgr.14140) 333 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:32.658 INFO:teuthology.orchestra.run.smithi059.stdout:133143986236 2022-04-28T19:43:33.268 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986230 got 133143986236 for osd.4 2022-04-28T19:43:33.268 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.5 2022-04-28T19:43:33.412 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:33.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[23786]: cluster 2022-04-28T19:43:32.045311+0000 mgr.y (mgr.14140) 334 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:33.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[23786]: audit 2022-04-28T19:43:32.655930+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.59:0/4045092402' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-04-28T19:43:33.514 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[29114]: cluster 2022-04-28T19:43:32.045311+0000 mgr.y (mgr.14140) 334 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:33.514 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[29114]: audit 2022-04-28T19:43:32.655930+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.59:0/4045092402' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-04-28T19:43:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:33 smithi139 bash[23439]: cluster 2022-04-28T19:43:32.045311+0000 mgr.y (mgr.14140) 334 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:33 smithi139 bash[23439]: audit 2022-04-28T19:43:32.655930+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.59:0/4045092402' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2022-04-28T19:43:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:33.512Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:43:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:33.512Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:43:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:33.512Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:43:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:35 smithi059 bash[23786]: cluster 2022-04-28T19:43:34.046226+0000 mgr.y (mgr.14140) 335 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:35.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:35 smithi059 bash[29114]: cluster 2022-04-28T19:43:34.046226+0000 mgr.y (mgr.14140) 335 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:35 smithi139 bash[23439]: cluster 2022-04-28T19:43:34.046226+0000 mgr.y (mgr.14140) 335 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:36.718 INFO:teuthology.orchestra.run.smithi059.stdout:154618822711 2022-04-28T19:43:37.440 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:37 smithi059 bash[23786]: cluster 2022-04-28T19:43:36.047140+0000 mgr.y (mgr.14140) 336 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:37.441 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:37 smithi059 bash[23786]: audit 2022-04-28T19:43:36.715636+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.59:0/2437697311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-04-28T19:43:37.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:37 smithi059 bash[29114]: cluster 2022-04-28T19:43:36.047140+0000 mgr.y (mgr.14140) 336 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:37.441 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:37 smithi059 bash[29114]: audit 2022-04-28T19:43:36.715636+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.59:0/2437697311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-04-28T19:43:37.442 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822705 got 154618822711 for osd.5 2022-04-28T19:43:37.442 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.6 2022-04-28T19:43:37.585 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:37 smithi139 bash[23439]: cluster 2022-04-28T19:43:36.047140+0000 mgr.y (mgr.14140) 336 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:37 smithi139 bash[23439]: audit 2022-04-28T19:43:36.715636+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.59:0/2437697311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2022-04-28T19:43:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:39 smithi059 bash[23786]: cluster 2022-04-28T19:43:38.048076+0000 mgr.y (mgr.14140) 337 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:39 smithi059 bash[29114]: cluster 2022-04-28T19:43:38.048076+0000 mgr.y (mgr.14140) 337 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:39 smithi139 bash[23439]: cluster 2022-04-28T19:43:38.048076+0000 mgr.y (mgr.14140) 337 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:40.742 INFO:teuthology.orchestra.run.smithi059.stdout:176093659188 2022-04-28T19:43:41.383 INFO:tasks.cephadm.ceph_manager.ceph:need seq 176093659181 got 176093659188 for osd.6 2022-04-28T19:43:41.384 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd last-stat-seq osd.7 2022-04-28T19:43:41.527 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:41 smithi059 bash[29114]: cluster 2022-04-28T19:43:40.048896+0000 mgr.y (mgr.14140) 338 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:41 smithi059 bash[29114]: audit 2022-04-28T19:43:40.740513+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.59:0/3494065978' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-04-28T19:43:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:41 smithi059 bash[23786]: cluster 2022-04-28T19:43:40.048896+0000 mgr.y (mgr.14140) 338 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:41.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:41 smithi059 bash[23786]: audit 2022-04-28T19:43:40.740513+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.59:0/3494065978' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-04-28T19:43:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:41 smithi139 bash[23439]: cluster 2022-04-28T19:43:40.048896+0000 mgr.y (mgr.14140) 338 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:41 smithi139 bash[23439]: audit 2022-04-28T19:43:40.740513+0000 mon.a (mon.0) 642 : audit [DBG] from='client.? 172.21.15.59:0/3494065978' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2022-04-28T19:43:43.514 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:43 smithi059 bash[23786]: cluster 2022-04-28T19:43:42.049756+0000 mgr.y (mgr.14140) 339 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:43.515 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:43 smithi059 bash[29114]: cluster 2022-04-28T19:43:42.049756+0000 mgr.y (mgr.14140) 339 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:43 smithi139 bash[23439]: cluster 2022-04-28T19:43:42.049756+0000 mgr.y (mgr.14140) 339 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:43.512Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:43:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:43.512Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:43:43.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:43.512Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:43:44.775 INFO:teuthology.orchestra.run.smithi059.stdout:197568495663 2022-04-28T19:43:45.411 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495657 got 197568495663 for osd.7 2022-04-28T19:43:45.411 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2022-04-28T19:43:45.412 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph pg dump --format=json 2022-04-28T19:43:45.424 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:45 smithi059 bash[29114]: cluster 2022-04-28T19:43:44.050773+0000 mgr.y (mgr.14140) 340 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:45.424 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:45 smithi059 bash[29114]: audit 2022-04-28T19:43:44.772951+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.59:0/1508761823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-04-28T19:43:45.425 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:45 smithi059 bash[23786]: cluster 2022-04-28T19:43:44.050773+0000 mgr.y (mgr.14140) 340 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:45.425 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:45 smithi059 bash[23786]: audit 2022-04-28T19:43:44.772951+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.59:0/1508761823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-04-28T19:43:45.555 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:45 smithi139 bash[23439]: cluster 2022-04-28T19:43:44.050773+0000 mgr.y (mgr.14140) 340 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:45 smithi139 bash[23439]: audit 2022-04-28T19:43:44.772951+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.59:0/1508761823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2022-04-28T19:43:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:47 smithi059 bash[29114]: cluster 2022-04-28T19:43:46.051742+0000 mgr.y (mgr.14140) 341 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:47 smithi059 bash[23786]: cluster 2022-04-28T19:43:46.051742+0000 mgr.y (mgr.14140) 341 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:47 smithi139 bash[23439]: cluster 2022-04-28T19:43:46.051742+0000 mgr.y (mgr.14140) 341 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:48.745 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:43:48.746 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2022-04-28T19:43:49.440 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:49 smithi059 bash[29114]: cluster 2022-04-28T19:43:48.052694+0000 mgr.y (mgr.14140) 342 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:49.440 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:49 smithi059 bash[23786]: cluster 2022-04-28T19:43:48.052694+0000 mgr.y (mgr.14140) 342 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:49.442 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":302,"stamp":"2022-04-28T19:43:48.052038+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2304,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8393088,"kb_used_data":4416,"kb_used_omap":0,"kb_used_meta":8388608,"kb_avail":741568128,"statfs":{"total":767960285184,"available":759365763072,"internally_reserved":8589934592,"allocated":4521984,"data_stored":1888448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8589934592},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005478"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263633+0000","last_change":"2022-04-28T19:42:28.263333+0000","last_active":"2022-04-28T19:42:28.263633+0000","last_peered":"2022-04-28T19:42:28.263633+0000","last_clean":"2022-04-28T19:42:28.263633+0000","last_became_active":"2022-04-28T19:42:28.263110+0000","last_became_peered":"2022-04-28T19:42:28.263110+0000","last_unstale":"2022-04-28T19:42:28.263633+0000","last_undegraded":"2022-04-28T19:42:28.263633+0000","last_fullsized":"2022-04-28T19:42:28.263633+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"56'4","reported_seq":16,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.189185+0000","last_change":"2022-04-28T19:42:27.239469+0000","last_active":"2022-04-28T19:42:36.189185+0000","last_peered":"2022-04-28T19:42:36.189185+0000","last_clean":"2022-04-28T19:42:36.189185+0000","last_became_active":"2022-04-28T19:42:27.238941+0000","last_became_peered":"2022-04-28T19:42:27.238941+0000","last_unstale":"2022-04-28T19:42:36.189185+0000","last_undegraded":"2022-04-28T19:42:36.189185+0000","last_fullsized":"2022-04-28T19:42:36.189185+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":553,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484946+0000","last_change":"2022-04-28T19:42:27.273624+0000","last_active":"2022-04-28T19:42:28.484946+0000","last_peered":"2022-04-28T19:42:28.484946+0000","last_clean":"2022-04-28T19:42:28.484946+0000","last_became_active":"2022-04-28T19:42:27.273115+0000","last_became_peered":"2022-04-28T19:42:27.273115+0000","last_unstale":"2022-04-28T19:42:28.484946+0000","last_undegraded":"2022-04-28T19:42:28.484946+0000","last_fullsized":"2022-04-28T19:42:28.484946+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.265364+0000","last_change":"2022-04-28T19:42:27.246294+0000","last_active":"2022-04-28T19:42:29.265364+0000","last_peered":"2022-04-28T19:42:29.265364+0000","last_clean":"2022-04-28T19:42:29.265364+0000","last_became_active":"2022-04-28T19:42:27.245818+0000","last_became_peered":"2022-04-28T19:42:27.245818+0000","last_unstale":"2022-04-28T19:42:29.265364+0000","last_undegraded":"2022-04-28T19:42:29.265364+0000","last_fullsized":"2022-04-28T19:42:29.265364+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.488158+0000","last_change":"2022-04-28T19:42:28.487652+0000","last_active":"2022-04-28T19:42:28.488158+0000","last_peered":"2022-04-28T19:42:28.488158+0000","last_clean":"2022-04-28T19:42:28.488158+0000","last_became_active":"2022-04-28T19:42:28.486395+0000","last_became_peered":"2022-04-28T19:42:28.486395+0000","last_unstale":"2022-04-28T19:42:28.488158+0000","last_undegraded":"2022-04-28T19:42:28.488158+0000","last_fullsized":"2022-04-28T19:42:28.488158+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.190605+0000","last_change":"2022-04-28T19:42:27.260709+0000","last_active":"2022-04-28T19:42:36.190605+0000","last_peered":"2022-04-28T19:42:36.190605+0000","last_clean":"2022-04-28T19:42:36.190605+0000","last_became_active":"2022-04-28T19:42:27.260331+0000","last_became_peered":"2022-04-28T19:42:27.260331+0000","last_unstale":"2022-04-28T19:42:36.190605+0000","last_undegraded":"2022-04-28T19:42:36.190605+0000","last_fullsized":"2022-04-28T19:42:36.190605+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.485611+0000","last_change":"2022-04-28T19:42:28.485023+0000","last_active":"2022-04-28T19:42:28.485611+0000","last_peered":"2022-04-28T19:42:28.485611+0000","last_clean":"2022-04-28T19:42:28.485611+0000","last_became_active":"2022-04-28T19:42:28.484361+0000","last_became_peered":"2022-04-28T19:42:28.484361+0000","last_unstale":"2022-04-28T19:42:28.485611+0000","last_undegraded":"2022-04-28T19:42:28.485611+0000","last_fullsized":"2022-04-28T19:42:28.485611+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.192638+0000","last_change":"2022-04-28T19:42:27.260251+0000","last_active":"2022-04-28T19:42:36.192638+0000","last_peered":"2022-04-28T19:42:36.192638+0000","last_clean":"2022-04-28T19:42:36.192638+0000","last_became_active":"2022-04-28T19:42:27.259782+0000","last_became_peered":"2022-04-28T19:42:27.259782+0000","last_unstale":"2022-04-28T19:42:36.192638+0000","last_undegraded":"2022-04-28T19:42:36.192638+0000","last_fullsized":"2022-04-28T19:42:36.192638+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.945557+0000","last_change":"2022-04-28T19:42:27.267928+0000","last_active":"2022-04-28T19:42:28.945557+0000","last_peered":"2022-04-28T19:42:28.945557+0000","last_clean":"2022-04-28T19:42:28.945557+0000","last_became_active":"2022-04-28T19:42:27.267443+0000","last_became_peered":"2022-04-28T19:42:27.267443+0000","last_unstale":"2022-04-28T19:42:28.945557+0000","last_undegraded":"2022-04-28T19:42:28.945557+0000","last_fullsized":"2022-04-28T19:42:28.945557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.267203+0000","last_change":"2022-04-28T19:42:27.256493+0000","last_active":"2022-04-28T19:42:29.267203+0000","last_peered":"2022-04-28T19:42:29.267203+0000","last_clean":"2022-04-28T19:42:29.267203+0000","last_became_active":"2022-04-28T19:42:27.250015+0000","last_became_peered":"2022-04-28T19:42:27.250015+0000","last_unstale":"2022-04-28T19:42:29.267203+0000","last_undegraded":"2022-04-28T19:42:29.267203+0000","last_fullsized":"2022-04-28T19:42:29.267203+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.565407+0000","last_change":"2022-04-28T19:42:28.564652+0000","last_active":"2022-04-28T19:42:28.565407+0000","last_peered":"2022-04-28T19:42:28.565407+0000","last_clean":"2022-04-28T19:42:28.565407+0000","last_became_active":"2022-04-28T19:42:28.564176+0000","last_became_peered":"2022-04-28T19:42:28.564176+0000","last_unstale":"2022-04-28T19:42:28.565407+0000","last_undegraded":"2022-04-28T19:42:28.565407+0000","last_fullsized":"2022-04-28T19:42:28.565407+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.264139+0000","last_change":"2022-04-28T19:42:27.259973+0000","last_active":"2022-04-28T19:42:29.264139+0000","last_peered":"2022-04-28T19:42:29.264139+0000","last_clean":"2022-04-28T19:42:29.264139+0000","last_became_active":"2022-04-28T19:42:27.256958+0000","last_became_peered":"2022-04-28T19:42:27.256958+0000","last_unstale":"2022-04-28T19:42:29.264139+0000","last_undegraded":"2022-04-28T19:42:29.264139+0000","last_fullsized":"2022-04-28T19:42:29.264139+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263982+0000","last_change":"2022-04-28T19:40:39.845995+0000","last_active":"2022-04-28T19:42:28.263982+0000","last_peered":"2022-04-28T19:42:28.263982+0000","last_clean":"2022-04-28T19:42:28.263982+0000","last_became_active":"2022-04-28T19:40:39.845574+0000","last_became_peered":"2022-04-28T19:40:39.845574+0000","last_unstale":"2022-04-28T19:42:28.263982+0000","last_undegraded":"2022-04-28T19:42:28.263982+0000","last_fullsized":"2022-04-28T19:42:28.263982+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483595+0000","last_change":"2022-04-28T19:42:27.327656+0000","last_active":"2022-04-28T19:42:28.483595+0000","last_peered":"2022-04-28T19:42:28.483595+0000","last_clean":"2022-04-28T19:42:28.483595+0000","last_became_active":"2022-04-28T19:42:27.327205+0000","last_became_peered":"2022-04-28T19:42:27.327205+0000","last_unstale":"2022-04-28T19:42:28.483595+0000","last_undegraded":"2022-04-28T19:42:28.483595+0000","last_fullsized":"2022-04-28T19:42:28.483595+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254256+0000","last_change":"2022-04-28T19:40:39.837533+0000","last_active":"2022-04-28T19:42:28.254256+0000","last_peered":"2022-04-28T19:42:28.254256+0000","last_clean":"2022-04-28T19:42:28.254256+0000","last_became_active":"2022-04-28T19:40:39.837213+0000","last_became_peered":"2022-04-28T19:40:39.837213+0000","last_unstale":"2022-04-28T19:42:28.254256+0000","last_undegraded":"2022-04-28T19:42:28.254256+0000","last_fullsized":"2022-04-28T19:42:28.254256+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.188366+0000","last_change":"2022-04-28T19:42:27.249044+0000","last_active":"2022-04-28T19:42:36.188366+0000","last_peered":"2022-04-28T19:42:36.188366+0000","last_clean":"2022-04-28T19:42:36.188366+0000","last_became_active":"2022-04-28T19:42:27.248558+0000","last_became_peered":"2022-04-28T19:42:27.248558+0000","last_unstale":"2022-04-28T19:42:36.188366+0000","last_undegraded":"2022-04-28T19:42:36.188366+0000","last_fullsized":"2022-04-28T19:42:36.188366+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254143+0000","last_change":"2022-04-28T19:42:27.328071+0000","last_active":"2022-04-28T19:42:28.254143+0000","last_peered":"2022-04-28T19:42:28.254143+0000","last_clean":"2022-04-28T19:42:28.254143+0000","last_became_active":"2022-04-28T19:42:27.327332+0000","last_became_peered":"2022-04-28T19:42:27.327332+0000","last_unstale":"2022-04-28T19:42:28.254143+0000","last_undegraded":"2022-04-28T19:42:28.254143+0000","last_fullsized":"2022-04-28T19:42:28.254143+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253816+0000","last_change":"2022-04-28T19:42:27.250305+0000","last_active":"2022-04-28T19:42:28.253816+0000","last_peered":"2022-04-28T19:42:28.253816+0000","last_clean":"2022-04-28T19:42:28.253816+0000","last_became_active":"2022-04-28T19:42:27.249816+0000","last_became_peered":"2022-04-28T19:42:27.249816+0000","last_unstale":"2022-04-28T19:42:28.253816+0000","last_undegraded":"2022-04-28T19:42:28.253816+0000","last_fullsized":"2022-04-28T19:42:28.253816+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907654+0000","last_change":"2022-04-28T19:40:39.846944+0000","last_active":"2022-04-28T19:42:28.907654+0000","last_peered":"2022-04-28T19:42:28.907654+0000","last_clean":"2022-04-28T19:42:28.907654+0000","last_became_active":"2022-04-28T19:40:39.846597+0000","last_became_peered":"2022-04-28T19:40:39.846597+0000","last_unstale":"2022-04-28T19:42:28.907654+0000","last_undegraded":"2022-04-28T19:42:28.907654+0000","last_fullsized":"2022-04-28T19:42:28.907654+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254270+0000","last_change":"2022-04-28T19:42:27.268475+0000","last_active":"2022-04-28T19:42:28.254270+0000","last_peered":"2022-04-28T19:42:28.254270+0000","last_clean":"2022-04-28T19:42:28.254270+0000","last_became_active":"2022-04-28T19:42:27.268077+0000","last_became_peered":"2022-04-28T19:42:27.268077+0000","last_unstale":"2022-04-28T19:42:28.254270+0000","last_undegraded":"2022-04-28T19:42:28.254270+0000","last_fullsized":"2022-04-28T19:42:28.254270+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.252615+0000","last_change":"2022-04-28T19:42:27.250732+0000","last_active":"2022-04-28T19:42:28.252615+0000","last_peered":"2022-04-28T19:42:28.252615+0000","last_clean":"2022-04-28T19:42:28.252615+0000","last_became_active":"2022-04-28T19:42:27.249950+0000","last_became_peered":"2022-04-28T19:42:27.249950+0000","last_unstale":"2022-04-28T19:42:28.252615+0000","last_undegraded":"2022-04-28T19:42:28.252615+0000","last_fullsized":"2022-04-28T19:42:28.252615+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907138+0000","last_change":"2022-04-28T19:40:39.848985+0000","last_active":"2022-04-28T19:42:28.907138+0000","last_peered":"2022-04-28T19:42:28.907138+0000","last_clean":"2022-04-28T19:42:28.907138+0000","last_became_active":"2022-04-28T19:40:39.848622+0000","last_became_peered":"2022-04-28T19:40:39.848622+0000","last_unstale":"2022-04-28T19:42:28.907138+0000","last_undegraded":"2022-04-28T19:42:28.907138+0000","last_fullsized":"2022-04-28T19:42:28.907138+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.909406+0000","last_change":"2022-04-28T19:42:28.908907+0000","last_active":"2022-04-28T19:42:28.909406+0000","last_peered":"2022-04-28T19:42:28.909406+0000","last_clean":"2022-04-28T19:42:28.909406+0000","last_became_active":"2022-04-28T19:42:28.908537+0000","last_became_peered":"2022-04-28T19:42:28.908537+0000","last_unstale":"2022-04-28T19:42:28.909406+0000","last_undegraded":"2022-04-28T19:42:28.909406+0000","last_fullsized":"2022-04-28T19:42:28.909406+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484713+0000","last_change":"2022-04-28T19:42:27.250116+0000","last_active":"2022-04-28T19:42:28.484713+0000","last_peered":"2022-04-28T19:42:28.484713+0000","last_clean":"2022-04-28T19:42:28.484713+0000","last_became_active":"2022-04-28T19:42:27.249710+0000","last_became_peered":"2022-04-28T19:42:27.249710+0000","last_unstale":"2022-04-28T19:42:28.484713+0000","last_undegraded":"2022-04-28T19:42:28.484713+0000","last_fullsized":"2022-04-28T19:42:28.484713+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":19,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907080+0000","last_change":"2022-04-28T19:40:02.912608+0000","last_active":"2022-04-28T19:42:28.907080+0000","last_peered":"2022-04-28T19:42:28.907080+0000","last_clean":"2022-04-28T19:42:28.907080+0000","last_became_active":"2022-04-28T19:40:02.911686+0000","last_became_peered":"2022-04-28T19:40:02.911686+0000","last_unstale":"2022-04-28T19:42:28.907080+0000","last_undegraded":"2022-04-28T19:42:28.907080+0000","last_fullsized":"2022-04-28T19:42:28.907080+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":5,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_clean_scrub_stamp":"2022-04-28T19:36:10.237993+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563616+0000","last_change":"2022-04-28T19:40:39.847019+0000","last_active":"2022-04-28T19:42:28.563616+0000","last_peered":"2022-04-28T19:42:28.563616+0000","last_clean":"2022-04-28T19:42:28.563616+0000","last_became_active":"2022-04-28T19:40:39.846557+0000","last_became_peered":"2022-04-28T19:40:39.846557+0000","last_unstale":"2022-04-28T19:42:28.563616+0000","last_undegraded":"2022-04-28T19:42:28.563616+0000","last_fullsized":"2022-04-28T19:42:28.563616+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254321+0000","last_change":"2022-04-28T19:42:27.271631+0000","last_active":"2022-04-28T19:42:28.254321+0000","last_peered":"2022-04-28T19:42:28.254321+0000","last_clean":"2022-04-28T19:42:28.254321+0000","last_became_active":"2022-04-28T19:42:27.271033+0000","last_became_peered":"2022-04-28T19:42:27.271033+0000","last_unstale":"2022-04-28T19:42:28.254321+0000","last_undegraded":"2022-04-28T19:42:28.254321+0000","last_fullsized":"2022-04-28T19:42:28.254321+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253959+0000","last_change":"2022-04-28T19:42:27.244744+0000","last_active":"2022-04-28T19:42:28.253959+0000","last_peered":"2022-04-28T19:42:28.253959+0000","last_clean":"2022-04-28T19:42:28.253959+0000","last_became_active":"2022-04-28T19:42:27.244334+0000","last_became_peered":"2022-04-28T19:42:27.244334+0000","last_unstale":"2022-04-28T19:42:28.253959+0000","last_undegraded":"2022-04-28T19:42:28.253959+0000","last_fullsized":"2022-04-28T19:42:28.253959+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.945411+0000","last_change":"2022-04-28T19:40:40.101205+0000","last_active":"2022-04-28T19:42:28.945411+0000","last_peered":"2022-04-28T19:42:28.945411+0000","last_clean":"2022-04-28T19:42:28.945411+0000","last_became_active":"2022-04-28T19:40:40.100696+0000","last_became_peered":"2022-04-28T19:40:40.100696+0000","last_unstale":"2022-04-28T19:42:28.945411+0000","last_undegraded":"2022-04-28T19:42:28.945411+0000","last_fullsized":"2022-04-28T19:42:28.945411+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253322+0000","last_change":"2022-04-28T19:42:27.270202+0000","last_active":"2022-04-28T19:42:28.253322+0000","last_peered":"2022-04-28T19:42:28.253322+0000","last_clean":"2022-04-28T19:42:28.253322+0000","last_became_active":"2022-04-28T19:42:27.269794+0000","last_became_peered":"2022-04-28T19:42:27.269794+0000","last_unstale":"2022-04-28T19:42:28.253322+0000","last_undegraded":"2022-04-28T19:42:28.253322+0000","last_fullsized":"2022-04-28T19:42:28.253322+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.259128+0000","last_change":"2022-04-28T19:42:27.250744+0000","last_active":"2022-04-28T19:42:28.259128+0000","last_peered":"2022-04-28T19:42:28.259128+0000","last_clean":"2022-04-28T19:42:28.259128+0000","last_became_active":"2022-04-28T19:42:27.248908+0000","last_became_peered":"2022-04-28T19:42:27.248908+0000","last_unstale":"2022-04-28T19:42:28.259128+0000","last_undegraded":"2022-04-28T19:42:28.259128+0000","last_fullsized":"2022-04-28T19:42:28.259128+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563661+0000","last_change":"2022-04-28T19:40:39.848860+0000","last_active":"2022-04-28T19:42:28.563661+0000","last_peered":"2022-04-28T19:42:28.563661+0000","last_clean":"2022-04-28T19:42:28.563661+0000","last_became_active":"2022-04-28T19:40:39.848587+0000","last_became_peered":"2022-04-28T19:40:39.848587+0000","last_unstale":"2022-04-28T19:42:28.563661+0000","last_undegraded":"2022-04-28T19:42:28.563661+0000","last_fullsized":"2022-04-28T19:42:28.563661+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253281+0000","last_change":"2022-04-28T19:42:27.326386+0000","last_active":"2022-04-28T19:42:28.253281+0000","last_peered":"2022-04-28T19:42:28.253281+0000","last_clean":"2022-04-28T19:42:28.253281+0000","last_became_active":"2022-04-28T19:42:27.325767+0000","last_became_peered":"2022-04-28T19:42:27.325767+0000","last_unstale":"2022-04-28T19:42:28.253281+0000","last_undegraded":"2022-04-28T19:42:28.253281+0000","last_fullsized":"2022-04-28T19:42:28.253281+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.564872+0000","last_change":"2022-04-28T19:42:27.247247+0000","last_active":"2022-04-28T19:42:28.564872+0000","last_peered":"2022-04-28T19:42:28.564872+0000","last_clean":"2022-04-28T19:42:28.564872+0000","last_became_active":"2022-04-28T19:42:27.246833+0000","last_became_peered":"2022-04-28T19:42:27.246833+0000","last_unstale":"2022-04-28T19:42:28.564872+0000","last_undegraded":"2022-04-28T19:42:28.564872+0000","last_fullsized":"2022-04-28T19:42:28.564872+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253206+0000","last_change":"2022-04-28T19:40:39.838356+0000","last_active":"2022-04-28T19:42:28.253206+0000","last_peered":"2022-04-28T19:42:28.253206+0000","last_clean":"2022-04-28T19:42:28.253206+0000","last_became_active":"2022-04-28T19:40:39.837847+0000","last_became_peered":"2022-04-28T19:40:39.837847+0000","last_unstale":"2022-04-28T19:42:28.253206+0000","last_undegraded":"2022-04-28T19:42:28.253206+0000","last_fullsized":"2022-04-28T19:42:28.253206+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254508+0000","last_change":"2022-04-28T19:42:27.325953+0000","last_active":"2022-04-28T19:42:28.254508+0000","last_peered":"2022-04-28T19:42:28.254508+0000","last_clean":"2022-04-28T19:42:28.254508+0000","last_became_active":"2022-04-28T19:42:27.325071+0000","last_became_peered":"2022-04-28T19:42:27.325071+0000","last_unstale":"2022-04-28T19:42:28.254508+0000","last_undegraded":"2022-04-28T19:42:28.254508+0000","last_fullsized":"2022-04-28T19:42:28.254508+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483765+0000","last_change":"2022-04-28T19:42:27.253821+0000","last_active":"2022-04-28T19:42:28.483765+0000","last_peered":"2022-04-28T19:42:28.483765+0000","last_clean":"2022-04-28T19:42:28.483765+0000","last_became_active":"2022-04-28T19:42:27.253401+0000","last_became_peered":"2022-04-28T19:42:27.253401+0000","last_unstale":"2022-04-28T19:42:28.483765+0000","last_undegraded":"2022-04-28T19:42:28.483765+0000","last_fullsized":"2022-04-28T19:42:28.483765+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.265051+0000","last_change":"2022-04-28T19:42:28.264810+0000","last_active":"2022-04-28T19:42:28.265051+0000","last_peered":"2022-04-28T19:42:28.265051+0000","last_clean":"2022-04-28T19:42:28.265051+0000","last_became_active":"2022-04-28T19:42:28.264499+0000","last_became_peered":"2022-04-28T19:42:28.264499+0000","last_unstale":"2022-04-28T19:42:28.265051+0000","last_undegraded":"2022-04-28T19:42:28.265051+0000","last_fullsized":"2022-04-28T19:42:28.265051+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.257866+0000","last_change":"2022-04-28T19:42:27.256454+0000","last_active":"2022-04-28T19:42:28.257866+0000","last_peered":"2022-04-28T19:42:28.257866+0000","last_clean":"2022-04-28T19:42:28.257866+0000","last_became_active":"2022-04-28T19:42:27.252186+0000","last_became_peered":"2022-04-28T19:42:27.252186+0000","last_unstale":"2022-04-28T19:42:28.257866+0000","last_undegraded":"2022-04-28T19:42:28.257866+0000","last_fullsized":"2022-04-28T19:42:28.257866+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.947298+0000","last_change":"2022-04-28T19:42:28.947039+0000","last_active":"2022-04-28T19:42:28.947298+0000","last_peered":"2022-04-28T19:42:28.947298+0000","last_clean":"2022-04-28T19:42:28.947298+0000","last_became_active":"2022-04-28T19:42:28.946558+0000","last_became_peered":"2022-04-28T19:42:28.946558+0000","last_unstale":"2022-04-28T19:42:28.947298+0000","last_undegraded":"2022-04-28T19:42:28.947298+0000","last_fullsized":"2022-04-28T19:42:28.947298+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263012+0000","last_change":"2022-04-28T19:42:27.253990+0000","last_active":"2022-04-28T19:42:28.263012+0000","last_peered":"2022-04-28T19:42:28.263012+0000","last_clean":"2022-04-28T19:42:28.263012+0000","last_became_active":"2022-04-28T19:42:27.253662+0000","last_became_peered":"2022-04-28T19:42:27.253662+0000","last_unstale":"2022-04-28T19:42:28.263012+0000","last_undegraded":"2022-04-28T19:42:28.263012+0000","last_fullsized":"2022-04-28T19:42:28.263012+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.487107+0000","last_change":"2022-04-28T19:42:28.486474+0000","last_active":"2022-04-28T19:42:28.487107+0000","last_peered":"2022-04-28T19:42:28.487107+0000","last_clean":"2022-04-28T19:42:28.487107+0000","last_became_active":"2022-04-28T19:42:28.485762+0000","last_became_peered":"2022-04-28T19:42:28.485762+0000","last_unstale":"2022-04-28T19:42:28.487107+0000","last_undegraded":"2022-04-28T19:42:28.487107+0000","last_fullsized":"2022-04-28T19:42:28.487107+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.189441+0000","last_change":"2022-04-28T19:42:27.254656+0000","last_active":"2022-04-28T19:42:36.189441+0000","last_peered":"2022-04-28T19:42:36.189441+0000","last_clean":"2022-04-28T19:42:36.189441+0000","last_became_active":"2022-04-28T19:42:27.254143+0000","last_became_peered":"2022-04-28T19:42:27.254143+0000","last_unstale":"2022-04-28T19:42:36.189441+0000","last_undegraded":"2022-04-28T19:42:36.189441+0000","last_fullsized":"2022-04-28T19:42:36.189441+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253913+0000","last_change":"2022-04-28T19:42:27.326241+0000","last_active":"2022-04-28T19:42:28.253913+0000","last_peered":"2022-04-28T19:42:28.253913+0000","last_clean":"2022-04-28T19:42:28.253913+0000","last_became_active":"2022-04-28T19:42:27.325542+0000","last_became_peered":"2022-04-28T19:42:27.325542+0000","last_unstale":"2022-04-28T19:42:28.253913+0000","last_undegraded":"2022-04-28T19:42:28.253913+0000","last_fullsized":"2022-04-28T19:42:28.253913+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563607+0000","last_change":"2022-04-28T19:42:27.246713+0000","last_active":"2022-04-28T19:42:28.563607+0000","last_peered":"2022-04-28T19:42:28.563607+0000","last_clean":"2022-04-28T19:42:28.563607+0000","last_became_active":"2022-04-28T19:42:27.246287+0000","last_became_peered":"2022-04-28T19:42:27.246287+0000","last_unstale":"2022-04-28T19:42:28.563607+0000","last_undegraded":"2022-04-28T19:42:28.563607+0000","last_fullsized":"2022-04-28T19:42:28.563607+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.909429+0000","last_change":"2022-04-28T19:42:28.908879+0000","last_active":"2022-04-28T19:42:28.909429+0000","last_peered":"2022-04-28T19:42:28.909429+0000","last_clean":"2022-04-28T19:42:28.909429+0000","last_became_active":"2022-04-28T19:42:28.908417+0000","last_became_peered":"2022-04-28T19:42:28.908417+0000","last_unstale":"2022-04-28T19:42:28.909429+0000","last_undegraded":"2022-04-28T19:42:28.909429+0000","last_fullsized":"2022-04-28T19:42:28.909429+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.856444+0000","last_change":"2022-04-28T19:42:27.249628+0000","last_active":"2022-04-28T19:42:29.856444+0000","last_peered":"2022-04-28T19:42:29.856444+0000","last_clean":"2022-04-28T19:42:29.856444+0000","last_became_active":"2022-04-28T19:42:27.249203+0000","last_became_peered":"2022-04-28T19:42:27.249203+0000","last_unstale":"2022-04-28T19:42:29.856444+0000","last_undegraded":"2022-04-28T19:42:29.856444+0000","last_fullsized":"2022-04-28T19:42:29.856444+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.487895+0000","last_change":"2022-04-28T19:42:28.487607+0000","last_active":"2022-04-28T19:42:28.487895+0000","last_peered":"2022-04-28T19:42:28.487895+0000","last_clean":"2022-04-28T19:42:28.487895+0000","last_became_active":"2022-04-28T19:42:28.486682+0000","last_became_peered":"2022-04-28T19:42:28.486682+0000","last_unstale":"2022-04-28T19:42:28.487895+0000","last_undegraded":"2022-04-28T19:42:28.487895+0000","last_fullsized":"2022-04-28T19:42:28.487895+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.188446+0000","last_change":"2022-04-28T19:42:27.255168+0000","last_active":"2022-04-28T19:42:36.188446+0000","last_peered":"2022-04-28T19:42:36.188446+0000","last_clean":"2022-04-28T19:42:36.188446+0000","last_became_active":"2022-04-28T19:42:27.254478+0000","last_became_peered":"2022-04-28T19:42:27.254478+0000","last_unstale":"2022-04-28T19:42:36.188446+0000","last_undegraded":"2022-04-28T19:42:36.188446+0000","last_fullsized":"2022-04-28T19:42:36.188446+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563583+0000","last_change":"2022-04-28T19:42:27.328189+0000","last_active":"2022-04-28T19:42:28.563583+0000","last_peered":"2022-04-28T19:42:28.563583+0000","last_clean":"2022-04-28T19:42:28.563583+0000","last_became_active":"2022-04-28T19:42:27.327231+0000","last_became_peered":"2022-04-28T19:42:27.327231+0000","last_unstale":"2022-04-28T19:42:28.563583+0000","last_undegraded":"2022-04-28T19:42:28.563583+0000","last_fullsized":"2022-04-28T19:42:28.563583+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.856150+0000","last_change":"2022-04-28T19:42:27.253531+0000","last_active":"2022-04-28T19:42:29.856150+0000","last_peered":"2022-04-28T19:42:29.856150+0000","last_clean":"2022-04-28T19:42:29.856150+0000","last_became_active":"2022-04-28T19:42:27.252949+0000","last_became_peered":"2022-04-28T19:42:27.252949+0000","last_unstale":"2022-04-28T19:42:29.856150+0000","last_undegraded":"2022-04-28T19:42:29.856150+0000","last_fullsized":"2022-04-28T19:42:29.856150+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483957+0000","last_change":"2022-04-28T19:42:27.330855+0000","last_active":"2022-04-28T19:42:28.483957+0000","last_peered":"2022-04-28T19:42:28.483957+0000","last_clean":"2022-04-28T19:42:28.483957+0000","last_became_active":"2022-04-28T19:42:27.330047+0000","last_became_peered":"2022-04-28T19:42:27.330047+0000","last_unstale":"2022-04-28T19:42:28.483957+0000","last_undegraded":"2022-04-28T19:42:28.483957+0000","last_fullsized":"2022-04-28T19:42:28.483957+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"56'8","reported_seq":20,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.194007+0000","last_change":"2022-04-28T19:42:27.249243+0000","last_active":"2022-04-28T19:42:36.194007+0000","last_peered":"2022-04-28T19:42:36.194007+0000","last_clean":"2022-04-28T19:42:36.194007+0000","last_became_active":"2022-04-28T19:42:27.248832+0000","last_became_peered":"2022-04-28T19:42:27.248832+0000","last_unstale":"2022-04-28T19:42:36.194007+0000","last_undegraded":"2022-04-28T19:42:36.194007+0000","last_fullsized":"2022-04-28T19:42:36.194007+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253898+0000","last_change":"2022-04-28T19:42:27.330900+0000","last_active":"2022-04-28T19:42:28.253898+0000","last_peered":"2022-04-28T19:42:28.253898+0000","last_clean":"2022-04-28T19:42:28.253898+0000","last_became_active":"2022-04-28T19:42:27.330465+0000","last_became_peered":"2022-04-28T19:42:27.330465+0000","last_unstale":"2022-04-28T19:42:28.253898+0000","last_undegraded":"2022-04-28T19:42:28.253898+0000","last_fullsized":"2022-04-28T19:42:28.253898+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907144+0000","last_change":"2022-04-28T19:42:27.256107+0000","last_active":"2022-04-28T19:42:28.907144+0000","last_peered":"2022-04-28T19:42:28.907144+0000","last_clean":"2022-04-28T19:42:28.907144+0000","last_became_active":"2022-04-28T19:42:27.255024+0000","last_became_peered":"2022-04-28T19:42:27.255024+0000","last_unstale":"2022-04-28T19:42:28.907144+0000","last_undegraded":"2022-04-28T19:42:28.907144+0000","last_fullsized":"2022-04-28T19:42:28.907144+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563887+0000","last_change":"2022-04-28T19:42:27.330064+0000","last_active":"2022-04-28T19:42:28.563887+0000","last_peered":"2022-04-28T19:42:28.563887+0000","last_clean":"2022-04-28T19:42:28.563887+0000","last_became_active":"2022-04-28T19:42:27.329658+0000","last_became_peered":"2022-04-28T19:42:27.329658+0000","last_unstale":"2022-04-28T19:42:28.563887+0000","last_undegraded":"2022-04-28T19:42:28.563887+0000","last_fullsized":"2022-04-28T19:42:28.563887+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.252610+0000","last_change":"2022-04-28T19:42:27.249334+0000","last_active":"2022-04-28T19:42:28.252610+0000","last_peered":"2022-04-28T19:42:28.252610+0000","last_clean":"2022-04-28T19:42:28.252610+0000","last_became_active":"2022-04-28T19:42:27.248594+0000","last_became_peered":"2022-04-28T19:42:27.248594+0000","last_unstale":"2022-04-28T19:42:28.252610+0000","last_undegraded":"2022-04-28T19:42:28.252610+0000","last_fullsized":"2022-04-28T19:42:28.252610+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484731+0000","last_change":"2022-04-28T19:42:27.329677+0000","last_active":"2022-04-28T19:42:28.484731+0000","last_peered":"2022-04-28T19:42:28.484731+0000","last_clean":"2022-04-28T19:42:28.484731+0000","last_became_active":"2022-04-28T19:42:27.329252+0000","last_became_peered":"2022-04-28T19:42:27.329252+0000","last_unstale":"2022-04-28T19:42:28.484731+0000","last_undegraded":"2022-04-28T19:42:28.484731+0000","last_fullsized":"2022-04-28T19:42:28.484731+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.192849+0000","last_change":"2022-04-28T19:42:27.255633+0000","last_active":"2022-04-28T19:42:36.192849+0000","last_peered":"2022-04-28T19:42:36.192849+0000","last_clean":"2022-04-28T19:42:36.192849+0000","last_became_active":"2022-04-28T19:42:27.254990+0000","last_became_peered":"2022-04-28T19:42:27.254990+0000","last_unstale":"2022-04-28T19:42:36.192849+0000","last_undegraded":"2022-04-28T19:42:36.192849+0000","last_fullsized":"2022-04-28T19:42:36.192849+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263076+0000","last_change":"2022-04-28T19:42:28.262801+0000","last_active":"2022-04-28T19:42:28.263076+0000","last_peered":"2022-04-28T19:42:28.263076+0000","last_clean":"2022-04-28T19:42:28.263076+0000","last_became_active":"2022-04-28T19:42:28.262437+0000","last_became_peered":"2022-04-28T19:42:28.262437+0000","last_unstale":"2022-04-28T19:42:28.263076+0000","last_undegraded":"2022-04-28T19:42:28.263076+0000","last_fullsized":"2022-04-28T19:42:28.263076+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"56'6","reported_seq":18,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.194557+0000","last_change":"2022-04-28T19:42:27.257775+0000","last_active":"2022-04-28T19:42:36.194557+0000","last_peered":"2022-04-28T19:42:36.194557+0000","last_clean":"2022-04-28T19:42:36.194557+0000","last_became_active":"2022-04-28T19:42:27.257418+0000","last_became_peered":"2022-04-28T19:42:27.257418+0000","last_unstale":"2022-04-28T19:42:36.194557+0000","last_undegraded":"2022-04-28T19:42:36.194557+0000","last_fullsized":"2022-04-28T19:42:36.194557+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1535,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253210+0000","last_change":"2022-04-28T19:42:27.262070+0000","last_active":"2022-04-28T19:42:28.253210+0000","last_peered":"2022-04-28T19:42:28.253210+0000","last_clean":"2022-04-28T19:42:28.253210+0000","last_became_active":"2022-04-28T19:42:27.261722+0000","last_became_peered":"2022-04-28T19:42:27.261722+0000","last_unstale":"2022-04-28T19:42:28.253210+0000","last_undegraded":"2022-04-28T19:42:28.253210+0000","last_fullsized":"2022-04-28T19:42:28.253210+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"56'3","reported_seq":15,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.192201+0000","last_change":"2022-04-28T19:42:27.255046+0000","last_active":"2022-04-28T19:42:36.192201+0000","last_peered":"2022-04-28T19:42:36.192201+0000","last_clean":"2022-04-28T19:42:36.192201+0000","last_became_active":"2022-04-28T19:42:27.254475+0000","last_became_peered":"2022-04-28T19:42:27.254475+0000","last_unstale":"2022-04-28T19:42:36.192201+0000","last_undegraded":"2022-04-28T19:42:36.192201+0000","last_fullsized":"2022-04-28T19:42:36.192201+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.485244+0000","last_change":"2022-04-28T19:42:27.329682+0000","last_active":"2022-04-28T19:42:28.485244+0000","last_peered":"2022-04-28T19:42:28.485244+0000","last_clean":"2022-04-28T19:42:28.485244+0000","last_became_active":"2022-04-28T19:42:27.329263+0000","last_became_peered":"2022-04-28T19:42:27.329263+0000","last_unstale":"2022-04-28T19:42:28.485244+0000","last_undegraded":"2022-04-28T19:42:28.485244+0000","last_fullsized":"2022-04-28T19:42:28.485244+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563758+0000","last_change":"2022-04-28T19:42:27.250559+0000","last_active":"2022-04-28T19:42:28.563758+0000","last_peered":"2022-04-28T19:42:28.563758+0000","last_clean":"2022-04-28T19:42:28.563758+0000","last_became_active":"2022-04-28T19:42:27.250088+0000","last_became_peered":"2022-04-28T19:42:27.250088+0000","last_unstale":"2022-04-28T19:42:28.563758+0000","last_undegraded":"2022-04-28T19:42:28.563758+0000","last_fullsized":"2022-04-28T19:42:28.563758+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483769+0000","last_change":"2022-04-28T19:42:27.271551+0000","last_active":"2022-04-28T19:42:28.483769+0000","last_peered":"2022-04-28T19:42:28.483769+0000","last_clean":"2022-04-28T19:42:28.483769+0000","last_became_active":"2022-04-28T19:42:27.271161+0000","last_became_peered":"2022-04-28T19:42:27.271161+0000","last_unstale":"2022-04-28T19:42:28.483769+0000","last_undegraded":"2022-04-28T19:42:28.483769+0000","last_fullsized":"2022-04-28T19:42:28.483769+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253390+0000","last_change":"2022-04-28T19:42:27.247683+0000","last_active":"2022-04-28T19:42:28.253390+0000","last_peered":"2022-04-28T19:42:28.253390+0000","last_clean":"2022-04-28T19:42:28.253390+0000","last_became_active":"2022-04-28T19:42:27.245987+0000","last_became_peered":"2022-04-28T19:42:27.245987+0000","last_unstale":"2022-04-28T19:42:28.253390+0000","last_undegraded":"2022-04-28T19:42:28.253390+0000","last_fullsized":"2022-04-28T19:42:28.253390+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.566416+0000","last_change":"2022-04-28T19:42:28.565733+0000","last_active":"2022-04-28T19:42:28.566416+0000","last_peered":"2022-04-28T19:42:28.566416+0000","last_clean":"2022-04-28T19:42:28.566416+0000","last_became_active":"2022-04-28T19:42:28.564968+0000","last_became_peered":"2022-04-28T19:42:28.564968+0000","last_unstale":"2022-04-28T19:42:28.566416+0000","last_undegraded":"2022-04-28T19:42:28.566416+0000","last_fullsized":"2022-04-28T19:42:28.566416+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483941+0000","last_change":"2022-04-28T19:42:27.251808+0000","last_active":"2022-04-28T19:42:28.483941+0000","last_peered":"2022-04-28T19:42:28.483941+0000","last_clean":"2022-04-28T19:42:28.483941+0000","last_became_active":"2022-04-28T19:42:27.250732+0000","last_became_peered":"2022-04-28T19:42:27.250732+0000","last_unstale":"2022-04-28T19:42:28.483941+0000","last_undegraded":"2022-04-28T19:42:28.483941+0000","last_fullsized":"2022-04-28T19:42:28.483941+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254025+0000","last_change":"2022-04-28T19:42:27.328400+0000","last_active":"2022-04-28T19:42:28.254025+0000","last_peered":"2022-04-28T19:42:28.254025+0000","last_clean":"2022-04-28T19:42:28.254025+0000","last_became_active":"2022-04-28T19:42:27.327976+0000","last_became_peered":"2022-04-28T19:42:27.327976+0000","last_unstale":"2022-04-28T19:42:28.254025+0000","last_undegraded":"2022-04-28T19:42:28.254025+0000","last_fullsized":"2022-04-28T19:42:28.254025+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.252922+0000","last_change":"2022-04-28T19:42:27.251754+0000","last_active":"2022-04-28T19:42:28.252922+0000","last_peered":"2022-04-28T19:42:28.252922+0000","last_clean":"2022-04-28T19:42:28.252922+0000","last_became_active":"2022-04-28T19:42:27.251030+0000","last_became_peered":"2022-04-28T19:42:27.251030+0000","last_unstale":"2022-04-28T19:42:28.252922+0000","last_undegraded":"2022-04-28T19:42:28.252922+0000","last_fullsized":"2022-04-28T19:42:28.252922+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484689+0000","last_change":"2022-04-28T19:42:27.326396+0000","last_active":"2022-04-28T19:42:28.484689+0000","last_peered":"2022-04-28T19:42:28.484689+0000","last_clean":"2022-04-28T19:42:28.484689+0000","last_became_active":"2022-04-28T19:42:27.325373+0000","last_became_peered":"2022-04-28T19:42:27.325373+0000","last_unstale":"2022-04-28T19:42:28.484689+0000","last_undegraded":"2022-04-28T19:42:28.484689+0000","last_fullsized":"2022-04-28T19:42:28.484689+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.265771+0000","last_change":"2022-04-28T19:42:27.250075+0000","last_active":"2022-04-28T19:42:29.265771+0000","last_peered":"2022-04-28T19:42:29.265771+0000","last_clean":"2022-04-28T19:42:29.265771+0000","last_became_active":"2022-04-28T19:42:27.248968+0000","last_became_peered":"2022-04-28T19:42:27.248968+0000","last_unstale":"2022-04-28T19:42:29.265771+0000","last_undegraded":"2022-04-28T19:42:29.265771+0000","last_fullsized":"2022-04-28T19:42:29.265771+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2304,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":46,"seq":197568495664,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049152,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696000,"statfs":{"total":95995035648,"available":94920704000,"internally_reserved":1073741824,"allocated":581632,"data_stored":236943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.81899999999999995,"15min":0.81999999999999995},"min":{"1min":0.498,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.1439999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82299999999999995,"15min":0.83599999999999997},"min":{"1min":0.54400000000000004,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.82399999999999995}]},{"osd":1,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.0489999999999999,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.84199999999999997,"15min":0.82099999999999995},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.8240000000000001,"5min":1.8240000000000001,"15min":1.8240000000000001},"last":0.98399999999999999}]},{"osd":2,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.84799999999999998,"15min":0.83799999999999997},"min":{"1min":0.65600000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":1.038},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.85999999999999999,"15min":0.84799999999999998},"min":{"1min":0.58299999999999996,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":2.298,"5min":2.298,"15min":2.298},"last":0.75}]},{"osd":3,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.89500000000000002,"15min":0.877},"min":{"1min":0.622,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.958,"5min":1.958,"15min":1.958},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.88200000000000001,"15min":0.86199999999999999},"min":{"1min":0.64600000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":2.165,"5min":2.165,"15min":2.165},"last":0.83899999999999997}]},{"osd":4,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.76600000000000001,"15min":0.75800000000000001},"min":{"1min":0.61799999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.80900000000000005,"15min":0.81000000000000005},"min":{"1min":0.54400000000000004,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"last":0.84699999999999998}]},{"osd":5,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.81100000000000005,"15min":0.79900000000000004},"min":{"1min":0.64900000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.0449999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.621},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.86299999999999999,"15min":0.86599999999999999},"min":{"1min":0.58199999999999996,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.109,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.77200000000000002}]},{"osd":6,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.64000000000000001,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.1539999999999999,"5min":1.409,"15min":1.409},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.92800000000000005,"15min":0.92700000000000005},"min":{"1min":0.67600000000000005,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":1.095}]}]},{"osd":6,"up_from":41,"seq":176093659189,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049140,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696012,"statfs":{"total":95995035648,"available":94920716288,"internally_reserved":1073741824,"allocated":569344,"data_stored":236797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.86699999999999999,"15min":0.879},"min":{"1min":0.61699999999999999,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.22,"5min":1.22,"15min":1.22},"last":0.622},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.88500000000000001,"15min":0.88900000000000001},"min":{"1min":0.627,"5min":0.502,"15min":0.502},"max":{"1min":1.165,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.76200000000000001}]},{"osd":1,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.86299999999999999,"15min":0.85899999999999999},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.0329999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.90400000000000003,"15min":0.92300000000000004},"min":{"1min":0.629,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1359999999999999,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.98699999999999999}]},{"osd":2,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.87,"15min":0.88200000000000001},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.91000000000000003,"15min":0.92400000000000004},"min":{"1min":0.65800000000000003,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.1799999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.94899999999999995}]},{"osd":3,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.85299999999999998,"15min":0.84799999999999998},"min":{"1min":0.61799999999999999,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.0649999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.96799999999999997,"15min":0.99199999999999999},"min":{"1min":0.71899999999999997,"5min":0.628,"15min":0.628},"max":{"1min":1.4059999999999999,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.91900000000000004}]},{"osd":4,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.81399999999999995,"15min":0.83399999999999996},"min":{"1min":0.46999999999999997,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3720000000000001,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.90400000000000003,"15min":0.92000000000000004},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.486,"5min":1.5,"15min":1.5},"last":0.81000000000000005}]},{"osd":5,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.88,"15min":0.88500000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.2869999999999999,"5min":1.663,"15min":1.663},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.878,"15min":0.88600000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1799999999999999,"5min":1.619,"15min":1.619},"last":1.0109999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:43:09 2022","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.92500000000000004,"15min":0.91300000000000003},"min":{"1min":0.56100000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.157,"5min":1.431,"15min":1.431},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.93500000000000005,"15min":0.91200000000000003},"min":{"1min":0.51700000000000002,"5min":0.495,"15min":0.495},"max":{"1min":1.1850000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.0649999999999999}]}]},{"osd":1,"up_from":16,"seq":68719476814,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049124,"kb_used_data":540,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696028,"statfs":{"total":95995035648,"available":94920732672,"internally_reserved":1073741824,"allocated":552960,"data_stored":235192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:54 2022","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.626,"15min":0.55600000000000005},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82599999999999996,"5min":1.736,"15min":1.736},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.68600000000000005,"15min":0.58599999999999997},"min":{"1min":0.54200000000000004,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.98299999999999998,"5min":1.544,"15min":1.544},"last":0.63}]},{"osd":2,"last update":"Thu Apr 28 19:43:14 2022","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.72899999999999998,"15min":0.71199999999999997},"min":{"1min":0.54400000000000004,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.101,"5min":1.377,"15min":1.377},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.70199999999999996,"15min":0.67400000000000004},"min":{"1min":0.47399999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0740000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.54000000000000004}]},{"osd":3,"last update":"Thu Apr 28 19:43:38 2022","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.78700000000000003,"15min":0.78400000000000003},"min":{"1min":0.54700000000000004,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.149,"5min":1.377,"15min":1.377},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.76800000000000002,"15min":0.76600000000000001},"min":{"1min":0.38200000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.99199999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Thu Apr 28 19:43:00 2022","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.873,"15min":0.84299999999999997},"min":{"1min":0.59999999999999998,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.175,"5min":1.446,"15min":1.446},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.95099999999999996,"15min":0.91200000000000003},"min":{"1min":0.66900000000000004,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.2350000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.89300000000000002}]},{"osd":5,"last update":"Thu Apr 28 19:43:19 2022","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.98999999999999999,"15min":0.998},"min":{"1min":0.71799999999999997,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.1839999999999999,"5min":1.788,"15min":1.788},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.996,"5min":0.98499999999999999,"15min":0.97499999999999998},"min":{"1min":0.77700000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.206,"5min":1.802,"15min":1.802},"last":0.91600000000000004}]},{"osd":6,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.05,"15min":1.0800000000000001},"min":{"1min":0.69299999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.3120000000000001,"5min":1.653,"15min":1.653},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.1020000000000001,"15min":1.141},"min":{"1min":0.73799999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.3069999999999999,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":1.0109999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:43:07 2022","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.1060000000000001,"15min":1.119},"min":{"1min":0.89100000000000001,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.5660000000000001,"5min":1.907,"15min":1.907},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1200000000000001,"15min":1.1259999999999999},"min":{"1min":0.84199999999999997,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.3360000000000001,"5min":1.673,"15min":1.673},"last":1.0580000000000001}]}]},{"osd":0,"up_from":10,"seq":42949673042,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049132,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696020,"statfs":{"total":95995035648,"available":94920724480,"internally_reserved":1073741824,"allocated":561152,"data_stored":235767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 28 19:42:52 2022","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.64700000000000002,"15min":0.58099999999999996},"min":{"1min":0.44,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.022,"5min":1.829,"15min":1.829},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.75800000000000001,"15min":0.89100000000000001},"min":{"1min":0.45800000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.081,"5min":9.4420000000000002,"15min":9.4420000000000002},"last":1.1180000000000001}]},{"osd":2,"last update":"Thu Apr 28 19:43:10 2022","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.67100000000000004,"15min":0.61099999999999999},"min":{"1min":0.48699999999999999,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.0860000000000001,"5min":1.2,"15min":1.2},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.75800000000000001,"15min":0.69899999999999995},"min":{"1min":0.52800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0649999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.81200000000000006}]},{"osd":3,"last update":"Thu Apr 28 19:43:40 2022","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.753,"15min":0.76000000000000001},"min":{"1min":0.442,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.117,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.82899999999999996,"15min":0.81299999999999994},"min":{"1min":0.54800000000000004,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.077,"5min":1.224,"15min":1.224},"last":1.0580000000000001}]},{"osd":4,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.95099999999999996,"15min":0.95999999999999996},"min":{"1min":0.69499999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.3240000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.012,"15min":0.96699999999999997},"min":{"1min":0.755,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":1.2150000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:43:20 2022","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.05,"15min":1.032},"min":{"1min":0.79800000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.425,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.298},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.96799999999999997,"15min":0.98399999999999999},"min":{"1min":0.626,"5min":0.62,"15min":0.62},"max":{"1min":1.3759999999999999,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":0.85599999999999998}]},{"osd":6,"last update":"Thu Apr 28 19:42:45 2022","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.01,"15min":1},"min":{"1min":0.71499999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.3200000000000001,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.0109999999999999,"15min":0.997},"min":{"1min":0.748,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.264,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.044}]},{"osd":7,"last update":"Thu Apr 28 19:43:10 2022","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.1319999999999999,"15min":1.1319999999999999},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3919999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.369},{"interface":"front","average":{"1min":1.03,"5min":1.048,"15min":1.0349999999999999},"min":{"1min":0.75600000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.385,"5min":1.504,"15min":1.504},"last":1.244}]}]},{"osd":2,"up_from":21,"seq":90194313289,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049132,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696020,"statfs":{"total":95995035648,"available":94920724480,"internally_reserved":1073741824,"allocated":561152,"data_stored":235779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:11 2022","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66500000000000004,"15min":0.61099999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93300000000000005,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.70499999999999996,"15min":0.66300000000000003},"min":{"1min":0.42599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":1.7330000000000001},"last":0.79000000000000004}]},{"osd":1,"last update":"Thu Apr 28 19:43:11 2022","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68999999999999995,"15min":0.66200000000000003},"min":{"1min":0.49299999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.627,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.76900000000000002,"15min":0.69999999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.335,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.57899999999999996}]},{"osd":3,"last update":"Thu Apr 28 19:42:40 2022","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.82099999999999995,"15min":0.78700000000000003},"min":{"1min":0.50800000000000001,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.216,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.73699999999999999,"15min":0.72999999999999998},"min":{"1min":0.437,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.0509999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.1259999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:42:55 2022","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.87,"15min":0.86399999999999999},"min":{"1min":0.65300000000000002,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.268,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.024,"15min":0.99399999999999999},"min":{"1min":0.77000000000000002,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.486,"5min":1.615,"15min":1.615},"last":0.626}]},{"osd":5,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0800000000000001,"15min":1.1040000000000001},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.512,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0149999999999999,"15min":1.0169999999999999},"min":{"1min":0.69199999999999995,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.885,"5min":1.885,"15min":1.885},"last":1.2569999999999999}]},{"osd":6,"last update":"Thu Apr 28 19:42:44 2022","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.03,"15min":1.0409999999999999},"min":{"1min":0.68100000000000005,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.194,"15min":1.2370000000000001},"min":{"1min":0.79400000000000004,"5min":0.72499999999999998,"15min":0.72499999999999998},"max":{"1min":1.619,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":0.69699999999999995}]},{"osd":7,"last update":"Thu Apr 28 19:43:11 2022","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0920000000000001,"15min":1.097},"min":{"1min":0.75600000000000001,"5min":0.623,"15min":0.623},"max":{"1min":2.5419999999999998,"5min":2.5419999999999998,"15min":2.5419999999999998},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.161,"5min":1.2,"15min":1.226},"min":{"1min":0.77900000000000003,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.5580000000000001,"5min":1.655,"15min":1.655},"last":0.78300000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149763,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049128,"kb_used_data":544,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696024,"statfs":{"total":95995035648,"available":94920728576,"internally_reserved":1073741824,"allocated":557056,"data_stored":235282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:35 2022","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.68400000000000005,"15min":0.66000000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.833,"5min":1.833,"15min":1.833},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.79000000000000004,"15min":0.78200000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.133,"5min":2.133,"15min":2.133},"last":0.90800000000000003}]},{"osd":1,"last update":"Thu Apr 28 19:43:35 2022","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.753,"15min":0.72299999999999998},"min":{"1min":0.497,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.214,"5min":2.2970000000000002,"15min":2.2970000000000002},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.72299999999999998,"15min":0.69499999999999995},"min":{"1min":0.48199999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.98599999999999999,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.75}]},{"osd":2,"last update":"Thu Apr 28 19:43:35 2022","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.75900000000000001,"15min":0.77400000000000002},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.92300000000000004,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.76200000000000001,"15min":0.72399999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":2.2400000000000002},"last":0.84799999999999998}]},{"osd":4,"last update":"Thu Apr 28 19:43:03 2022","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.92300000000000004,"15min":0.90500000000000003},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.1599999999999999,"5min":2.415,"15min":2.415},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.92900000000000005,"15min":0.90700000000000003},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2729999999999999,"5min":1.363,"15min":1.363},"last":0.877}]},{"osd":5,"last update":"Thu Apr 28 19:43:20 2022","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.97899999999999998,"15min":0.95099999999999996},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":2.0680000000000001,"5min":2.0680000000000001,"15min":2.0680000000000001},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.042,"15min":1.028},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":2.3580000000000001,"5min":2.5590000000000002,"15min":2.5590000000000002},"last":1.1779999999999999}]},{"osd":6,"last update":"Thu Apr 28 19:42:45 2022","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0229999999999999,"15min":1.0129999999999999},"min":{"1min":0.752,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.518,"5min":1.518,"15min":1.518},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.0660000000000001,"15min":1.0620000000000001},"min":{"1min":0.78300000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.083}]},{"osd":7,"last update":"Thu Apr 28 19:43:08 2022","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.101,"15min":1.103},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.121},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.1299999999999999,"15min":1.1499999999999999},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.351,"5min":2.4900000000000002,"15min":2.4900000000000002},"last":1.268}]}]},{"osd":4,"up_from":31,"seq":133143986240,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049148,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696004,"statfs":{"total":95995035648,"available":94920708096,"internally_reserved":1073741824,"allocated":577536,"data_stored":236873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.84399999999999997,"15min":0.82799999999999996},"min":{"1min":0.56999999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.81799999999999995},"min":{"1min":0.58899999999999997,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.145,"5min":1.456,"15min":1.456},"last":0.73499999999999999}]},{"osd":1,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.84299999999999997,"15min":0.81799999999999995},"min":{"1min":0.55100000000000005,"5min":0.44,"15min":0.44},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.79900000000000004,"15min":0.80000000000000004},"min":{"1min":0.63800000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":0.98999999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.94799999999999995}]},{"osd":2,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.81899999999999995,"15min":0.82099999999999995},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.96999999999999997,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.85499999999999998,"15min":0.86899999999999999},"min":{"1min":0.54800000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.095,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.53800000000000003}]},{"osd":3,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.83199999999999996,"15min":0.83599999999999997},"min":{"1min":0.58399999999999996,"5min":0.495,"15min":0.495},"max":{"1min":1.143,"5min":1.601,"15min":1.601},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.89500000000000002,"15min":0.879},"min":{"1min":0.68100000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.276,"5min":1.276,"15min":1.276},"last":0.755}]},{"osd":5,"last update":"Thu Apr 28 19:43:21 2022","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.82799999999999996,"15min":0.77000000000000002},"min":{"1min":0.54400000000000004,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.292,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.878,"15min":0.80300000000000005},"min":{"1min":0.67100000000000004,"5min":0.436,"15min":0.436},"max":{"1min":1.1839999999999999,"5min":1.399,"15min":1.399},"last":0.96699999999999997}]},{"osd":6,"last update":"Thu Apr 28 19:42:48 2022","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.90500000000000003,"15min":0.89500000000000002},"min":{"1min":0.68200000000000005,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.222,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.90800000000000003,"15min":0.90600000000000003},"min":{"1min":0.75,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.1539999999999999,"5min":1.427,"15min":1.427},"last":0.98799999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:43:04 2022","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.90700000000000003,"15min":0.89700000000000002},"min":{"1min":0.627,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.22,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.083,"5min":0.96599999999999997,"15min":0.93600000000000005},"min":{"1min":0.83899999999999997,"5min":0.498,"15min":0.498},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.89500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822713,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049132,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696020,"statfs":{"total":95995035648,"available":94920724480,"internally_reserved":1073741824,"allocated":561152,"data_stored":235815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.81699999999999995,"15min":0.81299999999999994},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.151,"5min":1.222,"15min":1.222},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.86699999999999999,"15min":0.83899999999999997},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.167,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.0289999999999999}]},{"osd":1,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.83799999999999997,"15min":0.81599999999999995},"min":{"1min":0.53700000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0960000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.878,"5min":0.85799999999999998,"15min":0.85199999999999998},"min":{"1min":0.625,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1399999999999999,"5min":1.8300000000000001,"15min":1.8300000000000001},"last":0.623}]},{"osd":2,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.80300000000000005,"15min":0.79200000000000004},"min":{"1min":0.56599999999999995,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.079,"5min":1.615,"15min":1.615},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.88900000000000001,"15min":0.89500000000000002},"min":{"1min":0.60899999999999999,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.1200000000000001,"5min":1.145,"15min":1.145},"last":0.998}]},{"osd":3,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.84499999999999997,"15min":0.84899999999999998},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1359999999999999,"5min":1.74,"15min":1.74},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.85399999999999998,"15min":0.84399999999999997},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.149,"5min":1.536,"15min":1.536},"last":0.86899999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.77600000000000002,"15min":0.746},"min":{"1min":0.42799999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.0429999999999999,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.78200000000000003,"15min":0.75700000000000001},"min":{"1min":0.58899999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1850000000000001,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":0.84199999999999997}]},{"osd":6,"last update":"Thu Apr 28 19:42:45 2022","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.871,"15min":0.86899999999999999},"min":{"1min":0.57199999999999995,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.6220000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.88,"5min":0.85199999999999998,"15min":0.82899999999999996},"min":{"1min":0.64200000000000002,"5min":0.504,"15min":0.504},"max":{"1min":1.5629999999999999,"5min":1.887,"15min":1.887},"last":0.97899999999999998}]},{"osd":7,"last update":"Thu Apr 28 19:43:08 2022","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.93000000000000005,"15min":0.93200000000000005},"min":{"1min":0.66700000000000004,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.1599999999999999,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.84999999999999998,"15min":0.83199999999999996},"min":{"1min":0.66300000000000003,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.7509999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.77600000000000002}]}]}]}} 2022-04-28T19:43:49.473 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph pg dump --format=json 2022-04-28T19:43:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:49 smithi139 bash[23439]: cluster 2022-04-28T19:43:48.052694+0000 mgr.y (mgr.14140) 342 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:49.708 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:50 smithi059 bash[23786]: audit 2022-04-28T19:43:48.722603+0000 mgr.y (mgr.14140) 343 : audit [DBG] from='client.24655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:43:50.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:50 smithi059 bash[29114]: audit 2022-04-28T19:43:48.722603+0000 mgr.y (mgr.14140) 343 : audit [DBG] from='client.24655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:43:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:50 smithi139 bash[23439]: audit 2022-04-28T19:43:48.722603+0000 mgr.y (mgr.14140) 343 : audit [DBG] from='client.24655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:43:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:51 smithi059 bash[23786]: cluster 2022-04-28T19:43:50.053510+0000 mgr.y (mgr.14140) 344 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:51.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:51 smithi059 bash[29114]: cluster 2022-04-28T19:43:50.053510+0000 mgr.y (mgr.14140) 344 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:51 smithi139 bash[23439]: cluster 2022-04-28T19:43:50.053510+0000 mgr.y (mgr.14140) 344 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:52.769 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:43:52.777 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2022-04-28T19:43:53.482 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:53 smithi059 bash[29114]: cluster 2022-04-28T19:43:52.054413+0000 mgr.y (mgr.14140) 345 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:53.483 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:53 smithi059 bash[23786]: cluster 2022-04-28T19:43:52.054413+0000 mgr.y (mgr.14140) 345 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:53.483 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":304,"stamp":"2022-04-28T19:43:52.053876+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2304,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8393088,"kb_used_data":4416,"kb_used_omap":0,"kb_used_meta":8388608,"kb_avail":741568128,"statfs":{"total":767960285184,"available":759365763072,"internally_reserved":8589934592,"allocated":4521984,"data_stored":1888448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":8589934592},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005491"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263633+0000","last_change":"2022-04-28T19:42:28.263333+0000","last_active":"2022-04-28T19:42:28.263633+0000","last_peered":"2022-04-28T19:42:28.263633+0000","last_clean":"2022-04-28T19:42:28.263633+0000","last_became_active":"2022-04-28T19:42:28.263110+0000","last_became_peered":"2022-04-28T19:42:28.263110+0000","last_unstale":"2022-04-28T19:42:28.263633+0000","last_undegraded":"2022-04-28T19:42:28.263633+0000","last_fullsized":"2022-04-28T19:42:28.263633+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"56'4","reported_seq":16,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.189185+0000","last_change":"2022-04-28T19:42:27.239469+0000","last_active":"2022-04-28T19:42:36.189185+0000","last_peered":"2022-04-28T19:42:36.189185+0000","last_clean":"2022-04-28T19:42:36.189185+0000","last_became_active":"2022-04-28T19:42:27.238941+0000","last_became_peered":"2022-04-28T19:42:27.238941+0000","last_unstale":"2022-04-28T19:42:36.189185+0000","last_undegraded":"2022-04-28T19:42:36.189185+0000","last_fullsized":"2022-04-28T19:42:36.189185+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":553,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484946+0000","last_change":"2022-04-28T19:42:27.273624+0000","last_active":"2022-04-28T19:42:28.484946+0000","last_peered":"2022-04-28T19:42:28.484946+0000","last_clean":"2022-04-28T19:42:28.484946+0000","last_became_active":"2022-04-28T19:42:27.273115+0000","last_became_peered":"2022-04-28T19:42:27.273115+0000","last_unstale":"2022-04-28T19:42:28.484946+0000","last_undegraded":"2022-04-28T19:42:28.484946+0000","last_fullsized":"2022-04-28T19:42:28.484946+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.265364+0000","last_change":"2022-04-28T19:42:27.246294+0000","last_active":"2022-04-28T19:42:29.265364+0000","last_peered":"2022-04-28T19:42:29.265364+0000","last_clean":"2022-04-28T19:42:29.265364+0000","last_became_active":"2022-04-28T19:42:27.245818+0000","last_became_peered":"2022-04-28T19:42:27.245818+0000","last_unstale":"2022-04-28T19:42:29.265364+0000","last_undegraded":"2022-04-28T19:42:29.265364+0000","last_fullsized":"2022-04-28T19:42:29.265364+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.488158+0000","last_change":"2022-04-28T19:42:28.487652+0000","last_active":"2022-04-28T19:42:28.488158+0000","last_peered":"2022-04-28T19:42:28.488158+0000","last_clean":"2022-04-28T19:42:28.488158+0000","last_became_active":"2022-04-28T19:42:28.486395+0000","last_became_peered":"2022-04-28T19:42:28.486395+0000","last_unstale":"2022-04-28T19:42:28.488158+0000","last_undegraded":"2022-04-28T19:42:28.488158+0000","last_fullsized":"2022-04-28T19:42:28.488158+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.190605+0000","last_change":"2022-04-28T19:42:27.260709+0000","last_active":"2022-04-28T19:42:36.190605+0000","last_peered":"2022-04-28T19:42:36.190605+0000","last_clean":"2022-04-28T19:42:36.190605+0000","last_became_active":"2022-04-28T19:42:27.260331+0000","last_became_peered":"2022-04-28T19:42:27.260331+0000","last_unstale":"2022-04-28T19:42:36.190605+0000","last_undegraded":"2022-04-28T19:42:36.190605+0000","last_fullsized":"2022-04-28T19:42:36.190605+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.485611+0000","last_change":"2022-04-28T19:42:28.485023+0000","last_active":"2022-04-28T19:42:28.485611+0000","last_peered":"2022-04-28T19:42:28.485611+0000","last_clean":"2022-04-28T19:42:28.485611+0000","last_became_active":"2022-04-28T19:42:28.484361+0000","last_became_peered":"2022-04-28T19:42:28.484361+0000","last_unstale":"2022-04-28T19:42:28.485611+0000","last_undegraded":"2022-04-28T19:42:28.485611+0000","last_fullsized":"2022-04-28T19:42:28.485611+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.192638+0000","last_change":"2022-04-28T19:42:27.260251+0000","last_active":"2022-04-28T19:42:36.192638+0000","last_peered":"2022-04-28T19:42:36.192638+0000","last_clean":"2022-04-28T19:42:36.192638+0000","last_became_active":"2022-04-28T19:42:27.259782+0000","last_became_peered":"2022-04-28T19:42:27.259782+0000","last_unstale":"2022-04-28T19:42:36.192638+0000","last_undegraded":"2022-04-28T19:42:36.192638+0000","last_fullsized":"2022-04-28T19:42:36.192638+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.945557+0000","last_change":"2022-04-28T19:42:27.267928+0000","last_active":"2022-04-28T19:42:28.945557+0000","last_peered":"2022-04-28T19:42:28.945557+0000","last_clean":"2022-04-28T19:42:28.945557+0000","last_became_active":"2022-04-28T19:42:27.267443+0000","last_became_peered":"2022-04-28T19:42:27.267443+0000","last_unstale":"2022-04-28T19:42:28.945557+0000","last_undegraded":"2022-04-28T19:42:28.945557+0000","last_fullsized":"2022-04-28T19:42:28.945557+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.267203+0000","last_change":"2022-04-28T19:42:27.256493+0000","last_active":"2022-04-28T19:42:29.267203+0000","last_peered":"2022-04-28T19:42:29.267203+0000","last_clean":"2022-04-28T19:42:29.267203+0000","last_became_active":"2022-04-28T19:42:27.250015+0000","last_became_peered":"2022-04-28T19:42:27.250015+0000","last_unstale":"2022-04-28T19:42:29.267203+0000","last_undegraded":"2022-04-28T19:42:29.267203+0000","last_fullsized":"2022-04-28T19:42:29.267203+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.565407+0000","last_change":"2022-04-28T19:42:28.564652+0000","last_active":"2022-04-28T19:42:28.565407+0000","last_peered":"2022-04-28T19:42:28.565407+0000","last_clean":"2022-04-28T19:42:28.565407+0000","last_became_active":"2022-04-28T19:42:28.564176+0000","last_became_peered":"2022-04-28T19:42:28.564176+0000","last_unstale":"2022-04-28T19:42:28.565407+0000","last_undegraded":"2022-04-28T19:42:28.565407+0000","last_fullsized":"2022-04-28T19:42:28.565407+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.264139+0000","last_change":"2022-04-28T19:42:27.259973+0000","last_active":"2022-04-28T19:42:29.264139+0000","last_peered":"2022-04-28T19:42:29.264139+0000","last_clean":"2022-04-28T19:42:29.264139+0000","last_became_active":"2022-04-28T19:42:27.256958+0000","last_became_peered":"2022-04-28T19:42:27.256958+0000","last_unstale":"2022-04-28T19:42:29.264139+0000","last_undegraded":"2022-04-28T19:42:29.264139+0000","last_fullsized":"2022-04-28T19:42:29.264139+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263982+0000","last_change":"2022-04-28T19:40:39.845995+0000","last_active":"2022-04-28T19:42:28.263982+0000","last_peered":"2022-04-28T19:42:28.263982+0000","last_clean":"2022-04-28T19:42:28.263982+0000","last_became_active":"2022-04-28T19:40:39.845574+0000","last_became_peered":"2022-04-28T19:40:39.845574+0000","last_unstale":"2022-04-28T19:42:28.263982+0000","last_undegraded":"2022-04-28T19:42:28.263982+0000","last_fullsized":"2022-04-28T19:42:28.263982+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483595+0000","last_change":"2022-04-28T19:42:27.327656+0000","last_active":"2022-04-28T19:42:28.483595+0000","last_peered":"2022-04-28T19:42:28.483595+0000","last_clean":"2022-04-28T19:42:28.483595+0000","last_became_active":"2022-04-28T19:42:27.327205+0000","last_became_peered":"2022-04-28T19:42:27.327205+0000","last_unstale":"2022-04-28T19:42:28.483595+0000","last_undegraded":"2022-04-28T19:42:28.483595+0000","last_fullsized":"2022-04-28T19:42:28.483595+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254256+0000","last_change":"2022-04-28T19:40:39.837533+0000","last_active":"2022-04-28T19:42:28.254256+0000","last_peered":"2022-04-28T19:42:28.254256+0000","last_clean":"2022-04-28T19:42:28.254256+0000","last_became_active":"2022-04-28T19:40:39.837213+0000","last_became_peered":"2022-04-28T19:40:39.837213+0000","last_unstale":"2022-04-28T19:42:28.254256+0000","last_undegraded":"2022-04-28T19:42:28.254256+0000","last_fullsized":"2022-04-28T19:42:28.254256+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.188366+0000","last_change":"2022-04-28T19:42:27.249044+0000","last_active":"2022-04-28T19:42:36.188366+0000","last_peered":"2022-04-28T19:42:36.188366+0000","last_clean":"2022-04-28T19:42:36.188366+0000","last_became_active":"2022-04-28T19:42:27.248558+0000","last_became_peered":"2022-04-28T19:42:27.248558+0000","last_unstale":"2022-04-28T19:42:36.188366+0000","last_undegraded":"2022-04-28T19:42:36.188366+0000","last_fullsized":"2022-04-28T19:42:36.188366+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254143+0000","last_change":"2022-04-28T19:42:27.328071+0000","last_active":"2022-04-28T19:42:28.254143+0000","last_peered":"2022-04-28T19:42:28.254143+0000","last_clean":"2022-04-28T19:42:28.254143+0000","last_became_active":"2022-04-28T19:42:27.327332+0000","last_became_peered":"2022-04-28T19:42:27.327332+0000","last_unstale":"2022-04-28T19:42:28.254143+0000","last_undegraded":"2022-04-28T19:42:28.254143+0000","last_fullsized":"2022-04-28T19:42:28.254143+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253816+0000","last_change":"2022-04-28T19:42:27.250305+0000","last_active":"2022-04-28T19:42:28.253816+0000","last_peered":"2022-04-28T19:42:28.253816+0000","last_clean":"2022-04-28T19:42:28.253816+0000","last_became_active":"2022-04-28T19:42:27.249816+0000","last_became_peered":"2022-04-28T19:42:27.249816+0000","last_unstale":"2022-04-28T19:42:28.253816+0000","last_undegraded":"2022-04-28T19:42:28.253816+0000","last_fullsized":"2022-04-28T19:42:28.253816+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907654+0000","last_change":"2022-04-28T19:40:39.846944+0000","last_active":"2022-04-28T19:42:28.907654+0000","last_peered":"2022-04-28T19:42:28.907654+0000","last_clean":"2022-04-28T19:42:28.907654+0000","last_became_active":"2022-04-28T19:40:39.846597+0000","last_became_peered":"2022-04-28T19:40:39.846597+0000","last_unstale":"2022-04-28T19:42:28.907654+0000","last_undegraded":"2022-04-28T19:42:28.907654+0000","last_fullsized":"2022-04-28T19:42:28.907654+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254270+0000","last_change":"2022-04-28T19:42:27.268475+0000","last_active":"2022-04-28T19:42:28.254270+0000","last_peered":"2022-04-28T19:42:28.254270+0000","last_clean":"2022-04-28T19:42:28.254270+0000","last_became_active":"2022-04-28T19:42:27.268077+0000","last_became_peered":"2022-04-28T19:42:27.268077+0000","last_unstale":"2022-04-28T19:42:28.254270+0000","last_undegraded":"2022-04-28T19:42:28.254270+0000","last_fullsized":"2022-04-28T19:42:28.254270+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.252615+0000","last_change":"2022-04-28T19:42:27.250732+0000","last_active":"2022-04-28T19:42:28.252615+0000","last_peered":"2022-04-28T19:42:28.252615+0000","last_clean":"2022-04-28T19:42:28.252615+0000","last_became_active":"2022-04-28T19:42:27.249950+0000","last_became_peered":"2022-04-28T19:42:27.249950+0000","last_unstale":"2022-04-28T19:42:28.252615+0000","last_undegraded":"2022-04-28T19:42:28.252615+0000","last_fullsized":"2022-04-28T19:42:28.252615+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907138+0000","last_change":"2022-04-28T19:40:39.848985+0000","last_active":"2022-04-28T19:42:28.907138+0000","last_peered":"2022-04-28T19:42:28.907138+0000","last_clean":"2022-04-28T19:42:28.907138+0000","last_became_active":"2022-04-28T19:40:39.848622+0000","last_became_peered":"2022-04-28T19:40:39.848622+0000","last_unstale":"2022-04-28T19:42:28.907138+0000","last_undegraded":"2022-04-28T19:42:28.907138+0000","last_fullsized":"2022-04-28T19:42:28.907138+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.909406+0000","last_change":"2022-04-28T19:42:28.908907+0000","last_active":"2022-04-28T19:42:28.909406+0000","last_peered":"2022-04-28T19:42:28.909406+0000","last_clean":"2022-04-28T19:42:28.909406+0000","last_became_active":"2022-04-28T19:42:28.908537+0000","last_became_peered":"2022-04-28T19:42:28.908537+0000","last_unstale":"2022-04-28T19:42:28.909406+0000","last_undegraded":"2022-04-28T19:42:28.909406+0000","last_fullsized":"2022-04-28T19:42:28.909406+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484713+0000","last_change":"2022-04-28T19:42:27.250116+0000","last_active":"2022-04-28T19:42:28.484713+0000","last_peered":"2022-04-28T19:42:28.484713+0000","last_clean":"2022-04-28T19:42:28.484713+0000","last_became_active":"2022-04-28T19:42:27.249710+0000","last_became_peered":"2022-04-28T19:42:27.249710+0000","last_unstale":"2022-04-28T19:42:28.484713+0000","last_undegraded":"2022-04-28T19:42:28.484713+0000","last_fullsized":"2022-04-28T19:42:28.484713+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":19,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907080+0000","last_change":"2022-04-28T19:40:02.912608+0000","last_active":"2022-04-28T19:42:28.907080+0000","last_peered":"2022-04-28T19:42:28.907080+0000","last_clean":"2022-04-28T19:42:28.907080+0000","last_became_active":"2022-04-28T19:40:02.911686+0000","last_became_peered":"2022-04-28T19:40:02.911686+0000","last_unstale":"2022-04-28T19:42:28.907080+0000","last_undegraded":"2022-04-28T19:42:28.907080+0000","last_fullsized":"2022-04-28T19:42:28.907080+0000","mapping_epoch":46,"log_start":"0'0","ondisk_log_start":"0'0","created":5,"last_epoch_clean":47,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_clean_scrub_stamp":"2022-04-28T19:36:10.237993+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563616+0000","last_change":"2022-04-28T19:40:39.847019+0000","last_active":"2022-04-28T19:42:28.563616+0000","last_peered":"2022-04-28T19:42:28.563616+0000","last_clean":"2022-04-28T19:42:28.563616+0000","last_became_active":"2022-04-28T19:40:39.846557+0000","last_became_peered":"2022-04-28T19:40:39.846557+0000","last_unstale":"2022-04-28T19:42:28.563616+0000","last_undegraded":"2022-04-28T19:42:28.563616+0000","last_fullsized":"2022-04-28T19:42:28.563616+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254321+0000","last_change":"2022-04-28T19:42:27.271631+0000","last_active":"2022-04-28T19:42:28.254321+0000","last_peered":"2022-04-28T19:42:28.254321+0000","last_clean":"2022-04-28T19:42:28.254321+0000","last_became_active":"2022-04-28T19:42:27.271033+0000","last_became_peered":"2022-04-28T19:42:27.271033+0000","last_unstale":"2022-04-28T19:42:28.254321+0000","last_undegraded":"2022-04-28T19:42:28.254321+0000","last_fullsized":"2022-04-28T19:42:28.254321+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253959+0000","last_change":"2022-04-28T19:42:27.244744+0000","last_active":"2022-04-28T19:42:28.253959+0000","last_peered":"2022-04-28T19:42:28.253959+0000","last_clean":"2022-04-28T19:42:28.253959+0000","last_became_active":"2022-04-28T19:42:27.244334+0000","last_became_peered":"2022-04-28T19:42:27.244334+0000","last_unstale":"2022-04-28T19:42:28.253959+0000","last_undegraded":"2022-04-28T19:42:28.253959+0000","last_fullsized":"2022-04-28T19:42:28.253959+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.945411+0000","last_change":"2022-04-28T19:40:40.101205+0000","last_active":"2022-04-28T19:42:28.945411+0000","last_peered":"2022-04-28T19:42:28.945411+0000","last_clean":"2022-04-28T19:42:28.945411+0000","last_became_active":"2022-04-28T19:40:40.100696+0000","last_became_peered":"2022-04-28T19:40:40.100696+0000","last_unstale":"2022-04-28T19:42:28.945411+0000","last_undegraded":"2022-04-28T19:42:28.945411+0000","last_fullsized":"2022-04-28T19:42:28.945411+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253322+0000","last_change":"2022-04-28T19:42:27.270202+0000","last_active":"2022-04-28T19:42:28.253322+0000","last_peered":"2022-04-28T19:42:28.253322+0000","last_clean":"2022-04-28T19:42:28.253322+0000","last_became_active":"2022-04-28T19:42:27.269794+0000","last_became_peered":"2022-04-28T19:42:27.269794+0000","last_unstale":"2022-04-28T19:42:28.253322+0000","last_undegraded":"2022-04-28T19:42:28.253322+0000","last_fullsized":"2022-04-28T19:42:28.253322+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.259128+0000","last_change":"2022-04-28T19:42:27.250744+0000","last_active":"2022-04-28T19:42:28.259128+0000","last_peered":"2022-04-28T19:42:28.259128+0000","last_clean":"2022-04-28T19:42:28.259128+0000","last_became_active":"2022-04-28T19:42:27.248908+0000","last_became_peered":"2022-04-28T19:42:27.248908+0000","last_unstale":"2022-04-28T19:42:28.259128+0000","last_undegraded":"2022-04-28T19:42:28.259128+0000","last_fullsized":"2022-04-28T19:42:28.259128+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563661+0000","last_change":"2022-04-28T19:40:39.848860+0000","last_active":"2022-04-28T19:42:28.563661+0000","last_peered":"2022-04-28T19:42:28.563661+0000","last_clean":"2022-04-28T19:42:28.563661+0000","last_became_active":"2022-04-28T19:40:39.848587+0000","last_became_peered":"2022-04-28T19:40:39.848587+0000","last_unstale":"2022-04-28T19:42:28.563661+0000","last_undegraded":"2022-04-28T19:42:28.563661+0000","last_fullsized":"2022-04-28T19:42:28.563661+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253281+0000","last_change":"2022-04-28T19:42:27.326386+0000","last_active":"2022-04-28T19:42:28.253281+0000","last_peered":"2022-04-28T19:42:28.253281+0000","last_clean":"2022-04-28T19:42:28.253281+0000","last_became_active":"2022-04-28T19:42:27.325767+0000","last_became_peered":"2022-04-28T19:42:27.325767+0000","last_unstale":"2022-04-28T19:42:28.253281+0000","last_undegraded":"2022-04-28T19:42:28.253281+0000","last_fullsized":"2022-04-28T19:42:28.253281+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.564872+0000","last_change":"2022-04-28T19:42:27.247247+0000","last_active":"2022-04-28T19:42:28.564872+0000","last_peered":"2022-04-28T19:42:28.564872+0000","last_clean":"2022-04-28T19:42:28.564872+0000","last_became_active":"2022-04-28T19:42:27.246833+0000","last_became_peered":"2022-04-28T19:42:27.246833+0000","last_unstale":"2022-04-28T19:42:28.564872+0000","last_undegraded":"2022-04-28T19:42:28.564872+0000","last_fullsized":"2022-04-28T19:42:28.564872+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253206+0000","last_change":"2022-04-28T19:40:39.838356+0000","last_active":"2022-04-28T19:42:28.253206+0000","last_peered":"2022-04-28T19:42:28.253206+0000","last_clean":"2022-04-28T19:42:28.253206+0000","last_became_active":"2022-04-28T19:40:39.837847+0000","last_became_peered":"2022-04-28T19:40:39.837847+0000","last_unstale":"2022-04-28T19:42:28.253206+0000","last_undegraded":"2022-04-28T19:42:28.253206+0000","last_fullsized":"2022-04-28T19:42:28.253206+0000","mapping_epoch":48,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":49,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254508+0000","last_change":"2022-04-28T19:42:27.325953+0000","last_active":"2022-04-28T19:42:28.254508+0000","last_peered":"2022-04-28T19:42:28.254508+0000","last_clean":"2022-04-28T19:42:28.254508+0000","last_became_active":"2022-04-28T19:42:27.325071+0000","last_became_peered":"2022-04-28T19:42:27.325071+0000","last_unstale":"2022-04-28T19:42:28.254508+0000","last_undegraded":"2022-04-28T19:42:28.254508+0000","last_fullsized":"2022-04-28T19:42:28.254508+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483765+0000","last_change":"2022-04-28T19:42:27.253821+0000","last_active":"2022-04-28T19:42:28.483765+0000","last_peered":"2022-04-28T19:42:28.483765+0000","last_clean":"2022-04-28T19:42:28.483765+0000","last_became_active":"2022-04-28T19:42:27.253401+0000","last_became_peered":"2022-04-28T19:42:27.253401+0000","last_unstale":"2022-04-28T19:42:28.483765+0000","last_undegraded":"2022-04-28T19:42:28.483765+0000","last_fullsized":"2022-04-28T19:42:28.483765+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.265051+0000","last_change":"2022-04-28T19:42:28.264810+0000","last_active":"2022-04-28T19:42:28.265051+0000","last_peered":"2022-04-28T19:42:28.265051+0000","last_clean":"2022-04-28T19:42:28.265051+0000","last_became_active":"2022-04-28T19:42:28.264499+0000","last_became_peered":"2022-04-28T19:42:28.264499+0000","last_unstale":"2022-04-28T19:42:28.265051+0000","last_undegraded":"2022-04-28T19:42:28.265051+0000","last_fullsized":"2022-04-28T19:42:28.265051+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.257866+0000","last_change":"2022-04-28T19:42:27.256454+0000","last_active":"2022-04-28T19:42:28.257866+0000","last_peered":"2022-04-28T19:42:28.257866+0000","last_clean":"2022-04-28T19:42:28.257866+0000","last_became_active":"2022-04-28T19:42:27.252186+0000","last_became_peered":"2022-04-28T19:42:27.252186+0000","last_unstale":"2022-04-28T19:42:28.257866+0000","last_undegraded":"2022-04-28T19:42:28.257866+0000","last_fullsized":"2022-04-28T19:42:28.257866+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.947298+0000","last_change":"2022-04-28T19:42:28.947039+0000","last_active":"2022-04-28T19:42:28.947298+0000","last_peered":"2022-04-28T19:42:28.947298+0000","last_clean":"2022-04-28T19:42:28.947298+0000","last_became_active":"2022-04-28T19:42:28.946558+0000","last_became_peered":"2022-04-28T19:42:28.946558+0000","last_unstale":"2022-04-28T19:42:28.947298+0000","last_undegraded":"2022-04-28T19:42:28.947298+0000","last_fullsized":"2022-04-28T19:42:28.947298+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263012+0000","last_change":"2022-04-28T19:42:27.253990+0000","last_active":"2022-04-28T19:42:28.263012+0000","last_peered":"2022-04-28T19:42:28.263012+0000","last_clean":"2022-04-28T19:42:28.263012+0000","last_became_active":"2022-04-28T19:42:27.253662+0000","last_became_peered":"2022-04-28T19:42:27.253662+0000","last_unstale":"2022-04-28T19:42:28.263012+0000","last_undegraded":"2022-04-28T19:42:28.263012+0000","last_fullsized":"2022-04-28T19:42:28.263012+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.487107+0000","last_change":"2022-04-28T19:42:28.486474+0000","last_active":"2022-04-28T19:42:28.487107+0000","last_peered":"2022-04-28T19:42:28.487107+0000","last_clean":"2022-04-28T19:42:28.487107+0000","last_became_active":"2022-04-28T19:42:28.485762+0000","last_became_peered":"2022-04-28T19:42:28.485762+0000","last_unstale":"2022-04-28T19:42:28.487107+0000","last_undegraded":"2022-04-28T19:42:28.487107+0000","last_fullsized":"2022-04-28T19:42:28.487107+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.189441+0000","last_change":"2022-04-28T19:42:27.254656+0000","last_active":"2022-04-28T19:42:36.189441+0000","last_peered":"2022-04-28T19:42:36.189441+0000","last_clean":"2022-04-28T19:42:36.189441+0000","last_became_active":"2022-04-28T19:42:27.254143+0000","last_became_peered":"2022-04-28T19:42:27.254143+0000","last_unstale":"2022-04-28T19:42:36.189441+0000","last_undegraded":"2022-04-28T19:42:36.189441+0000","last_fullsized":"2022-04-28T19:42:36.189441+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253913+0000","last_change":"2022-04-28T19:42:27.326241+0000","last_active":"2022-04-28T19:42:28.253913+0000","last_peered":"2022-04-28T19:42:28.253913+0000","last_clean":"2022-04-28T19:42:28.253913+0000","last_became_active":"2022-04-28T19:42:27.325542+0000","last_became_peered":"2022-04-28T19:42:27.325542+0000","last_unstale":"2022-04-28T19:42:28.253913+0000","last_undegraded":"2022-04-28T19:42:28.253913+0000","last_fullsized":"2022-04-28T19:42:28.253913+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563607+0000","last_change":"2022-04-28T19:42:27.246713+0000","last_active":"2022-04-28T19:42:28.563607+0000","last_peered":"2022-04-28T19:42:28.563607+0000","last_clean":"2022-04-28T19:42:28.563607+0000","last_became_active":"2022-04-28T19:42:27.246287+0000","last_became_peered":"2022-04-28T19:42:27.246287+0000","last_unstale":"2022-04-28T19:42:28.563607+0000","last_undegraded":"2022-04-28T19:42:28.563607+0000","last_fullsized":"2022-04-28T19:42:28.563607+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.909429+0000","last_change":"2022-04-28T19:42:28.908879+0000","last_active":"2022-04-28T19:42:28.909429+0000","last_peered":"2022-04-28T19:42:28.909429+0000","last_clean":"2022-04-28T19:42:28.909429+0000","last_became_active":"2022-04-28T19:42:28.908417+0000","last_became_peered":"2022-04-28T19:42:28.908417+0000","last_unstale":"2022-04-28T19:42:28.909429+0000","last_undegraded":"2022-04-28T19:42:28.909429+0000","last_fullsized":"2022-04-28T19:42:28.909429+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.856444+0000","last_change":"2022-04-28T19:42:27.249628+0000","last_active":"2022-04-28T19:42:29.856444+0000","last_peered":"2022-04-28T19:42:29.856444+0000","last_clean":"2022-04-28T19:42:29.856444+0000","last_became_active":"2022-04-28T19:42:27.249203+0000","last_became_peered":"2022-04-28T19:42:27.249203+0000","last_unstale":"2022-04-28T19:42:29.856444+0000","last_undegraded":"2022-04-28T19:42:29.856444+0000","last_fullsized":"2022-04-28T19:42:29.856444+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.487895+0000","last_change":"2022-04-28T19:42:28.487607+0000","last_active":"2022-04-28T19:42:28.487895+0000","last_peered":"2022-04-28T19:42:28.487895+0000","last_clean":"2022-04-28T19:42:28.487895+0000","last_became_active":"2022-04-28T19:42:28.486682+0000","last_became_peered":"2022-04-28T19:42:28.486682+0000","last_unstale":"2022-04-28T19:42:28.487895+0000","last_undegraded":"2022-04-28T19:42:28.487895+0000","last_fullsized":"2022-04-28T19:42:28.487895+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.188446+0000","last_change":"2022-04-28T19:42:27.255168+0000","last_active":"2022-04-28T19:42:36.188446+0000","last_peered":"2022-04-28T19:42:36.188446+0000","last_clean":"2022-04-28T19:42:36.188446+0000","last_became_active":"2022-04-28T19:42:27.254478+0000","last_became_peered":"2022-04-28T19:42:27.254478+0000","last_unstale":"2022-04-28T19:42:36.188446+0000","last_undegraded":"2022-04-28T19:42:36.188446+0000","last_fullsized":"2022-04-28T19:42:36.188446+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563583+0000","last_change":"2022-04-28T19:42:27.328189+0000","last_active":"2022-04-28T19:42:28.563583+0000","last_peered":"2022-04-28T19:42:28.563583+0000","last_clean":"2022-04-28T19:42:28.563583+0000","last_became_active":"2022-04-28T19:42:27.327231+0000","last_became_peered":"2022-04-28T19:42:27.327231+0000","last_unstale":"2022-04-28T19:42:28.563583+0000","last_undegraded":"2022-04-28T19:42:28.563583+0000","last_fullsized":"2022-04-28T19:42:28.563583+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.856150+0000","last_change":"2022-04-28T19:42:27.253531+0000","last_active":"2022-04-28T19:42:29.856150+0000","last_peered":"2022-04-28T19:42:29.856150+0000","last_clean":"2022-04-28T19:42:29.856150+0000","last_became_active":"2022-04-28T19:42:27.252949+0000","last_became_peered":"2022-04-28T19:42:27.252949+0000","last_unstale":"2022-04-28T19:42:29.856150+0000","last_undegraded":"2022-04-28T19:42:29.856150+0000","last_fullsized":"2022-04-28T19:42:29.856150+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483957+0000","last_change":"2022-04-28T19:42:27.330855+0000","last_active":"2022-04-28T19:42:28.483957+0000","last_peered":"2022-04-28T19:42:28.483957+0000","last_clean":"2022-04-28T19:42:28.483957+0000","last_became_active":"2022-04-28T19:42:27.330047+0000","last_became_peered":"2022-04-28T19:42:27.330047+0000","last_unstale":"2022-04-28T19:42:28.483957+0000","last_undegraded":"2022-04-28T19:42:28.483957+0000","last_fullsized":"2022-04-28T19:42:28.483957+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"56'8","reported_seq":20,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.194007+0000","last_change":"2022-04-28T19:42:27.249243+0000","last_active":"2022-04-28T19:42:36.194007+0000","last_peered":"2022-04-28T19:42:36.194007+0000","last_clean":"2022-04-28T19:42:36.194007+0000","last_became_active":"2022-04-28T19:42:27.248832+0000","last_became_peered":"2022-04-28T19:42:27.248832+0000","last_unstale":"2022-04-28T19:42:36.194007+0000","last_undegraded":"2022-04-28T19:42:36.194007+0000","last_fullsized":"2022-04-28T19:42:36.194007+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253898+0000","last_change":"2022-04-28T19:42:27.330900+0000","last_active":"2022-04-28T19:42:28.253898+0000","last_peered":"2022-04-28T19:42:28.253898+0000","last_clean":"2022-04-28T19:42:28.253898+0000","last_became_active":"2022-04-28T19:42:27.330465+0000","last_became_peered":"2022-04-28T19:42:27.330465+0000","last_unstale":"2022-04-28T19:42:28.253898+0000","last_undegraded":"2022-04-28T19:42:28.253898+0000","last_fullsized":"2022-04-28T19:42:28.253898+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.907144+0000","last_change":"2022-04-28T19:42:27.256107+0000","last_active":"2022-04-28T19:42:28.907144+0000","last_peered":"2022-04-28T19:42:28.907144+0000","last_clean":"2022-04-28T19:42:28.907144+0000","last_became_active":"2022-04-28T19:42:27.255024+0000","last_became_peered":"2022-04-28T19:42:27.255024+0000","last_unstale":"2022-04-28T19:42:28.907144+0000","last_undegraded":"2022-04-28T19:42:28.907144+0000","last_fullsized":"2022-04-28T19:42:28.907144+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563887+0000","last_change":"2022-04-28T19:42:27.330064+0000","last_active":"2022-04-28T19:42:28.563887+0000","last_peered":"2022-04-28T19:42:28.563887+0000","last_clean":"2022-04-28T19:42:28.563887+0000","last_became_active":"2022-04-28T19:42:27.329658+0000","last_became_peered":"2022-04-28T19:42:27.329658+0000","last_unstale":"2022-04-28T19:42:28.563887+0000","last_undegraded":"2022-04-28T19:42:28.563887+0000","last_fullsized":"2022-04-28T19:42:28.563887+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.252610+0000","last_change":"2022-04-28T19:42:27.249334+0000","last_active":"2022-04-28T19:42:28.252610+0000","last_peered":"2022-04-28T19:42:28.252610+0000","last_clean":"2022-04-28T19:42:28.252610+0000","last_became_active":"2022-04-28T19:42:27.248594+0000","last_became_peered":"2022-04-28T19:42:27.248594+0000","last_unstale":"2022-04-28T19:42:28.252610+0000","last_undegraded":"2022-04-28T19:42:28.252610+0000","last_fullsized":"2022-04-28T19:42:28.252610+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484731+0000","last_change":"2022-04-28T19:42:27.329677+0000","last_active":"2022-04-28T19:42:28.484731+0000","last_peered":"2022-04-28T19:42:28.484731+0000","last_clean":"2022-04-28T19:42:28.484731+0000","last_became_active":"2022-04-28T19:42:27.329252+0000","last_became_peered":"2022-04-28T19:42:27.329252+0000","last_unstale":"2022-04-28T19:42:28.484731+0000","last_undegraded":"2022-04-28T19:42:28.484731+0000","last_fullsized":"2022-04-28T19:42:28.484731+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"56'2","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.192849+0000","last_change":"2022-04-28T19:42:27.255633+0000","last_active":"2022-04-28T19:42:36.192849+0000","last_peered":"2022-04-28T19:42:36.192849+0000","last_clean":"2022-04-28T19:42:36.192849+0000","last_became_active":"2022-04-28T19:42:27.254990+0000","last_became_peered":"2022-04-28T19:42:27.254990+0000","last_unstale":"2022-04-28T19:42:36.192849+0000","last_undegraded":"2022-04-28T19:42:36.192849+0000","last_fullsized":"2022-04-28T19:42:36.192849+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.263076+0000","last_change":"2022-04-28T19:42:28.262801+0000","last_active":"2022-04-28T19:42:28.263076+0000","last_peered":"2022-04-28T19:42:28.263076+0000","last_clean":"2022-04-28T19:42:28.263076+0000","last_became_active":"2022-04-28T19:42:28.262437+0000","last_became_peered":"2022-04-28T19:42:28.262437+0000","last_unstale":"2022-04-28T19:42:28.263076+0000","last_undegraded":"2022-04-28T19:42:28.263076+0000","last_fullsized":"2022-04-28T19:42:28.263076+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"56'6","reported_seq":18,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.194557+0000","last_change":"2022-04-28T19:42:27.257775+0000","last_active":"2022-04-28T19:42:36.194557+0000","last_peered":"2022-04-28T19:42:36.194557+0000","last_clean":"2022-04-28T19:42:36.194557+0000","last_became_active":"2022-04-28T19:42:27.257418+0000","last_became_peered":"2022-04-28T19:42:27.257418+0000","last_unstale":"2022-04-28T19:42:36.194557+0000","last_undegraded":"2022-04-28T19:42:36.194557+0000","last_fullsized":"2022-04-28T19:42:36.194557+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1535,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253210+0000","last_change":"2022-04-28T19:42:27.262070+0000","last_active":"2022-04-28T19:42:28.253210+0000","last_peered":"2022-04-28T19:42:28.253210+0000","last_clean":"2022-04-28T19:42:28.253210+0000","last_became_active":"2022-04-28T19:42:27.261722+0000","last_became_peered":"2022-04-28T19:42:27.261722+0000","last_unstale":"2022-04-28T19:42:28.253210+0000","last_undegraded":"2022-04-28T19:42:28.253210+0000","last_fullsized":"2022-04-28T19:42:28.253210+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"56'3","reported_seq":15,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:36.192201+0000","last_change":"2022-04-28T19:42:27.255046+0000","last_active":"2022-04-28T19:42:36.192201+0000","last_peered":"2022-04-28T19:42:36.192201+0000","last_clean":"2022-04-28T19:42:36.192201+0000","last_became_active":"2022-04-28T19:42:27.254475+0000","last_became_peered":"2022-04-28T19:42:27.254475+0000","last_unstale":"2022-04-28T19:42:36.192201+0000","last_undegraded":"2022-04-28T19:42:36.192201+0000","last_fullsized":"2022-04-28T19:42:36.192201+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.485244+0000","last_change":"2022-04-28T19:42:27.329682+0000","last_active":"2022-04-28T19:42:28.485244+0000","last_peered":"2022-04-28T19:42:28.485244+0000","last_clean":"2022-04-28T19:42:28.485244+0000","last_became_active":"2022-04-28T19:42:27.329263+0000","last_became_peered":"2022-04-28T19:42:27.329263+0000","last_unstale":"2022-04-28T19:42:28.485244+0000","last_undegraded":"2022-04-28T19:42:28.485244+0000","last_fullsized":"2022-04-28T19:42:28.485244+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.563758+0000","last_change":"2022-04-28T19:42:27.250559+0000","last_active":"2022-04-28T19:42:28.563758+0000","last_peered":"2022-04-28T19:42:28.563758+0000","last_clean":"2022-04-28T19:42:28.563758+0000","last_became_active":"2022-04-28T19:42:27.250088+0000","last_became_peered":"2022-04-28T19:42:27.250088+0000","last_unstale":"2022-04-28T19:42:28.563758+0000","last_undegraded":"2022-04-28T19:42:28.563758+0000","last_fullsized":"2022-04-28T19:42:28.563758+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483769+0000","last_change":"2022-04-28T19:42:27.271551+0000","last_active":"2022-04-28T19:42:28.483769+0000","last_peered":"2022-04-28T19:42:28.483769+0000","last_clean":"2022-04-28T19:42:28.483769+0000","last_became_active":"2022-04-28T19:42:27.271161+0000","last_became_peered":"2022-04-28T19:42:27.271161+0000","last_unstale":"2022-04-28T19:42:28.483769+0000","last_undegraded":"2022-04-28T19:42:28.483769+0000","last_fullsized":"2022-04-28T19:42:28.483769+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.253390+0000","last_change":"2022-04-28T19:42:27.247683+0000","last_active":"2022-04-28T19:42:28.253390+0000","last_peered":"2022-04-28T19:42:28.253390+0000","last_clean":"2022-04-28T19:42:28.253390+0000","last_became_active":"2022-04-28T19:42:27.245987+0000","last_became_peered":"2022-04-28T19:42:27.245987+0000","last_unstale":"2022-04-28T19:42:28.253390+0000","last_undegraded":"2022-04-28T19:42:28.253390+0000","last_fullsized":"2022-04-28T19:42:28.253390+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.566416+0000","last_change":"2022-04-28T19:42:28.565733+0000","last_active":"2022-04-28T19:42:28.566416+0000","last_peered":"2022-04-28T19:42:28.566416+0000","last_clean":"2022-04-28T19:42:28.566416+0000","last_became_active":"2022-04-28T19:42:28.564968+0000","last_became_peered":"2022-04-28T19:42:28.564968+0000","last_unstale":"2022-04-28T19:42:28.566416+0000","last_undegraded":"2022-04-28T19:42:28.566416+0000","last_fullsized":"2022-04-28T19:42:28.566416+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.483941+0000","last_change":"2022-04-28T19:42:27.251808+0000","last_active":"2022-04-28T19:42:28.483941+0000","last_peered":"2022-04-28T19:42:28.483941+0000","last_clean":"2022-04-28T19:42:28.483941+0000","last_became_active":"2022-04-28T19:42:27.250732+0000","last_became_peered":"2022-04-28T19:42:27.250732+0000","last_unstale":"2022-04-28T19:42:28.483941+0000","last_undegraded":"2022-04-28T19:42:28.483941+0000","last_fullsized":"2022-04-28T19:42:28.483941+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.254025+0000","last_change":"2022-04-28T19:42:27.328400+0000","last_active":"2022-04-28T19:42:28.254025+0000","last_peered":"2022-04-28T19:42:28.254025+0000","last_clean":"2022-04-28T19:42:28.254025+0000","last_became_active":"2022-04-28T19:42:27.327976+0000","last_became_peered":"2022-04-28T19:42:27.327976+0000","last_unstale":"2022-04-28T19:42:28.254025+0000","last_undegraded":"2022-04-28T19:42:28.254025+0000","last_fullsized":"2022-04-28T19:42:28.254025+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":12,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.252922+0000","last_change":"2022-04-28T19:42:27.251754+0000","last_active":"2022-04-28T19:42:28.252922+0000","last_peered":"2022-04-28T19:42:28.252922+0000","last_clean":"2022-04-28T19:42:28.252922+0000","last_became_active":"2022-04-28T19:42:27.251030+0000","last_became_peered":"2022-04-28T19:42:27.251030+0000","last_unstale":"2022-04-28T19:42:28.252922+0000","last_undegraded":"2022-04-28T19:42:28.252922+0000","last_fullsized":"2022-04-28T19:42:28.252922+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":11,"reported_epoch":55,"state":"active+clean","last_fresh":"2022-04-28T19:42:28.484689+0000","last_change":"2022-04-28T19:42:27.326396+0000","last_active":"2022-04-28T19:42:28.484689+0000","last_peered":"2022-04-28T19:42:28.484689+0000","last_clean":"2022-04-28T19:42:28.484689+0000","last_became_active":"2022-04-28T19:42:27.325373+0000","last_became_peered":"2022-04-28T19:42:27.325373+0000","last_unstale":"2022-04-28T19:42:28.484689+0000","last_undegraded":"2022-04-28T19:42:28.484689+0000","last_fullsized":"2022-04-28T19:42:28.484689+0000","mapping_epoch":53,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":54,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"56'1","reported_seq":13,"reported_epoch":56,"state":"active+clean","last_fresh":"2022-04-28T19:42:29.265771+0000","last_change":"2022-04-28T19:42:27.250075+0000","last_active":"2022-04-28T19:42:29.265771+0000","last_peered":"2022-04-28T19:42:29.265771+0000","last_clean":"2022-04-28T19:42:29.265771+0000","last_became_active":"2022-04-28T19:42:27.248968+0000","last_became_peered":"2022-04-28T19:42:27.248968+0000","last_unstale":"2022-04-28T19:42:29.265771+0000","last_undegraded":"2022-04-28T19:42:29.265771+0000","last_fullsized":"2022-04-28T19:42:29.265771+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2304,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":46,"seq":197568495665,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049152,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696000,"statfs":{"total":95995035648,"available":94920704000,"internally_reserved":1073741824,"allocated":581632,"data_stored":236943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.81899999999999995,"15min":0.81999999999999995},"min":{"1min":0.498,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.1439999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":1.042},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.82299999999999995,"15min":0.83599999999999997},"min":{"1min":0.54400000000000004,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.2150000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.70499999999999996}]},{"osd":1,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"min":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.0489999999999999,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.84199999999999997,"15min":0.82099999999999995},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.8240000000000001,"5min":1.8240000000000001,"15min":1.8240000000000001},"last":0.77100000000000002}]},{"osd":2,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.84799999999999998,"15min":0.83799999999999997},"min":{"1min":0.65600000000000003,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.85999999999999999,"15min":0.84799999999999998},"min":{"1min":0.58299999999999996,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":2.298,"5min":2.298,"15min":2.298},"last":0.86899999999999999}]},{"osd":3,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.89500000000000002,"15min":0.877},"min":{"1min":0.622,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.958,"5min":1.958,"15min":1.958},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.88200000000000001,"15min":0.86199999999999999},"min":{"1min":0.64600000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":2.165,"5min":2.165,"15min":2.165},"last":0.91500000000000004}]},{"osd":4,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.76600000000000001,"15min":0.75800000000000001},"min":{"1min":0.61799999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.80900000000000005,"15min":0.81000000000000005},"min":{"1min":0.54400000000000004,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"last":0.83299999999999996}]},{"osd":5,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.81100000000000005,"15min":0.79900000000000004},"min":{"1min":0.64900000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.0449999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.86299999999999999,"15min":0.86599999999999999},"min":{"1min":0.58199999999999996,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.109,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.86199999999999999}]},{"osd":6,"last update":"Thu Apr 28 19:43:02 2022","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.64000000000000001,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.1539999999999999,"5min":1.409,"15min":1.409},"last":0.93899999999999995},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.92800000000000005,"15min":0.92700000000000005},"min":{"1min":0.67600000000000005,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.381,"5min":1.381,"15min":1.381},"last":1.026}]}]},{"osd":6,"up_from":41,"seq":176093659190,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049140,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696012,"statfs":{"total":95995035648,"available":94920716288,"internally_reserved":1073741824,"allocated":569344,"data_stored":236797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:48 2022","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.84099999999999997,"15min":0.871},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.0649999999999999,"5min":1.22,"15min":1.22},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.876,"15min":0.88600000000000001},"min":{"1min":0.66800000000000004,"5min":0.502,"15min":0.502},"max":{"1min":0.995,"5min":1.2470000000000001,"15min":1.2470000000000001},"last":0.79200000000000004}]},{"osd":1,"last update":"Thu Apr 28 19:43:48 2022","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.86199999999999999,"15min":0.85899999999999999},"min":{"1min":0.71799999999999997,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.105,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.875,"5min":0.89200000000000002,"15min":0.91900000000000004},"min":{"1min":0.63600000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.1240000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.68999999999999995}]},{"osd":2,"last update":"Thu Apr 28 19:43:48 2022","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.85799999999999998,"15min":0.878},"min":{"1min":0.628,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.022,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.628},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.90100000000000002,"15min":0.92100000000000004},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.119,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.93100000000000005}]},{"osd":3,"last update":"Thu Apr 28 19:43:48 2022","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.85699999999999998,"15min":0.84899999999999998},"min":{"1min":0.63100000000000001,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.0580000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.95199999999999996,"15min":0.98599999999999999},"min":{"1min":0.69599999999999995,"5min":0.628,"15min":0.628},"max":{"1min":1.163,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.73699999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:43:48 2022","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.82399999999999995,"15min":0.83699999999999997},"min":{"1min":0.66400000000000003,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1479999999999999,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.90700000000000003,"15min":0.92100000000000004},"min":{"1min":0.67500000000000004,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1739999999999999,"5min":1.5,"15min":1.5},"last":0.82799999999999996}]},{"osd":5,"last update":"Thu Apr 28 19:43:48 2022","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.88,"15min":0.88500000000000001},"min":{"1min":0.61399999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.0740000000000001,"5min":1.663,"15min":1.663},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.89300000000000002,"15min":0.89100000000000001},"min":{"1min":0.69699999999999995,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2290000000000001,"5min":1.619,"15min":1.619},"last":0.75800000000000001}]},{"osd":7,"last update":"Thu Apr 28 19:43:09 2022","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.92500000000000004,"15min":0.91300000000000003},"min":{"1min":0.56100000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.157,"5min":1.431,"15min":1.431},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.93500000000000005,"15min":0.91200000000000003},"min":{"1min":0.51700000000000002,"5min":0.495,"15min":0.495},"max":{"1min":1.1850000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.91100000000000003}]}]},{"osd":1,"up_from":16,"seq":68719476814,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049124,"kb_used_data":540,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696028,"statfs":{"total":95995035648,"available":94920732672,"internally_reserved":1073741824,"allocated":552960,"data_stored":235192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:54 2022","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.626,"15min":0.55600000000000005},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82599999999999996,"5min":1.736,"15min":1.736},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.68600000000000005,"15min":0.58599999999999997},"min":{"1min":0.54200000000000004,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.98299999999999998,"5min":1.544,"15min":1.544},"last":0.63}]},{"osd":2,"last update":"Thu Apr 28 19:43:14 2022","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.72899999999999998,"15min":0.71199999999999997},"min":{"1min":0.54400000000000004,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.101,"5min":1.377,"15min":1.377},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.70199999999999996,"15min":0.67400000000000004},"min":{"1min":0.47399999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0740000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.54000000000000004}]},{"osd":3,"last update":"Thu Apr 28 19:43:38 2022","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.78700000000000003,"15min":0.78400000000000003},"min":{"1min":0.54700000000000004,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.149,"5min":1.377,"15min":1.377},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.76800000000000002,"15min":0.76600000000000001},"min":{"1min":0.38200000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.99199999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Thu Apr 28 19:43:00 2022","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.873,"15min":0.84299999999999997},"min":{"1min":0.59999999999999998,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.175,"5min":1.446,"15min":1.446},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.95099999999999996,"15min":0.91200000000000003},"min":{"1min":0.66900000000000004,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.2350000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.89300000000000002}]},{"osd":5,"last update":"Thu Apr 28 19:43:19 2022","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.98999999999999999,"15min":0.998},"min":{"1min":0.71799999999999997,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.1839999999999999,"5min":1.788,"15min":1.788},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.996,"5min":0.98499999999999999,"15min":0.97499999999999998},"min":{"1min":0.77700000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.206,"5min":1.802,"15min":1.802},"last":0.91600000000000004}]},{"osd":6,"last update":"Thu Apr 28 19:42:46 2022","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.05,"15min":1.0800000000000001},"min":{"1min":0.69299999999999995,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.3120000000000001,"5min":1.653,"15min":1.653},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.1020000000000001,"15min":1.141},"min":{"1min":0.73799999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.3069999999999999,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":1.0109999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:43:07 2022","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.1060000000000001,"15min":1.119},"min":{"1min":0.89100000000000001,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.5660000000000001,"5min":1.907,"15min":1.907},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1200000000000001,"15min":1.1259999999999999},"min":{"1min":0.84199999999999997,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.3360000000000001,"5min":1.673,"15min":1.673},"last":1.0580000000000001}]}]},{"osd":0,"up_from":10,"seq":42949673042,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049132,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696020,"statfs":{"total":95995035648,"available":94920724480,"internally_reserved":1073741824,"allocated":561152,"data_stored":235767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 28 19:42:52 2022","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.64700000000000002,"15min":0.58099999999999996},"min":{"1min":0.44,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.022,"5min":1.829,"15min":1.829},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.75800000000000001,"15min":0.89100000000000001},"min":{"1min":0.45800000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.081,"5min":9.4420000000000002,"15min":9.4420000000000002},"last":1.1180000000000001}]},{"osd":2,"last update":"Thu Apr 28 19:43:10 2022","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.67100000000000004,"15min":0.61099999999999999},"min":{"1min":0.48699999999999999,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.0860000000000001,"5min":1.2,"15min":1.2},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.75800000000000001,"15min":0.69899999999999995},"min":{"1min":0.52800000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0649999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.81200000000000006}]},{"osd":3,"last update":"Thu Apr 28 19:43:40 2022","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.753,"15min":0.76000000000000001},"min":{"1min":0.442,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.117,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.82899999999999996,"15min":0.81299999999999994},"min":{"1min":0.54800000000000004,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.077,"5min":1.224,"15min":1.224},"last":1.0580000000000001}]},{"osd":4,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.95099999999999996,"15min":0.95999999999999996},"min":{"1min":0.69499999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.3240000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.012,"15min":0.96699999999999997},"min":{"1min":0.755,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":1.2150000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:43:20 2022","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.05,"15min":1.032},"min":{"1min":0.79800000000000004,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.425,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":1.298},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.96799999999999997,"15min":0.98399999999999999},"min":{"1min":0.626,"5min":0.62,"15min":0.62},"max":{"1min":1.3759999999999999,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":0.85599999999999998}]},{"osd":6,"last update":"Thu Apr 28 19:42:45 2022","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.01,"15min":1},"min":{"1min":0.71499999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.3200000000000001,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.0109999999999999,"15min":0.997},"min":{"1min":0.748,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.264,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.044}]},{"osd":7,"last update":"Thu Apr 28 19:43:10 2022","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.1319999999999999,"15min":1.1319999999999999},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3919999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.369},{"interface":"front","average":{"1min":1.03,"5min":1.048,"15min":1.0349999999999999},"min":{"1min":0.75600000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.385,"5min":1.504,"15min":1.504},"last":1.244}]}]},{"osd":2,"up_from":21,"seq":90194313290,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049132,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696020,"statfs":{"total":95995035648,"available":94920724480,"internally_reserved":1073741824,"allocated":561152,"data_stored":235779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:11 2022","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66500000000000004,"15min":0.61099999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93300000000000005,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.70499999999999996,"15min":0.66300000000000003},"min":{"1min":0.42599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.7330000000000001,"5min":1.7330000000000001,"15min":1.7330000000000001},"last":0.68000000000000005}]},{"osd":1,"last update":"Thu Apr 28 19:43:11 2022","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.68999999999999995,"15min":0.66200000000000003},"min":{"1min":0.49299999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.627,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.76900000000000002,"15min":0.69999999999999996},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.335,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.749}]},{"osd":3,"last update":"Thu Apr 28 19:43:45 2022","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.82499999999999996,"15min":0.78800000000000003},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1299999999999999,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.76100000000000001,"15min":0.73799999999999999},"min":{"1min":0.57599999999999996,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.2229999999999999,"5min":2.2229999999999999,"15min":2.2229999999999999},"last":0.77000000000000002}]},{"osd":4,"last update":"Thu Apr 28 19:42:55 2022","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.87,"15min":0.86399999999999999},"min":{"1min":0.65300000000000002,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.268,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.024,"15min":0.99399999999999999},"min":{"1min":0.77000000000000002,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":1.486,"5min":1.615,"15min":1.615},"last":1.143}]},{"osd":5,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0800000000000001,"15min":1.1040000000000001},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.512,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.266},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0149999999999999,"15min":1.0169999999999999},"min":{"1min":0.69199999999999995,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.885,"5min":1.885,"15min":1.885},"last":1.1870000000000001}]},{"osd":6,"last update":"Thu Apr 28 19:43:45 2022","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.0449999999999999,"15min":1.046},"min":{"1min":0.72099999999999997,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":2.6080000000000001,"5min":2.6080000000000001,"15min":2.6080000000000001},"last":1.119},{"interface":"front","average":{"1min":1.105,"5min":1.163,"15min":1.2270000000000001},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.601,"5min":1.9399999999999999,"15min":1.9399999999999999},"last":1.173}]},{"osd":7,"last update":"Thu Apr 28 19:43:11 2022","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.0920000000000001,"15min":1.097},"min":{"1min":0.75600000000000001,"5min":0.623,"15min":0.623},"max":{"1min":2.5419999999999998,"5min":2.5419999999999998,"15min":2.5419999999999998},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.161,"5min":1.2,"15min":1.226},"min":{"1min":0.77900000000000003,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.5580000000000001,"5min":1.655,"15min":1.655},"last":1.2090000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149764,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049128,"kb_used_data":544,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696024,"statfs":{"total":95995035648,"available":94920728576,"internally_reserved":1073741824,"allocated":557056,"data_stored":235282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:35 2022","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.68400000000000005,"15min":0.66000000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.833,"5min":1.833,"15min":1.833},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.79000000000000004,"15min":0.78200000000000003},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.133,"5min":2.133,"15min":2.133},"last":0.85099999999999998}]},{"osd":1,"last update":"Thu Apr 28 19:43:35 2022","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.753,"15min":0.72299999999999998},"min":{"1min":0.497,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.214,"5min":2.2970000000000002,"15min":2.2970000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.72299999999999998,"15min":0.69499999999999995},"min":{"1min":0.48199999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.98599999999999999,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.751}]},{"osd":2,"last update":"Thu Apr 28 19:43:35 2022","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.75900000000000001,"15min":0.77400000000000002},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.92300000000000004,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.76200000000000001,"15min":0.72399999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.2400000000000002,"5min":2.2400000000000002,"15min":2.2400000000000002},"last":0.93899999999999995}]},{"osd":4,"last update":"Thu Apr 28 19:43:03 2022","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.92300000000000004,"15min":0.90500000000000003},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.1599999999999999,"5min":2.415,"15min":2.415},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.92900000000000005,"15min":0.90700000000000003},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2729999999999999,"5min":1.363,"15min":1.363},"last":0.82899999999999996}]},{"osd":5,"last update":"Thu Apr 28 19:43:20 2022","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.97899999999999998,"15min":0.95099999999999996},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":2.0680000000000001,"5min":2.0680000000000001,"15min":2.0680000000000001},"last":1.1930000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.042,"15min":1.028},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":2.3580000000000001,"5min":2.5590000000000002,"15min":2.5590000000000002},"last":1.071}]},{"osd":6,"last update":"Thu Apr 28 19:43:47 2022","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0409999999999999,"15min":1.0189999999999999},"min":{"1min":0.69899999999999995,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":2.109,"5min":2.109,"15min":2.109},"last":1.018},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.075,"15min":1.0649999999999999},"min":{"1min":0.77000000000000002,"5min":0.66000000000000003,"15min":0.66000000000000003},"max":{"1min":2.161,"5min":2.161,"15min":2.161},"last":1.1240000000000001}]},{"osd":7,"last update":"Thu Apr 28 19:43:08 2022","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.101,"15min":1.103},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.234},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.1299999999999999,"15min":1.1499999999999999},"min":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"max":{"1min":1.351,"5min":2.4900000000000002,"15min":2.4900000000000002},"last":1.165}]}]},{"osd":4,"up_from":31,"seq":133143986240,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049148,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696004,"statfs":{"total":95995035648,"available":94920708096,"internally_reserved":1073741824,"allocated":577536,"data_stored":236873,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.84399999999999997,"15min":0.82799999999999996},"min":{"1min":0.56999999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.81799999999999995},"min":{"1min":0.58899999999999997,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.145,"5min":1.456,"15min":1.456},"last":0.73499999999999999}]},{"osd":1,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.84299999999999997,"15min":0.81799999999999995},"min":{"1min":0.55100000000000005,"5min":0.44,"15min":0.44},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.79900000000000004,"15min":0.80000000000000004},"min":{"1min":0.63800000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":0.98999999999999999,"5min":1.1759999999999999,"15min":1.1759999999999999},"last":0.94799999999999995}]},{"osd":2,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.81899999999999995,"15min":0.82099999999999995},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.96999999999999997,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.85499999999999998,"15min":0.86899999999999999},"min":{"1min":0.54800000000000004,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.095,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.53800000000000003}]},{"osd":3,"last update":"Thu Apr 28 19:42:58 2022","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.83199999999999996,"15min":0.83599999999999997},"min":{"1min":0.58399999999999996,"5min":0.495,"15min":0.495},"max":{"1min":1.143,"5min":1.601,"15min":1.601},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.89500000000000002,"15min":0.879},"min":{"1min":0.68100000000000005,"5min":0.622,"15min":0.622},"max":{"1min":1.276,"5min":1.276,"15min":1.276},"last":0.755}]},{"osd":5,"last update":"Thu Apr 28 19:43:21 2022","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.82799999999999996,"15min":0.77000000000000002},"min":{"1min":0.54400000000000004,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.292,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.878,"15min":0.80300000000000005},"min":{"1min":0.67100000000000004,"5min":0.436,"15min":0.436},"max":{"1min":1.1839999999999999,"5min":1.399,"15min":1.399},"last":0.96699999999999997}]},{"osd":6,"last update":"Thu Apr 28 19:42:48 2022","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.90500000000000003,"15min":0.89500000000000002},"min":{"1min":0.68200000000000005,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.222,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.90800000000000003,"15min":0.90600000000000003},"min":{"1min":0.75,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.1539999999999999,"5min":1.427,"15min":1.427},"last":0.98799999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:43:04 2022","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.90700000000000003,"15min":0.89700000000000002},"min":{"1min":0.627,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.22,"5min":1.4930000000000001,"15min":1.4930000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.083,"5min":0.96599999999999997,"15min":0.93600000000000005},"min":{"1min":0.83899999999999997,"5min":0.498,"15min":0.498},"max":{"1min":1.385,"5min":1.385,"15min":1.385},"last":0.89500000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822714,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1049132,"kb_used_data":548,"kb_used_omap":0,"kb_used_meta":1048576,"kb_avail":92696020,"statfs":{"total":95995035648,"available":94920724480,"internally_reserved":1073741824,"allocated":561152,"data_stored":235815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1073741824},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.81699999999999995,"15min":0.81299999999999994},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.151,"5min":1.222,"15min":1.222},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.86699999999999999,"15min":0.83899999999999997},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.167,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.70699999999999996}]},{"osd":1,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.83799999999999997,"15min":0.81599999999999995},"min":{"1min":0.53700000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0960000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.878,"5min":0.85799999999999998,"15min":0.85199999999999998},"min":{"1min":0.625,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1399999999999999,"5min":1.8300000000000001,"15min":1.8300000000000001},"last":1.026}]},{"osd":2,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.80300000000000005,"15min":0.79200000000000004},"min":{"1min":0.56599999999999995,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.079,"5min":1.615,"15min":1.615},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.88900000000000001,"15min":0.89500000000000002},"min":{"1min":0.60899999999999999,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.1200000000000001,"5min":1.145,"15min":1.145},"last":0.80900000000000005}]},{"osd":3,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.84499999999999997,"15min":0.84899999999999998},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1359999999999999,"5min":1.74,"15min":1.74},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.83499999999999996,"5min":0.85399999999999998,"15min":0.84399999999999997},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.149,"5min":1.536,"15min":1.536},"last":0.61699999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:43:22 2022","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.77600000000000002,"15min":0.746},"min":{"1min":0.42799999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.0429999999999999,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.78200000000000003,"15min":0.75700000000000001},"min":{"1min":0.58899999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1850000000000001,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.0760000000000001}]},{"osd":6,"last update":"Thu Apr 28 19:43:45 2022","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.879,"15min":0.871},"min":{"1min":0.70799999999999996,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.1899999999999999,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.87,"15min":0.83499999999999996},"min":{"1min":0.59899999999999998,"5min":0.504,"15min":0.504},"max":{"1min":1.2430000000000001,"5min":1.887,"15min":1.887},"last":1.115}]},{"osd":7,"last update":"Thu Apr 28 19:43:08 2022","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.93000000000000005,"15min":0.93200000000000005},"min":{"1min":0.66700000000000004,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.1599999999999999,"5min":1.7869999999999999,"15min":1.7869999999999999},"last":1.151},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.84999999999999998,"15min":0.83199999999999996},"min":{"1min":0.66300000000000003,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.7509999999999999,"5min":1.7509999999999999,"15min":1.7509999999999999},"last":0.86299999999999999}]}]}]}} 2022-04-28T19:43:53.513 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2022-04-28T19:43:53.513 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2022-04-28T19:43:53.514 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2022-04-28T19:43:53.514 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph health --format=json 2022-04-28T19:43:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:53 smithi139 bash[23439]: cluster 2022-04-28T19:43:52.054413+0000 mgr.y (mgr.14140) 345 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:53.658 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:43:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:53.512Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:43:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:53.512Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:43:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:43:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:43:53.513Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:43:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:54 smithi059 bash[29114]: audit 2022-04-28T19:43:52.752469+0000 mgr.y (mgr.14140) 346 : audit [DBG] from='client.24661 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:43:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:54 smithi059 bash[23786]: audit 2022-04-28T19:43:52.752469+0000 mgr.y (mgr.14140) 346 : audit [DBG] from='client.24661 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:43:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:54 smithi139 bash[23439]: audit 2022-04-28T19:43:52.752469+0000 mgr.y (mgr.14140) 346 : audit [DBG] from='client.24661 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:43:55.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:55 smithi059 bash[29114]: cluster 2022-04-28T19:43:54.055424+0000 mgr.y (mgr.14140) 347 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:55.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:55 smithi059 bash[23786]: cluster 2022-04-28T19:43:54.055424+0000 mgr.y (mgr.14140) 347 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:55 smithi139 bash[23439]: cluster 2022-04-28T19:43:54.055424+0000 mgr.y (mgr.14140) 347 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:56.894 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:43:57.581 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:57 smithi059 bash[29114]: cluster 2022-04-28T19:43:56.056362+0000 mgr.y (mgr.14140) 348 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:57.581 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:57 smithi059 bash[29114]: audit 2022-04-28T19:43:56.892767+0000 mon.a (mon.0) 643 : audit [DBG] from='client.? 172.21.15.59:0/2207476527' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-04-28T19:43:57.581 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:57 smithi059 bash[23786]: cluster 2022-04-28T19:43:56.056362+0000 mgr.y (mgr.14140) 348 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:57.582 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:57 smithi059 bash[23786]: audit 2022-04-28T19:43:56.892767+0000 mon.a (mon.0) 643 : audit [DBG] from='client.? 172.21.15.59:0/2207476527' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-04-28T19:43:57.582 INFO:teuthology.orchestra.run.smithi059.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2022-04-28T19:43:57.583 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2022-04-28T19:43:57.583 INFO:teuthology.run_tasks:Running task print... 2022-04-28T19:43:57.593 INFO:teuthology.task.print:**** done creating new fs 2022-04-28T19:43:57.594 INFO:teuthology.run_tasks:Running task cephadm.shell... 2022-04-28T19:43:57.623 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi059.front.sepia.ceph.com 2022-04-28T19:43:57.623 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2022-04-28T19:43:57.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:57 smithi139 bash[23439]: cluster 2022-04-28T19:43:56.056362+0000 mgr.y (mgr.14140) 348 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:57.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:57 smithi139 bash[23439]: audit 2022-04-28T19:43:56.892767+0000 mon.a (mon.0) 643 : audit [DBG] from='client.? 172.21.15.59:0/2207476527' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2022-04-28T19:43:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:43:59 smithi059 bash[23786]: cluster 2022-04-28T19:43:58.057344+0000 mgr.y (mgr.14140) 349 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:43:59 smithi059 bash[29114]: cluster 2022-04-28T19:43:58.057344+0000 mgr.y (mgr.14140) 349 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:43:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:43:59 smithi139 bash[23439]: cluster 2022-04-28T19:43:58.057344+0000 mgr.y (mgr.14140) 349 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:01 smithi059 bash[23786]: cluster 2022-04-28T19:44:00.057956+0000 mgr.y (mgr.14140) 350 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:01 smithi059 bash[23786]: audit 2022-04-28T19:44:01.020392+0000 mon.a (mon.0) 644 : audit [INF] from='client.? 172.21.15.59:0/1742531475' entity='client.admin' 2022-04-28T19:44:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:01 smithi059 bash[23786]: audit 2022-04-28T19:44:01.045162+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:01.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:01 smithi059 bash[29114]: cluster 2022-04-28T19:44:00.057956+0000 mgr.y (mgr.14140) 350 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:01.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:01 smithi059 bash[29114]: audit 2022-04-28T19:44:01.020392+0000 mon.a (mon.0) 644 : audit [INF] from='client.? 172.21.15.59:0/1742531475' entity='client.admin' 2022-04-28T19:44:01.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:01 smithi059 bash[29114]: audit 2022-04-28T19:44:01.045162+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:01 smithi139 bash[23439]: cluster 2022-04-28T19:44:00.057956+0000 mgr.y (mgr.14140) 350 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:01 smithi139 bash[23439]: audit 2022-04-28T19:44:01.020392+0000 mon.a (mon.0) 644 : audit [INF] from='client.? 172.21.15.59:0/1742531475' entity='client.admin' 2022-04-28T19:44:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:01 smithi139 bash[23439]: audit 2022-04-28T19:44:01.045162+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:01.760 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2022-04-28T19:44:03.515 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:03 smithi059 bash[29114]: cluster 2022-04-28T19:44:02.058821+0000 mgr.y (mgr.14140) 351 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:03.515 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:03 smithi059 bash[23786]: cluster 2022-04-28T19:44:02.058821+0000 mgr.y (mgr.14140) 351 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:03 smithi139 bash[23439]: cluster 2022-04-28T19:44:02.058821+0000 mgr.y (mgr.14140) 351 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:03.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:44:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:03.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:44:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:03.513Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:44:05.509 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:05 smithi059 bash[29114]: cluster 2022-04-28T19:44:04.059736+0000 mgr.y (mgr.14140) 352 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:05.510 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:05 smithi059 bash[23786]: cluster 2022-04-28T19:44:04.059736+0000 mgr.y (mgr.14140) 352 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:05 smithi139 bash[23439]: cluster 2022-04-28T19:44:04.059736+0000 mgr.y (mgr.14140) 352 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:06.489 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2022-04-28T19:44:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[29114]: cluster 2022-04-28T19:44:06.060661+0000 mgr.y (mgr.14140) 353 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[29114]: audit 2022-04-28T19:44:06.400257+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[29114]: audit 2022-04-28T19:44:06.791031+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[29114]: audit 2022-04-28T19:44:06.796373+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[29114]: audit 2022-04-28T19:44:06.804944+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[23786]: cluster 2022-04-28T19:44:06.060661+0000 mgr.y (mgr.14140) 353 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[23786]: audit 2022-04-28T19:44:06.400257+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[23786]: audit 2022-04-28T19:44:06.791031+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[23786]: audit 2022-04-28T19:44:06.796373+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:07.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:07 smithi059 bash[23786]: audit 2022-04-28T19:44:06.804944+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:07 smithi139 bash[23439]: cluster 2022-04-28T19:44:06.060661+0000 mgr.y (mgr.14140) 353 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:07 smithi139 bash[23439]: audit 2022-04-28T19:44:06.400257+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:07 smithi139 bash[23439]: audit 2022-04-28T19:44:06.791031+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:07 smithi139 bash[23439]: audit 2022-04-28T19:44:06.796373+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:07 smithi139 bash[23439]: audit 2022-04-28T19:44:06.804944+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:08 smithi059 bash[29114]: cephadm 2022-04-28T19:44:06.403778+0000 mgr.y (mgr.14140) 354 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:44:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:08 smithi059 bash[29114]: cephadm 2022-04-28T19:44:06.404611+0000 mgr.y (mgr.14140) 355 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:44:08.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:08 smithi059 bash[23786]: cephadm 2022-04-28T19:44:06.403778+0000 mgr.y (mgr.14140) 354 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:44:08.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:08 smithi059 bash[23786]: cephadm 2022-04-28T19:44:06.404611+0000 mgr.y (mgr.14140) 355 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:44:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:08 smithi139 bash[23439]: cephadm 2022-04-28T19:44:06.403778+0000 mgr.y (mgr.14140) 354 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:44:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:08 smithi139 bash[23439]: cephadm 2022-04-28T19:44:06.404611+0000 mgr.y (mgr.14140) 355 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:44:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:09 smithi059 bash[23786]: cluster 2022-04-28T19:44:08.061601+0000 mgr.y (mgr.14140) 356 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:09 smithi059 bash[29114]: cluster 2022-04-28T19:44:08.061601+0000 mgr.y (mgr.14140) 356 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:09 smithi139 bash[23439]: cluster 2022-04-28T19:44:08.061601+0000 mgr.y (mgr.14140) 356 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:10.462 INFO:teuthology.run_tasks:Running task print... 2022-04-28T19:44:10.470 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2022-04-28T19:44:10.470 INFO:teuthology.run_tasks:Running task print... 2022-04-28T19:44:10.508 INFO:teuthology.task.print:**** done start parallel 2022-04-28T19:44:10.508 INFO:teuthology.run_tasks:Running task parallel... 2022-04-28T19:44:10.542 INFO:teuthology.task.parallel:starting parallel... 2022-04-28T19:44:10.543 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2022-04-28T19:44:10.543 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2022-04-28T19:44:10.543 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2022-04-28T19:44:10.543 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2022-04-28T19:44:10.545 INFO:tasks.rados:Beginning rados... 2022-04-28T19:44:10.546 INFO:tasks.rados:joining rados 2022-04-28T19:44:10.546 INFO:teuthology.task.parallel:In parallel, running task sequential... 2022-04-28T19:44:10.546 INFO:teuthology.task.sequential:In sequential, running task print... 2022-04-28T19:44:10.546 INFO:teuthology.task.print:**** done start upgrade, wait 2022-04-28T19:44:10.547 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2022-04-28T19:44:10.547 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi059.front.sepia.ceph.com 2022-04-28T19:44:10.547 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -e sha1=4fa079ba14503defa8dc257d7c2d506ebefcfe6d -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2022-04-28T19:44:10.548 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2022-04-28T19:44:10.548 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2022-04-28T19:44:10.698 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:44:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:11 smithi059 bash[29114]: cluster 2022-04-28T19:44:10.062342+0000 mgr.y (mgr.14140) 357 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:11 smithi059 bash[23786]: cluster 2022-04-28T19:44:10.062342+0000 mgr.y (mgr.14140) 357 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:11 smithi139 bash[23439]: cluster 2022-04-28T19:44:10.062342+0000 mgr.y (mgr.14140) 357 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:13 smithi059 bash[29114]: cluster 2022-04-28T19:44:12.063260+0000 mgr.y (mgr.14140) 358 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:13 smithi059 bash[23786]: cluster 2022-04-28T19:44:12.063260+0000 mgr.y (mgr.14140) 358 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:13.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:13.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:44:13.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:13.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:44:13.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:13.513Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:44:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:13 smithi139 bash[23439]: cluster 2022-04-28T19:44:12.063260+0000 mgr.y (mgr.14140) 358 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:15.122 INFO:teuthology.orchestra.run.smithi059.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:44:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[23786]: cluster 2022-04-28T19:44:14.064204+0000 mgr.y (mgr.14140) 359 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[23786]: audit 2022-04-28T19:44:14.927166+0000 mon.c (mon.1) 29 : audit [INF] from='client.? 172.21.15.59:0/668668950' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[23786]: audit 2022-04-28T19:44:14.927655+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[23786]: audit 2022-04-28T19:44:15.119395+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[23786]: audit 2022-04-28T19:44:15.148495+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[23786]: audit 2022-04-28T19:44:15.157720+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[29114]: cluster 2022-04-28T19:44:14.064204+0000 mgr.y (mgr.14140) 359 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[29114]: audit 2022-04-28T19:44:14.927166+0000 mon.c (mon.1) 29 : audit [INF] from='client.? 172.21.15.59:0/668668950' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[29114]: audit 2022-04-28T19:44:14.927655+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T19:44:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[29114]: audit 2022-04-28T19:44:15.119395+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:15.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[29114]: audit 2022-04-28T19:44:15.148495+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:15.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:15 smithi059 bash[29114]: audit 2022-04-28T19:44:15.157720+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:15 smithi139 bash[23439]: cluster 2022-04-28T19:44:14.064204+0000 mgr.y (mgr.14140) 359 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:15 smithi139 bash[23439]: audit 2022-04-28T19:44:14.927166+0000 mon.c (mon.1) 29 : audit [INF] from='client.? 172.21.15.59:0/668668950' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T19:44:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:15 smithi139 bash[23439]: audit 2022-04-28T19:44:14.927655+0000 mon.a (mon.0) 650 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T19:44:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:15 smithi139 bash[23439]: audit 2022-04-28T19:44:15.119395+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:15 smithi139 bash[23439]: audit 2022-04-28T19:44:15.148495+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:44:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:15 smithi139 bash[23439]: audit 2022-04-28T19:44:15.157720+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:44:16.238 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -e sha1=4fa079ba14503defa8dc257d7c2d506ebefcfe6d -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done' 2022-04-28T19:44:16.344 INFO:tasks.rados:starting run 0 out of 1 2022-04-28T19:44:16.344 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2022-04-28T19:44:16.344 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2022-04-28T19:44:16.483 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:44:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[29114]: audit 2022-04-28T19:44:15.111916+0000 mgr.y (mgr.14140) 360 : audit [DBG] from='client.24697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[29114]: cephadm 2022-04-28T19:44:15.168485+0000 mgr.y (mgr.14140) 361 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[29114]: audit 2022-04-28T19:44:15.278181+0000 mon.a (mon.0) 654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[29114]: cluster 2022-04-28T19:44:15.278447+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[29114]: audit 2022-04-28T19:44:16.134174+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[29114]: audit 2022-04-28T19:44:16.136107+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[23786]: audit 2022-04-28T19:44:15.111916+0000 mgr.y (mgr.14140) 360 : audit [DBG] from='client.24697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[23786]: cephadm 2022-04-28T19:44:15.168485+0000 mgr.y (mgr.14140) 361 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[23786]: audit 2022-04-28T19:44:15.278181+0000 mon.a (mon.0) 654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T19:44:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[23786]: cluster 2022-04-28T19:44:15.278447+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2022-04-28T19:44:16.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[23786]: audit 2022-04-28T19:44:16.134174+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:44:16.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:16 smithi059 bash[23786]: audit 2022-04-28T19:44:16.136107+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:16 smithi139 bash[23439]: audit 2022-04-28T19:44:15.111916+0000 mgr.y (mgr.14140) 360 : audit [DBG] from='client.24697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:16 smithi139 bash[23439]: cephadm 2022-04-28T19:44:15.168485+0000 mgr.y (mgr.14140) 361 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:4fa079ba14503defa8dc257d7c2d506ebefcfe6d 2022-04-28T19:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:16 smithi139 bash[23439]: audit 2022-04-28T19:44:15.278181+0000 mon.a (mon.0) 654 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T19:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:16 smithi139 bash[23439]: cluster 2022-04-28T19:44:15.278447+0000 mon.a (mon.0) 655 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2022-04-28T19:44:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:16 smithi139 bash[23439]: audit 2022-04-28T19:44:16.134174+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:44:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:16 smithi139 bash[23439]: audit 2022-04-28T19:44:16.136107+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:44:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:17 smithi059 bash[23786]: cluster 2022-04-28T19:44:16.064883+0000 mgr.y (mgr.14140) 362 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:17 smithi059 bash[29114]: cluster 2022-04-28T19:44:16.064883+0000 mgr.y (mgr.14140) 362 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:17 smithi139 bash[23439]: cluster 2022-04-28T19:44:16.064883+0000 mgr.y (mgr.14140) 362 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:19 smithi059 bash[23786]: cluster 2022-04-28T19:44:18.065544+0000 mgr.y (mgr.14140) 363 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:19.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:19 smithi059 bash[29114]: cluster 2022-04-28T19:44:18.065544+0000 mgr.y (mgr.14140) 363 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:19 smithi139 bash[23439]: cluster 2022-04-28T19:44:18.065544+0000 mgr.y (mgr.14140) 363 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 2.2 KiB data, 4.4 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:21.160 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST STATUS REFRESHED AGE VERSION IMAGE NAME IMAGE ID CONTAINER ID 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (3m) 106s ago 3m 0.20.0 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (3m) 106s ago 3m 6.7.4 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (107s) 106s ago 108s 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 44062639a78b 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (111s) 106s ago 111s 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 3516bf4894c9 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (7m) 106s ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b8c424c2e210 2022-04-28T19:44:21.473 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (9m) 106s ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 320b27bb479e 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (9m) 106s ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 239497cf6769 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (7m) 106s ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0e12092e0bc3 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (8m) 106s ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (3m) 106s ago 3m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (3m) 106s ago 3m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (7m) 106s ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 547b5580e073 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (6m) 106s ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (6m) 106s ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 35cdc0c6604a 2022-04-28T19:44:21.474 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (5m) 106s ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 2344bb9dfbae 2022-04-28T19:44:21.475 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (5m) 106s ago 5m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 5976654882b1 2022-04-28T19:44:21.475 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (5m) 106s ago 5m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 e68cffb4766c 2022-04-28T19:44:21.475 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (4m) 106s ago 4m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 c36b8738a469 2022-04-28T19:44:21.475 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (4m) 106s ago 4m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:44:21.475 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (3m) 106s ago 3m 2.18.1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 c763409947e4 2022-04-28T19:44:21.801 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:44:21.802 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 15 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:44:21.803 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:44:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:21 smithi059 bash[23786]: cluster 2022-04-28T19:44:20.066273+0000 mgr.y (mgr.14140) 364 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:21 smithi059 bash[23786]: audit 2022-04-28T19:44:21.286633+0000 mon.c (mon.1) 30 : audit [INF] from='client.? 172.21.15.59:0/1727722092' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2022-04-28T19:44:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:21 smithi059 bash[23786]: audit 2022-04-28T19:44:21.287195+0000 mon.a (mon.0) 658 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2022-04-28T19:44:22.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:21 smithi059 bash[29114]: cluster 2022-04-28T19:44:20.066273+0000 mgr.y (mgr.14140) 364 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:22.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:21 smithi059 bash[29114]: audit 2022-04-28T19:44:21.286633+0000 mon.c (mon.1) 30 : audit [INF] from='client.? 172.21.15.59:0/1727722092' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2022-04-28T19:44:22.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:21 smithi059 bash[29114]: audit 2022-04-28T19:44:21.287195+0000 mon.a (mon.0) 658 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2022-04-28T19:44:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:21 smithi139 bash[23439]: cluster 2022-04-28T19:44:20.066273+0000 mgr.y (mgr.14140) 364 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:21 smithi139 bash[23439]: audit 2022-04-28T19:44:21.286633+0000 mon.c (mon.1) 30 : audit [INF] from='client.? 172.21.15.59:0/1727722092' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2022-04-28T19:44:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:21 smithi139 bash[23439]: audit 2022-04-28T19:44:21.287195+0000 mon.a (mon.0) 658 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2022-04-28T19:44:22.328 INFO:teuthology.orchestra.run.smithi059.stderr:pool 'unique_pool_0' created 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: audit 2022-04-28T19:44:21.145813+0000 mgr.y (mgr.14140) 365 : audit [DBG] from='client.24703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: audit 2022-04-28T19:44:21.450715+0000 mgr.y (mgr.14140) 366 : audit [DBG] from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: cluster 2022-04-28T19:44:21.750958+0000 mon.a (mon.0) 659 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: audit 2022-04-28T19:44:21.799522+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.59:0/2230723966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: cluster 2022-04-28T19:44:22.066871+0000 mgr.y (mgr.14140) 367 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: audit 2022-04-28T19:44:22.315869+0000 mon.a (mon.0) 660 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2022-04-28T19:44:23.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[29114]: cluster 2022-04-28T19:44:22.316114+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: audit 2022-04-28T19:44:21.145813+0000 mgr.y (mgr.14140) 365 : audit [DBG] from='client.24703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: audit 2022-04-28T19:44:21.450715+0000 mgr.y (mgr.14140) 366 : audit [DBG] from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: cluster 2022-04-28T19:44:21.750958+0000 mon.a (mon.0) 659 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: audit 2022-04-28T19:44:21.799522+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.59:0/2230723966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: cluster 2022-04-28T19:44:22.066871+0000 mgr.y (mgr.14140) 367 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: audit 2022-04-28T19:44:22.315869+0000 mon.a (mon.0) 660 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2022-04-28T19:44:23.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:22 smithi059 bash[23786]: cluster 2022-04-28T19:44:22.316114+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2022-04-28T19:44:23.059 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: audit 2022-04-28T19:44:21.145813+0000 mgr.y (mgr.14140) 365 : audit [DBG] from='client.24703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: audit 2022-04-28T19:44:21.450715+0000 mgr.y (mgr.14140) 366 : audit [DBG] from='client.14805 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: cluster 2022-04-28T19:44:21.750958+0000 mon.a (mon.0) 659 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: audit 2022-04-28T19:44:21.799522+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.59:0/2230723966' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: cluster 2022-04-28T19:44:22.066871+0000 mgr.y (mgr.14140) 367 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: audit 2022-04-28T19:44:22.315869+0000 mon.a (mon.0) 660 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2022-04-28T19:44:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:22 smithi139 bash[23439]: cluster 2022-04-28T19:44:22.316114+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2022-04-28T19:44:23.206 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:44:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:23.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:44:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:23.513Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:44:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:23.513Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:44:24.537 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:24 smithi059 bash[29114]: cluster 2022-04-28T19:44:23.317617+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2022-04-28T19:44:24.537 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:24 smithi059 bash[23786]: cluster 2022-04-28T19:44:23.317617+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2022-04-28T19:44:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:24 smithi139 bash[23439]: cluster 2022-04-28T19:44:23.317617+0000 mon.a (mon.0) 662 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2022-04-28T19:44:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:25 smithi059 bash[29114]: cluster 2022-04-28T19:44:24.067656+0000 mgr.y (mgr.14140) 368 : cluster [DBG] pgmap v324: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:25 smithi059 bash[29114]: cluster 2022-04-28T19:44:24.327031+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2022-04-28T19:44:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:25 smithi059 bash[23786]: cluster 2022-04-28T19:44:24.067656+0000 mgr.y (mgr.14140) 368 : cluster [DBG] pgmap v324: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:25 smithi059 bash[23786]: cluster 2022-04-28T19:44:24.327031+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2022-04-28T19:44:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:25 smithi139 bash[23439]: cluster 2022-04-28T19:44:24.067656+0000 mgr.y (mgr.14140) 368 : cluster [DBG] pgmap v324: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 2.2 KiB data, 4.5 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:25 smithi139 bash[23439]: cluster 2022-04-28T19:44:24.327031+0000 mon.a (mon.0) 663 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2022-04-28T19:44:27.354 INFO:teuthology.orchestra.run.smithi059.stderr:enabled application 'rados' on pool 'unique_pool_0' 2022-04-28T19:44:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:27 smithi059 bash[29114]: cluster 2022-04-28T19:44:26.068472+0000 mgr.y (mgr.14140) 369 : cluster [DBG] pgmap v326: 89 pgs: 2 creating+peering, 5 unknown, 82 active+clean; 2.2 KiB data, 4.6 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:27 smithi059 bash[29114]: audit 2022-04-28T19:44:26.480471+0000 mon.c (mon.1) 32 : audit [INF] from='client.? 172.21.15.59:0/1748318287' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:44:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:27 smithi059 bash[29114]: audit 2022-04-28T19:44:26.481306+0000 mon.a (mon.0) 664 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:44:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:27 smithi059 bash[23786]: cluster 2022-04-28T19:44:26.068472+0000 mgr.y (mgr.14140) 369 : cluster [DBG] pgmap v326: 89 pgs: 2 creating+peering, 5 unknown, 82 active+clean; 2.2 KiB data, 4.6 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:27 smithi059 bash[23786]: audit 2022-04-28T19:44:26.480471+0000 mon.c (mon.1) 32 : audit [INF] from='client.? 172.21.15.59:0/1748318287' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:44:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:27 smithi059 bash[23786]: audit 2022-04-28T19:44:26.481306+0000 mon.a (mon.0) 664 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:44:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:27 smithi139 bash[23439]: cluster 2022-04-28T19:44:26.068472+0000 mgr.y (mgr.14140) 369 : cluster [DBG] pgmap v326: 89 pgs: 2 creating+peering, 5 unknown, 82 active+clean; 2.2 KiB data, 4.6 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:27 smithi139 bash[23439]: audit 2022-04-28T19:44:26.480471+0000 mon.c (mon.1) 32 : audit [INF] from='client.? 172.21.15.59:0/1748318287' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:44:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:27 smithi139 bash[23439]: audit 2022-04-28T19:44:26.481306+0000 mon.a (mon.0) 664 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:44:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:28 smithi139 bash[23439]: audit 2022-04-28T19:44:27.343579+0000 mon.a (mon.0) 665 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2022-04-28T19:44:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:28 smithi139 bash[23439]: cluster 2022-04-28T19:44:27.343665+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2022-04-28T19:44:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:28 smithi059 bash[23786]: audit 2022-04-28T19:44:27.343579+0000 mon.a (mon.0) 665 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2022-04-28T19:44:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:28 smithi059 bash[23786]: cluster 2022-04-28T19:44:27.343665+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2022-04-28T19:44:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:28 smithi059 bash[29114]: audit 2022-04-28T19:44:27.343579+0000 mon.a (mon.0) 665 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2022-04-28T19:44:28.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:28 smithi059 bash[29114]: cluster 2022-04-28T19:44:27.343665+0000 mon.a (mon.0) 666 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2022-04-28T19:44:29.057 DEBUG:teuthology.orchestra.run.smithi059:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight read -> 100 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight delete -> 50 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight snap_create -> 50 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight snap_remove -> 50 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight rollback -> 50 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight setattr -> 25 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight rmattr -> 25 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight copy_from -> 50 2022-04-28T19:44:29.077 INFO:tasks.rados.rados.0.smithi059.stdout:adding op weight append -> 100 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout:ceph version 15.2.16 (d46a73d6d0a67a79558054a3a5a72cb561724974) octopus (stable) 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout:Configuration: 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout: Number of operations: 4000 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout: Number of objects: 50 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout: Max in flight operations: 16 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout: Object size (in bytes): 4000000 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout: Write stride min: 400000 2022-04-28T19:44:29.078 INFO:tasks.rados.rados.0.smithi059.stdout: Write stride max: 800000 2022-04-28T19:44:29.367 INFO:tasks.rados.rados.0.smithi059.stdout:1: write initial oid 1 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:1: seq_num 0 ranges {0=24576} 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:1: writing smithi05955964-1 from 0 to 24576 tid 1 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:2: write initial oid 2 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:2: seq_num 1 ranges {0=24576} 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:2: writing smithi05955964-2 from 0 to 24576 tid 1 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:3: write initial oid 3 2022-04-28T19:44:29.368 INFO:tasks.rados.rados.0.smithi059.stdout:3: seq_num 2 ranges {0=40960} 2022-04-28T19:44:29.369 INFO:tasks.rados.rados.0.smithi059.stdout:3: writing smithi05955964-3 from 0 to 40960 tid 1 2022-04-28T19:44:29.369 INFO:tasks.rados.rados.0.smithi059.stdout:4: write initial oid 4 2022-04-28T19:44:29.369 INFO:tasks.rados.rados.0.smithi059.stdout:4: seq_num 3 ranges {0=57344} 2022-04-28T19:44:29.369 INFO:tasks.rados.rados.0.smithi059.stdout:4: writing smithi05955964-4 from 0 to 57344 tid 1 2022-04-28T19:44:29.369 INFO:tasks.rados.rados.0.smithi059.stdout:5: write initial oid 5 2022-04-28T19:44:29.370 INFO:tasks.rados.rados.0.smithi059.stdout:5: seq_num 4 ranges {0=73728} 2022-04-28T19:44:29.370 INFO:tasks.rados.rados.0.smithi059.stdout:5: writing smithi05955964-5 from 0 to 73728 tid 1 2022-04-28T19:44:29.370 INFO:tasks.rados.rados.0.smithi059.stdout:6: write initial oid 6 2022-04-28T19:44:29.370 INFO:tasks.rados.rados.0.smithi059.stdout:6: seq_num 5 ranges {0=90112} 2022-04-28T19:44:29.371 INFO:tasks.rados.rados.0.smithi059.stdout:6: writing smithi05955964-6 from 0 to 90112 tid 1 2022-04-28T19:44:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:7: write initial oid 7 2022-04-28T19:44:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:7: seq_num 6 ranges {0=106496} 2022-04-28T19:44:29.373 INFO:tasks.rados.rados.0.smithi059.stdout:7: writing smithi05955964-7 from 0 to 106496 tid 1 2022-04-28T19:44:29.373 INFO:tasks.rados.rados.0.smithi059.stdout:8: write initial oid 8 2022-04-28T19:44:29.374 INFO:tasks.rados.rados.0.smithi059.stdout:8: seq_num 7 ranges {0=122880} 2022-04-28T19:44:29.375 INFO:tasks.rados.rados.0.smithi059.stdout:8: writing smithi05955964-8 from 0 to 122880 tid 1 2022-04-28T19:44:29.375 INFO:tasks.rados.rados.0.smithi059.stdout:1: finishing write tid 1 to smithi05955964-1 2022-04-28T19:44:29.375 INFO:tasks.rados.rados.0.smithi059.stdout:4: finishing write tid 1 to smithi05955964-4 2022-04-28T19:44:29.375 INFO:tasks.rados.rados.0.smithi059.stdout:3: finishing write tid 1 to smithi05955964-3 2022-04-28T19:44:29.375 INFO:tasks.rados.rados.0.smithi059.stdout:9: write initial oid 9 2022-04-28T19:44:29.375 INFO:tasks.rados.rados.0.smithi059.stdout:9: seq_num 8 ranges {0=139264} 2022-04-28T19:44:29.376 INFO:tasks.rados.rados.0.smithi059.stdout:9: writing smithi05955964-9 from 0 to 139264 tid 1 2022-04-28T19:44:29.376 INFO:tasks.rados.rados.0.smithi059.stdout:5: finishing write tid 1 to smithi05955964-5 2022-04-28T19:44:29.376 INFO:tasks.rados.rados.0.smithi059.stdout:2: finishing write tid 1 to smithi05955964-2 2022-04-28T19:44:29.377 INFO:tasks.rados.rados.0.smithi059.stdout:10: write initial oid 10 2022-04-28T19:44:29.377 INFO:tasks.rados.rados.0.smithi059.stdout:10: seq_num 9 ranges {0=155648} 2022-04-28T19:44:29.378 INFO:tasks.rados.rados.0.smithi059.stdout:10: writing smithi05955964-10 from 0 to 155648 tid 1 2022-04-28T19:44:29.378 INFO:tasks.rados.rados.0.smithi059.stdout:11: write initial oid 11 2022-04-28T19:44:29.378 INFO:tasks.rados.rados.0.smithi059.stdout:11: seq_num 10 ranges {0=172032} 2022-04-28T19:44:29.379 INFO:tasks.rados.rados.0.smithi059.stdout:11: writing smithi05955964-11 from 0 to 172032 tid 1 2022-04-28T19:44:29.379 INFO:tasks.rados.rados.0.smithi059.stdout:1: finishing write tid 3 to smithi05955964-1 2022-04-28T19:44:29.379 INFO:tasks.rados.rados.0.smithi059.stdout:4: finishing write tid 3 to smithi05955964-4 2022-04-28T19:44:29.379 INFO:tasks.rados.rados.0.smithi059.stdout:12: write initial oid 12 2022-04-28T19:44:29.379 INFO:tasks.rados.rados.0.smithi059.stdout:12: seq_num 11 ranges {0=188416} 2022-04-28T19:44:29.381 INFO:tasks.rados.rados.0.smithi059.stdout:12: writing smithi05955964-12 from 0 to 188416 tid 1 2022-04-28T19:44:29.381 INFO:tasks.rados.rados.0.smithi059.stdout:13: write initial oid 13 2022-04-28T19:44:29.381 INFO:tasks.rados.rados.0.smithi059.stdout:13: seq_num 12 ranges {0=204800} 2022-04-28T19:44:29.382 INFO:tasks.rados.rados.0.smithi059.stdout:13: writing smithi05955964-13 from 0 to 204800 tid 1 2022-04-28T19:44:29.382 INFO:tasks.rados.rados.0.smithi059.stdout:14: write initial oid 14 2022-04-28T19:44:29.382 INFO:tasks.rados.rados.0.smithi059.stdout:14: seq_num 13 ranges {0=221184} 2022-04-28T19:44:29.384 INFO:tasks.rados.rados.0.smithi059.stdout:14: writing smithi05955964-14 from 0 to 221184 tid 1 2022-04-28T19:44:29.384 INFO:tasks.rados.rados.0.smithi059.stdout:15: write initial oid 15 2022-04-28T19:44:29.384 INFO:tasks.rados.rados.0.smithi059.stdout:15: seq_num 14 ranges {0=237568} 2022-04-28T19:44:29.386 INFO:tasks.rados.rados.0.smithi059.stdout:15: writing smithi05955964-15 from 0 to 237568 tid 1 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:7: finishing write tid 1 to smithi05955964-7 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:5: finishing write tid 3 to smithi05955964-5 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:8: finishing write tid 1 to smithi05955964-8 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:4: finishing write tid 4 to smithi05955964-4 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2022-04-28T19:44:29.387 INFO:tasks.rados.rados.0.smithi059.stdout:9: finishing write tid 1 to smithi05955964-9 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:5: finishing write tid 4 to smithi05955964-5 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:7: finishing write tid 3 to smithi05955964-7 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:12: finishing write tid 1 to smithi05955964-12 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:9: finishing write tid 3 to smithi05955964-9 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:10: finishing write tid 1 to smithi05955964-10 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:8: finishing write tid 3 to smithi05955964-8 2022-04-28T19:44:29.388 INFO:tasks.rados.rados.0.smithi059.stdout:7: finishing write tid 4 to smithi05955964-7 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:4: done (14 left) 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:5: done (13 left) 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:7: done (12 left) 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:16: write initial oid 16 2022-04-28T19:44:29.389 INFO:tasks.rados.rados.0.smithi059.stdout:16: seq_num 15 ranges {0=253952} 2022-04-28T19:44:29.391 INFO:tasks.rados.rados.0.smithi059.stdout:16: writing smithi05955964-16 from 0 to 253952 tid 1 2022-04-28T19:44:29.391 INFO:tasks.rados.rados.0.smithi059.stdout:17: write initial oid 17 2022-04-28T19:44:29.391 INFO:tasks.rados.rados.0.smithi059.stdout:17: seq_num 16 ranges {0=270336} 2022-04-28T19:44:29.391 INFO:tasks.rados.rados.0.smithi059.stdout:17: writing smithi05955964-17 from 0 to 270336 tid 1 2022-04-28T19:44:29.391 INFO:tasks.rados.rados.0.smithi059.stdout:18: write initial oid 18 2022-04-28T19:44:29.391 INFO:tasks.rados.rados.0.smithi059.stdout:18: seq_num 17 ranges {0=286720} 2022-04-28T19:44:29.393 INFO:tasks.rados.rados.0.smithi059.stdout:18: writing smithi05955964-18 from 0 to 286720 tid 1 2022-04-28T19:44:29.393 INFO:tasks.rados.rados.0.smithi059.stdout:8: finishing write tid 4 to smithi05955964-8 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 4 (ObjNum 7 snap 0 seq_num 7) dirty exists 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:12: finishing write tid 3 to smithi05955964-12 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:9: finishing write tid 4 to smithi05955964-9 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 2 (ObjNum 8 snap 0 seq_num 8) dirty exists 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:3: finishing write tid 3 to smithi05955964-3 2022-04-28T19:44:29.394 INFO:tasks.rados.rados.0.smithi059.stdout:1: finishing write tid 4 to smithi05955964-1 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:14: finishing write tid 1 to smithi05955964-14 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:10: finishing write tid 3 to smithi05955964-10 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:6: finishing write tid 1 to smithi05955964-6 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:12: finishing write tid 4 to smithi05955964-12 2022-04-28T19:44:29.395 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 4 (ObjNum 11 snap 0 seq_num 11) dirty exists 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:15: finishing write tid 1 to smithi05955964-15 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:13: finishing write tid 1 to smithi05955964-13 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:11: finishing write tid 1 to smithi05955964-11 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:2: finishing write tid 3 to smithi05955964-2 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:14: finishing write tid 3 to smithi05955964-14 2022-04-28T19:44:29.396 INFO:tasks.rados.rados.0.smithi059.stdout:15: finishing write tid 3 to smithi05955964-15 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:13: finishing write tid 3 to smithi05955964-13 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:1: done (14 left) 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:8: done (13 left) 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:9: done (12 left) 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:12: done (11 left) 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:19: write initial oid 19 2022-04-28T19:44:29.397 INFO:tasks.rados.rados.0.smithi059.stdout:19: seq_num 18 ranges {0=303104} 2022-04-28T19:44:29.398 INFO:tasks.rados.rados.0.smithi059.stdout:19: writing smithi05955964-19 from 0 to 303104 tid 1 2022-04-28T19:44:29.398 INFO:tasks.rados.rados.0.smithi059.stdout:20: write initial oid 20 2022-04-28T19:44:29.398 INFO:tasks.rados.rados.0.smithi059.stdout:20: seq_num 19 ranges {0=319488} 2022-04-28T19:44:29.398 INFO:tasks.rados.rados.0.smithi059.stdout:20: writing smithi05955964-20 from 0 to 319488 tid 1 2022-04-28T19:44:29.399 INFO:tasks.rados.rados.0.smithi059.stdout:21: write initial oid 21 2022-04-28T19:44:29.399 INFO:tasks.rados.rados.0.smithi059.stdout:21: seq_num 20 ranges {0=344064} 2022-04-28T19:44:29.400 INFO:tasks.rados.rados.0.smithi059.stdout:21: writing smithi05955964-21 from 0 to 344064 tid 1 2022-04-28T19:44:29.400 INFO:tasks.rados.rados.0.smithi059.stdout:22: write initial oid 22 2022-04-28T19:44:29.400 INFO:tasks.rados.rados.0.smithi059.stdout:22: seq_num 21 ranges {0=360448} 2022-04-28T19:44:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:22: writing smithi05955964-22 from 0 to 360448 tid 1 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:23: write initial oid 23 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:23: seq_num 22 ranges {0=376832} 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:23: writing smithi05955964-23 from 0 to 376832 tid 1 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 16 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:11: finishing write tid 3 to smithi05955964-11 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:11: finishing write tid 4 to smithi05955964-11 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2022-04-28T19:44:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:10: finishing write tid 4 to smithi05955964-10 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 2 (ObjNum 9 snap 0 seq_num 9) dirty exists 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:2: finishing write tid 4 to smithi05955964-2 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:3: finishing write tid 4 to smithi05955964-3 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 2 (ObjNum 2 snap 0 seq_num 2) dirty exists 2022-04-28T19:44:29.490 INFO:tasks.rados.rados.0.smithi059.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:6: finishing write tid 3 to smithi05955964-6 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:17: finishing write tid 1 to smithi05955964-17 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:2: done (15 left) 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:3: done (14 left) 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:10: done (13 left) 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:11: done (12 left) 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:24: write initial oid 24 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:24: seq_num 23 ranges {0=393216} 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:24: writing smithi05955964-24 from 0 to 393216 tid 1 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:25: write initial oid 25 2022-04-28T19:44:29.491 INFO:tasks.rados.rados.0.smithi059.stdout:25: seq_num 24 ranges {0=409600} 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:25: writing smithi05955964-25 from 0 to 409600 tid 1 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:26: write initial oid 26 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:26: seq_num 25 ranges {0=425984} 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:26: writing smithi05955964-26 from 0 to 425984 tid 1 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:27: write initial oid 27 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:27: seq_num 26 ranges {0=442368} 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:27: writing smithi05955964-27 from 0 to 442368 tid 1 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 16 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:14: finishing write tid 4 to smithi05955964-14 2022-04-28T19:44:29.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 2 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:16: finishing write tid 1 to smithi05955964-16 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:13: finishing write tid 4 to smithi05955964-13 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 2 (ObjNum 12 snap 0 seq_num 12) dirty exists 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:15: finishing write tid 4 to smithi05955964-15 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 6 (ObjNum 14 snap 0 seq_num 14) dirty exists 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2022-04-28T19:44:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:6: finishing write tid 4 to smithi05955964-6 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 4 (ObjNum 5 snap 0 seq_num 5) dirty exists 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:18: finishing write tid 1 to smithi05955964-18 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:17: finishing write tid 3 to smithi05955964-17 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:16: finishing write tid 3 to smithi05955964-16 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:17: finishing write tid 4 to smithi05955964-17 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 4 (ObjNum 16 snap 0 seq_num 16) dirty exists 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:18: finishing write tid 3 to smithi05955964-18 2022-04-28T19:44:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:19: finishing write tid 1 to smithi05955964-19 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:16: finishing write tid 4 to smithi05955964-16 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 2 (ObjNum 15 snap 0 seq_num 15) dirty exists 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:18: finishing write tid 4 to smithi05955964-18 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 2 (ObjNum 17 snap 0 seq_num 17) dirty exists 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:20: finishing write tid 1 to smithi05955964-20 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:19: finishing write tid 3 to smithi05955964-19 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:19: finishing write tid 4 to smithi05955964-19 2022-04-28T19:44:29.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 4 (ObjNum 18 snap 0 seq_num 18) dirty exists 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:21: finishing write tid 1 to smithi05955964-21 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:20: finishing write tid 3 to smithi05955964-20 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:22: finishing write tid 1 to smithi05955964-22 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:20: finishing write tid 4 to smithi05955964-20 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 6 (ObjNum 19 snap 0 seq_num 19) dirty exists 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:21: finishing write tid 3 to smithi05955964-21 2022-04-28T19:44:29.496 INFO:tasks.rados.rados.0.smithi059.stdout:23: finishing write tid 1 to smithi05955964-23 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:22: finishing write tid 3 to smithi05955964-22 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:21: finishing write tid 4 to smithi05955964-21 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 2 (ObjNum 20 snap 0 seq_num 20) dirty exists 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:22: finishing write tid 4 to smithi05955964-22 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2022-04-28T19:44:29.497 INFO:tasks.rados.rados.0.smithi059.stdout:23: finishing write tid 3 to smithi05955964-23 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:24: finishing write tid 1 to smithi05955964-24 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:23: finishing write tid 4 to smithi05955964-23 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 4 (ObjNum 22 snap 0 seq_num 22) dirty exists 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:24: finishing write tid 3 to smithi05955964-24 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:25: finishing write tid 1 to smithi05955964-25 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:25: finishing write tid 3 to smithi05955964-25 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:24: finishing write tid 4 to smithi05955964-24 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2022-04-28T19:44:29.498 INFO:tasks.rados.rados.0.smithi059.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:25: finishing write tid 4 to smithi05955964-25 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:26: finishing write tid 1 to smithi05955964-26 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:6: done (15 left) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:13: done (14 left) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:14: done (13 left) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:15: done (12 left) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:16: done (11 left) 2022-04-28T19:44:29.499 INFO:tasks.rados.rados.0.smithi059.stdout:17: done (10 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:18: done (9 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:19: done (8 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:20: done (7 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:21: done (6 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:22: done (5 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:23: done (4 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:24: done (3 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:25: done (2 left) 2022-04-28T19:44:29.500 INFO:tasks.rados.rados.0.smithi059.stdout:28: write initial oid 28 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:28: seq_num 27 ranges {0=458752} 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:28: writing smithi05955964-28 from 0 to 458752 tid 1 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:26: finishing write tid 3 to smithi05955964-26 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:29: write initial oid 29 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:29: seq_num 28 ranges {0=475136} 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:29: writing smithi05955964-29 from 0 to 475136 tid 1 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:30: write initial oid 30 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:27: finishing write tid 1 to smithi05955964-27 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:26: finishing write tid 4 to smithi05955964-26 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 8 (ObjNum 25 snap 0 seq_num 25) dirty exists 2022-04-28T19:44:29.501 INFO:tasks.rados.rados.0.smithi059.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:30: seq_num 29 ranges {0=491520} 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:30: writing smithi05955964-30 from 0 to 491520 tid 1 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:26: done (4 left) 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:31: write initial oid 31 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:31: seq_num 30 ranges {0=507904} 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:31: writing smithi05955964-31 from 0 to 507904 tid 1 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:32: write initial oid 32 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:32: seq_num 31 ranges {0=524288} 2022-04-28T19:44:29.502 INFO:tasks.rados.rados.0.smithi059.stdout:32: writing smithi05955964-32 from 0 to 524288 tid 1 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:33: write initial oid 33 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:33: seq_num 32 ranges {0=540672} 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:33: writing smithi05955964-33 from 0 to 540672 tid 1 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:34: write initial oid 34 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:34: seq_num 33 ranges {0=557056} 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:34: writing smithi05955964-34 from 0 to 557056 tid 1 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:35: write initial oid 35 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:35: seq_num 34 ranges {0=573440} 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:35: writing smithi05955964-35 from 0 to 573440 tid 1 2022-04-28T19:44:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:36: write initial oid 36 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:36: seq_num 35 ranges {0=589824} 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:36: writing smithi05955964-36 from 0 to 589824 tid 1 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:37: write initial oid 37 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:27: finishing write tid 3 to smithi05955964-27 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:27: finishing write tid 4 to smithi05955964-27 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 6 (ObjNum 26 snap 0 seq_num 26) dirty exists 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:28: finishing write tid 1 to smithi05955964-28 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:28: finishing write tid 3 to smithi05955964-28 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:28: finishing write tid 4 to smithi05955964-28 2022-04-28T19:44:29.504 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 4 (ObjNum 27 snap 0 seq_num 27) dirty exists 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:29: finishing write tid 1 to smithi05955964-29 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:29: finishing write tid 3 to smithi05955964-29 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:29: finishing write tid 4 to smithi05955964-29 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 8 (ObjNum 28 snap 0 seq_num 28) dirty exists 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:30: finishing write tid 1 to smithi05955964-30 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:30: finishing write tid 3 to smithi05955964-30 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:31: finishing write tid 1 to smithi05955964-31 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:30: finishing write tid 4 to smithi05955964-30 2022-04-28T19:44:29.505 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 4 (ObjNum 29 snap 0 seq_num 29) dirty exists 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:31: finishing write tid 3 to smithi05955964-31 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:37: seq_num 36 ranges {0=606208} 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:37: writing smithi05955964-37 from 0 to 606208 tid 1 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:27: done (10 left) 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:28: done (9 left) 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:29: done (8 left) 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:30: done (7 left) 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:38: write initial oid 38 2022-04-28T19:44:29.506 INFO:tasks.rados.rados.0.smithi059.stdout:38: seq_num 37 ranges {0=622592} 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:38: writing smithi05955964-38 from 0 to 622592 tid 1 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:39: write initial oid 39 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:39: seq_num 38 ranges {0=638976} 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:39: writing smithi05955964-39 from 0 to 638976 tid 1 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:40: write initial oid 40 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:40: seq_num 39 ranges {0=663552} 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:40: writing smithi05955964-40 from 0 to 663552 tid 1 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:41: write initial oid 41 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:41: seq_num 40 ranges {0=679936} 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:41: writing smithi05955964-41 from 0 to 679936 tid 1 2022-04-28T19:44:29.507 INFO:tasks.rados.rados.0.smithi059.stdout:42: write initial oid 42 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:42: seq_num 41 ranges {0=696320} 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:42: writing smithi05955964-42 from 0 to 696320 tid 1 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:43: write initial oid 43 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:43: seq_num 42 ranges {0=712704} 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:43: writing smithi05955964-43 from 0 to 712704 tid 1 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:44: write initial oid 44 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:44: seq_num 43 ranges {0=729088} 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:44: writing smithi05955964-44 from 0 to 729088 tid 1 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:45: write initial oid 45 2022-04-28T19:44:29.508 INFO:tasks.rados.rados.0.smithi059.stdout:45: seq_num 44 ranges {0=745472} 2022-04-28T19:44:29.509 INFO:tasks.rados.rados.0.smithi059.stdout:45: writing smithi05955964-45 from 0 to 745472 tid 1 2022-04-28T19:44:29.509 INFO:tasks.rados.rados.0.smithi059.stdout:46: write initial oid 46 2022-04-28T19:44:29.509 INFO:tasks.rados.rados.0.smithi059.stdout:46: seq_num 45 ranges {0=761856} 2022-04-28T19:44:29.511 INFO:tasks.rados.rados.0.smithi059.stdout:46: writing smithi05955964-46 from 0 to 761856 tid 1 2022-04-28T19:44:29.511 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 16 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:32: finishing write tid 1 to smithi05955964-32 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:31: finishing write tid 4 to smithi05955964-31 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 8 (ObjNum 30 snap 0 seq_num 30) dirty exists 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:32: finishing write tid 3 to smithi05955964-32 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:33: finishing write tid 1 to smithi05955964-33 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:32: finishing write tid 4 to smithi05955964-32 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 2 (ObjNum 31 snap 0 seq_num 31) dirty exists 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2022-04-28T19:44:29.512 INFO:tasks.rados.rados.0.smithi059.stdout:33: finishing write tid 3 to smithi05955964-33 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:33: finishing write tid 4 to smithi05955964-33 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 4 (ObjNum 32 snap 0 seq_num 32) dirty exists 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:34: finishing write tid 1 to smithi05955964-34 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:34: finishing write tid 3 to smithi05955964-34 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:34: finishing write tid 4 to smithi05955964-34 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:35: finishing write tid 1 to smithi05955964-35 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:31: done (15 left) 2022-04-28T19:44:29.513 INFO:tasks.rados.rados.0.smithi059.stdout:32: done (14 left) 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:33: done (13 left) 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:34: done (12 left) 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:47: write initial oid 47 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:47: seq_num 46 ranges {0=778240} 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:47: writing smithi05955964-47 from 0 to 778240 tid 1 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:48: write initial oid 48 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:48: seq_num 47 ranges {0=794624} 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:48: writing smithi05955964-48 from 0 to 794624 tid 1 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:49: write initial oid 49 2022-04-28T19:44:29.514 INFO:tasks.rados.rados.0.smithi059.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2022-04-28T19:44:29.515 INFO:tasks.rados.rados.0.smithi059.stdout:49: writing smithi05955964-49 from 0 to 409600 tid 1 2022-04-28T19:44:29.515 INFO:tasks.rados.rados.0.smithi059.stdout:49: writing smithi05955964-49 from 409600 to 811008 tid 2 2022-04-28T19:44:29.515 INFO:tasks.rados.rados.0.smithi059.stdout:50: write initial oid 50 2022-04-28T19:44:29.515 INFO:tasks.rados.rados.0.smithi059.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2022-04-28T19:44:29.516 INFO:tasks.rados.rados.0.smithi059.stdout:50: writing smithi05955964-50 from 0 to 425984 tid 1 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:50: writing smithi05955964-50 from 425984 to 827392 tid 2 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 16 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:35: finishing write tid 3 to smithi05955964-35 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:35: finishing write tid 4 to smithi05955964-35 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 2 (ObjNum 34 snap 0 seq_num 34) dirty exists 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:36: finishing write tid 1 to smithi05955964-36 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:36: finishing write tid 3 to smithi05955964-36 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:35: done (15 left) 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:51: copy_from oid 2 from oid 11 current snap is 0 2022-04-28T19:44:29.517 INFO:tasks.rados.rados.0.smithi059.stdout:36: finishing write tid 4 to smithi05955964-36 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 4 (ObjNum 35 snap 0 seq_num 35) dirty exists 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:37: finishing write tid 1 to smithi05955964-37 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:37: finishing write tid 3 to smithi05955964-37 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:37: finishing write tid 4 to smithi05955964-37 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:36: done (15 left) 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:37: done (14 left) 2022-04-28T19:44:29.518 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 0 2022-04-28T19:44:29.519 INFO:tasks.rados.rados.0.smithi059.stdout:52: seq_num 50 ranges {286720=442368,729088=401408} 2022-04-28T19:44:29.519 INFO:tasks.rados.rados.0.smithi059.stdout:52: writing smithi05955964-18 from 286720 to 729088 tid 1 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:52: writing smithi05955964-18 from 729088 to 1130496 tid 2 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:53: rmattr oid 30 current snap is 0 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:53: done (15 left) 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:54: delete oid 3 current snap is 0 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:38: finishing write tid 1 to smithi05955964-38 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:38: finishing write tid 3 to smithi05955964-38 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:38: finishing write tid 4 to smithi05955964-38 2022-04-28T19:44:29.523 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 8 (ObjNum 37 snap 0 seq_num 37) dirty exists 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:39: finishing write tid 1 to smithi05955964-39 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:39: finishing write tid 3 to smithi05955964-39 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:39: finishing write tid 4 to smithi05955964-39 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 6 (ObjNum 38 snap 0 seq_num 38) dirty exists 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:40: finishing write tid 1 to smithi05955964-40 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:40: finishing write tid 3 to smithi05955964-40 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:40: finishing write tid 4 to smithi05955964-40 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 6 (ObjNum 39 snap 0 seq_num 39) dirty exists 2022-04-28T19:44:29.524 INFO:tasks.rados.rados.0.smithi059.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:41: finishing write tid 1 to smithi05955964-41 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:41: finishing write tid 3 to smithi05955964-41 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:41: finishing write tid 4 to smithi05955964-41 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 2 (ObjNum 40 snap 0 seq_num 40) dirty exists 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:42: finishing write tid 1 to smithi05955964-42 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:42: finishing write tid 3 to smithi05955964-42 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:42: finishing write tid 4 to smithi05955964-42 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 10 (ObjNum 41 snap 0 seq_num 41) dirty exists 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2022-04-28T19:44:29.525 INFO:tasks.rados.rados.0.smithi059.stdout:43: finishing write tid 1 to smithi05955964-43 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:43: finishing write tid 3 to smithi05955964-43 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:43: finishing write tid 4 to smithi05955964-43 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 8 (ObjNum 42 snap 0 seq_num 42) dirty exists 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:44: finishing write tid 1 to smithi05955964-44 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:44: finishing write tid 3 to smithi05955964-44 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:44: finishing write tid 4 to smithi05955964-44 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 4 (ObjNum 43 snap 0 seq_num 43) dirty exists 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2022-04-28T19:44:29.526 INFO:tasks.rados.rados.0.smithi059.stdout:45: finishing write tid 1 to smithi05955964-45 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:45: finishing write tid 3 to smithi05955964-45 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:45: finishing write tid 4 to smithi05955964-45 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 8 (ObjNum 44 snap 0 seq_num 44) dirty exists 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:46: finishing write tid 1 to smithi05955964-46 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:46: finishing write tid 3 to smithi05955964-46 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:46: finishing write tid 4 to smithi05955964-46 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 10 (ObjNum 45 snap 0 seq_num 45) dirty exists 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:47: finishing write tid 1 to smithi05955964-47 2022-04-28T19:44:29.527 INFO:tasks.rados.rados.0.smithi059.stdout:47: finishing write tid 3 to smithi05955964-47 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:47: finishing write tid 4 to smithi05955964-47 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:48: finishing write tid 1 to smithi05955964-48 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:48: finishing write tid 3 to smithi05955964-48 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:49: finishing write tid 1 to smithi05955964-49 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:48: finishing write tid 4 to smithi05955964-48 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 10 (ObjNum 47 snap 0 seq_num 47) dirty exists 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2022-04-28T19:44:29.528 INFO:tasks.rados.rados.0.smithi059.stdout:49: finishing write tid 2 to smithi05955964-49 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:49: finishing write tid 4 to smithi05955964-49 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:49: finishing write tid 5 to smithi05955964-49 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 9 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:50: finishing write tid 1 to smithi05955964-50 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:50: finishing write tid 2 to smithi05955964-50 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:50: finishing write tid 4 to smithi05955964-50 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:50: finishing write tid 5 to smithi05955964-50 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 11 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:44:29.529 INFO:tasks.rados.rados.0.smithi059.stdout:51: finishing copy_from to smithi05955964-2 2022-04-28T19:44:29.530 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2022-04-28T19:44:29.530 INFO:tasks.rados.rados.0.smithi059.stdout:51: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:44:29.530 INFO:tasks.rados.rados.0.smithi059.stdout:52: finishing write tid 1 to smithi05955964-18 2022-04-28T19:44:29.531 INFO:tasks.rados.rados.0.smithi059.stdout:38: done (15 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:39: done (14 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:40: done (13 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:41: done (12 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:42: done (11 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:43: done (10 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:44: done (9 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:45: done (8 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:46: done (7 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:47: done (6 left) 2022-04-28T19:44:29.532 INFO:tasks.rados.rados.0.smithi059.stdout:48: done (5 left) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:49: done (4 left) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:50: done (3 left) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:51: done (2 left) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:54: done (1 left) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:55: read oid 15 snap -1 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:56: rollback oid 31 current snap is 0 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:56: done (2 left) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:57: read oid 28 snap -1 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2022-04-28T19:44:29.533 INFO:tasks.rados.rados.0.smithi059.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:59: rollback oid 45 current snap is 0 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:59: done (4 left) 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:60: copy_from oid 23 from oid 48 current snap is 0 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:61: delete oid 1 current snap is 0 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:52: finishing write tid 2 to smithi05955964-18 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:52: finishing write tid 4 to smithi05955964-18 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:61: done (5 left) 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:62: read oid 17 snap -1 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:62: expect (ObjNum 16 snap 0 seq_num 16) 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:63: snap_create 2022-04-28T19:44:29.534 INFO:tasks.rados.rados.0.smithi059.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2022-04-28T19:44:29.539 INFO:tasks.rados.rados.0.smithi059.stdout:52: finishing write tid 5 to smithi05955964-18 2022-04-28T19:44:29.539 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 13 (ObjNum 50 snap 0 seq_num 50) dirty exists 2022-04-28T19:44:29.539 INFO:tasks.rados.rados.0.smithi059.stdout:52: left oid 18 (ObjNum 50 snap 0 seq_num 50) 2022-04-28T19:44:29.539 INFO:tasks.rados.rados.0.smithi059.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2022-04-28T19:44:29.545 INFO:tasks.rados.rados.0.smithi059.stdout:60: finishing copy_from to smithi05955964-23 2022-04-28T19:44:29.545 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 10 (ObjNum 47 snap 0 seq_num 47) dirty exists 2022-04-28T19:44:29.545 INFO:tasks.rados.rados.0.smithi059.stdout:62: expect (ObjNum 16 snap 0 seq_num 16) 2022-04-28T19:44:29.551 INFO:tasks.rados.rados.0.smithi059.stdout:60: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:44:29.551 INFO:tasks.rados.rados.0.smithi059.stdout:58: finishing copy_from to smithi05955964-29 2022-04-28T19:44:29.551 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2022-04-28T19:44:29.551 INFO:tasks.rados.rados.0.smithi059.stdout:58: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:44:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:29 smithi139 bash[23439]: cluster 2022-04-28T19:44:28.069323+0000 mgr.y (mgr.14140) 370 : cluster [DBG] pgmap v328: 89 pgs: 89 active+clean; 2.2 KiB data, 4.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:29 smithi139 bash[23439]: audit 2022-04-28T19:44:29.089015+0000 mon.c (mon.1) 33 : audit [INF] from='client.? 172.21.15.59:0/1877043218' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2022-04-28T19:44:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:29 smithi139 bash[23439]: audit 2022-04-28T19:44:29.089644+0000 mon.a (mon.0) 667 : audit [INF] from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2022-04-28T19:44:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:29 smithi059 bash[29114]: cluster 2022-04-28T19:44:28.069323+0000 mgr.y (mgr.14140) 370 : cluster [DBG] pgmap v328: 89 pgs: 89 active+clean; 2.2 KiB data, 4.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:29 smithi059 bash[29114]: audit 2022-04-28T19:44:29.089015+0000 mon.c (mon.1) 33 : audit [INF] from='client.? 172.21.15.59:0/1877043218' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2022-04-28T19:44:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:29 smithi059 bash[29114]: audit 2022-04-28T19:44:29.089644+0000 mon.a (mon.0) 667 : audit [INF] from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2022-04-28T19:44:29.875 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:29 smithi059 bash[23786]: cluster 2022-04-28T19:44:28.069323+0000 mgr.y (mgr.14140) 370 : cluster [DBG] pgmap v328: 89 pgs: 89 active+clean; 2.2 KiB data, 4.8 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:44:29.875 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:29 smithi059 bash[23786]: audit 2022-04-28T19:44:29.089015+0000 mon.c (mon.1) 33 : audit [INF] from='client.? 172.21.15.59:0/1877043218' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2022-04-28T19:44:29.875 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:29 smithi059 bash[23786]: audit 2022-04-28T19:44:29.089644+0000 mon.a (mon.0) 667 : audit [INF] from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2022-04-28T19:44:30.378 INFO:tasks.rados.rados.0.smithi059.stdout:52: done (6 left) 2022-04-28T19:44:30.378 INFO:tasks.rados.rados.0.smithi059.stdout:55: done (5 left) 2022-04-28T19:44:30.378 INFO:tasks.rados.rados.0.smithi059.stdout:57: done (4 left) 2022-04-28T19:44:30.378 INFO:tasks.rados.rados.0.smithi059.stdout:58: done (3 left) 2022-04-28T19:44:30.378 INFO:tasks.rados.rados.0.smithi059.stdout:60: done (2 left) 2022-04-28T19:44:30.379 INFO:tasks.rados.rados.0.smithi059.stdout:62: done (1 left) 2022-04-28T19:44:30.379 INFO:tasks.rados.rados.0.smithi059.stdout:63: done (0 left) 2022-04-28T19:44:30.379 INFO:tasks.rados.rados.0.smithi059.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2022-04-28T19:44:30.379 INFO:tasks.rados.rados.0.smithi059.stdout:65: read oid 30 snap -1 2022-04-28T19:44:30.379 INFO:tasks.rados.rados.0.smithi059.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2022-04-28T19:44:30.380 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 1 2022-04-28T19:44:30.380 INFO:tasks.rados.rados.0.smithi059.stdout:66: seq_num 53 ranges {794624=491520,1286144=401408} 2022-04-28T19:44:30.385 INFO:tasks.rados.rados.0.smithi059.stdout:66: writing smithi05955964-23 from 794624 to 1286144 tid 1 2022-04-28T19:44:30.387 INFO:tasks.rados.rados.0.smithi059.stdout:66: writing smithi05955964-23 from 1286144 to 1687552 tid 2 2022-04-28T19:44:30.388 INFO:tasks.rados.rados.0.smithi059.stdout:67: rollback oid 10 current snap is 1 2022-04-28T19:44:30.388 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 0 2022-04-28T19:44:30.388 INFO:tasks.rados.rados.0.smithi059.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2022-04-28T19:44:30.401 INFO:tasks.rados.rados.0.smithi059.stdout:64: finishing copy_from to smithi05955964-15 2022-04-28T19:44:30.401 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 4 (ObjNum 10 snap 0 seq_num 10) dirty exists 2022-04-28T19:44:30.401 INFO:tasks.rados.rados.0.smithi059.stdout:64: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:44:30.401 INFO:tasks.rados.rados.0.smithi059.stdout:66: finishing write tid 1 to smithi05955964-23 2022-04-28T19:44:30.401 INFO:tasks.rados.rados.0.smithi059.stdout:64: done (3 left) 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:65: done (2 left) 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:68: setattr oid 20 current snap is 1 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:66: finishing write tid 2 to smithi05955964-23 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:66: finishing write tid 4 to smithi05955964-23 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:67: finishing rollback tid 0 to smithi05955964-10 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:67: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:67: finishing rollback tid 2 to smithi05955964-10 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 17 (ObjNum 9 snap 0 seq_num 9) dirty exists 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:66: finishing write tid 5 to smithi05955964-23 2022-04-28T19:44:30.402 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 13 (ObjNum 53 snap 1 seq_num 53) dirty exists 2022-04-28T19:44:30.403 INFO:tasks.rados.rados.0.smithi059.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:44:30.404 INFO:tasks.rados.rados.0.smithi059.stdout:66: done (2 left) 2022-04-28T19:44:30.404 INFO:tasks.rados.rados.0.smithi059.stdout:67: done (1 left) 2022-04-28T19:44:30.404 INFO:tasks.rados.rados.0.smithi059.stdout:69: copy_from oid 45 from oid 34 current snap is 1 2022-04-28T19:44:30.404 INFO:tasks.rados.rados.0.smithi059.stdout:70: read oid 24 snap 0 2022-04-28T19:44:30.404 INFO:tasks.rados.rados.0.smithi059.stdout:70: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:44:30.404 INFO:tasks.rados.rados.0.smithi059.stdout:71: copy_from oid 36 from oid 28 current snap is 1 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:72: rollback oid 49 current snap is 1 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 0 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:73: read oid 4 snap 0 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:74: rmattr oid 7 current snap is 1 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:74: done (6 left) 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:75: copy_from oid 21 from oid 12 current snap is 1 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:76: read oid 39 snap -1 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:77: copy_from oid 6 from oid 37 current snap is 1 2022-04-28T19:44:30.405 INFO:tasks.rados.rados.0.smithi059.stdout:78: read oid 2 snap -1 2022-04-28T19:44:30.406 INFO:tasks.rados.rados.0.smithi059.stdout:78: expect (ObjNum 10 snap 0 seq_num 10) 2022-04-28T19:44:30.406 INFO:tasks.rados.rados.0.smithi059.stdout:79: setattr oid 41 current snap is 1 2022-04-28T19:44:30.406 INFO:tasks.rados.rados.0.smithi059.stdout:80: snap_create 2022-04-28T19:44:30.407 INFO:tasks.rados.rados.0.smithi059.stdout:78: expect (ObjNum 10 snap 0 seq_num 10) 2022-04-28T19:44:30.413 INFO:tasks.rados.rados.0.smithi059.stdout:70: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:44:30.424 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 11 (ObjNum 19 snap 0 seq_num 19) dirty exists 2022-04-28T19:44:30.425 INFO:tasks.rados.rados.0.smithi059.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:75: finishing copy_from to smithi05955964-21 2022-04-28T19:44:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 4 (ObjNum 11 snap 0 seq_num 11) dirty exists 2022-04-28T19:44:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:75: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:44:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:73: expect (ObjNum 3 snap 0 seq_num 3) 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 4 (ObjNum 40 snap 0 seq_num 40) dirty exists 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:72: finishing rollback tid 0 to smithi05955964-49 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:72: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:72: finishing rollback tid 2 to smithi05955964-49 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 14 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:71: finishing copy_from to smithi05955964-36 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 4 (ObjNum 27 snap 0 seq_num 27) dirty exists 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:71: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:44:30.441 INFO:tasks.rados.rados.0.smithi059.stdout:69: finishing copy_from to smithi05955964-45 2022-04-28T19:44:30.442 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2022-04-28T19:44:30.442 INFO:tasks.rados.rados.0.smithi059.stdout:69: finishing copy_from racing read to smithi05955964-45 2022-04-28T19:44:30.442 INFO:tasks.rados.rados.0.smithi059.stdout:77: finishing copy_from to smithi05955964-6 2022-04-28T19:44:30.442 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 8 (ObjNum 36 snap 0 seq_num 36) dirty exists 2022-04-28T19:44:30.442 INFO:tasks.rados.rados.0.smithi059.stdout:77: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:44:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:30 smithi139 bash[23439]: audit 2022-04-28T19:44:29.363558+0000 mon.a (mon.0) 668 : audit [INF] from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2022-04-28T19:44:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:30 smithi139 bash[23439]: cluster 2022-04-28T19:44:29.363680+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-04-28T19:44:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:30 smithi139 bash[23439]: cluster 2022-04-28T19:44:30.070179+0000 mgr.y (mgr.14140) 371 : cluster [DBG] pgmap v330: 89 pgs: 89 active+clean; 6.6 MiB data, 4.9 MiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 MiB/s wr, 10 op/s 2022-04-28T19:44:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:30 smithi059 bash[23786]: audit 2022-04-28T19:44:29.363558+0000 mon.a (mon.0) 668 : audit [INF] from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2022-04-28T19:44:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:30 smithi059 bash[23786]: cluster 2022-04-28T19:44:29.363680+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-04-28T19:44:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:30 smithi059 bash[23786]: cluster 2022-04-28T19:44:30.070179+0000 mgr.y (mgr.14140) 371 : cluster [DBG] pgmap v330: 89 pgs: 89 active+clean; 6.6 MiB data, 4.9 MiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 MiB/s wr, 10 op/s 2022-04-28T19:44:30.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:30 smithi059 bash[29114]: audit 2022-04-28T19:44:29.363558+0000 mon.a (mon.0) 668 : audit [INF] from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2022-04-28T19:44:30.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:30 smithi059 bash[29114]: cluster 2022-04-28T19:44:29.363680+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2022-04-28T19:44:30.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:30 smithi059 bash[29114]: cluster 2022-04-28T19:44:30.070179+0000 mgr.y (mgr.14140) 371 : cluster [DBG] pgmap v330: 89 pgs: 89 active+clean; 6.6 MiB data, 4.9 MiB used, 707 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.2 MiB/s wr, 10 op/s 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:68: done (11 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:69: done (10 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:70: done (9 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:71: done (8 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:72: done (7 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:73: done (6 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:75: done (5 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:76: done (4 left) 2022-04-28T19:44:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:77: done (3 left) 2022-04-28T19:44:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:78: done (2 left) 2022-04-28T19:44:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:79: done (1 left) 2022-04-28T19:44:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:80: done (0 left) 2022-04-28T19:44:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:81: rollback oid 40 current snap is 2 2022-04-28T19:44:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 1 2022-04-28T19:44:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:82: snap_create 2022-04-28T19:44:31.392 INFO:tasks.rados.rados.0.smithi059.stdout:81: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:44:31.392 INFO:tasks.rados.rados.0.smithi059.stdout:81: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:44:31.393 INFO:tasks.rados.rados.0.smithi059.stdout:81: finishing rollback tid 2 to smithi05955964-40 2022-04-28T19:44:31.393 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 20 (ObjNum 39 snap 0 seq_num 39) dirty exists 2022-04-28T19:44:31.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:31 smithi059 bash[29114]: cluster 2022-04-28T19:44:30.375691+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2022-04-28T19:44:31.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:31 smithi059 bash[23786]: cluster 2022-04-28T19:44:30.375691+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2022-04-28T19:44:31.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:31 smithi139 bash[23439]: cluster 2022-04-28T19:44:30.375691+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2022-04-28T19:44:32.387 INFO:tasks.rados.rados.0.smithi059.stdout:81: done (1 left) 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:82: done (0 left) 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:83: rollback oid 9 current snap is 3 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 0 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:84: read oid 20 snap 1 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:85: read oid 25 snap -1 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:85: expect (ObjNum 24 snap 0 seq_num 24) 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:86: copy_from oid 34 from oid 28 current snap is 3 2022-04-28T19:44:32.388 INFO:tasks.rados.rados.0.smithi059.stdout:87: setattr oid 48 current snap is 3 2022-04-28T19:44:32.389 INFO:tasks.rados.rados.0.smithi059.stdout:88: snap_remove snap 2 2022-04-28T19:44:32.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:32 smithi059 bash[29114]: cluster 2022-04-28T19:44:31.382566+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2022-04-28T19:44:32.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:32 smithi059 bash[29114]: cluster 2022-04-28T19:44:32.070920+0000 mgr.y (mgr.14140) 372 : cluster [DBG] pgmap v333: 89 pgs: 89 active+clean; 14 MiB data, 8.1 MiB used, 707 GiB / 715 GiB avail; 402 KiB/s rd, 3.1 MiB/s wr, 27 op/s 2022-04-28T19:44:32.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:32 smithi059 bash[29114]: cluster 2022-04-28T19:44:32.380162+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2022-04-28T19:44:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:32 smithi059 bash[23786]: cluster 2022-04-28T19:44:31.382566+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2022-04-28T19:44:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:32 smithi059 bash[23786]: cluster 2022-04-28T19:44:32.070920+0000 mgr.y (mgr.14140) 372 : cluster [DBG] pgmap v333: 89 pgs: 89 active+clean; 14 MiB data, 8.1 MiB used, 707 GiB / 715 GiB avail; 402 KiB/s rd, 3.1 MiB/s wr, 27 op/s 2022-04-28T19:44:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:32 smithi059 bash[23786]: cluster 2022-04-28T19:44:32.380162+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2022-04-28T19:44:32.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:32 smithi139 bash[23439]: cluster 2022-04-28T19:44:31.382566+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2022-04-28T19:44:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:32 smithi139 bash[23439]: cluster 2022-04-28T19:44:32.070920+0000 mgr.y (mgr.14140) 372 : cluster [DBG] pgmap v333: 89 pgs: 89 active+clean; 14 MiB data, 8.1 MiB used, 707 GiB / 715 GiB avail; 402 KiB/s rd, 3.1 MiB/s wr, 27 op/s 2022-04-28T19:44:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:32 smithi139 bash[23439]: cluster 2022-04-28T19:44:32.380162+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2022-04-28T19:44:33.387 INFO:tasks.rados.rados.0.smithi059.stdout:88: done (5 left) 2022-04-28T19:44:33.387 INFO:tasks.rados.rados.0.smithi059.stdout:89: read oid 38 snap 1 2022-04-28T19:44:33.387 INFO:tasks.rados.rados.0.smithi059.stdout:89: expect (ObjNum 37 snap 0 seq_num 37) 2022-04-28T19:44:33.387 INFO:tasks.rados.rados.0.smithi059.stdout:90: read oid 18 snap -1 2022-04-28T19:44:33.387 INFO:tasks.rados.rados.0.smithi059.stdout:90: expect (ObjNum 50 snap 0 seq_num 50) 2022-04-28T19:44:33.387 INFO:tasks.rados.rados.0.smithi059.stdout:91: copy_from oid 2 from oid 29 current snap is 3 2022-04-28T19:44:33.388 INFO:tasks.rados.rados.0.smithi059.stdout:92: rmattr oid 43 current snap is 3 2022-04-28T19:44:33.388 INFO:tasks.rados.rados.0.smithi059.stdout:92: done (8 left) 2022-04-28T19:44:33.388 INFO:tasks.rados.rados.0.smithi059.stdout:93: delete oid 14 current snap is 3 2022-04-28T19:44:33.388 INFO:tasks.rados.rados.0.smithi059.stdout:85: expect (ObjNum 24 snap 0 seq_num 24) 2022-04-28T19:44:33.399 INFO:tasks.rados.rados.0.smithi059.stdout:83: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:44:33.399 INFO:tasks.rados.rados.0.smithi059.stdout:84: expect (ObjNum 19 snap 0 seq_num 19) 2022-04-28T19:44:33.408 INFO:tasks.rados.rados.0.smithi059.stdout:83: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:83: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 8 (ObjNum 8 snap 0 seq_num 8) dirty exists 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 12 (ObjNum 47 snap 0 seq_num 47) dirty exists 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:86: finishing copy_from to smithi05955964-34 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 4 (ObjNum 27 snap 0 seq_num 27) dirty exists 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:86: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:83: done (8 left) 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:84: done (7 left) 2022-04-28T19:44:33.409 INFO:tasks.rados.rados.0.smithi059.stdout:85: done (6 left) 2022-04-28T19:44:33.410 INFO:tasks.rados.rados.0.smithi059.stdout:86: done (5 left) 2022-04-28T19:44:33.410 INFO:tasks.rados.rados.0.smithi059.stdout:87: done (4 left) 2022-04-28T19:44:33.410 INFO:tasks.rados.rados.0.smithi059.stdout:93: done (3 left) 2022-04-28T19:44:33.410 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 3 2022-04-28T19:44:33.410 INFO:tasks.rados.rados.0.smithi059.stdout:94: seq_num 55 ranges {507904=524288,1032192=401408} 2022-04-28T19:44:33.415 INFO:tasks.rados.rados.0.smithi059.stdout:94: writing smithi05955964-31 from 507904 to 1032192 tid 1 2022-04-28T19:44:33.418 INFO:tasks.rados.rados.0.smithi059.stdout:94: writing smithi05955964-31 from 1032192 to 1433600 tid 2 2022-04-28T19:44:33.418 INFO:tasks.rados.rados.0.smithi059.stdout:95: read oid 40 snap 0 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:95: expect (ObjNum 39 snap 0 seq_num 39) 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:96: rmattr oid 28 current snap is 3 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:96: done (5 left) 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:97: read oid 33 snap 0 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:97: expect (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 3 2022-04-28T19:44:33.419 INFO:tasks.rados.rados.0.smithi059.stdout:98: seq_num 56 ranges {606208=540672,1146880=401408} 2022-04-28T19:44:33.426 INFO:tasks.rados.rados.0.smithi059.stdout:98: writing smithi05955964-6 from 606208 to 1146880 tid 1 2022-04-28T19:44:33.428 INFO:tasks.rados.rados.0.smithi059.stdout:98: writing smithi05955964-6 from 1146880 to 1548288 tid 2 2022-04-28T19:44:33.429 INFO:tasks.rados.rados.0.smithi059.stdout:99: copy_from oid 32 from oid 27 current snap is 3 2022-04-28T19:44:33.429 INFO:tasks.rados.rados.0.smithi059.stdout:90: expect (ObjNum 50 snap 0 seq_num 50) 2022-04-28T19:44:33.462 INFO:tasks.rados.rados.0.smithi059.stdout:89: expect (ObjNum 37 snap 0 seq_num 37) 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:91: finishing copy_from to smithi05955964-2 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:91: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:89: done (7 left) 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:90: done (6 left) 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:91: done (5 left) 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:100: copy_from oid 42 from oid 39 current snap is 3 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:94: finishing write tid 1 to smithi05955964-31 2022-04-28T19:44:33.477 INFO:tasks.rados.rados.0.smithi059.stdout:94: finishing write tid 2 to smithi05955964-31 2022-04-28T19:44:33.478 INFO:tasks.rados.rados.0.smithi059.stdout:94: finishing write tid 4 to smithi05955964-31 2022-04-28T19:44:33.478 INFO:tasks.rados.rados.0.smithi059.stdout:97: expect (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:94: finishing write tid 5 to smithi05955964-31 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 14 (ObjNum 55 snap 3 seq_num 55) dirty exists 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:94: left oid 31 (ObjNum 55 snap 3 seq_num 55) 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:94: done (5 left) 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:97: done (4 left) 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 3 2022-04-28T19:44:33.492 INFO:tasks.rados.rados.0.smithi059.stdout:101: seq_num 57 ranges {679936=557056,1236992=401408} 2022-04-28T19:44:33.499 INFO:tasks.rados.rados.0.smithi059.stdout:101: writing smithi05955964-41 from 679936 to 1236992 tid 1 2022-04-28T19:44:33.503 INFO:tasks.rados.rados.0.smithi059.stdout:101: writing smithi05955964-41 from 1236992 to 1638400 tid 2 2022-04-28T19:44:33.503 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 3 2022-04-28T19:44:33.503 INFO:tasks.rados.rados.0.smithi059.stdout:102: seq_num 58 ranges {458752=573440,1032192=401408} 2022-04-28T19:44:33.510 INFO:tasks.rados.rados.0.smithi059.stdout:102: writing smithi05955964-36 from 458752 to 1032192 tid 1 2022-04-28T19:44:33.512 INFO:tasks.rados.rados.0.smithi059.stdout:102: writing smithi05955964-36 from 1032192 to 1433600 tid 2 2022-04-28T19:44:33.512 INFO:tasks.rados.rados.0.smithi059.stdout:103: rollback oid 31 current snap is 3 2022-04-28T19:44:33.513 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 0 2022-04-28T19:44:33.513 INFO:tasks.rados.rados.0.smithi059.stdout:95: expect (ObjNum 39 snap 0 seq_num 39) 2022-04-28T19:44:33.529 INFO:tasks.rados.rados.0.smithi059.stdout:98: finishing write tid 1 to smithi05955964-6 2022-04-28T19:44:33.529 INFO:tasks.rados.rados.0.smithi059.stdout:98: finishing write tid 2 to smithi05955964-6 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:98: finishing write tid 4 to smithi05955964-6 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:98: finishing write tid 5 to smithi05955964-6 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 19 (ObjNum 56 snap 3 seq_num 56) dirty exists 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:98: left oid 6 (ObjNum 56 snap 3 seq_num 56) 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:99: finishing copy_from to smithi05955964-32 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 6 (ObjNum 26 snap 0 seq_num 26) dirty exists 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:99: finishing copy_from racing read to smithi05955964-32 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:100: finishing copy_from to smithi05955964-42 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 6 (ObjNum 38 snap 0 seq_num 38) dirty exists 2022-04-28T19:44:33.530 INFO:tasks.rados.rados.0.smithi059.stdout:100: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:101: finishing write tid 1 to smithi05955964-41 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:101: finishing write tid 2 to smithi05955964-41 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:95: done (6 left) 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:98: done (5 left) 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:99: done (4 left) 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:100: done (3 left) 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:104: delete oid 43 current snap is 3 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:101: finishing write tid 4 to smithi05955964-41 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:101: finishing write tid 5 to smithi05955964-41 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 8 (ObjNum 57 snap 3 seq_num 57) dirty exists 2022-04-28T19:44:33.531 INFO:tasks.rados.rados.0.smithi059.stdout:101: left oid 41 (ObjNum 57 snap 3 seq_num 57) 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:102: finishing write tid 1 to smithi05955964-36 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:102: finishing write tid 2 to smithi05955964-36 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:103: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:102: finishing write tid 4 to smithi05955964-36 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:103: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:103: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 17 (ObjNum 30 snap 0 seq_num 30) dirty exists 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:102: finishing write tid 5 to smithi05955964-36 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 24 (ObjNum 58 snap 3 seq_num 58) dirty exists 2022-04-28T19:44:33.532 INFO:tasks.rados.rados.0.smithi059.stdout:102: left oid 36 (ObjNum 58 snap 3 seq_num 58) 2022-04-28T19:44:33.533 INFO:tasks.rados.rados.0.smithi059.stdout:101: done (3 left) 2022-04-28T19:44:33.533 INFO:tasks.rados.rados.0.smithi059.stdout:102: done (2 left) 2022-04-28T19:44:33.533 INFO:tasks.rados.rados.0.smithi059.stdout:103: done (1 left) 2022-04-28T19:44:33.534 INFO:tasks.rados.rados.0.smithi059.stdout:104: done (0 left) 2022-04-28T19:44:33.534 INFO:tasks.rados.rados.0.smithi059.stdout:105: read oid 43 snap -1 2022-04-28T19:44:33.534 INFO:tasks.rados.rados.0.smithi059.stdout:105: expect deleted 2022-04-28T19:44:33.534 INFO:tasks.rados.rados.0.smithi059.stdout:106: rollback oid 38 current snap is 3 2022-04-28T19:44:33.534 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 0 2022-04-28T19:44:33.534 INFO:tasks.rados.rados.0.smithi059.stdout:107: delete oid 12 current snap is 3 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:105: done (2 left) 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:107: done (1 left) 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:108: read oid 9 snap 1 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:108: expect (ObjNum 8 snap 0 seq_num 8) 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:109: rollback oid 14 current snap is 3 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 0 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:106: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:44:33.535 INFO:tasks.rados.rados.0.smithi059.stdout:110: delete oid 13 current snap is 3 2022-04-28T19:44:33.536 INFO:tasks.rados.rados.0.smithi059.stdout:106: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:44:33.536 INFO:tasks.rados.rados.0.smithi059.stdout:106: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:44:33.536 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 12 (ObjNum 37 snap 0 seq_num 37) dirty exists 2022-04-28T19:44:33.536 INFO:tasks.rados.rados.0.smithi059.stdout:106: done (3 left) 2022-04-28T19:44:33.536 INFO:tasks.rados.rados.0.smithi059.stdout:110: done (2 left) 2022-04-28T19:44:33.536 INFO:tasks.rados.rados.0.smithi059.stdout:111: setattr oid 32 current snap is 3 2022-04-28T19:44:33.537 INFO:tasks.rados.rados.0.smithi059.stdout:109: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:44:33.537 INFO:tasks.rados.rados.0.smithi059.stdout:109: finishing rollback tid 2 to smithi05955964-14 2022-04-28T19:44:33.537 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 18 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:44:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:108: expect (ObjNum 8 snap 0 seq_num 8) 2022-04-28T19:44:33.541 INFO:tasks.rados.rados.0.smithi059.stdout:108: done (2 left) 2022-04-28T19:44:33.541 INFO:tasks.rados.rados.0.smithi059.stdout:109: done (1 left) 2022-04-28T19:44:33.541 INFO:tasks.rados.rados.0.smithi059.stdout:112: snap_create 2022-04-28T19:44:33.543 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 16 (ObjNum 26 snap 0 seq_num 26) dirty exists 2022-04-28T19:44:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:33.514Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:44:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:33.514Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:44:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:33.514Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:44:34.393 INFO:tasks.rados.rados.0.smithi059.stdout:111: done (1 left) 2022-04-28T19:44:34.393 INFO:tasks.rados.rados.0.smithi059.stdout:112: done (0 left) 2022-04-28T19:44:34.394 INFO:tasks.rados.rados.0.smithi059.stdout:113: copy_from oid 25 from oid 6 current snap is 4 2022-04-28T19:44:34.394 INFO:tasks.rados.rados.0.smithi059.stdout:114: setattr oid 23 current snap is 4 2022-04-28T19:44:34.395 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 4 2022-04-28T19:44:34.396 INFO:tasks.rados.rados.0.smithi059.stdout:115: seq_num 62 ranges {172032=647168,819200=401408} 2022-04-28T19:44:34.402 INFO:tasks.rados.rados.0.smithi059.stdout:115: writing smithi05955964-15 from 172032 to 819200 tid 1 2022-04-28T19:44:34.405 INFO:tasks.rados.rados.0.smithi059.stdout:115: writing smithi05955964-15 from 819200 to 1220608 tid 2 2022-04-28T19:44:34.405 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 4 2022-04-28T19:44:34.405 INFO:tasks.rados.rados.0.smithi059.stdout:116: seq_num 63 ranges {319488=663552,983040=401408} 2022-04-28T19:44:34.411 INFO:tasks.rados.rados.0.smithi059.stdout:116: writing smithi05955964-20 from 319488 to 983040 tid 1 2022-04-28T19:44:34.414 INFO:tasks.rados.rados.0.smithi059.stdout:116: writing smithi05955964-20 from 983040 to 1384448 tid 2 2022-04-28T19:44:34.414 INFO:tasks.rados.rados.0.smithi059.stdout:117: rollback oid 31 current snap is 4 2022-04-28T19:44:34.414 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 1 2022-04-28T19:44:34.415 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 19 (ObjNum 53 snap 1 seq_num 53) dirty exists 2022-04-28T19:44:34.415 INFO:tasks.rados.rados.0.smithi059.stdout:114: done (4 left) 2022-04-28T19:44:34.415 INFO:tasks.rados.rados.0.smithi059.stdout:118: read oid 30 snap -1 2022-04-28T19:44:34.415 INFO:tasks.rados.rados.0.smithi059.stdout:118: expect (ObjNum 29 snap 0 seq_num 29) 2022-04-28T19:44:34.415 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 4 2022-04-28T19:44:34.415 INFO:tasks.rados.rados.0.smithi059.stdout:119: seq_num 64 ranges {811008=679936,1490944=401408} 2022-04-28T19:44:34.422 INFO:tasks.rados.rados.0.smithi059.stdout:119: writing smithi05955964-49 from 811008 to 1490944 tid 1 2022-04-28T19:44:34.425 INFO:tasks.rados.rados.0.smithi059.stdout:119: writing smithi05955964-49 from 1490944 to 1892352 tid 2 2022-04-28T19:44:34.425 INFO:tasks.rados.rados.0.smithi059.stdout:120: snap_create 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:115: finishing write tid 1 to smithi05955964-15 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:115: finishing write tid 2 to smithi05955964-15 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:115: finishing write tid 4 to smithi05955964-15 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:115: finishing write tid 5 to smithi05955964-15 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 23 (ObjNum 62 snap 4 seq_num 62) dirty exists 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:115: left oid 15 (ObjNum 62 snap 4 seq_num 62) 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:113: finishing copy_from to smithi05955964-25 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 19 (ObjNum 56 snap 3 seq_num 56) dirty exists 2022-04-28T19:44:34.426 INFO:tasks.rados.rados.0.smithi059.stdout:113: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:116: finishing write tid 1 to smithi05955964-20 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:116: finishing write tid 2 to smithi05955964-20 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:117: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:116: finishing write tid 4 to smithi05955964-20 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:117: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:117: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 27 (ObjNum 30 snap 0 seq_num 30) dirty exists 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:116: finishing write tid 5 to smithi05955964-20 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 23 (ObjNum 63 snap 4 seq_num 63) dirty exists 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:116: left oid 20 (ObjNum 63 snap 4 seq_num 63) 2022-04-28T19:44:34.427 INFO:tasks.rados.rados.0.smithi059.stdout:118: expect (ObjNum 29 snap 0 seq_num 29) 2022-04-28T19:44:34.438 INFO:tasks.rados.rados.0.smithi059.stdout:119: finishing write tid 1 to smithi05955964-49 2022-04-28T19:44:34.438 INFO:tasks.rados.rados.0.smithi059.stdout:119: finishing write tid 2 to smithi05955964-49 2022-04-28T19:44:34.438 INFO:tasks.rados.rados.0.smithi059.stdout:119: finishing write tid 4 to smithi05955964-49 2022-04-28T19:44:34.438 INFO:tasks.rados.rados.0.smithi059.stdout:119: finishing write tid 5 to smithi05955964-49 2022-04-28T19:44:34.438 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 28 (ObjNum 64 snap 4 seq_num 64) dirty exists 2022-04-28T19:44:34.439 INFO:tasks.rados.rados.0.smithi059.stdout:119: left oid 49 (ObjNum 64 snap 4 seq_num 64) 2022-04-28T19:44:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:34 smithi059 bash[23786]: cluster 2022-04-28T19:44:33.383587+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2022-04-28T19:44:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:34 smithi059 bash[23786]: cluster 2022-04-28T19:44:34.071671+0000 mgr.y (mgr.14140) 373 : cluster [DBG] pgmap v336: 89 pgs: 89 active+clean; 17 MiB data, 23 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.6 MiB/s wr, 30 op/s 2022-04-28T19:44:34.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:34 smithi059 bash[29114]: cluster 2022-04-28T19:44:33.383587+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2022-04-28T19:44:34.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:34 smithi059 bash[29114]: cluster 2022-04-28T19:44:34.071671+0000 mgr.y (mgr.14140) 373 : cluster [DBG] pgmap v336: 89 pgs: 89 active+clean; 17 MiB data, 23 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.6 MiB/s wr, 30 op/s 2022-04-28T19:44:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:34 smithi139 bash[23439]: cluster 2022-04-28T19:44:33.383587+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2022-04-28T19:44:34.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:34 smithi139 bash[23439]: cluster 2022-04-28T19:44:34.071671+0000 mgr.y (mgr.14140) 373 : cluster [DBG] pgmap v336: 89 pgs: 89 active+clean; 17 MiB data, 23 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.6 MiB/s wr, 30 op/s 2022-04-28T19:44:35.398 INFO:tasks.rados.rados.0.smithi059.stdout:113: done (6 left) 2022-04-28T19:44:35.398 INFO:tasks.rados.rados.0.smithi059.stdout:115: done (5 left) 2022-04-28T19:44:35.398 INFO:tasks.rados.rados.0.smithi059.stdout:116: done (4 left) 2022-04-28T19:44:35.398 INFO:tasks.rados.rados.0.smithi059.stdout:117: done (3 left) 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:118: done (2 left) 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:119: done (1 left) 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:120: done (0 left) 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:121: read oid 48 snap 1 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:121: expect (ObjNum 47 snap 0 seq_num 47) 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:122: read oid 49 snap 0 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:122: expect (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:44:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:123: delete oid 44 current snap is 5 2022-04-28T19:44:35.407 INFO:tasks.rados.rados.0.smithi059.stdout:122: expect (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:44:35.426 INFO:tasks.rados.rados.0.smithi059.stdout:121: expect (ObjNum 47 snap 0 seq_num 47) 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:121: done (2 left) 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:122: done (1 left) 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:123: done (0 left) 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:124: rollback oid 49 current snap is 5 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 3 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:125: read oid 27 snap -1 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:125: expect (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:44:35.521 INFO:tasks.rados.rados.0.smithi059.stdout:126: setattr oid 41 current snap is 5 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:127: delete oid 28 current snap is 5 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:125: expect (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:124: finishing rollback tid 0 to smithi05955964-49 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:124: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:124: finishing rollback tid 2 to smithi05955964-49 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 32 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 10 (ObjNum 57 snap 3 seq_num 57) dirty exists 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:124: done (3 left) 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:125: done (2 left) 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:126: done (1 left) 2022-04-28T19:44:35.522 INFO:tasks.rados.rados.0.smithi059.stdout:127: done (0 left) 2022-04-28T19:44:35.523 INFO:tasks.rados.rados.0.smithi059.stdout:128: snap_remove snap 3 2022-04-28T19:44:35.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:35 smithi059 bash[29114]: cluster 2022-04-28T19:44:34.391206+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2022-04-28T19:44:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:35 smithi059 bash[23786]: cluster 2022-04-28T19:44:34.391206+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2022-04-28T19:44:35.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:35 smithi139 bash[23439]: cluster 2022-04-28T19:44:34.391206+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2022-04-28T19:44:36.413 INFO:tasks.rados.rados.0.smithi059.stdout:128: done (0 left) 2022-04-28T19:44:36.414 INFO:tasks.rados.rados.0.smithi059.stdout:129: setattr oid 28 current snap is 5 2022-04-28T19:44:36.416 INFO:tasks.rados.rados.0.smithi059.stdout:130: snap_remove snap 1 2022-04-28T19:44:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:36 smithi059 bash[23786]: cluster 2022-04-28T19:44:35.393920+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2022-04-28T19:44:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:36 smithi059 bash[23786]: cluster 2022-04-28T19:44:36.072349+0000 mgr.y (mgr.14140) 374 : cluster [DBG] pgmap v339: 89 pgs: 89 active+clean; 30 MiB data, 45 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.3 MiB/s wr, 66 op/s 2022-04-28T19:44:36.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:36 smithi059 bash[29114]: cluster 2022-04-28T19:44:35.393920+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2022-04-28T19:44:36.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:36 smithi059 bash[29114]: cluster 2022-04-28T19:44:36.072349+0000 mgr.y (mgr.14140) 374 : cluster [DBG] pgmap v339: 89 pgs: 89 active+clean; 30 MiB data, 45 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.3 MiB/s wr, 66 op/s 2022-04-28T19:44:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:36 smithi139 bash[23439]: cluster 2022-04-28T19:44:35.393920+0000 mon.a (mon.0) 675 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2022-04-28T19:44:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:36 smithi139 bash[23439]: cluster 2022-04-28T19:44:36.072349+0000 mgr.y (mgr.14140) 374 : cluster [DBG] pgmap v339: 89 pgs: 89 active+clean; 30 MiB data, 45 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 4.3 MiB/s wr, 66 op/s 2022-04-28T19:44:37.427 INFO:tasks.rados.rados.0.smithi059.stdout:130: done (1 left) 2022-04-28T19:44:37.427 INFO:tasks.rados.rados.0.smithi059.stdout:131: delete oid 6 current snap is 5 2022-04-28T19:44:37.427 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 7 (ObjNum 0 snap 32762 seq_num 4161489424) dirty exists 2022-04-28T19:44:37.433 INFO:tasks.rados.rados.0.smithi059.stdout:129: done (1 left) 2022-04-28T19:44:37.433 INFO:tasks.rados.rados.0.smithi059.stdout:131: done (0 left) 2022-04-28T19:44:37.433 INFO:tasks.rados.rados.0.smithi059.stdout:132: snap_remove snap 0 2022-04-28T19:44:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:37 smithi059 bash[23786]: cluster 2022-04-28T19:44:36.410332+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2022-04-28T19:44:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:37 smithi059 bash[29114]: cluster 2022-04-28T19:44:36.410332+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2022-04-28T19:44:37.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:37 smithi139 bash[23439]: cluster 2022-04-28T19:44:36.410332+0000 mon.a (mon.0) 676 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2022-04-28T19:44:38.436 INFO:tasks.rados.rados.0.smithi059.stdout:132: done (0 left) 2022-04-28T19:44:38.437 INFO:tasks.rados.rados.0.smithi059.stdout:133: rollback oid 17 current snap is 5 2022-04-28T19:44:38.437 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 4 2022-04-28T19:44:38.437 INFO:tasks.rados.rados.0.smithi059.stdout:134: delete oid 18 current snap is 5 2022-04-28T19:44:38.439 INFO:tasks.rados.rados.0.smithi059.stdout:133: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:44:38.441 INFO:tasks.rados.rados.0.smithi059.stdout:133: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:44:38.441 INFO:tasks.rados.rados.0.smithi059.stdout:133: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:44:38.442 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 14 (ObjNum 16 snap 0 seq_num 16) dirty exists 2022-04-28T19:44:38.446 INFO:tasks.rados.rados.0.smithi059.stdout:133: done (1 left) 2022-04-28T19:44:38.447 INFO:tasks.rados.rados.0.smithi059.stdout:134: done (0 left) 2022-04-28T19:44:38.447 INFO:tasks.rados.rados.0.smithi059.stdout:135: delete oid 42 current snap is 5 2022-04-28T19:44:38.450 INFO:tasks.rados.rados.0.smithi059.stdout:135: done (0 left) 2022-04-28T19:44:38.450 INFO:tasks.rados.rados.0.smithi059.stdout:136: copy_from oid 36 from oid 1 current snap is 5 2022-04-28T19:44:38.450 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 5 2022-04-28T19:44:38.451 INFO:tasks.rados.rados.0.smithi059.stdout:137: seq_num 70 ranges {507904=778240,1286144=401408} 2022-04-28T19:44:38.458 INFO:tasks.rados.rados.0.smithi059.stdout:137: writing smithi05955964-31 from 507904 to 1286144 tid 1 2022-04-28T19:44:38.461 INFO:tasks.rados.rados.0.smithi059.stdout:137: writing smithi05955964-31 from 1286144 to 1687552 tid 2 2022-04-28T19:44:38.461 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 5 2022-04-28T19:44:38.461 INFO:tasks.rados.rados.0.smithi059.stdout:138: seq_num 71 ranges {122880=794624,917504=401408} 2022-04-28T19:44:38.469 INFO:tasks.rados.rados.0.smithi059.stdout:138: writing smithi05955964-8 from 122880 to 917504 tid 1 2022-04-28T19:44:38.469 INFO:tasks.rados.rados.0.smithi059.stdout:138: writing smithi05955964-8 from 917504 to 1318912 tid 2 2022-04-28T19:44:38.469 INFO:tasks.rados.rados.0.smithi059.stdout:139: copy_from oid 41 from oid 17 current snap is 5 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:136: finishing copy_from to smithi05955964-36 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:136: got expected ENOENT (src dne) 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:136: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:137: finishing write tid 1 to smithi05955964-31 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:137: finishing write tid 2 to smithi05955964-31 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:137: finishing write tid 4 to smithi05955964-31 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:137: finishing write tid 5 to smithi05955964-31 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 47 (ObjNum 70 snap 5 seq_num 70) dirty exists 2022-04-28T19:44:38.470 INFO:tasks.rados.rados.0.smithi059.stdout:137: left oid 31 (ObjNum 70 snap 5 seq_num 70) 2022-04-28T19:44:38.471 INFO:tasks.rados.rados.0.smithi059.stdout:136: done (3 left) 2022-04-28T19:44:38.471 INFO:tasks.rados.rados.0.smithi059.stdout:137: done (2 left) 2022-04-28T19:44:38.471 INFO:tasks.rados.rados.0.smithi059.stdout:140: copy_from oid 29 from oid 10 current snap is 5 2022-04-28T19:44:38.471 INFO:tasks.rados.rados.0.smithi059.stdout:141: snap_create 2022-04-28T19:44:38.474 INFO:tasks.rados.rados.0.smithi059.stdout:138: finishing write tid 1 to smithi05955964-8 2022-04-28T19:44:38.475 INFO:tasks.rados.rados.0.smithi059.stdout:138: finishing write tid 2 to smithi05955964-8 2022-04-28T19:44:38.475 INFO:tasks.rados.rados.0.smithi059.stdout:138: finishing write tid 4 to smithi05955964-8 2022-04-28T19:44:38.476 INFO:tasks.rados.rados.0.smithi059.stdout:138: finishing write tid 5 to smithi05955964-8 2022-04-28T19:44:38.476 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 23 (ObjNum 71 snap 5 seq_num 71) dirty exists 2022-04-28T19:44:38.476 INFO:tasks.rados.rados.0.smithi059.stdout:138: left oid 8 (ObjNum 71 snap 5 seq_num 71) 2022-04-28T19:44:38.477 INFO:tasks.rados.rados.0.smithi059.stdout:139: finishing copy_from to smithi05955964-41 2022-04-28T19:44:38.477 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 14 (ObjNum 16 snap 0 seq_num 16) dirty exists 2022-04-28T19:44:38.478 INFO:tasks.rados.rados.0.smithi059.stdout:139: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:44:38.478 INFO:tasks.rados.rados.0.smithi059.stdout:140: finishing copy_from to smithi05955964-29 2022-04-28T19:44:38.478 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 17 (ObjNum 9 snap 0 seq_num 9) dirty exists 2022-04-28T19:44:38.479 INFO:tasks.rados.rados.0.smithi059.stdout:140: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:44:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:38 smithi059 bash[23786]: cluster 2022-04-28T19:44:37.420964+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2022-04-28T19:44:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:38 smithi059 bash[23786]: cluster 2022-04-28T19:44:38.073245+0000 mgr.y (mgr.14140) 375 : cluster [DBG] pgmap v342: 89 pgs: 89 active+clean; 33 MiB data, 53 MiB used, 707 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.2 MiB/s wr, 62 op/s 2022-04-28T19:44:38.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:38 smithi059 bash[29114]: cluster 2022-04-28T19:44:37.420964+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2022-04-28T19:44:38.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:38 smithi059 bash[29114]: cluster 2022-04-28T19:44:38.073245+0000 mgr.y (mgr.14140) 375 : cluster [DBG] pgmap v342: 89 pgs: 89 active+clean; 33 MiB data, 53 MiB used, 707 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.2 MiB/s wr, 62 op/s 2022-04-28T19:44:38.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:38 smithi139 bash[23439]: cluster 2022-04-28T19:44:37.420964+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2022-04-28T19:44:38.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:38 smithi139 bash[23439]: cluster 2022-04-28T19:44:38.073245+0000 mgr.y (mgr.14140) 375 : cluster [DBG] pgmap v342: 89 pgs: 89 active+clean; 33 MiB data, 53 MiB used, 707 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.2 MiB/s wr, 62 op/s 2022-04-28T19:44:39.439 INFO:tasks.rados.rados.0.smithi059.stdout:138: done (3 left) 2022-04-28T19:44:39.440 INFO:tasks.rados.rados.0.smithi059.stdout:139: done (2 left) 2022-04-28T19:44:39.440 INFO:tasks.rados.rados.0.smithi059.stdout:140: done (1 left) 2022-04-28T19:44:39.440 INFO:tasks.rados.rados.0.smithi059.stdout:141: done (0 left) 2022-04-28T19:44:39.440 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 6 2022-04-28T19:44:39.440 INFO:tasks.rados.rados.0.smithi059.stdout:142: seq_num 72 ranges {139264=409600,548864=524288,1073152=278528} 2022-04-28T19:44:39.443 INFO:tasks.rados.rados.0.smithi059.stdout:142: writing smithi05955964-9 from 139264 to 548864 tid 1 2022-04-28T19:44:39.446 INFO:tasks.rados.rados.0.smithi059.stdout:142: writing smithi05955964-9 from 548864 to 1073152 tid 2 2022-04-28T19:44:39.448 INFO:tasks.rados.rados.0.smithi059.stdout:142: writing smithi05955964-9 from 1073152 to 1351680 tid 3 2022-04-28T19:44:39.448 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 6 2022-04-28T19:44:39.448 INFO:tasks.rados.rados.0.smithi059.stdout:143: seq_num 73 ranges {0=425984,425984=409600,835584=393216} 2022-04-28T19:44:39.451 INFO:tasks.rados.rados.0.smithi059.stdout:143: writing smithi05955964-12 from 0 to 425984 tid 1 2022-04-28T19:44:39.453 INFO:tasks.rados.rados.0.smithi059.stdout:143: writing smithi05955964-12 from 425984 to 835584 tid 2 2022-04-28T19:44:39.456 INFO:tasks.rados.rados.0.smithi059.stdout:143: writing smithi05955964-12 from 835584 to 1228800 tid 3 2022-04-28T19:44:39.456 INFO:tasks.rados.rados.0.smithi059.stdout:144: rollback oid 22 current snap is 6 2022-04-28T19:44:39.456 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 4 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: finishing write tid 1 to smithi05955964-9 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: finishing write tid 2 to smithi05955964-9 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: finishing write tid 3 to smithi05955964-9 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: finishing write tid 5 to smithi05955964-9 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: finishing write tid 6 to smithi05955964-9 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 27 (ObjNum 72 snap 6 seq_num 72) dirty exists 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: left oid 9 (ObjNum 72 snap 6 seq_num 72) 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:142: done (2 left) 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:145: snap_create 2022-04-28T19:44:39.457 INFO:tasks.rados.rados.0.smithi059.stdout:143: finishing write tid 1 to smithi05955964-12 2022-04-28T19:44:39.458 INFO:tasks.rados.rados.0.smithi059.stdout:143: finishing write tid 2 to smithi05955964-12 2022-04-28T19:44:39.459 INFO:tasks.rados.rados.0.smithi059.stdout:143: finishing write tid 3 to smithi05955964-12 2022-04-28T19:44:39.459 INFO:tasks.rados.rados.0.smithi059.stdout:143: finishing write tid 5 to smithi05955964-12 2022-04-28T19:44:39.460 INFO:tasks.rados.rados.0.smithi059.stdout:144: finishing rollback tid 0 to smithi05955964-22 2022-04-28T19:44:39.461 INFO:tasks.rados.rados.0.smithi059.stdout:144: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:44:39.461 INFO:tasks.rados.rados.0.smithi059.stdout:144: finishing rollback tid 2 to smithi05955964-22 2022-04-28T19:44:39.461 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 30 (ObjNum 21 snap 0 seq_num 21) dirty exists 2022-04-28T19:44:39.463 INFO:tasks.rados.rados.0.smithi059.stdout:143: finishing write tid 6 to smithi05955964-12 2022-04-28T19:44:39.463 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 51 (ObjNum 73 snap 6 seq_num 73) dirty exists 2022-04-28T19:44:39.463 INFO:tasks.rados.rados.0.smithi059.stdout:143: left oid 12 (ObjNum 73 snap 6 seq_num 73) 2022-04-28T19:44:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:39 smithi059 bash[29114]: cluster 2022-04-28T19:44:38.432843+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2022-04-28T19:44:39.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:39 smithi059 bash[23786]: cluster 2022-04-28T19:44:38.432843+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2022-04-28T19:44:39.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:39 smithi139 bash[23439]: cluster 2022-04-28T19:44:38.432843+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2022-04-28T19:44:40.457 INFO:tasks.rados.rados.0.smithi059.stdout:143: done (2 left) 2022-04-28T19:44:40.457 INFO:tasks.rados.rados.0.smithi059.stdout:144: done (1 left) 2022-04-28T19:44:40.457 INFO:tasks.rados.rados.0.smithi059.stdout:145: done (0 left) 2022-04-28T19:44:40.457 INFO:tasks.rados.rados.0.smithi059.stdout:146: read oid 33 snap -1 2022-04-28T19:44:40.457 INFO:tasks.rados.rados.0.smithi059.stdout:146: expect (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:40.457 INFO:tasks.rados.rados.0.smithi059.stdout:147: read oid 45 snap -1 2022-04-28T19:44:40.458 INFO:tasks.rados.rados.0.smithi059.stdout:147: expect (ObjNum 33 snap 0 seq_num 33) 2022-04-28T19:44:40.458 INFO:tasks.rados.rados.0.smithi059.stdout:148: snap_create 2022-04-28T19:44:40.463 INFO:tasks.rados.rados.0.smithi059.stdout:147: expect (ObjNum 33 snap 0 seq_num 33) 2022-04-28T19:44:40.477 INFO:tasks.rados.rados.0.smithi059.stdout:146: expect (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:40 smithi059 bash[29114]: cluster 2022-04-28T19:44:39.436395+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2022-04-28T19:44:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:40 smithi059 bash[29114]: cluster 2022-04-28T19:44:40.074018+0000 mgr.y (mgr.14140) 376 : cluster [DBG] pgmap v345: 89 pgs: 89 active+clean; 33 MiB data, 55 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 30 op/s 2022-04-28T19:44:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:40 smithi059 bash[23786]: cluster 2022-04-28T19:44:39.436395+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2022-04-28T19:44:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:40 smithi059 bash[23786]: cluster 2022-04-28T19:44:40.074018+0000 mgr.y (mgr.14140) 376 : cluster [DBG] pgmap v345: 89 pgs: 89 active+clean; 33 MiB data, 55 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 30 op/s 2022-04-28T19:44:40.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:40 smithi139 bash[23439]: cluster 2022-04-28T19:44:39.436395+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2022-04-28T19:44:40.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:40 smithi139 bash[23439]: cluster 2022-04-28T19:44:40.074018+0000 mgr.y (mgr.14140) 376 : cluster [DBG] pgmap v345: 89 pgs: 89 active+clean; 33 MiB data, 55 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 30 op/s 2022-04-28T19:44:41.464 INFO:tasks.rados.rados.0.smithi059.stdout:146: done (2 left) 2022-04-28T19:44:41.464 INFO:tasks.rados.rados.0.smithi059.stdout:147: done (1 left) 2022-04-28T19:44:41.464 INFO:tasks.rados.rados.0.smithi059.stdout:148: done (0 left) 2022-04-28T19:44:41.464 INFO:tasks.rados.rados.0.smithi059.stdout:149: copy_from oid 48 from oid 42 current snap is 8 2022-04-28T19:44:41.464 INFO:tasks.rados.rados.0.smithi059.stdout:150: delete oid 29 current snap is 8 2022-04-28T19:44:41.472 INFO:tasks.rados.rados.0.smithi059.stdout:150: done (1 left) 2022-04-28T19:44:41.472 INFO:tasks.rados.rados.0.smithi059.stdout:151: snap_create 2022-04-28T19:44:41.473 INFO:tasks.rados.rados.0.smithi059.stdout:149: finishing copy_from to smithi05955964-48 2022-04-28T19:44:41.473 INFO:tasks.rados.rados.0.smithi059.stdout:149: got expected ENOENT (src dne) 2022-04-28T19:44:41.473 INFO:tasks.rados.rados.0.smithi059.stdout:149: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:44:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:41 smithi059 bash[29114]: cluster 2022-04-28T19:44:40.454574+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2022-04-28T19:44:41.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:41 smithi059 bash[23786]: cluster 2022-04-28T19:44:40.454574+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2022-04-28T19:44:41.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:41 smithi139 bash[23439]: cluster 2022-04-28T19:44:40.454574+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2022-04-28T19:44:42.144 INFO:tasks.rados.rados.0.smithi059.stdout:149: done (1 left) 2022-04-28T19:44:42.144 INFO:tasks.rados.rados.0.smithi059.stdout:151: done (0 left) 2022-04-28T19:44:42.144 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 9 2022-04-28T19:44:42.144 INFO:tasks.rados.rados.0.smithi059.stdout:152: seq_num 75 ranges {0=458752,458752=581632,1040384=221184} 2022-04-28T19:44:42.146 INFO:tasks.rados.rados.0.smithi059.stdout:152: writing smithi05955964-43 from 0 to 458752 tid 1 2022-04-28T19:44:42.150 INFO:tasks.rados.rados.0.smithi059.stdout:152: writing smithi05955964-43 from 458752 to 1040384 tid 2 2022-04-28T19:44:42.152 INFO:tasks.rados.rados.0.smithi059.stdout:152: writing smithi05955964-43 from 1040384 to 1261568 tid 3 2022-04-28T19:44:42.152 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 9 2022-04-28T19:44:42.152 INFO:tasks.rados.rados.0.smithi059.stdout:153: seq_num 76 ranges {155648=475136,630784=458752,1089536=344064} 2022-04-28T19:44:42.155 INFO:tasks.rados.rados.0.smithi059.stdout:153: writing smithi05955964-10 from 155648 to 630784 tid 1 2022-04-28T19:44:42.159 INFO:tasks.rados.rados.0.smithi059.stdout:153: writing smithi05955964-10 from 630784 to 1089536 tid 2 2022-04-28T19:44:42.161 INFO:tasks.rados.rados.0.smithi059.stdout:153: writing smithi05955964-10 from 1089536 to 1433600 tid 3 2022-04-28T19:44:42.161 INFO:tasks.rados.rados.0.smithi059.stdout:154: snap_create 2022-04-28T19:44:42.162 INFO:tasks.rados.rados.0.smithi059.stdout:152: finishing write tid 1 to smithi05955964-43 2022-04-28T19:44:42.162 INFO:tasks.rados.rados.0.smithi059.stdout:152: finishing write tid 2 to smithi05955964-43 2022-04-28T19:44:42.162 INFO:tasks.rados.rados.0.smithi059.stdout:152: finishing write tid 3 to smithi05955964-43 2022-04-28T19:44:42.162 INFO:tasks.rados.rados.0.smithi059.stdout:152: finishing write tid 5 to smithi05955964-43 2022-04-28T19:44:42.163 INFO:tasks.rados.rados.0.smithi059.stdout:152: finishing write tid 6 to smithi05955964-43 2022-04-28T19:44:42.163 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 28 (ObjNum 75 snap 9 seq_num 75) dirty exists 2022-04-28T19:44:42.163 INFO:tasks.rados.rados.0.smithi059.stdout:152: left oid 43 (ObjNum 75 snap 9 seq_num 75) 2022-04-28T19:44:42.163 INFO:tasks.rados.rados.0.smithi059.stdout:153: finishing write tid 1 to smithi05955964-10 2022-04-28T19:44:42.164 INFO:tasks.rados.rados.0.smithi059.stdout:153: finishing write tid 2 to smithi05955964-10 2022-04-28T19:44:42.164 INFO:tasks.rados.rados.0.smithi059.stdout:153: finishing write tid 3 to smithi05955964-10 2022-04-28T19:44:42.165 INFO:tasks.rados.rados.0.smithi059.stdout:153: finishing write tid 5 to smithi05955964-10 2022-04-28T19:44:42.166 INFO:tasks.rados.rados.0.smithi059.stdout:153: finishing write tid 6 to smithi05955964-10 2022-04-28T19:44:42.166 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 32 (ObjNum 76 snap 9 seq_num 76) dirty exists 2022-04-28T19:44:42.166 INFO:tasks.rados.rados.0.smithi059.stdout:153: left oid 10 (ObjNum 76 snap 9 seq_num 76) 2022-04-28T19:44:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:42 smithi059 bash[23786]: cluster 2022-04-28T19:44:41.460802+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2022-04-28T19:44:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:42 smithi059 bash[23786]: cluster 2022-04-28T19:44:42.074748+0000 mgr.y (mgr.14140) 377 : cluster [DBG] pgmap v348: 89 pgs: 89 active+clean; 33 MiB data, 55 MiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 23 op/s 2022-04-28T19:44:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:42 smithi059 bash[23786]: cluster 2022-04-28T19:44:42.142047+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2022-04-28T19:44:42.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:42 smithi059 bash[29114]: cluster 2022-04-28T19:44:41.460802+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2022-04-28T19:44:42.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:42 smithi059 bash[29114]: cluster 2022-04-28T19:44:42.074748+0000 mgr.y (mgr.14140) 377 : cluster [DBG] pgmap v348: 89 pgs: 89 active+clean; 33 MiB data, 55 MiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 23 op/s 2022-04-28T19:44:42.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:42 smithi059 bash[29114]: cluster 2022-04-28T19:44:42.142047+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2022-04-28T19:44:42.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:42 smithi139 bash[23439]: cluster 2022-04-28T19:44:41.460802+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2022-04-28T19:44:42.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:42 smithi139 bash[23439]: cluster 2022-04-28T19:44:42.074748+0000 mgr.y (mgr.14140) 377 : cluster [DBG] pgmap v348: 89 pgs: 89 active+clean; 33 MiB data, 55 MiB used, 707 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.2 MiB/s wr, 23 op/s 2022-04-28T19:44:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:42 smithi139 bash[23439]: cluster 2022-04-28T19:44:42.142047+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2022-04-28T19:44:43.158 INFO:tasks.rados.rados.0.smithi059.stdout:152: done (2 left) 2022-04-28T19:44:43.158 INFO:tasks.rados.rados.0.smithi059.stdout:153: done (1 left) 2022-04-28T19:44:43.158 INFO:tasks.rados.rados.0.smithi059.stdout:154: done (0 left) 2022-04-28T19:44:43.159 INFO:tasks.rados.rados.0.smithi059.stdout:155: snap_create 2022-04-28T19:44:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:43.514Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:44:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:43.514Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:44:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:43.514Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:44:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:155: done (0 left) 2022-04-28T19:44:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:156: read oid 42 snap -1 2022-04-28T19:44:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:156: expect deleted 2022-04-28T19:44:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:157: rmattr oid 17 current snap is 11 2022-04-28T19:44:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:157: done (1 left) 2022-04-28T19:44:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:158: setattr oid 28 current snap is 11 2022-04-28T19:44:44.163 INFO:tasks.rados.rados.0.smithi059.stdout:156: done (1 left) 2022-04-28T19:44:44.163 INFO:tasks.rados.rados.0.smithi059.stdout:159: copy_from oid 7 from oid 24 current snap is 11 2022-04-28T19:44:44.163 INFO:tasks.rados.rados.0.smithi059.stdout:160: setattr oid 10 current snap is 11 2022-04-28T19:44:44.166 INFO:tasks.rados.rados.0.smithi059.stdout:161: snap_remove snap 6 2022-04-28T19:44:44.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:44 smithi059 bash[29114]: cluster 2022-04-28T19:44:43.154480+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2022-04-28T19:44:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:44 smithi059 bash[23786]: cluster 2022-04-28T19:44:43.154480+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2022-04-28T19:44:44.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:44 smithi139 bash[23439]: cluster 2022-04-28T19:44:43.154480+0000 mon.a (mon.0) 683 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2022-04-28T19:44:45.175 INFO:tasks.rados.rados.0.smithi059.stdout:161: done (3 left) 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 11 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 34 (ObjNum 76 snap 9 seq_num 76) dirty exists 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:159: finishing copy_from to smithi05955964-7 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 15 (ObjNum -133601760 snap 21893 seq_num 4161839664) dirty exists 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:159: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:44:45.176 INFO:tasks.rados.rados.0.smithi059.stdout:162: seq_num 77 ranges {761856=491520,1253376=794624,2048000=8192} 2022-04-28T19:44:45.182 INFO:tasks.rados.rados.0.smithi059.stdout:162: writing smithi05955964-46 from 761856 to 1253376 tid 1 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:162: writing smithi05955964-46 from 1253376 to 2048000 tid 2 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:162: writing smithi05955964-46 from 2048000 to 2056192 tid 3 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:158: done (3 left) 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:159: done (2 left) 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:160: done (1 left) 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 11 2022-04-28T19:44:45.188 INFO:tasks.rados.rados.0.smithi059.stdout:163: seq_num 78 ranges {0=516096,516096=679936,1196032=122880} 2022-04-28T19:44:45.191 INFO:tasks.rados.rados.0.smithi059.stdout:163: writing smithi05955964-44 from 0 to 516096 tid 1 2022-04-28T19:44:45.195 INFO:tasks.rados.rados.0.smithi059.stdout:163: writing smithi05955964-44 from 516096 to 1196032 tid 2 2022-04-28T19:44:45.196 INFO:tasks.rados.rados.0.smithi059.stdout:163: writing smithi05955964-44 from 1196032 to 1318912 tid 3 2022-04-28T19:44:45.196 INFO:tasks.rados.rados.0.smithi059.stdout:164: snap_remove snap 4 2022-04-28T19:44:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:45 smithi059 bash[29114]: cluster 2022-04-28T19:44:44.075524+0000 mgr.y (mgr.14140) 378 : cluster [DBG] pgmap v351: 89 pgs: 89 active+clean; 32 MiB data, 54 MiB used, 707 GiB / 715 GiB avail; 978 KiB/s rd, 330 KiB/s wr, 4 op/s 2022-04-28T19:44:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:45 smithi059 bash[29114]: cluster 2022-04-28T19:44:44.158096+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2022-04-28T19:44:45.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:45 smithi059 bash[23786]: cluster 2022-04-28T19:44:44.075524+0000 mgr.y (mgr.14140) 378 : cluster [DBG] pgmap v351: 89 pgs: 89 active+clean; 32 MiB data, 54 MiB used, 707 GiB / 715 GiB avail; 978 KiB/s rd, 330 KiB/s wr, 4 op/s 2022-04-28T19:44:45.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:45 smithi059 bash[23786]: cluster 2022-04-28T19:44:44.158096+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2022-04-28T19:44:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:45 smithi139 bash[23439]: cluster 2022-04-28T19:44:44.075524+0000 mgr.y (mgr.14140) 378 : cluster [DBG] pgmap v351: 89 pgs: 89 active+clean; 32 MiB data, 54 MiB used, 707 GiB / 715 GiB avail; 978 KiB/s rd, 330 KiB/s wr, 4 op/s 2022-04-28T19:44:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:45 smithi139 bash[23439]: cluster 2022-04-28T19:44:44.158096+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2022-04-28T19:44:46.204 INFO:tasks.rados.rados.0.smithi059.stdout:164: done (2 left) 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:165: delete oid 22 current snap is 11 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:162: finishing write tid 1 to smithi05955964-46 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:162: finishing write tid 2 to smithi05955964-46 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:162: finishing write tid 3 to smithi05955964-46 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:162: finishing write tid 5 to smithi05955964-46 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:162: finishing write tid 6 to smithi05955964-46 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 34 (ObjNum 77 snap 11 seq_num 77) dirty exists 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:162: left oid 46 (ObjNum 77 snap 11 seq_num 77) 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:163: finishing write tid 1 to smithi05955964-44 2022-04-28T19:44:46.205 INFO:tasks.rados.rados.0.smithi059.stdout:163: finishing write tid 2 to smithi05955964-44 2022-04-28T19:44:46.206 INFO:tasks.rados.rados.0.smithi059.stdout:163: finishing write tid 3 to smithi05955964-44 2022-04-28T19:44:46.206 INFO:tasks.rados.rados.0.smithi059.stdout:163: finishing write tid 5 to smithi05955964-44 2022-04-28T19:44:46.206 INFO:tasks.rados.rados.0.smithi059.stdout:163: finishing write tid 6 to smithi05955964-44 2022-04-28T19:44:46.206 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 31 (ObjNum 78 snap 11 seq_num 78) dirty exists 2022-04-28T19:44:46.206 INFO:tasks.rados.rados.0.smithi059.stdout:163: left oid 44 (ObjNum 78 snap 11 seq_num 78) 2022-04-28T19:44:46.209 INFO:tasks.rados.rados.0.smithi059.stdout:162: done (2 left) 2022-04-28T19:44:46.209 INFO:tasks.rados.rados.0.smithi059.stdout:163: done (1 left) 2022-04-28T19:44:46.209 INFO:tasks.rados.rados.0.smithi059.stdout:165: done (0 left) 2022-04-28T19:44:46.209 INFO:tasks.rados.rados.0.smithi059.stdout:166: snap_create 2022-04-28T19:44:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:46 smithi059 bash[29114]: cluster 2022-04-28T19:44:45.169687+0000 mon.a (mon.0) 685 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2022-04-28T19:44:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:46 smithi059 bash[23786]: cluster 2022-04-28T19:44:45.169687+0000 mon.a (mon.0) 685 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2022-04-28T19:44:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:46 smithi139 bash[23439]: cluster 2022-04-28T19:44:45.169687+0000 mon.a (mon.0) 685 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2022-04-28T19:44:47.149 INFO:tasks.rados.rados.0.smithi059.stdout:166: done (0 left) 2022-04-28T19:44:47.149 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 12 2022-04-28T19:44:47.149 INFO:tasks.rados.rados.0.smithi059.stdout:167: seq_num 80 ranges {1687552=548864,2236416=450560,2686976=352256} 2022-04-28T19:44:47.166 INFO:tasks.rados.rados.0.smithi059.stdout:167: writing smithi05955964-31 from 1687552 to 2236416 tid 1 2022-04-28T19:44:47.169 INFO:tasks.rados.rados.0.smithi059.stdout:167: writing smithi05955964-31 from 2236416 to 2686976 tid 2 2022-04-28T19:44:47.172 INFO:tasks.rados.rados.0.smithi059.stdout:167: writing smithi05955964-31 from 2686976 to 3039232 tid 3 2022-04-28T19:44:47.172 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 12 2022-04-28T19:44:47.172 INFO:tasks.rados.rados.0.smithi059.stdout:168: seq_num 81 ranges {253952=565248,819200=737280,1556480=65536} 2022-04-28T19:44:47.176 INFO:tasks.rados.rados.0.smithi059.stdout:168: writing smithi05955964-16 from 253952 to 819200 tid 1 2022-04-28T19:44:47.181 INFO:tasks.rados.rados.0.smithi059.stdout:168: writing smithi05955964-16 from 819200 to 1556480 tid 2 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:168: writing smithi05955964-16 from 1556480 to 1622016 tid 3 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:169: copy_from oid 10 from oid 39 current snap is 12 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:167: finishing write tid 1 to smithi05955964-31 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:167: finishing write tid 2 to smithi05955964-31 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:167: finishing write tid 3 to smithi05955964-31 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:167: finishing write tid 5 to smithi05955964-31 2022-04-28T19:44:47.182 INFO:tasks.rados.rados.0.smithi059.stdout:167: finishing write tid 6 to smithi05955964-31 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 64 (ObjNum 80 snap 12 seq_num 80) dirty exists 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:167: left oid 31 (ObjNum 80 snap 12 seq_num 80) 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:168: finishing write tid 1 to smithi05955964-16 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:167: done (2 left) 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:170: read oid 25 snap 8 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:170: expect (ObjNum 56 snap 3 seq_num 56) 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:171: copy_from oid 21 from oid 35 current snap is 12 2022-04-28T19:44:47.183 INFO:tasks.rados.rados.0.smithi059.stdout:172: snap_remove snap 7 2022-04-28T19:44:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:47 smithi059 bash[23786]: cluster 2022-04-28T19:44:46.076359+0000 mgr.y (mgr.14140) 379 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 35 MiB data, 59 MiB used, 707 GiB / 715 GiB avail; 409 KiB/s rd, 708 KiB/s wr, 14 op/s 2022-04-28T19:44:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:47 smithi059 bash[23786]: cluster 2022-04-28T19:44:46.194276+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2022-04-28T19:44:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:47 smithi059 bash[23786]: cluster 2022-04-28T19:44:47.144984+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2022-04-28T19:44:47.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:47 smithi059 bash[29114]: cluster 2022-04-28T19:44:46.076359+0000 mgr.y (mgr.14140) 379 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 35 MiB data, 59 MiB used, 707 GiB / 715 GiB avail; 409 KiB/s rd, 708 KiB/s wr, 14 op/s 2022-04-28T19:44:47.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:47 smithi059 bash[29114]: cluster 2022-04-28T19:44:46.194276+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2022-04-28T19:44:47.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:47 smithi059 bash[29114]: cluster 2022-04-28T19:44:47.144984+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2022-04-28T19:44:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:47 smithi139 bash[23439]: cluster 2022-04-28T19:44:46.076359+0000 mgr.y (mgr.14140) 379 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 35 MiB data, 59 MiB used, 707 GiB / 715 GiB avail; 409 KiB/s rd, 708 KiB/s wr, 14 op/s 2022-04-28T19:44:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:47 smithi139 bash[23439]: cluster 2022-04-28T19:44:46.194276+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2022-04-28T19:44:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:47 smithi139 bash[23439]: cluster 2022-04-28T19:44:47.144984+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2022-04-28T19:44:48.163 INFO:tasks.rados.rados.0.smithi059.stdout:172: done (4 left) 2022-04-28T19:44:48.163 INFO:tasks.rados.rados.0.smithi059.stdout:173: delete oid 8 current snap is 12 2022-04-28T19:44:48.163 INFO:tasks.rados.rados.0.smithi059.stdout:168: finishing write tid 2 to smithi05955964-16 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:168: finishing write tid 3 to smithi05955964-16 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:168: finishing write tid 5 to smithi05955964-16 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:168: finishing write tid 6 to smithi05955964-16 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 24 (ObjNum 81 snap 12 seq_num 81) dirty exists 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:168: left oid 16 (ObjNum 81 snap 12 seq_num 81) 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:171: finishing copy_from to smithi05955964-21 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 2 (ObjNum 34 snap 0 seq_num 34) dirty exists 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:171: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:169: finishing copy_from to smithi05955964-10 2022-04-28T19:44:48.164 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 6 (ObjNum 38 snap 0 seq_num 38) dirty exists 2022-04-28T19:44:48.165 INFO:tasks.rados.rados.0.smithi059.stdout:170: expect (ObjNum 56 snap 3 seq_num 56) 2022-04-28T19:44:48.213 INFO:tasks.rados.rados.0.smithi059.stdout:169: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:44:48.213 INFO:tasks.rados.rados.0.smithi059.stdout:168: done (4 left) 2022-04-28T19:44:48.214 INFO:tasks.rados.rados.0.smithi059.stdout:169: done (3 left) 2022-04-28T19:44:48.214 INFO:tasks.rados.rados.0.smithi059.stdout:170: done (2 left) 2022-04-28T19:44:48.214 INFO:tasks.rados.rados.0.smithi059.stdout:171: done (1 left) 2022-04-28T19:44:48.214 INFO:tasks.rados.rados.0.smithi059.stdout:173: done (0 left) 2022-04-28T19:44:48.214 INFO:tasks.rados.rados.0.smithi059.stdout:174: snap_remove snap 10 2022-04-28T19:44:49.205 INFO:tasks.rados.rados.0.smithi059.stdout:174: done (0 left) 2022-04-28T19:44:49.206 INFO:tasks.rados.rados.0.smithi059.stdout:175: snap_create 2022-04-28T19:44:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:49 smithi059 bash[29114]: cluster 2022-04-28T19:44:48.077214+0000 mgr.y (mgr.14140) 380 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 37 MiB data, 61 MiB used, 707 GiB / 715 GiB avail; 409 KiB/s rd, 1.3 MiB/s wr, 16 op/s 2022-04-28T19:44:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:49 smithi059 bash[29114]: cluster 2022-04-28T19:44:48.160935+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2022-04-28T19:44:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:49 smithi059 bash[23786]: cluster 2022-04-28T19:44:48.077214+0000 mgr.y (mgr.14140) 380 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 37 MiB data, 61 MiB used, 707 GiB / 715 GiB avail; 409 KiB/s rd, 1.3 MiB/s wr, 16 op/s 2022-04-28T19:44:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:49 smithi059 bash[23786]: cluster 2022-04-28T19:44:48.160935+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2022-04-28T19:44:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:49 smithi139 bash[23439]: cluster 2022-04-28T19:44:48.077214+0000 mgr.y (mgr.14140) 380 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 37 MiB data, 61 MiB used, 707 GiB / 715 GiB avail; 409 KiB/s rd, 1.3 MiB/s wr, 16 op/s 2022-04-28T19:44:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:49 smithi139 bash[23439]: cluster 2022-04-28T19:44:48.160935+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2022-04-28T19:44:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:175: done (0 left) 2022-04-28T19:44:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:176: read oid 46 snap -1 2022-04-28T19:44:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:176: expect (ObjNum 77 snap 11 seq_num 77) 2022-04-28T19:44:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:177: read oid 7 snap 9 2022-04-28T19:44:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:177: expect (ObjNum 6 snap 0 seq_num 6) 2022-04-28T19:44:50.220 INFO:tasks.rados.rados.0.smithi059.stdout:178: snap_create 2022-04-28T19:44:50.228 INFO:tasks.rados.rados.0.smithi059.stdout:177: expect (ObjNum 6 snap 0 seq_num 6) 2022-04-28T19:44:50.233 INFO:tasks.rados.rados.0.smithi059.stdout:176: expect (ObjNum 77 snap 11 seq_num 77) 2022-04-28T19:44:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:50 smithi059 bash[29114]: cluster 2022-04-28T19:44:49.202003+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2022-04-28T19:44:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:50 smithi059 bash[23786]: cluster 2022-04-28T19:44:49.202003+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2022-04-28T19:44:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:50 smithi139 bash[23439]: cluster 2022-04-28T19:44:49.202003+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2022-04-28T19:44:51.220 INFO:tasks.rados.rados.0.smithi059.stdout:176: done (2 left) 2022-04-28T19:44:51.220 INFO:tasks.rados.rados.0.smithi059.stdout:177: done (1 left) 2022-04-28T19:44:51.221 INFO:tasks.rados.rados.0.smithi059.stdout:178: done (0 left) 2022-04-28T19:44:51.221 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 14 2022-04-28T19:44:51.221 INFO:tasks.rados.rados.0.smithi059.stdout:179: seq_num 83 ranges {0=598016,598016=499712,1097728=303104} 2022-04-28T19:44:51.223 INFO:tasks.rados.rados.0.smithi059.stdout:179: writing smithi05955964-13 from 0 to 598016 tid 1 2022-04-28T19:44:51.227 INFO:tasks.rados.rados.0.smithi059.stdout:179: writing smithi05955964-13 from 598016 to 1097728 tid 2 2022-04-28T19:44:51.229 INFO:tasks.rados.rados.0.smithi059.stdout:179: writing smithi05955964-13 from 1097728 to 1400832 tid 3 2022-04-28T19:44:51.229 INFO:tasks.rados.rados.0.smithi059.stdout:180: copy_from oid 15 from oid 5 current snap is 14 2022-04-28T19:44:51.229 INFO:tasks.rados.rados.0.smithi059.stdout:181: rollback oid 19 current snap is 14 2022-04-28T19:44:51.229 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 19 to 8 2022-04-28T19:44:51.230 INFO:tasks.rados.rados.0.smithi059.stdout:182: copy_from oid 21 from oid 24 current snap is 14 2022-04-28T19:44:51.230 INFO:tasks.rados.rados.0.smithi059.stdout:183: copy_from oid 3 from oid 7 current snap is 14 2022-04-28T19:44:51.230 INFO:tasks.rados.rados.0.smithi059.stdout:184: read oid 33 snap -1 2022-04-28T19:44:51.230 INFO:tasks.rados.rados.0.smithi059.stdout:184: expect (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:51.232 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 14 2022-04-28T19:44:51.232 INFO:tasks.rados.rados.0.smithi059.stdout:185: seq_num 84 ranges {1622016=614400,2236416=434176,2670592=368640} 2022-04-28T19:44:51.243 INFO:tasks.rados.rados.0.smithi059.stdout:185: writing smithi05955964-16 from 1622016 to 2236416 tid 1 2022-04-28T19:44:51.245 INFO:tasks.rados.rados.0.smithi059.stdout:185: writing smithi05955964-16 from 2236416 to 2670592 tid 2 2022-04-28T19:44:51.248 INFO:tasks.rados.rados.0.smithi059.stdout:185: writing smithi05955964-16 from 2670592 to 3039232 tid 3 2022-04-28T19:44:51.248 INFO:tasks.rados.rados.0.smithi059.stdout:179: finishing write tid 1 to smithi05955964-13 2022-04-28T19:44:51.248 INFO:tasks.rados.rados.0.smithi059.stdout:179: finishing write tid 2 to smithi05955964-13 2022-04-28T19:44:51.248 INFO:tasks.rados.rados.0.smithi059.stdout:179: finishing write tid 3 to smithi05955964-13 2022-04-28T19:44:51.249 INFO:tasks.rados.rados.0.smithi059.stdout:186: rmattr oid 34 current snap is 14 2022-04-28T19:44:51.249 INFO:tasks.rados.rados.0.smithi059.stdout:186: done (7 left) 2022-04-28T19:44:51.249 INFO:tasks.rados.rados.0.smithi059.stdout:187: snap_remove snap 12 2022-04-28T19:44:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:51 smithi059 bash[29114]: cluster 2022-04-28T19:44:50.078143+0000 mgr.y (mgr.14140) 381 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 35 MiB data, 61 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.7 MiB/s wr, 6 op/s 2022-04-28T19:44:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:51 smithi059 bash[29114]: cluster 2022-04-28T19:44:50.216309+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2022-04-28T19:44:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:51 smithi059 bash[23786]: cluster 2022-04-28T19:44:50.078143+0000 mgr.y (mgr.14140) 381 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 35 MiB data, 61 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.7 MiB/s wr, 6 op/s 2022-04-28T19:44:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:51 smithi059 bash[23786]: cluster 2022-04-28T19:44:50.216309+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2022-04-28T19:44:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:51 smithi139 bash[23439]: cluster 2022-04-28T19:44:50.078143+0000 mgr.y (mgr.14140) 381 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 35 MiB data, 61 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.7 MiB/s wr, 6 op/s 2022-04-28T19:44:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:51 smithi139 bash[23439]: cluster 2022-04-28T19:44:50.216309+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2022-04-28T19:44:52.128 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:44:52.147 INFO:tasks.rados.rados.0.smithi059.stdout:187: done (7 left) 2022-04-28T19:44:52.147 INFO:tasks.rados.rados.0.smithi059.stdout:188: read oid 39 snap -1 2022-04-28T19:44:52.147 INFO:tasks.rados.rados.0.smithi059.stdout:188: expect (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:52.147 INFO:tasks.rados.rados.0.smithi059.stdout:189: rmattr oid 4 current snap is 14 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:189: done (8 left) 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:190: snap_create 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:179: finishing write tid 5 to smithi05955964-13 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:181: finishing rollback tid 0 to smithi05955964-19 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:181: finishing rollback tid 1 to smithi05955964-19 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:181: finishing rollback tid 2 to smithi05955964-19 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 42 (ObjNum 18 snap 0 seq_num 18) dirty exists 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:179: finishing write tid 6 to smithi05955964-13 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 20 (ObjNum 83 snap 14 seq_num 83) dirty exists 2022-04-28T19:44:52.148 INFO:tasks.rados.rados.0.smithi059.stdout:179: left oid 13 (ObjNum 83 snap 14 seq_num 83) 2022-04-28T19:44:52.149 INFO:tasks.rados.rados.0.smithi059.stdout:184: expect (ObjNum 32 snap 0 seq_num 32) 2022-04-28T19:44:52.161 INFO:tasks.rados.rados.0.smithi059.stdout:180: finishing copy_from to smithi05955964-15 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:180: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:183: finishing copy_from to smithi05955964-3 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:183: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:182: finishing copy_from to smithi05955964-21 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2022-04-28T19:44:52.162 INFO:tasks.rados.rados.0.smithi059.stdout:182: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:185: finishing write tid 1 to smithi05955964-16 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:185: finishing write tid 2 to smithi05955964-16 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:185: finishing write tid 3 to smithi05955964-16 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:185: finishing write tid 5 to smithi05955964-16 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:185: finishing write tid 6 to smithi05955964-16 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 37 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:185: left oid 16 (ObjNum 84 snap 14 seq_num 84) 2022-04-28T19:44:52.163 INFO:tasks.rados.rados.0.smithi059.stdout:188: expect (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:52.443 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST STATUS REFRESHED AGE VERSION IMAGE NAME IMAGE ID CONTAINER ID 2022-04-28T19:44:52.443 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (3m) 2m ago 4m 0.20.0 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:44:52.443 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (3m) 2m ago 3m 6.7.4 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:44:52.443 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (2m) 2m ago 2m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 44062639a78b 2022-04-28T19:44:52.443 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (2m) 2m ago 2m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 3516bf4894c9 2022-04-28T19:44:52.443 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (8m) 2m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b8c424c2e210 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (10m) 2m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 320b27bb479e 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (10m) 2m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 239497cf6769 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (8m) 2m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0e12092e0bc3 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (8m) 2m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (4m) 2m ago 4m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (4m) 2m ago 4m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (7m) 2m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 547b5580e073 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (7m) 2m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (6m) 2m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 35cdc0c6604a 2022-04-28T19:44:52.444 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (6m) 2m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 2344bb9dfbae 2022-04-28T19:44:52.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (6m) 2m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 5976654882b1 2022-04-28T19:44:52.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (5m) 2m ago 5m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 e68cffb4766c 2022-04-28T19:44:52.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (5m) 2m ago 5m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 c36b8738a469 2022-04-28T19:44:52.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (4m) 2m ago 4m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:44:52.445 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (3m) 2m ago 4m 2.18.1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 c763409947e4 2022-04-28T19:44:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:52 smithi059 bash[29114]: cluster 2022-04-28T19:44:51.214534+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2022-04-28T19:44:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:52 smithi059 bash[29114]: cluster 2022-04-28T19:44:52.145017+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2022-04-28T19:44:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:52 smithi059 bash[23786]: cluster 2022-04-28T19:44:51.214534+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2022-04-28T19:44:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:52 smithi059 bash[23786]: cluster 2022-04-28T19:44:52.145017+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2022-04-28T19:44:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:52 smithi139 bash[23439]: cluster 2022-04-28T19:44:51.214534+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2022-04-28T19:44:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:52 smithi139 bash[23439]: cluster 2022-04-28T19:44:52.145017+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2022-04-28T19:44:52.776 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:44:52.776 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:44:52.776 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:44:52.776 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:52.776 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:44:52.776 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:44:52.777 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 15 2022-04-28T19:44:52.778 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:44:52.778 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:44:53.157 INFO:tasks.rados.rados.0.smithi059.stdout:179: done (8 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:180: done (7 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:181: done (6 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:182: done (5 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:183: done (4 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:184: done (3 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:185: done (2 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:188: done (1 left) 2022-04-28T19:44:53.158 INFO:tasks.rados.rados.0.smithi059.stdout:190: done (0 left) 2022-04-28T19:44:53.159 INFO:tasks.rados.rados.0.smithi059.stdout:191: copy_from oid 17 from oid 8 current snap is 15 2022-04-28T19:44:53.159 INFO:tasks.rados.rados.0.smithi059.stdout:192: read oid 32 snap -1 2022-04-28T19:44:53.159 INFO:tasks.rados.rados.0.smithi059.stdout:192: expect (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:44:53.159 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 15 2022-04-28T19:44:53.159 INFO:tasks.rados.rados.0.smithi059.stdout:193: seq_num 85 ranges {1261568=630784,1892352=720896,2613248=81920} 2022-04-28T19:44:53.173 INFO:tasks.rados.rados.0.smithi059.stdout:193: writing smithi05955964-43 from 1261568 to 1892352 tid 1 2022-04-28T19:44:53.178 INFO:tasks.rados.rados.0.smithi059.stdout:193: writing smithi05955964-43 from 1892352 to 2613248 tid 2 2022-04-28T19:44:53.178 INFO:tasks.rados.rados.0.smithi059.stdout:193: writing smithi05955964-43 from 2613248 to 2695168 tid 3 2022-04-28T19:44:53.179 INFO:tasks.rados.rados.0.smithi059.stdout:194: setattr oid 39 current snap is 15 2022-04-28T19:44:53.179 INFO:tasks.rados.rados.0.smithi059.stdout:191: finishing copy_from to smithi05955964-17 2022-04-28T19:44:53.179 INFO:tasks.rados.rados.0.smithi059.stdout:191: got expected ENOENT (src dne) 2022-04-28T19:44:53.179 INFO:tasks.rados.rados.0.smithi059.stdout:191: finishing copy_from racing read to smithi05955964-17 2022-04-28T19:44:53.179 INFO:tasks.rados.rados.0.smithi059.stdout:192: expect (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:44:53.196 INFO:tasks.rados.rados.0.smithi059.stdout:193: finishing write tid 1 to smithi05955964-43 2022-04-28T19:44:53.196 INFO:tasks.rados.rados.0.smithi059.stdout:193: finishing write tid 2 to smithi05955964-43 2022-04-28T19:44:53.196 INFO:tasks.rados.rados.0.smithi059.stdout:193: finishing write tid 3 to smithi05955964-43 2022-04-28T19:44:53.196 INFO:tasks.rados.rados.0.smithi059.stdout:193: finishing write tid 5 to smithi05955964-43 2022-04-28T19:44:53.196 INFO:tasks.rados.rados.0.smithi059.stdout:193: finishing write tid 6 to smithi05955964-43 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 33 (ObjNum 85 snap 15 seq_num 85) dirty exists 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:193: left oid 43 (ObjNum 85 snap 15 seq_num 85) 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:191: done (3 left) 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:192: done (2 left) 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:193: done (1 left) 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:195: read oid 21 snap 13 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:195: expect (ObjNum 34 snap 0 seq_num 34) 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 15 2022-04-28T19:44:53.197 INFO:tasks.rados.rados.0.smithi059.stdout:196: seq_num 86 ranges {458752=647168,1105920=606208,1712128=196608} 2022-04-28T19:44:53.205 INFO:tasks.rados.rados.0.smithi059.stdout:196: writing smithi05955964-34 from 458752 to 1105920 tid 1 2022-04-28T19:44:53.209 INFO:tasks.rados.rados.0.smithi059.stdout:196: writing smithi05955964-34 from 1105920 to 1712128 tid 2 2022-04-28T19:44:53.211 INFO:tasks.rados.rados.0.smithi059.stdout:196: writing smithi05955964-34 from 1712128 to 1908736 tid 3 2022-04-28T19:44:53.211 INFO:tasks.rados.rados.0.smithi059.stdout:197: snap_create 2022-04-28T19:44:53.212 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 35 (ObjNum 38 snap 0 seq_num 38) dirty exists 2022-04-28T19:44:53.212 INFO:tasks.rados.rados.0.smithi059.stdout:195: expect (ObjNum 34 snap 0 seq_num 34) 2022-04-28T19:44:53.226 INFO:tasks.rados.rados.0.smithi059.stdout:196: finishing write tid 1 to smithi05955964-34 2022-04-28T19:44:53.226 INFO:tasks.rados.rados.0.smithi059.stdout:196: finishing write tid 2 to smithi05955964-34 2022-04-28T19:44:53.226 INFO:tasks.rados.rados.0.smithi059.stdout:196: finishing write tid 3 to smithi05955964-34 2022-04-28T19:44:53.226 INFO:tasks.rados.rados.0.smithi059.stdout:196: finishing write tid 5 to smithi05955964-34 2022-04-28T19:44:53.226 INFO:tasks.rados.rados.0.smithi059.stdout:196: finishing write tid 6 to smithi05955964-34 2022-04-28T19:44:53.227 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 40 (ObjNum 86 snap 15 seq_num 86) dirty exists 2022-04-28T19:44:53.227 INFO:tasks.rados.rados.0.smithi059.stdout:196: left oid 34 (ObjNum 86 snap 15 seq_num 86) 2022-04-28T19:44:53.516 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[29114]: cluster 2022-04-28T19:44:52.078833+0000 mgr.y (mgr.14140) 382 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 34 MiB data, 60 MiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2022-04-28T19:44:53.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[29114]: audit 2022-04-28T19:44:52.113254+0000 mgr.y (mgr.14140) 383 : audit [DBG] from='client.24739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:53.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[29114]: audit 2022-04-28T19:44:52.774217+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.59:0/3033425224' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:44:53.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[29114]: cluster 2022-04-28T19:44:53.154127+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2022-04-28T19:44:53.517 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[23786]: cluster 2022-04-28T19:44:52.078833+0000 mgr.y (mgr.14140) 382 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 34 MiB data, 60 MiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2022-04-28T19:44:53.517 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[23786]: audit 2022-04-28T19:44:52.113254+0000 mgr.y (mgr.14140) 383 : audit [DBG] from='client.24739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:53.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[23786]: audit 2022-04-28T19:44:52.774217+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.59:0/3033425224' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:44:53.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[23786]: cluster 2022-04-28T19:44:53.154127+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2022-04-28T19:44:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:53 smithi139 bash[23439]: cluster 2022-04-28T19:44:52.078833+0000 mgr.y (mgr.14140) 382 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 34 MiB data, 60 MiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2022-04-28T19:44:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:53 smithi139 bash[23439]: audit 2022-04-28T19:44:52.113254+0000 mgr.y (mgr.14140) 383 : audit [DBG] from='client.24739 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:53 smithi139 bash[23439]: audit 2022-04-28T19:44:52.774217+0000 mon.a (mon.0) 693 : audit [DBG] from='client.? 172.21.15.59:0/3033425224' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:44:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:53 smithi139 bash[23439]: cluster 2022-04-28T19:44:53.154127+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2022-04-28T19:44:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:53.514Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:44:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:53.514Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:44:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:44:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:44:53.514Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:44:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:194: done (3 left) 2022-04-28T19:44:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:195: done (2 left) 2022-04-28T19:44:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:196: done (1 left) 2022-04-28T19:44:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:197: done (0 left) 2022-04-28T19:44:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:198: snap_create 2022-04-28T19:44:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:54 smithi059 bash[29114]: audit 2022-04-28T19:44:52.428594+0000 mgr.y (mgr.14140) 384 : audit [DBG] from='client.24745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:54 smithi059 bash[29114]: cluster 2022-04-28T19:44:54.157682+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2022-04-28T19:44:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:54 smithi059 bash[23786]: audit 2022-04-28T19:44:52.428594+0000 mgr.y (mgr.14140) 384 : audit [DBG] from='client.24745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:54.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:54 smithi059 bash[23786]: cluster 2022-04-28T19:44:54.157682+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2022-04-28T19:44:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:54 smithi139 bash[23439]: audit 2022-04-28T19:44:52.428594+0000 mgr.y (mgr.14140) 384 : audit [DBG] from='client.24745 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:44:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:54 smithi139 bash[23439]: cluster 2022-04-28T19:44:54.157682+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2022-04-28T19:44:55.165 INFO:tasks.rados.rados.0.smithi059.stdout:198: done (0 left) 2022-04-28T19:44:55.165 INFO:tasks.rados.rados.0.smithi059.stdout:199: read oid 45 snap -1 2022-04-28T19:44:55.165 INFO:tasks.rados.rados.0.smithi059.stdout:199: expect (ObjNum 33 snap 0 seq_num 33) 2022-04-28T19:44:55.165 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 17 2022-04-28T19:44:55.165 INFO:tasks.rados.rados.0.smithi059.stdout:200: seq_num 87 ranges {3039232=663552,3702784=491520,4194304=311296} 2022-04-28T19:44:55.188 INFO:tasks.rados.rados.0.smithi059.stdout:200: writing smithi05955964-31 from 3039232 to 3702784 tid 1 2022-04-28T19:44:55.191 INFO:tasks.rados.rados.0.smithi059.stdout:200: writing smithi05955964-31 from 3702784 to 4194304 tid 2 2022-04-28T19:44:55.193 INFO:tasks.rados.rados.0.smithi059.stdout:200: writing smithi05955964-31 from 4194304 to 4505600 tid 3 2022-04-28T19:44:55.193 INFO:tasks.rados.rados.0.smithi059.stdout:201: read oid 49 snap -1 2022-04-28T19:44:55.193 INFO:tasks.rados.rados.0.smithi059.stdout:201: expect (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:44:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:202: read oid 38 snap -1 2022-04-28T19:44:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:202: expect (ObjNum 37 snap 0 seq_num 37) 2022-04-28T19:44:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:203: read oid 40 snap -1 2022-04-28T19:44:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:203: expect (ObjNum 39 snap 0 seq_num 39) 2022-04-28T19:44:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:204: snap_create 2022-04-28T19:44:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:199: expect (ObjNum 33 snap 0 seq_num 33) 2022-04-28T19:44:55.213 INFO:tasks.rados.rados.0.smithi059.stdout:200: finishing write tid 1 to smithi05955964-31 2022-04-28T19:44:55.213 INFO:tasks.rados.rados.0.smithi059.stdout:200: finishing write tid 2 to smithi05955964-31 2022-04-28T19:44:55.213 INFO:tasks.rados.rados.0.smithi059.stdout:200: finishing write tid 3 to smithi05955964-31 2022-04-28T19:44:55.213 INFO:tasks.rados.rados.0.smithi059.stdout:200: finishing write tid 5 to smithi05955964-31 2022-04-28T19:44:55.213 INFO:tasks.rados.rados.0.smithi059.stdout:202: expect (ObjNum 37 snap 0 seq_num 37) 2022-04-28T19:44:55.227 INFO:tasks.rados.rados.0.smithi059.stdout:201: expect (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:44:55.247 INFO:tasks.rados.rados.0.smithi059.stdout:200: finishing write tid 6 to smithi05955964-31 2022-04-28T19:44:55.247 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 81 (ObjNum 87 snap 17 seq_num 87) dirty exists 2022-04-28T19:44:55.247 INFO:tasks.rados.rados.0.smithi059.stdout:200: left oid 31 (ObjNum 87 snap 17 seq_num 87) 2022-04-28T19:44:55.247 INFO:tasks.rados.rados.0.smithi059.stdout:203: expect (ObjNum 39 snap 0 seq_num 39) 2022-04-28T19:44:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:55 smithi059 bash[23786]: cluster 2022-04-28T19:44:54.079540+0000 mgr.y (mgr.14140) 385 : cluster [DBG] pgmap v366: 89 pgs: 89 active+clean; 37 MiB data, 62 MiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 578 KiB/s wr, 4 op/s 2022-04-28T19:44:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:55 smithi059 bash[23786]: cluster 2022-04-28T19:44:55.160763+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2022-04-28T19:44:55.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:55 smithi059 bash[29114]: cluster 2022-04-28T19:44:54.079540+0000 mgr.y (mgr.14140) 385 : cluster [DBG] pgmap v366: 89 pgs: 89 active+clean; 37 MiB data, 62 MiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 578 KiB/s wr, 4 op/s 2022-04-28T19:44:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:55 smithi059 bash[29114]: cluster 2022-04-28T19:44:55.160763+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2022-04-28T19:44:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:55 smithi139 bash[23439]: cluster 2022-04-28T19:44:54.079540+0000 mgr.y (mgr.14140) 385 : cluster [DBG] pgmap v366: 89 pgs: 89 active+clean; 37 MiB data, 62 MiB used, 707 GiB / 715 GiB avail; 2.6 MiB/s rd, 578 KiB/s wr, 4 op/s 2022-04-28T19:44:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:55 smithi139 bash[23439]: cluster 2022-04-28T19:44:55.160763+0000 mon.a (mon.0) 696 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:199: done (5 left) 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:200: done (4 left) 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:201: done (3 left) 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:202: done (2 left) 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:203: done (1 left) 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:204: done (0 left) 2022-04-28T19:44:56.165 INFO:tasks.rados.rados.0.smithi059.stdout:205: snap_remove snap 5 2022-04-28T19:44:57.149 INFO:tasks.rados.rados.0.smithi059.stdout:205: done (0 left) 2022-04-28T19:44:57.149 INFO:tasks.rados.rados.0.smithi059.stdout:206: read oid 36 snap -1 2022-04-28T19:44:57.149 INFO:tasks.rados.rados.0.smithi059.stdout:206: expect (ObjNum 58 snap 3 seq_num 58) 2022-04-28T19:44:57.149 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 18 2022-04-28T19:44:57.150 INFO:tasks.rados.rados.0.smithi059.stdout:207: seq_num 88 ranges {0=679936,679936=778240,1458176=24576} 2022-04-28T19:44:57.153 INFO:tasks.rados.rados.0.smithi059.stdout:207: writing smithi05955964-1 from 0 to 679936 tid 1 2022-04-28T19:44:57.159 INFO:tasks.rados.rados.0.smithi059.stdout:207: writing smithi05955964-1 from 679936 to 1458176 tid 2 2022-04-28T19:44:57.160 INFO:tasks.rados.rados.0.smithi059.stdout:207: writing smithi05955964-1 from 1458176 to 1482752 tid 3 2022-04-28T19:44:57.160 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 18 2022-04-28T19:44:57.160 INFO:tasks.rados.rados.0.smithi059.stdout:208: seq_num 89 ranges {1351680=696320,2048000=655360,2703360=147456} 2022-04-28T19:44:57.172 INFO:tasks.rados.rados.0.smithi059.stdout:208: writing smithi05955964-9 from 1351680 to 2048000 tid 1 2022-04-28T19:44:57.176 INFO:tasks.rados.rados.0.smithi059.stdout:208: writing smithi05955964-9 from 2048000 to 2703360 tid 2 2022-04-28T19:44:57.177 INFO:tasks.rados.rados.0.smithi059.stdout:208: writing smithi05955964-9 from 2703360 to 2850816 tid 3 2022-04-28T19:44:57.178 INFO:tasks.rados.rados.0.smithi059.stdout:209: rmattr oid 11 current snap is 18 2022-04-28T19:44:57.178 INFO:tasks.rados.rados.0.smithi059.stdout:209: done (3 left) 2022-04-28T19:44:57.178 INFO:tasks.rados.rados.0.smithi059.stdout:210: snap_remove snap 13 2022-04-28T19:44:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:57 smithi059 bash[23786]: cluster 2022-04-28T19:44:56.080338+0000 mgr.y (mgr.14140) 386 : cluster [DBG] pgmap v369: 89 pgs: 89 active+clean; 40 MiB data, 66 MiB used, 707 GiB / 715 GiB avail; 79 KiB/s rd, 1.4 MiB/s wr, 7 op/s 2022-04-28T19:44:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:57 smithi059 bash[23786]: cluster 2022-04-28T19:44:56.162534+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2022-04-28T19:44:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:57 smithi059 bash[23786]: cluster 2022-04-28T19:44:57.145911+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2022-04-28T19:44:57.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:57 smithi059 bash[29114]: cluster 2022-04-28T19:44:56.080338+0000 mgr.y (mgr.14140) 386 : cluster [DBG] pgmap v369: 89 pgs: 89 active+clean; 40 MiB data, 66 MiB used, 707 GiB / 715 GiB avail; 79 KiB/s rd, 1.4 MiB/s wr, 7 op/s 2022-04-28T19:44:57.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:57 smithi059 bash[29114]: cluster 2022-04-28T19:44:56.162534+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2022-04-28T19:44:57.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:57 smithi059 bash[29114]: cluster 2022-04-28T19:44:57.145911+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2022-04-28T19:44:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:57 smithi139 bash[23439]: cluster 2022-04-28T19:44:56.080338+0000 mgr.y (mgr.14140) 386 : cluster [DBG] pgmap v369: 89 pgs: 89 active+clean; 40 MiB data, 66 MiB used, 707 GiB / 715 GiB avail; 79 KiB/s rd, 1.4 MiB/s wr, 7 op/s 2022-04-28T19:44:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:57 smithi139 bash[23439]: cluster 2022-04-28T19:44:56.162534+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2022-04-28T19:44:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:57 smithi139 bash[23439]: cluster 2022-04-28T19:44:57.145911+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2022-04-28T19:44:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:210: done (3 left) 2022-04-28T19:44:58.199 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 18 2022-04-28T19:44:58.199 INFO:tasks.rados.rados.0.smithi059.stdout:211: seq_num 90 ranges {778240=712704,1490944=540672,2031616=262144} 2022-04-28T19:44:58.200 INFO:tasks.rados.rados.0.smithi059.stdout:211: writing smithi05955964-2 from 778240 to 1490944 tid 1 2022-04-28T19:44:58.200 INFO:tasks.rados.rados.0.smithi059.stdout:211: writing smithi05955964-2 from 1490944 to 2031616 tid 2 2022-04-28T19:44:58.200 INFO:tasks.rados.rados.0.smithi059.stdout:211: writing smithi05955964-2 from 2031616 to 2293760 tid 3 2022-04-28T19:44:58.201 INFO:tasks.rados.rados.0.smithi059.stdout:212: copy_from oid 38 from oid 46 current snap is 18 2022-04-28T19:44:58.201 INFO:tasks.rados.rados.0.smithi059.stdout:206: expect (ObjNum 58 snap 3 seq_num 58) 2022-04-28T19:44:58.238 INFO:tasks.rados.rados.0.smithi059.stdout:207: finishing write tid 1 to smithi05955964-1 2022-04-28T19:44:58.238 INFO:tasks.rados.rados.0.smithi059.stdout:207: finishing write tid 2 to smithi05955964-1 2022-04-28T19:44:58.238 INFO:tasks.rados.rados.0.smithi059.stdout:207: finishing write tid 3 to smithi05955964-1 2022-04-28T19:44:58.238 INFO:tasks.rados.rados.0.smithi059.stdout:206: done (4 left) 2022-04-28T19:44:58.239 INFO:tasks.rados.rados.0.smithi059.stdout:213: snap_remove snap 16 2022-04-28T19:44:59.188 INFO:tasks.rados.rados.0.smithi059.stdout:213: done (4 left) 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:214: rmattr oid 20 current snap is 18 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:207: finishing write tid 5 to smithi05955964-1 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:207: finishing write tid 6 to smithi05955964-1 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 52 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:207: left oid 1 (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:208: finishing write tid 1 to smithi05955964-9 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:208: finishing write tid 2 to smithi05955964-9 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:208: finishing write tid 3 to smithi05955964-9 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:208: finishing write tid 5 to smithi05955964-9 2022-04-28T19:44:59.189 INFO:tasks.rados.rados.0.smithi059.stdout:208: finishing write tid 6 to smithi05955964-9 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 42 (ObjNum 89 snap 18 seq_num 89) dirty exists 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:208: left oid 9 (ObjNum 89 snap 18 seq_num 89) 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:211: finishing write tid 1 to smithi05955964-2 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:211: finishing write tid 2 to smithi05955964-2 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:211: finishing write tid 3 to smithi05955964-2 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:211: finishing write tid 5 to smithi05955964-2 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:211: finishing write tid 6 to smithi05955964-2 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 14 (ObjNum 90 snap 18 seq_num 90) dirty exists 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:211: left oid 2 (ObjNum 90 snap 18 seq_num 90) 2022-04-28T19:44:59.190 INFO:tasks.rados.rados.0.smithi059.stdout:212: finishing copy_from to smithi05955964-38 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 34 (ObjNum 77 snap 11 seq_num 77) dirty exists 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:212: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:207: done (4 left) 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:208: done (3 left) 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:211: done (2 left) 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:212: done (1 left) 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:215: rmattr oid 8 current snap is 18 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:215: done (1 left) 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:216: rmattr oid 21 current snap is 18 2022-04-28T19:44:59.191 INFO:tasks.rados.rados.0.smithi059.stdout:216: done (1 left) 2022-04-28T19:44:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:217: read oid 5 snap -1 2022-04-28T19:44:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:217: expect (ObjNum 4 snap 0 seq_num 4) 2022-04-28T19:44:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:218: read oid 48 snap -1 2022-04-28T19:44:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:218: expect (ObjNum 47 snap 0 seq_num 47) 2022-04-28T19:44:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:219: delete oid 11 current snap is 18 2022-04-28T19:44:59.194 INFO:tasks.rados.rados.0.smithi059.stdout:219: done (3 left) 2022-04-28T19:44:59.194 INFO:tasks.rados.rados.0.smithi059.stdout:220: read oid 39 snap -1 2022-04-28T19:44:59.194 INFO:tasks.rados.rados.0.smithi059.stdout:220: expect (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:59.194 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 18 2022-04-28T19:44:59.194 INFO:tasks.rados.rados.0.smithi059.stdout:221: seq_num 92 ranges {425984=745472,1171456=761856,1933312=40960} 2022-04-28T19:44:59.201 INFO:tasks.rados.rados.0.smithi059.stdout:221: writing smithi05955964-26 from 425984 to 1171456 tid 1 2022-04-28T19:44:59.206 INFO:tasks.rados.rados.0.smithi059.stdout:221: writing smithi05955964-26 from 1171456 to 1933312 tid 2 2022-04-28T19:44:59.206 INFO:tasks.rados.rados.0.smithi059.stdout:221: writing smithi05955964-26 from 1933312 to 1974272 tid 3 2022-04-28T19:44:59.206 INFO:tasks.rados.rados.0.smithi059.stdout:222: delete oid 18 current snap is 18 2022-04-28T19:44:59.206 INFO:tasks.rados.rados.0.smithi059.stdout:217: expect (ObjNum 4 snap 0 seq_num 4) 2022-04-28T19:44:59.208 INFO:tasks.rados.rados.0.smithi059.stdout:218: expect (ObjNum 47 snap 0 seq_num 47) 2022-04-28T19:44:59.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 42 (ObjNum 63 snap 4 seq_num 63) dirty exists 2022-04-28T19:44:59.228 INFO:tasks.rados.rados.0.smithi059.stdout:220: expect (ObjNum 38 snap 0 seq_num 38) 2022-04-28T19:44:59.244 INFO:tasks.rados.rados.0.smithi059.stdout:221: finishing write tid 1 to smithi05955964-26 2022-04-28T19:44:59.244 INFO:tasks.rados.rados.0.smithi059.stdout:221: finishing write tid 2 to smithi05955964-26 2022-04-28T19:44:59.244 INFO:tasks.rados.rados.0.smithi059.stdout:221: finishing write tid 3 to smithi05955964-26 2022-04-28T19:44:59.244 INFO:tasks.rados.rados.0.smithi059.stdout:221: finishing write tid 5 to smithi05955964-26 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:221: finishing write tid 6 to smithi05955964-26 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 55 (ObjNum 92 snap 18 seq_num 92) dirty exists 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:221: left oid 26 (ObjNum 92 snap 18 seq_num 92) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:214: done (5 left) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:217: done (4 left) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:218: done (3 left) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:220: done (2 left) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:221: done (1 left) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:222: done (0 left) 2022-04-28T19:44:59.245 INFO:tasks.rados.rados.0.smithi059.stdout:223: delete oid 35 current snap is 18 2022-04-28T19:44:59.247 INFO:tasks.rados.rados.0.smithi059.stdout:223: done (0 left) 2022-04-28T19:44:59.247 INFO:tasks.rados.rados.0.smithi059.stdout:224: setattr oid 29 current snap is 18 2022-04-28T19:44:59.249 INFO:tasks.rados.rados.0.smithi059.stdout:225: snap_remove snap 8 2022-04-28T19:44:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:59 smithi059 bash[23786]: cluster 2022-04-28T19:44:58.081236+0000 mgr.y (mgr.14140) 387 : cluster [DBG] pgmap v372: 89 pgs: 89 active+clean; 41 MiB data, 70 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2022-04-28T19:44:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:44:59 smithi059 bash[23786]: cluster 2022-04-28T19:44:58.173831+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2022-04-28T19:44:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:59 smithi059 bash[29114]: cluster 2022-04-28T19:44:58.081236+0000 mgr.y (mgr.14140) 387 : cluster [DBG] pgmap v372: 89 pgs: 89 active+clean; 41 MiB data, 70 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2022-04-28T19:44:59.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:44:59 smithi059 bash[29114]: cluster 2022-04-28T19:44:58.173831+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2022-04-28T19:44:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:59 smithi139 bash[23439]: cluster 2022-04-28T19:44:58.081236+0000 mgr.y (mgr.14140) 387 : cluster [DBG] pgmap v372: 89 pgs: 89 active+clean; 41 MiB data, 70 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2022-04-28T19:44:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:44:59 smithi139 bash[23439]: cluster 2022-04-28T19:44:58.173831+0000 mon.a (mon.0) 699 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2022-04-28T19:45:00.190 INFO:tasks.rados.rados.0.smithi059.stdout:225: done (1 left) 2022-04-28T19:45:00.190 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 18 2022-04-28T19:45:00.190 INFO:tasks.rados.rados.0.smithi059.stdout:226: seq_num 95 ranges {2056192=794624,2850816=409600,3260416=393216} 2022-04-28T19:45:00.207 INFO:tasks.rados.rados.0.smithi059.stdout:226: writing smithi05955964-38 from 2056192 to 2850816 tid 1 2022-04-28T19:45:00.210 INFO:tasks.rados.rados.0.smithi059.stdout:226: writing smithi05955964-38 from 2850816 to 3260416 tid 2 2022-04-28T19:45:00.213 INFO:tasks.rados.rados.0.smithi059.stdout:226: writing smithi05955964-38 from 3260416 to 3653632 tid 3 2022-04-28T19:45:00.213 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 18 2022-04-28T19:45:00.213 INFO:tasks.rados.rados.0.smithi059.stdout:227: seq_num 96 ranges {811008=409600,1220608=696320,1916928=507904} 2022-04-28T19:45:00.219 INFO:tasks.rados.rados.0.smithi059.stdout:227: writing smithi05955964-49 from 811008 to 1220608 tid 1 2022-04-28T19:45:00.223 INFO:tasks.rados.rados.0.smithi059.stdout:227: writing smithi05955964-49 from 1220608 to 1916928 tid 2 2022-04-28T19:45:00.227 INFO:tasks.rados.rados.0.smithi059.stdout:227: writing smithi05955964-49 from 1916928 to 2424832 tid 3 2022-04-28T19:45:00.227 INFO:tasks.rados.rados.0.smithi059.stdout:228: copy_from oid 26 from oid 14 current snap is 18 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 57 (ObjNum -133641296 snap 21893 seq_num 4161326000) dirty exists 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:226: finishing write tid 1 to smithi05955964-38 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:226: finishing write tid 2 to smithi05955964-38 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:226: finishing write tid 3 to smithi05955964-38 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:226: finishing write tid 5 to smithi05955964-38 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:226: finishing write tid 6 to smithi05955964-38 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 39 (ObjNum 95 snap 18 seq_num 95) dirty exists 2022-04-28T19:45:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:226: left oid 38 (ObjNum 95 snap 18 seq_num 95) 2022-04-28T19:45:00.229 INFO:tasks.rados.rados.0.smithi059.stdout:227: finishing write tid 1 to smithi05955964-49 2022-04-28T19:45:00.229 INFO:tasks.rados.rados.0.smithi059.stdout:227: finishing write tid 2 to smithi05955964-49 2022-04-28T19:45:00.229 INFO:tasks.rados.rados.0.smithi059.stdout:224: done (3 left) 2022-04-28T19:45:00.229 INFO:tasks.rados.rados.0.smithi059.stdout:226: done (2 left) 2022-04-28T19:45:00.229 INFO:tasks.rados.rados.0.smithi059.stdout:229: snap_create 2022-04-28T19:45:00.231 INFO:tasks.rados.rados.0.smithi059.stdout:227: finishing write tid 3 to smithi05955964-49 2022-04-28T19:45:00.231 INFO:tasks.rados.rados.0.smithi059.stdout:227: finishing write tid 5 to smithi05955964-49 2022-04-28T19:45:00.231 INFO:tasks.rados.rados.0.smithi059.stdout:228: finishing copy_from to smithi05955964-26 2022-04-28T19:45:00.232 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 18 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:45:00.232 INFO:tasks.rados.rados.0.smithi059.stdout:228: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:45:00.233 INFO:tasks.rados.rados.0.smithi059.stdout:227: finishing write tid 6 to smithi05955964-49 2022-04-28T19:45:00.233 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 57 (ObjNum 96 snap 18 seq_num 96) dirty exists 2022-04-28T19:45:00.233 INFO:tasks.rados.rados.0.smithi059.stdout:227: left oid 49 (ObjNum 96 snap 18 seq_num 96) 2022-04-28T19:45:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:00 smithi059 bash[23786]: cluster 2022-04-28T19:44:59.185337+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2022-04-28T19:45:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:00 smithi059 bash[29114]: cluster 2022-04-28T19:44:59.185337+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2022-04-28T19:45:00.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:00 smithi139 bash[23439]: cluster 2022-04-28T19:44:59.185337+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2022-04-28T19:45:01.206 INFO:tasks.rados.rados.0.smithi059.stdout:227: done (2 left) 2022-04-28T19:45:01.207 INFO:tasks.rados.rados.0.smithi059.stdout:228: done (1 left) 2022-04-28T19:45:01.207 INFO:tasks.rados.rados.0.smithi059.stdout:229: done (0 left) 2022-04-28T19:45:01.207 INFO:tasks.rados.rados.0.smithi059.stdout:230: snap_create 2022-04-28T19:45:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:01 smithi059 bash[29114]: cluster 2022-04-28T19:45:00.082173+0000 mgr.y (mgr.14140) 388 : cluster [DBG] pgmap v375: 89 pgs: 89 active+clean; 45 MiB data, 76 MiB used, 707 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.4 MiB/s wr, 20 op/s 2022-04-28T19:45:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:01 smithi059 bash[29114]: cluster 2022-04-28T19:45:00.185423+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2022-04-28T19:45:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:01 smithi059 bash[23786]: cluster 2022-04-28T19:45:00.082173+0000 mgr.y (mgr.14140) 388 : cluster [DBG] pgmap v375: 89 pgs: 89 active+clean; 45 MiB data, 76 MiB used, 707 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.4 MiB/s wr, 20 op/s 2022-04-28T19:45:01.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:01 smithi059 bash[23786]: cluster 2022-04-28T19:45:00.185423+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2022-04-28T19:45:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:01 smithi139 bash[23439]: cluster 2022-04-28T19:45:00.082173+0000 mgr.y (mgr.14140) 388 : cluster [DBG] pgmap v375: 89 pgs: 89 active+clean; 45 MiB data, 76 MiB used, 707 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.4 MiB/s wr, 20 op/s 2022-04-28T19:45:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:01 smithi139 bash[23439]: cluster 2022-04-28T19:45:00.185423+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2022-04-28T19:45:02.151 INFO:tasks.rados.rados.0.smithi059.stdout:230: done (0 left) 2022-04-28T19:45:02.151 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 20 2022-04-28T19:45:02.151 INFO:tasks.rados.rados.0.smithi059.stdout:231: seq_num 97 ranges {270336=434176,704512=581632,1286144=540672,1826816=81920} 2022-04-28T19:45:02.154 INFO:tasks.rados.rados.0.smithi059.stdout:231: writing smithi05955964-17 from 270336 to 704512 tid 1 2022-04-28T19:45:02.158 INFO:tasks.rados.rados.0.smithi059.stdout:231: writing smithi05955964-17 from 704512 to 1286144 tid 2 2022-04-28T19:45:02.161 INFO:tasks.rados.rados.0.smithi059.stdout:231: writing smithi05955964-17 from 1286144 to 1826816 tid 3 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:231: writing smithi05955964-17 from 1826816 to 1908736 tid 4 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:232: rollback oid 8 current snap is 20 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 8 to 19 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:231: finishing write tid 1 to smithi05955964-17 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:233: rmattr oid 26 current snap is 20 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:233: done (2 left) 2022-04-28T19:45:02.162 INFO:tasks.rados.rados.0.smithi059.stdout:234: delete oid 2 current snap is 20 2022-04-28T19:45:02.163 INFO:tasks.rados.rados.0.smithi059.stdout:231: finishing write tid 2 to smithi05955964-17 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:231: finishing write tid 3 to smithi05955964-17 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:234: done (2 left) 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:235: rmattr oid 24 current snap is 20 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:235: done (2 left) 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:236: snap_create 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:231: finishing write tid 4 to smithi05955964-17 2022-04-28T19:45:02.169 INFO:tasks.rados.rados.0.smithi059.stdout:232: finishing rollback tid 1 to smithi05955964-8 2022-04-28T19:45:02.170 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 42 (ObjNum -133355536 snap 21893 seq_num 0) dirty dne 2022-04-28T19:45:02.170 INFO:tasks.rados.rados.0.smithi059.stdout:231: finishing write tid 6 to smithi05955964-17 2022-04-28T19:45:02.170 INFO:tasks.rados.rados.0.smithi059.stdout:231: finishing write tid 7 to smithi05955964-17 2022-04-28T19:45:02.170 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 26 (ObjNum 97 snap 20 seq_num 97) dirty exists 2022-04-28T19:45:02.170 INFO:tasks.rados.rados.0.smithi059.stdout:231: left oid 17 (ObjNum 97 snap 20 seq_num 97) 2022-04-28T19:45:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:02 smithi059 bash[23786]: cluster 2022-04-28T19:45:01.204457+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2022-04-28T19:45:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:02 smithi059 bash[23786]: cluster 2022-04-28T19:45:02.148252+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2022-04-28T19:45:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:02 smithi059 bash[29114]: cluster 2022-04-28T19:45:01.204457+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2022-04-28T19:45:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:02 smithi059 bash[29114]: cluster 2022-04-28T19:45:02.148252+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2022-04-28T19:45:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:02 smithi139 bash[23439]: cluster 2022-04-28T19:45:01.204457+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2022-04-28T19:45:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:02 smithi139 bash[23439]: cluster 2022-04-28T19:45:02.148252+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2022-04-28T19:45:03.166 INFO:tasks.rados.rados.0.smithi059.stdout:231: done (2 left) 2022-04-28T19:45:03.166 INFO:tasks.rados.rados.0.smithi059.stdout:232: done (1 left) 2022-04-28T19:45:03.167 INFO:tasks.rados.rados.0.smithi059.stdout:236: done (0 left) 2022-04-28T19:45:03.167 INFO:tasks.rados.rados.0.smithi059.stdout:237: copy_from oid 48 from oid 47 current snap is 21 2022-04-28T19:45:03.167 INFO:tasks.rados.rados.0.smithi059.stdout:238: read oid 34 snap -1 2022-04-28T19:45:03.167 INFO:tasks.rados.rados.0.smithi059.stdout:238: expect (ObjNum 86 snap 15 seq_num 86) 2022-04-28T19:45:03.167 INFO:tasks.rados.rados.0.smithi059.stdout:239: snap_create 2022-04-28T19:45:03.179 INFO:tasks.rados.rados.0.smithi059.stdout:237: finishing copy_from to smithi05955964-48 2022-04-28T19:45:03.179 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2022-04-28T19:45:03.180 INFO:tasks.rados.rados.0.smithi059.stdout:237: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:45:03.181 INFO:tasks.rados.rados.0.smithi059.stdout:238: expect (ObjNum 86 snap 15 seq_num 86) 2022-04-28T19:45:03.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[23786]: cluster 2022-04-28T19:45:02.082937+0000 mgr.y (mgr.14140) 389 : cluster [DBG] pgmap v378: 89 pgs: 89 active+clean; 47 MiB data, 78 MiB used, 707 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2022-04-28T19:45:03.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[23786]: cluster 2022-04-28T19:45:03.163180+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2022-04-28T19:45:03.519 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[29114]: cluster 2022-04-28T19:45:02.082937+0000 mgr.y (mgr.14140) 389 : cluster [DBG] pgmap v378: 89 pgs: 89 active+clean; 47 MiB data, 78 MiB used, 707 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2022-04-28T19:45:03.519 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[29114]: cluster 2022-04-28T19:45:03.163180+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2022-04-28T19:45:03.519 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:03.515Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:45:03.520 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:03.515Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:45:03.520 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:03.515Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:45:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:03 smithi139 bash[23439]: cluster 2022-04-28T19:45:02.082937+0000 mgr.y (mgr.14140) 389 : cluster [DBG] pgmap v378: 89 pgs: 89 active+clean; 47 MiB data, 78 MiB used, 707 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2022-04-28T19:45:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:03 smithi139 bash[23439]: cluster 2022-04-28T19:45:03.163180+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2022-04-28T19:45:04.166 INFO:tasks.rados.rados.0.smithi059.stdout:237: done (2 left) 2022-04-28T19:45:04.166 INFO:tasks.rados.rados.0.smithi059.stdout:238: done (1 left) 2022-04-28T19:45:04.166 INFO:tasks.rados.rados.0.smithi059.stdout:239: done (0 left) 2022-04-28T19:45:04.166 INFO:tasks.rados.rados.0.smithi059.stdout:240: read oid 47 snap -1 2022-04-28T19:45:04.166 INFO:tasks.rados.rados.0.smithi059.stdout:240: expect (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:45:04.167 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 22 2022-04-28T19:45:04.167 INFO:tasks.rados.rados.0.smithi059.stdout:241: seq_num 99 ranges {557056=466944,1024000=802816,1826816=401408} 2022-04-28T19:45:04.171 INFO:tasks.rados.rados.0.smithi059.stdout:241: writing smithi05955964-45 from 557056 to 1024000 tid 1 2022-04-28T19:45:04.177 INFO:tasks.rados.rados.0.smithi059.stdout:241: writing smithi05955964-45 from 1024000 to 1826816 tid 2 2022-04-28T19:45:04.179 INFO:tasks.rados.rados.0.smithi059.stdout:241: writing smithi05955964-45 from 1826816 to 2228224 tid 3 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:242: read oid 49 snap 9 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:242: expect (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:243: read oid 28 snap -1 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:243: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:244: rollback oid 29 current snap is 22 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 15 2022-04-28T19:45:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:245: snap_remove snap 21 2022-04-28T19:45:05.173 INFO:tasks.rados.rados.0.smithi059.stdout:245: done (5 left) 2022-04-28T19:45:05.173 INFO:tasks.rados.rados.0.smithi059.stdout:246: rollback oid 11 current snap is 22 2022-04-28T19:45:05.173 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 14 2022-04-28T19:45:05.173 INFO:tasks.rados.rados.0.smithi059.stdout:240: expect (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:45:05.192 INFO:tasks.rados.rados.0.smithi059.stdout:241: finishing write tid 1 to smithi05955964-45 2022-04-28T19:45:05.192 INFO:tasks.rados.rados.0.smithi059.stdout:241: finishing write tid 2 to smithi05955964-45 2022-04-28T19:45:05.192 INFO:tasks.rados.rados.0.smithi059.stdout:240: done (5 left) 2022-04-28T19:45:05.192 INFO:tasks.rados.rados.0.smithi059.stdout:247: snap_remove snap 11 2022-04-28T19:45:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:05 smithi059 bash[29114]: cluster 2022-04-28T19:45:04.083661+0000 mgr.y (mgr.14140) 390 : cluster [DBG] pgmap v381: 89 pgs: 89 active+clean; 51 MiB data, 83 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2022-04-28T19:45:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:05 smithi059 bash[29114]: cluster 2022-04-28T19:45:04.162842+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2022-04-28T19:45:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:05 smithi059 bash[23786]: cluster 2022-04-28T19:45:04.083661+0000 mgr.y (mgr.14140) 390 : cluster [DBG] pgmap v381: 89 pgs: 89 active+clean; 51 MiB data, 83 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2022-04-28T19:45:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:05 smithi059 bash[23786]: cluster 2022-04-28T19:45:04.162842+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2022-04-28T19:45:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:05 smithi139 bash[23439]: cluster 2022-04-28T19:45:04.083661+0000 mgr.y (mgr.14140) 390 : cluster [DBG] pgmap v381: 89 pgs: 89 active+clean; 51 MiB data, 83 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2022-04-28T19:45:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:05 smithi139 bash[23439]: cluster 2022-04-28T19:45:04.162842+0000 mon.a (mon.0) 705 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2022-04-28T19:45:06.176 INFO:tasks.rados.rados.0.smithi059.stdout:247: done (5 left) 2022-04-28T19:45:06.176 INFO:tasks.rados.rados.0.smithi059.stdout:248: snap_create 2022-04-28T19:45:06.176 INFO:tasks.rados.rados.0.smithi059.stdout:241: finishing write tid 3 to smithi05955964-45 2022-04-28T19:45:06.176 INFO:tasks.rados.rados.0.smithi059.stdout:241: finishing write tid 5 to smithi05955964-45 2022-04-28T19:45:06.176 INFO:tasks.rados.rados.0.smithi059.stdout:242: expect (ObjNum 48 snap 0 seq_num 48) 2022-04-28T19:45:06.202 INFO:tasks.rados.rados.0.smithi059.stdout:241: finishing write tid 6 to smithi05955964-45 2022-04-28T19:45:06.202 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 49 (ObjNum 99 snap 22 seq_num 99) dirty exists 2022-04-28T19:45:06.203 INFO:tasks.rados.rados.0.smithi059.stdout:241: left oid 45 (ObjNum 99 snap 22 seq_num 99) 2022-04-28T19:45:06.203 INFO:tasks.rados.rados.0.smithi059.stdout:243: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:45:06.208 INFO:tasks.rados.rados.0.smithi059.stdout:244: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:45:06.208 INFO:tasks.rados.rados.0.smithi059.stdout:244: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:45:06.208 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 65 (ObjNum 188342176 snap 32762 seq_num 188342176) dirty dne 2022-04-28T19:45:06.208 INFO:tasks.rados.rados.0.smithi059.stdout:246: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:45:06.208 INFO:tasks.rados.rados.0.smithi059.stdout:246: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:45:06.208 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 41 (ObjNum 10 snap 0 seq_num 10) dirty exists 2022-04-28T19:45:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:06 smithi059 bash[29114]: cluster 2022-04-28T19:45:05.170400+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2022-04-28T19:45:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:06 smithi059 bash[23786]: cluster 2022-04-28T19:45:05.170400+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2022-04-28T19:45:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:06 smithi139 bash[23439]: cluster 2022-04-28T19:45:05.170400+0000 mon.a (mon.0) 706 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2022-04-28T19:45:07.158 INFO:tasks.rados.rados.0.smithi059.stdout:241: done (5 left) 2022-04-28T19:45:07.158 INFO:tasks.rados.rados.0.smithi059.stdout:242: done (4 left) 2022-04-28T19:45:07.158 INFO:tasks.rados.rados.0.smithi059.stdout:243: done (3 left) 2022-04-28T19:45:07.158 INFO:tasks.rados.rados.0.smithi059.stdout:244: done (2 left) 2022-04-28T19:45:07.158 INFO:tasks.rados.rados.0.smithi059.stdout:246: done (1 left) 2022-04-28T19:45:07.158 INFO:tasks.rados.rados.0.smithi059.stdout:248: done (0 left) 2022-04-28T19:45:07.159 INFO:tasks.rados.rados.0.smithi059.stdout:249: snap_create 2022-04-28T19:45:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:07 smithi059 bash[29114]: cluster 2022-04-28T19:45:06.084509+0000 mgr.y (mgr.14140) 391 : cluster [DBG] pgmap v384: 89 pgs: 89 active+clean; 54 MiB data, 88 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2022-04-28T19:45:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:07 smithi059 bash[29114]: cluster 2022-04-28T19:45:06.173832+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2022-04-28T19:45:07.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:07 smithi059 bash[29114]: cluster 2022-04-28T19:45:07.155932+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2022-04-28T19:45:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:07 smithi059 bash[23786]: cluster 2022-04-28T19:45:06.084509+0000 mgr.y (mgr.14140) 391 : cluster [DBG] pgmap v384: 89 pgs: 89 active+clean; 54 MiB data, 88 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2022-04-28T19:45:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:07 smithi059 bash[23786]: cluster 2022-04-28T19:45:06.173832+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2022-04-28T19:45:07.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:07 smithi059 bash[23786]: cluster 2022-04-28T19:45:07.155932+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2022-04-28T19:45:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:07 smithi139 bash[23439]: cluster 2022-04-28T19:45:06.084509+0000 mgr.y (mgr.14140) 391 : cluster [DBG] pgmap v384: 89 pgs: 89 active+clean; 54 MiB data, 88 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2022-04-28T19:45:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:07 smithi139 bash[23439]: cluster 2022-04-28T19:45:06.173832+0000 mon.a (mon.0) 707 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2022-04-28T19:45:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:07 smithi139 bash[23439]: cluster 2022-04-28T19:45:07.155932+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2022-04-28T19:45:08.172 INFO:tasks.rados.rados.0.smithi059.stdout:249: done (0 left) 2022-04-28T19:45:08.172 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 24 2022-04-28T19:45:08.172 INFO:tasks.rados.rados.0.smithi059.stdout:250: seq_num 100 ranges {491520=483328,974848=688128,1662976=516096} 2022-04-28T19:45:08.180 INFO:tasks.rados.rados.0.smithi059.stdout:250: writing smithi05955964-30 from 491520 to 974848 tid 1 2022-04-28T19:45:08.185 INFO:tasks.rados.rados.0.smithi059.stdout:250: writing smithi05955964-30 from 974848 to 1662976 tid 2 2022-04-28T19:45:08.189 INFO:tasks.rados.rados.0.smithi059.stdout:250: writing smithi05955964-30 from 1662976 to 2179072 tid 3 2022-04-28T19:45:08.190 INFO:tasks.rados.rados.0.smithi059.stdout:251: rmattr oid 17 current snap is 24 2022-04-28T19:45:08.190 INFO:tasks.rados.rados.0.smithi059.stdout:251: done (1 left) 2022-04-28T19:45:08.190 INFO:tasks.rados.rados.0.smithi059.stdout:252: snap_remove snap 9 2022-04-28T19:45:09.198 INFO:tasks.rados.rados.0.smithi059.stdout:252: done (1 left) 2022-04-28T19:45:09.199 INFO:tasks.rados.rados.0.smithi059.stdout:253: snap_remove snap 19 2022-04-28T19:45:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:09 smithi059 bash[29114]: cluster 2022-04-28T19:45:08.085455+0000 mgr.y (mgr.14140) 392 : cluster [DBG] pgmap v387: 89 pgs: 89 active+clean; 55 MiB data, 91 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2022-04-28T19:45:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:09 smithi059 bash[29114]: cluster 2022-04-28T19:45:08.166730+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2022-04-28T19:45:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:09 smithi059 bash[23786]: cluster 2022-04-28T19:45:08.085455+0000 mgr.y (mgr.14140) 392 : cluster [DBG] pgmap v387: 89 pgs: 89 active+clean; 55 MiB data, 91 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2022-04-28T19:45:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:09 smithi059 bash[23786]: cluster 2022-04-28T19:45:08.166730+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2022-04-28T19:45:09.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:09 smithi139 bash[23439]: cluster 2022-04-28T19:45:08.085455+0000 mgr.y (mgr.14140) 392 : cluster [DBG] pgmap v387: 89 pgs: 89 active+clean; 55 MiB data, 91 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2022-04-28T19:45:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:09 smithi139 bash[23439]: cluster 2022-04-28T19:45:08.166730+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2022-04-28T19:45:10.206 INFO:tasks.rados.rados.0.smithi059.stdout:253: done (1 left) 2022-04-28T19:45:10.206 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 24 2022-04-28T19:45:10.206 INFO:tasks.rados.rados.0.smithi059.stdout:254: seq_num 101 ranges {0=499712,499712=573440,1073152=524288,1597440=106496} 2022-04-28T19:45:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:254: writing smithi05955964-6 from 0 to 499712 tid 1 2022-04-28T19:45:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:254: writing smithi05955964-6 from 499712 to 1073152 tid 2 2022-04-28T19:45:10.216 INFO:tasks.rados.rados.0.smithi059.stdout:254: writing smithi05955964-6 from 1073152 to 1597440 tid 3 2022-04-28T19:45:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:254: writing smithi05955964-6 from 1597440 to 1703936 tid 4 2022-04-28T19:45:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:250: finishing write tid 1 to smithi05955964-30 2022-04-28T19:45:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:255: read oid 11 snap -1 2022-04-28T19:45:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:255: expect (ObjNum 10 snap 0 seq_num 10) 2022-04-28T19:45:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 24 2022-04-28T19:45:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:256: seq_num 102 ranges {0=516096,516096=450560,966656=729088,1695744=24576} 2022-04-28T19:45:10.220 INFO:tasks.rados.rados.0.smithi059.stdout:256: writing smithi05955964-28 from 0 to 516096 tid 1 2022-04-28T19:45:10.224 INFO:tasks.rados.rados.0.smithi059.stdout:256: writing smithi05955964-28 from 516096 to 966656 tid 2 2022-04-28T19:45:10.229 INFO:tasks.rados.rados.0.smithi059.stdout:256: writing smithi05955964-28 from 966656 to 1695744 tid 3 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:256: writing smithi05955964-28 from 1695744 to 1720320 tid 4 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:257: snap_create 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:250: finishing write tid 2 to smithi05955964-30 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:250: finishing write tid 3 to smithi05955964-30 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:250: finishing write tid 5 to smithi05955964-30 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:250: finishing write tid 6 to smithi05955964-30 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 54 (ObjNum 100 snap 24 seq_num 100) dirty exists 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:250: left oid 30 (ObjNum 100 snap 24 seq_num 100) 2022-04-28T19:45:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:254: finishing write tid 1 to smithi05955964-6 2022-04-28T19:45:10.231 INFO:tasks.rados.rados.0.smithi059.stdout:254: finishing write tid 2 to smithi05955964-6 2022-04-28T19:45:10.231 INFO:tasks.rados.rados.0.smithi059.stdout:254: finishing write tid 3 to smithi05955964-6 2022-04-28T19:45:10.231 INFO:tasks.rados.rados.0.smithi059.stdout:254: finishing write tid 4 to smithi05955964-6 2022-04-28T19:45:10.231 INFO:tasks.rados.rados.0.smithi059.stdout:255: expect (ObjNum 10 snap 0 seq_num 10) 2022-04-28T19:45:10.233 INFO:tasks.rados.rados.0.smithi059.stdout:254: finishing write tid 6 to smithi05955964-6 2022-04-28T19:45:10.234 INFO:tasks.rados.rados.0.smithi059.stdout:254: finishing write tid 7 to smithi05955964-6 2022-04-28T19:45:10.234 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 66 (ObjNum 101 snap 24 seq_num 101) dirty exists 2022-04-28T19:45:10.234 INFO:tasks.rados.rados.0.smithi059.stdout:254: left oid 6 (ObjNum 101 snap 24 seq_num 101) 2022-04-28T19:45:10.234 INFO:tasks.rados.rados.0.smithi059.stdout:256: finishing write tid 1 to smithi05955964-28 2022-04-28T19:45:10.234 INFO:tasks.rados.rados.0.smithi059.stdout:256: finishing write tid 2 to smithi05955964-28 2022-04-28T19:45:10.235 INFO:tasks.rados.rados.0.smithi059.stdout:256: finishing write tid 3 to smithi05955964-28 2022-04-28T19:45:10.235 INFO:tasks.rados.rados.0.smithi059.stdout:256: finishing write tid 4 to smithi05955964-28 2022-04-28T19:45:10.235 INFO:tasks.rados.rados.0.smithi059.stdout:256: finishing write tid 6 to smithi05955964-28 2022-04-28T19:45:10.240 INFO:tasks.rados.rados.0.smithi059.stdout:256: finishing write tid 7 to smithi05955964-28 2022-04-28T19:45:10.240 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 61 (ObjNum 102 snap 24 seq_num 102) dirty exists 2022-04-28T19:45:10.240 INFO:tasks.rados.rados.0.smithi059.stdout:256: left oid 28 (ObjNum 102 snap 24 seq_num 102) 2022-04-28T19:45:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:10 smithi059 bash[29114]: cluster 2022-04-28T19:45:09.196418+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2022-04-28T19:45:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:10 smithi059 bash[23786]: cluster 2022-04-28T19:45:09.196418+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2022-04-28T19:45:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:10 smithi139 bash[23439]: cluster 2022-04-28T19:45:09.196418+0000 mon.a (mon.0) 710 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2022-04-28T19:45:11.216 INFO:tasks.rados.rados.0.smithi059.stdout:250: done (4 left) 2022-04-28T19:45:11.217 INFO:tasks.rados.rados.0.smithi059.stdout:254: done (3 left) 2022-04-28T19:45:11.217 INFO:tasks.rados.rados.0.smithi059.stdout:255: done (2 left) 2022-04-28T19:45:11.217 INFO:tasks.rados.rados.0.smithi059.stdout:256: done (1 left) 2022-04-28T19:45:11.217 INFO:tasks.rados.rados.0.smithi059.stdout:257: done (0 left) 2022-04-28T19:45:11.217 INFO:tasks.rados.rados.0.smithi059.stdout:258: delete oid 41 current snap is 25 2022-04-28T19:45:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:258: done (0 left) 2022-04-28T19:45:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:259: read oid 13 snap -1 2022-04-28T19:45:11.223 INFO:tasks.rados.rados.0.smithi059.stdout:259: expect (ObjNum 83 snap 14 seq_num 83) 2022-04-28T19:45:11.223 INFO:tasks.rados.rados.0.smithi059.stdout:260: setattr oid 26 current snap is 25 2022-04-28T19:45:11.224 INFO:tasks.rados.rados.0.smithi059.stdout:261: copy_from oid 32 from oid 7 current snap is 25 2022-04-28T19:45:11.224 INFO:tasks.rados.rados.0.smithi059.stdout:262: read oid 22 snap -1 2022-04-28T19:45:11.224 INFO:tasks.rados.rados.0.smithi059.stdout:262: expect deleted 2022-04-28T19:45:11.224 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 25 2022-04-28T19:45:11.225 INFO:tasks.rados.rados.0.smithi059.stdout:263: seq_num 104 ranges {73728=548864,622592=622592,1245184=581632} 2022-04-28T19:45:11.228 INFO:tasks.rados.rados.0.smithi059.stdout:263: writing smithi05955964-15 from 73728 to 622592 tid 1 2022-04-28T19:45:11.232 INFO:tasks.rados.rados.0.smithi059.stdout:263: writing smithi05955964-15 from 622592 to 1245184 tid 2 2022-04-28T19:45:11.236 INFO:tasks.rados.rados.0.smithi059.stdout:263: writing smithi05955964-15 from 1245184 to 1826816 tid 3 2022-04-28T19:45:11.236 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 25 2022-04-28T19:45:11.236 INFO:tasks.rados.rados.0.smithi059.stdout:264: seq_num 105 ranges {2424832=565248,2990080=507904,3497984=507904,4005888=188416} 2022-04-28T19:45:11.252 INFO:tasks.rados.rados.0.smithi059.stdout:264: writing smithi05955964-49 from 2424832 to 2990080 tid 1 2022-04-28T19:45:11.255 INFO:tasks.rados.rados.0.smithi059.stdout:264: writing smithi05955964-49 from 2990080 to 3497984 tid 2 2022-04-28T19:45:11.258 INFO:tasks.rados.rados.0.smithi059.stdout:264: writing smithi05955964-49 from 3497984 to 4005888 tid 3 2022-04-28T19:45:11.260 INFO:tasks.rados.rados.0.smithi059.stdout:264: writing smithi05955964-49 from 4005888 to 4194304 tid 4 2022-04-28T19:45:11.260 INFO:tasks.rados.rados.0.smithi059.stdout:265: rollback oid 33 current snap is 25 2022-04-28T19:45:11.260 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 20 2022-04-28T19:45:11.260 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 82 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:45:11.261 INFO:tasks.rados.rados.0.smithi059.stdout:261: finishing copy_from to smithi05955964-32 2022-04-28T19:45:11.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2022-04-28T19:45:11.261 INFO:tasks.rados.rados.0.smithi059.stdout:261: finishing copy_from racing read to smithi05955964-32 2022-04-28T19:45:11.261 INFO:tasks.rados.rados.0.smithi059.stdout:259: expect (ObjNum 83 snap 14 seq_num 83) 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:263: finishing write tid 1 to smithi05955964-15 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:263: finishing write tid 2 to smithi05955964-15 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:263: finishing write tid 3 to smithi05955964-15 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:263: finishing write tid 5 to smithi05955964-15 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:263: finishing write tid 6 to smithi05955964-15 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 114 (ObjNum 104 snap 25 seq_num 104) dirty exists 2022-04-28T19:45:11.294 INFO:tasks.rados.rados.0.smithi059.stdout:263: left oid 15 (ObjNum 104 snap 25 seq_num 104) 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:264: finishing write tid 1 to smithi05955964-49 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:259: done (6 left) 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:260: done (5 left) 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:261: done (4 left) 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:262: done (3 left) 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:263: done (2 left) 2022-04-28T19:45:11.295 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 25 2022-04-28T19:45:11.296 INFO:tasks.rados.rados.0.smithi059.stdout:266: seq_num 106 ranges {638976=581632,1220608=794624,2015232=409600} 2022-04-28T19:45:11.306 INFO:tasks.rados.rados.0.smithi059.stdout:266: writing smithi05955964-10 from 638976 to 1220608 tid 1 2022-04-28T19:45:11.312 INFO:tasks.rados.rados.0.smithi059.stdout:266: writing smithi05955964-10 from 1220608 to 2015232 tid 2 2022-04-28T19:45:11.315 INFO:tasks.rados.rados.0.smithi059.stdout:266: writing smithi05955964-10 from 2015232 to 2424832 tid 3 2022-04-28T19:45:11.315 INFO:tasks.rados.rados.0.smithi059.stdout:267: read oid 13 snap -1 2022-04-28T19:45:11.315 INFO:tasks.rados.rados.0.smithi059.stdout:267: expect (ObjNum 83 snap 14 seq_num 83) 2022-04-28T19:45:11.315 INFO:tasks.rados.rados.0.smithi059.stdout:268: rollback oid 26 current snap is 25 2022-04-28T19:45:11.316 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 18 2022-04-28T19:45:11.316 INFO:tasks.rados.rados.0.smithi059.stdout:269: rmattr oid 42 current snap is 25 2022-04-28T19:45:11.316 INFO:tasks.rados.rados.0.smithi059.stdout:269: done (5 left) 2022-04-28T19:45:11.316 INFO:tasks.rados.rados.0.smithi059.stdout:270: copy_from oid 46 from oid 16 current snap is 25 2022-04-28T19:45:11.316 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 25 2022-04-28T19:45:11.316 INFO:tasks.rados.rados.0.smithi059.stdout:271: seq_num 107 ranges {0=598016,598016=729088,1327104=466944,1794048=8192} 2022-04-28T19:45:11.318 INFO:tasks.rados.rados.0.smithi059.stdout:271: writing smithi05955964-18 from 0 to 598016 tid 1 2022-04-28T19:45:11.323 INFO:tasks.rados.rados.0.smithi059.stdout:271: writing smithi05955964-18 from 598016 to 1327104 tid 2 2022-04-28T19:45:11.326 INFO:tasks.rados.rados.0.smithi059.stdout:271: writing smithi05955964-18 from 1327104 to 1794048 tid 3 2022-04-28T19:45:11.326 INFO:tasks.rados.rados.0.smithi059.stdout:271: writing smithi05955964-18 from 1794048 to 1802240 tid 4 2022-04-28T19:45:11.327 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 25 2022-04-28T19:45:11.327 INFO:tasks.rados.rados.0.smithi059.stdout:272: seq_num 108 ranges {606208=614400,1220608=606208,1826816=598016} 2022-04-28T19:45:11.333 INFO:tasks.rados.rados.0.smithi059.stdout:272: writing smithi05955964-37 from 606208 to 1220608 tid 1 2022-04-28T19:45:11.337 INFO:tasks.rados.rados.0.smithi059.stdout:272: writing smithi05955964-37 from 1220608 to 1826816 tid 2 2022-04-28T19:45:11.341 INFO:tasks.rados.rados.0.smithi059.stdout:272: writing smithi05955964-37 from 1826816 to 2424832 tid 3 2022-04-28T19:45:11.341 INFO:tasks.rados.rados.0.smithi059.stdout:273: read oid 29 snap 23 2022-04-28T19:45:11.341 INFO:tasks.rados.rados.0.smithi059.stdout:273: expect deleted 2022-04-28T19:45:11.342 INFO:tasks.rados.rados.0.smithi059.stdout:274: rmattr oid 38 current snap is 25 2022-04-28T19:45:11.342 INFO:tasks.rados.rados.0.smithi059.stdout:274: done (9 left) 2022-04-28T19:45:11.342 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 25 2022-04-28T19:45:11.342 INFO:tasks.rados.rados.0.smithi059.stdout:275: seq_num 109 ranges {1318912=630784,1949696=491520,2441216=483328,2924544=229376} 2022-04-28T19:45:11.352 INFO:tasks.rados.rados.0.smithi059.stdout:275: writing smithi05955964-44 from 1318912 to 1949696 tid 1 2022-04-28T19:45:11.355 INFO:tasks.rados.rados.0.smithi059.stdout:275: writing smithi05955964-44 from 1949696 to 2441216 tid 2 2022-04-28T19:45:11.358 INFO:tasks.rados.rados.0.smithi059.stdout:275: writing smithi05955964-44 from 2441216 to 2924544 tid 3 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:275: writing smithi05955964-44 from 2924544 to 3153920 tid 4 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:276: read oid 48 snap -1 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:276: expect (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:277: read oid 21 snap 17 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:277: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 25 2022-04-28T19:45:11.360 INFO:tasks.rados.rados.0.smithi059.stdout:278: seq_num 110 ranges {393216=647168,1040384=778240,1818624=425984} 2022-04-28T19:45:11.366 INFO:tasks.rados.rados.0.smithi059.stdout:278: writing smithi05955964-32 from 393216 to 1040384 tid 1 2022-04-28T19:45:11.370 INFO:tasks.rados.rados.0.smithi059.stdout:278: writing smithi05955964-32 from 1040384 to 1818624 tid 2 2022-04-28T19:45:11.373 INFO:tasks.rados.rados.0.smithi059.stdout:278: writing smithi05955964-32 from 1818624 to 2244608 tid 3 2022-04-28T19:45:11.373 INFO:tasks.rados.rados.0.smithi059.stdout:279: snap_remove snap 14 2022-04-28T19:45:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:11 smithi059 bash[29114]: cluster 2022-04-28T19:45:10.086463+0000 mgr.y (mgr.14140) 393 : cluster [DBG] pgmap v390: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 54 MiB data, 92 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 812 KiB/s wr, 3 op/s 2022-04-28T19:45:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:11 smithi059 bash[29114]: cluster 2022-04-28T19:45:10.203868+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2022-04-28T19:45:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:11 smithi059 bash[23786]: cluster 2022-04-28T19:45:10.086463+0000 mgr.y (mgr.14140) 393 : cluster [DBG] pgmap v390: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 54 MiB data, 92 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 812 KiB/s wr, 3 op/s 2022-04-28T19:45:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:11 smithi059 bash[23786]: cluster 2022-04-28T19:45:10.203868+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2022-04-28T19:45:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:11 smithi139 bash[23439]: cluster 2022-04-28T19:45:10.086463+0000 mgr.y (mgr.14140) 393 : cluster [DBG] pgmap v390: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 54 MiB data, 92 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 812 KiB/s wr, 3 op/s 2022-04-28T19:45:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:11 smithi139 bash[23439]: cluster 2022-04-28T19:45:10.203868+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2022-04-28T19:45:12.160 INFO:tasks.rados.rados.0.smithi059.stdout:279: done (13 left) 2022-04-28T19:45:12.160 INFO:tasks.rados.rados.0.smithi059.stdout:280: rollback oid 35 current snap is 25 2022-04-28T19:45:12.160 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 35 to 15 2022-04-28T19:45:12.160 INFO:tasks.rados.rados.0.smithi059.stdout:264: finishing write tid 2 to smithi05955964-49 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:264: finishing write tid 3 to smithi05955964-49 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:264: finishing write tid 4 to smithi05955964-49 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:264: finishing write tid 6 to smithi05955964-49 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:264: finishing write tid 7 to smithi05955964-49 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 67 (ObjNum 105 snap 25 seq_num 105) dirty exists 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:264: left oid 49 (ObjNum 105 snap 25 seq_num 105) 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:265: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:265: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:265: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:45:12.161 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 57 (ObjNum 32 snap 0 seq_num 32) dirty exists 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:266: finishing write tid 1 to smithi05955964-10 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:264: done (13 left) 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:265: done (12 left) 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:281: rollback oid 38 current snap is 25 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 17 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:266: finishing write tid 2 to smithi05955964-10 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:266: finishing write tid 3 to smithi05955964-10 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:266: finishing write tid 5 to smithi05955964-10 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:268: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:45:12.162 INFO:tasks.rados.rados.0.smithi059.stdout:268: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:45:12.163 INFO:tasks.rados.rados.0.smithi059.stdout:268: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:45:12.163 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 85 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:45:12.163 INFO:tasks.rados.rados.0.smithi059.stdout:266: finishing write tid 6 to smithi05955964-10 2022-04-28T19:45:12.163 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 57 (ObjNum 106 snap 25 seq_num 106) dirty exists 2022-04-28T19:45:12.163 INFO:tasks.rados.rados.0.smithi059.stdout:266: left oid 10 (ObjNum 106 snap 25 seq_num 106) 2022-04-28T19:45:12.163 INFO:tasks.rados.rados.0.smithi059.stdout:267: expect (ObjNum 83 snap 14 seq_num 83) 2022-04-28T19:45:12.200 INFO:tasks.rados.rados.0.smithi059.stdout:271: finishing write tid 1 to smithi05955964-18 2022-04-28T19:45:12.200 INFO:tasks.rados.rados.0.smithi059.stdout:271: finishing write tid 2 to smithi05955964-18 2022-04-28T19:45:12.200 INFO:tasks.rados.rados.0.smithi059.stdout:271: finishing write tid 3 to smithi05955964-18 2022-04-28T19:45:12.201 INFO:tasks.rados.rados.0.smithi059.stdout:271: finishing write tid 4 to smithi05955964-18 2022-04-28T19:45:12.201 INFO:tasks.rados.rados.0.smithi059.stdout:266: done (12 left) 2022-04-28T19:45:12.201 INFO:tasks.rados.rados.0.smithi059.stdout:267: done (11 left) 2022-04-28T19:45:12.201 INFO:tasks.rados.rados.0.smithi059.stdout:268: done (10 left) 2022-04-28T19:45:12.201 INFO:tasks.rados.rados.0.smithi059.stdout:282: rmattr oid 50 current snap is 25 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:282: done (10 left) 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:283: rollback oid 23 current snap is 25 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 20 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:271: finishing write tid 6 to smithi05955964-18 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:271: finishing write tid 7 to smithi05955964-18 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 65 (ObjNum 107 snap 25 seq_num 107) dirty exists 2022-04-28T19:45:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:271: left oid 18 (ObjNum 107 snap 25 seq_num 107) 2022-04-28T19:45:12.203 INFO:tasks.rados.rados.0.smithi059.stdout:270: finishing copy_from to smithi05955964-46 2022-04-28T19:45:12.203 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 37 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:45:12.203 INFO:tasks.rados.rados.0.smithi059.stdout:270: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:45:12.203 INFO:tasks.rados.rados.0.smithi059.stdout:272: finishing write tid 1 to smithi05955964-37 2022-04-28T19:45:12.203 INFO:tasks.rados.rados.0.smithi059.stdout:272: finishing write tid 2 to smithi05955964-37 2022-04-28T19:45:12.203 INFO:tasks.rados.rados.0.smithi059.stdout:272: finishing write tid 3 to smithi05955964-37 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:272: finishing write tid 5 to smithi05955964-37 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:272: finishing write tid 6 to smithi05955964-37 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 70 (ObjNum 108 snap 25 seq_num 108) dirty exists 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:272: left oid 37 (ObjNum 108 snap 25 seq_num 108) 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:275: finishing write tid 1 to smithi05955964-44 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:275: finishing write tid 2 to smithi05955964-44 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:275: finishing write tid 3 to smithi05955964-44 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:275: finishing write tid 4 to smithi05955964-44 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:275: finishing write tid 6 to smithi05955964-44 2022-04-28T19:45:12.204 INFO:tasks.rados.rados.0.smithi059.stdout:275: finishing write tid 7 to smithi05955964-44 2022-04-28T19:45:12.205 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 58 (ObjNum 109 snap 25 seq_num 109) dirty exists 2022-04-28T19:45:12.205 INFO:tasks.rados.rados.0.smithi059.stdout:275: left oid 44 (ObjNum 109 snap 25 seq_num 109) 2022-04-28T19:45:12.205 INFO:tasks.rados.rados.0.smithi059.stdout:276: expect (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:45:12.220 INFO:tasks.rados.rados.0.smithi059.stdout:277: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:278: finishing write tid 1 to smithi05955964-32 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:278: finishing write tid 2 to smithi05955964-32 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:278: finishing write tid 3 to smithi05955964-32 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:278: finishing write tid 5 to smithi05955964-32 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:278: finishing write tid 6 to smithi05955964-32 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 61 (ObjNum 110 snap 25 seq_num 110) dirty exists 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:278: left oid 32 (ObjNum 110 snap 25 seq_num 110) 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:280: finishing rollback tid 1 to smithi05955964-35 2022-04-28T19:45:12.231 INFO:tasks.rados.rados.0.smithi059.stdout:280: finishing rollback tid 2 to smithi05955964-35 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 14 (ObjNum 34 snap 0 seq_num 34) dirty exists 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:281: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:281: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:281: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 44 (ObjNum 37 snap 0 seq_num 37) dirty exists 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:283: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:283: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:283: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 63 (ObjNum 53 snap 1 seq_num 53) dirty exists 2022-04-28T19:45:12.232 INFO:tasks.rados.rados.0.smithi059.stdout:270: done (10 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:271: done (9 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:272: done (8 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:273: done (7 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:275: done (6 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:276: done (5 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:277: done (4 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:278: done (3 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:280: done (2 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:281: done (1 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:283: done (0 left) 2022-04-28T19:45:12.233 INFO:tasks.rados.rados.0.smithi059.stdout:284: rmattr oid 29 current snap is 25 2022-04-28T19:45:12.234 INFO:tasks.rados.rados.0.smithi059.stdout:284: done (0 left) 2022-04-28T19:45:12.234 INFO:tasks.rados.rados.0.smithi059.stdout:285: copy_from oid 7 from oid 29 current snap is 25 2022-04-28T19:45:12.234 INFO:tasks.rados.rados.0.smithi059.stdout:286: rollback oid 6 current snap is 25 2022-04-28T19:45:12.234 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 17 2022-04-28T19:45:12.234 INFO:tasks.rados.rados.0.smithi059.stdout:287: snap_remove snap 18 2022-04-28T19:45:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:12 smithi059 bash[29114]: cluster 2022-04-28T19:45:11.212663+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2022-04-28T19:45:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:12 smithi059 bash[29114]: cluster 2022-04-28T19:45:12.155905+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2022-04-28T19:45:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:12 smithi059 bash[23786]: cluster 2022-04-28T19:45:11.212663+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2022-04-28T19:45:12.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:12 smithi059 bash[23786]: cluster 2022-04-28T19:45:12.155905+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2022-04-28T19:45:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:12 smithi139 bash[23439]: cluster 2022-04-28T19:45:11.212663+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2022-04-28T19:45:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:12 smithi139 bash[23439]: cluster 2022-04-28T19:45:12.155905+0000 mon.a (mon.0) 713 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2022-04-28T19:45:13.228 INFO:tasks.rados.rados.0.smithi059.stdout:287: done (2 left) 2022-04-28T19:45:13.228 INFO:tasks.rados.rados.0.smithi059.stdout:288: snap_create 2022-04-28T19:45:13.228 INFO:tasks.rados.rados.0.smithi059.stdout:285: finishing copy_from to smithi05955964-7 2022-04-28T19:45:13.229 INFO:tasks.rados.rados.0.smithi059.stdout:285: got expected ENOENT (src dne) 2022-04-28T19:45:13.229 INFO:tasks.rados.rados.0.smithi059.stdout:285: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:45:13.229 INFO:tasks.rados.rados.0.smithi059.stdout:286: finishing rollback tid 0 to smithi05955964-6 2022-04-28T19:45:13.229 INFO:tasks.rados.rados.0.smithi059.stdout:286: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:45:13.229 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 73 (ObjNum -133639504 snap 21893 seq_num 4026534096) dirty dne 2022-04-28T19:45:13.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:13 smithi059 bash[29114]: cluster 2022-04-28T19:45:12.087260+0000 mgr.y (mgr.14140) 394 : cluster [DBG] pgmap v393: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 53 MiB data, 90 MiB used, 707 GiB / 715 GiB avail; 255 B/s rd, 412 KiB/s wr, 1 op/s 2022-04-28T19:45:13.517 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:13 smithi059 bash[23786]: cluster 2022-04-28T19:45:12.087260+0000 mgr.y (mgr.14140) 394 : cluster [DBG] pgmap v393: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 53 MiB data, 90 MiB used, 707 GiB / 715 GiB avail; 255 B/s rd, 412 KiB/s wr, 1 op/s 2022-04-28T19:45:13.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:13 smithi139 bash[23439]: cluster 2022-04-28T19:45:12.087260+0000 mgr.y (mgr.14140) 394 : cluster [DBG] pgmap v393: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 53 MiB data, 90 MiB used, 707 GiB / 715 GiB avail; 255 B/s rd, 412 KiB/s wr, 1 op/s 2022-04-28T19:45:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:13.515Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:45:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:13.515Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:45:13.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:13.515Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:45:14.240 INFO:tasks.rados.rados.0.smithi059.stdout:285: done (2 left) 2022-04-28T19:45:14.240 INFO:tasks.rados.rados.0.smithi059.stdout:286: done (1 left) 2022-04-28T19:45:14.240 INFO:tasks.rados.rados.0.smithi059.stdout:288: done (0 left) 2022-04-28T19:45:14.240 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 26 2022-04-28T19:45:14.241 INFO:tasks.rados.rados.0.smithi059.stdout:289: seq_num 111 ranges {3039232=663552,3702784=663552,4366336=450560,4816896=90112} 2022-04-28T19:45:14.265 INFO:tasks.rados.rados.0.smithi059.stdout:289: writing smithi05955964-16 from 3039232 to 3702784 tid 1 2022-04-28T19:45:14.268 INFO:tasks.rados.rados.0.smithi059.stdout:289: writing smithi05955964-16 from 3702784 to 4366336 tid 2 2022-04-28T19:45:14.271 INFO:tasks.rados.rados.0.smithi059.stdout:289: writing smithi05955964-16 from 4366336 to 4816896 tid 3 2022-04-28T19:45:14.272 INFO:tasks.rados.rados.0.smithi059.stdout:289: writing smithi05955964-16 from 4816896 to 4907008 tid 4 2022-04-28T19:45:14.272 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 26 2022-04-28T19:45:14.272 INFO:tasks.rados.rados.0.smithi059.stdout:290: seq_num 112 ranges {1802240=679936,2482176=548864,3031040=655360} 2022-04-28T19:45:14.285 INFO:tasks.rados.rados.0.smithi059.stdout:290: writing smithi05955964-18 from 1802240 to 2482176 tid 1 2022-04-28T19:45:14.289 INFO:tasks.rados.rados.0.smithi059.stdout:290: writing smithi05955964-18 from 2482176 to 3031040 tid 2 2022-04-28T19:45:14.293 INFO:tasks.rados.rados.0.smithi059.stdout:290: writing smithi05955964-18 from 3031040 to 3686400 tid 3 2022-04-28T19:45:14.293 INFO:tasks.rados.rados.0.smithi059.stdout:291: setattr oid 45 current snap is 26 2022-04-28T19:45:14.293 INFO:tasks.rados.rados.0.smithi059.stdout:289: finishing write tid 1 to smithi05955964-16 2022-04-28T19:45:14.293 INFO:tasks.rados.rados.0.smithi059.stdout:289: finishing write tid 2 to smithi05955964-16 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:289: finishing write tid 3 to smithi05955964-16 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:289: finishing write tid 4 to smithi05955964-16 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:289: finishing write tid 6 to smithi05955964-16 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:289: finishing write tid 7 to smithi05955964-16 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 71 (ObjNum 111 snap 26 seq_num 111) dirty exists 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:289: left oid 16 (ObjNum 111 snap 26 seq_num 111) 2022-04-28T19:45:14.294 INFO:tasks.rados.rados.0.smithi059.stdout:290: finishing write tid 1 to smithi05955964-18 2022-04-28T19:45:14.295 INFO:tasks.rados.rados.0.smithi059.stdout:290: finishing write tid 2 to smithi05955964-18 2022-04-28T19:45:14.295 INFO:tasks.rados.rados.0.smithi059.stdout:289: done (2 left) 2022-04-28T19:45:14.295 INFO:tasks.rados.rados.0.smithi059.stdout:292: snap_remove snap 15 2022-04-28T19:45:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:14 smithi059 bash[23786]: cluster 2022-04-28T19:45:13.223964+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2022-04-28T19:45:14.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:14 smithi059 bash[29114]: cluster 2022-04-28T19:45:13.223964+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2022-04-28T19:45:14.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:14 smithi139 bash[23439]: cluster 2022-04-28T19:45:13.223964+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2022-04-28T19:45:15.258 INFO:tasks.rados.rados.0.smithi059.stdout:290: finishing write tid 3 to smithi05955964-18 2022-04-28T19:45:15.258 INFO:tasks.rados.rados.0.smithi059.stdout:290: finishing write tid 5 to smithi05955964-18 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:290: finishing write tid 6 to smithi05955964-18 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 85 (ObjNum 112 snap 26 seq_num 112) dirty exists 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:290: left oid 18 (ObjNum 112 snap 26 seq_num 112) 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 77 (ObjNum 99 snap 22 seq_num 99) dirty exists 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:290: done (2 left) 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:291: done (1 left) 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:292: done (0 left) 2022-04-28T19:45:15.259 INFO:tasks.rados.rados.0.smithi059.stdout:293: setattr oid 40 current snap is 26 2022-04-28T19:45:15.260 INFO:tasks.rados.rados.0.smithi059.stdout:294: snap_create 2022-04-28T19:45:15.270 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 73 (ObjNum 39 snap 0 seq_num 39) dirty exists 2022-04-28T19:45:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:15 smithi059 bash[23786]: cluster 2022-04-28T19:45:14.088111+0000 mgr.y (mgr.14140) 395 : cluster [DBG] pgmap v396: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 53 MiB data, 96 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 B/s wr, 4 op/s 2022-04-28T19:45:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:15 smithi059 bash[23786]: cluster 2022-04-28T19:45:14.235257+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2022-04-28T19:45:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:15 smithi059 bash[29114]: cluster 2022-04-28T19:45:14.088111+0000 mgr.y (mgr.14140) 395 : cluster [DBG] pgmap v396: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 53 MiB data, 96 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 B/s wr, 4 op/s 2022-04-28T19:45:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:15 smithi059 bash[29114]: cluster 2022-04-28T19:45:14.235257+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2022-04-28T19:45:15.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:15 smithi139 bash[23439]: cluster 2022-04-28T19:45:14.088111+0000 mgr.y (mgr.14140) 395 : cluster [DBG] pgmap v396: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 53 MiB data, 96 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 0 B/s wr, 4 op/s 2022-04-28T19:45:15.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:15 smithi139 bash[23439]: cluster 2022-04-28T19:45:14.235257+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2022-04-28T19:45:16.267 INFO:tasks.rados.rados.0.smithi059.stdout:293: done (1 left) 2022-04-28T19:45:16.267 INFO:tasks.rados.rados.0.smithi059.stdout:294: done (0 left) 2022-04-28T19:45:16.268 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 27 2022-04-28T19:45:16.268 INFO:tasks.rados.rados.0.smithi059.stdout:295: seq_num 113 ranges {540672=696320,1236992=434176,1671168=466944,2138112=303104} 2022-04-28T19:45:16.275 INFO:tasks.rados.rados.0.smithi059.stdout:295: writing smithi05955964-33 from 540672 to 1236992 tid 1 2022-04-28T19:45:16.278 INFO:tasks.rados.rados.0.smithi059.stdout:295: writing smithi05955964-33 from 1236992 to 1671168 tid 2 2022-04-28T19:45:16.281 INFO:tasks.rados.rados.0.smithi059.stdout:295: writing smithi05955964-33 from 1671168 to 2138112 tid 3 2022-04-28T19:45:16.283 INFO:tasks.rados.rados.0.smithi059.stdout:295: writing smithi05955964-33 from 2138112 to 2441216 tid 4 2022-04-28T19:45:16.283 INFO:tasks.rados.rados.0.smithi059.stdout:296: copy_from oid 18 from oid 13 current snap is 27 2022-04-28T19:45:16.283 INFO:tasks.rados.rados.0.smithi059.stdout:297: snap_create 2022-04-28T19:45:16.286 INFO:tasks.rados.rados.0.smithi059.stdout:295: finishing write tid 1 to smithi05955964-33 2022-04-28T19:45:16.287 INFO:tasks.rados.rados.0.smithi059.stdout:295: finishing write tid 2 to smithi05955964-33 2022-04-28T19:45:16.287 INFO:tasks.rados.rados.0.smithi059.stdout:295: finishing write tid 3 to smithi05955964-33 2022-04-28T19:45:16.287 INFO:tasks.rados.rados.0.smithi059.stdout:295: finishing write tid 4 to smithi05955964-33 2022-04-28T19:45:16.288 INFO:tasks.rados.rados.0.smithi059.stdout:295: finishing write tid 6 to smithi05955964-33 2022-04-28T19:45:16.288 INFO:tasks.rados.rados.0.smithi059.stdout:295: finishing write tid 7 to smithi05955964-33 2022-04-28T19:45:16.288 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 83 (ObjNum 113 snap 27 seq_num 113) dirty exists 2022-04-28T19:45:16.289 INFO:tasks.rados.rados.0.smithi059.stdout:295: left oid 33 (ObjNum 113 snap 27 seq_num 113) 2022-04-28T19:45:16.295 INFO:tasks.rados.rados.0.smithi059.stdout:296: finishing copy_from to smithi05955964-18 2022-04-28T19:45:16.295 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 20 (ObjNum 83 snap 14 seq_num 83) dirty exists 2022-04-28T19:45:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:296: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:45:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:16 smithi059 bash[23786]: cluster 2022-04-28T19:45:15.254138+0000 mon.a (mon.0) 716 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2022-04-28T19:45:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:16 smithi059 bash[23786]: audit 2022-04-28T19:45:16.142905+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:45:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:16 smithi059 bash[23786]: audit 2022-04-28T19:45:16.143361+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:45:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:16 smithi059 bash[29114]: cluster 2022-04-28T19:45:15.254138+0000 mon.a (mon.0) 716 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2022-04-28T19:45:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:16 smithi059 bash[29114]: audit 2022-04-28T19:45:16.142905+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:45:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:16 smithi059 bash[29114]: audit 2022-04-28T19:45:16.143361+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:45:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:16 smithi139 bash[23439]: cluster 2022-04-28T19:45:15.254138+0000 mon.a (mon.0) 716 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2022-04-28T19:45:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:16 smithi139 bash[23439]: audit 2022-04-28T19:45:16.142905+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:45:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:16 smithi139 bash[23439]: audit 2022-04-28T19:45:16.143361+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:45:17.158 INFO:tasks.rados.rados.0.smithi059.stdout:295: done (2 left) 2022-04-28T19:45:17.159 INFO:tasks.rados.rados.0.smithi059.stdout:296: done (1 left) 2022-04-28T19:45:17.159 INFO:tasks.rados.rados.0.smithi059.stdout:297: done (0 left) 2022-04-28T19:45:17.159 INFO:tasks.rados.rados.0.smithi059.stdout:298: snap_create 2022-04-28T19:45:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:17 smithi059 bash[29114]: cluster 2022-04-28T19:45:16.088854+0000 mgr.y (mgr.14140) 396 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 67 MiB data, 109 MiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.4 MiB/s wr, 28 op/s 2022-04-28T19:45:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:17 smithi059 bash[29114]: cluster 2022-04-28T19:45:16.263914+0000 mon.a (mon.0) 719 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2022-04-28T19:45:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:17 smithi059 bash[29114]: cluster 2022-04-28T19:45:17.154167+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2022-04-28T19:45:17.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:17 smithi059 bash[23786]: cluster 2022-04-28T19:45:16.088854+0000 mgr.y (mgr.14140) 396 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 67 MiB data, 109 MiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.4 MiB/s wr, 28 op/s 2022-04-28T19:45:17.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:17 smithi059 bash[23786]: cluster 2022-04-28T19:45:16.263914+0000 mon.a (mon.0) 719 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2022-04-28T19:45:17.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:17 smithi059 bash[23786]: cluster 2022-04-28T19:45:17.154167+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2022-04-28T19:45:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:17 smithi139 bash[23439]: cluster 2022-04-28T19:45:16.088854+0000 mgr.y (mgr.14140) 396 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 67 MiB data, 109 MiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 3.4 MiB/s wr, 28 op/s 2022-04-28T19:45:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:17 smithi139 bash[23439]: cluster 2022-04-28T19:45:16.263914+0000 mon.a (mon.0) 719 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2022-04-28T19:45:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:17 smithi139 bash[23439]: cluster 2022-04-28T19:45:17.154167+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2022-04-28T19:45:18.169 INFO:tasks.rados.rados.0.smithi059.stdout:298: done (0 left) 2022-04-28T19:45:18.169 INFO:tasks.rados.rados.0.smithi059.stdout:299: read oid 15 snap -1 2022-04-28T19:45:18.169 INFO:tasks.rados.rados.0.smithi059.stdout:299: expect (ObjNum 104 snap 25 seq_num 104) 2022-04-28T19:45:18.169 INFO:tasks.rados.rados.0.smithi059.stdout:300: read oid 1 snap -1 2022-04-28T19:45:18.170 INFO:tasks.rados.rados.0.smithi059.stdout:300: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:45:18.170 INFO:tasks.rados.rados.0.smithi059.stdout:301: read oid 10 snap -1 2022-04-28T19:45:18.170 INFO:tasks.rados.rados.0.smithi059.stdout:301: expect (ObjNum 106 snap 25 seq_num 106) 2022-04-28T19:45:18.170 INFO:tasks.rados.rados.0.smithi059.stdout:302: delete oid 49 current snap is 29 2022-04-28T19:45:18.184 INFO:tasks.rados.rados.0.smithi059.stdout:300: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:45:18.219 INFO:tasks.rados.rados.0.smithi059.stdout:301: expect (ObjNum 106 snap 25 seq_num 106) 2022-04-28T19:45:18.286 INFO:tasks.rados.rados.0.smithi059.stdout:299: expect (ObjNum 104 snap 25 seq_num 104) 2022-04-28T19:45:18.329 INFO:tasks.rados.rados.0.smithi059.stdout:299: done (3 left) 2022-04-28T19:45:18.329 INFO:tasks.rados.rados.0.smithi059.stdout:300: done (2 left) 2022-04-28T19:45:18.329 INFO:tasks.rados.rados.0.smithi059.stdout:301: done (1 left) 2022-04-28T19:45:18.330 INFO:tasks.rados.rados.0.smithi059.stdout:302: done (0 left) 2022-04-28T19:45:18.330 INFO:tasks.rados.rados.0.smithi059.stdout:303: delete oid 44 current snap is 29 2022-04-28T19:45:18.334 INFO:tasks.rados.rados.0.smithi059.stdout:303: done (0 left) 2022-04-28T19:45:18.334 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 29 2022-04-28T19:45:18.334 INFO:tasks.rados.rados.0.smithi059.stdout:304: seq_num 116 ranges {1548288=745472,2293760=532480,2826240=638976,3465216=32768} 2022-04-28T19:45:18.347 INFO:tasks.rados.rados.0.smithi059.stdout:304: writing smithi05955964-25 from 1548288 to 2293760 tid 1 2022-04-28T19:45:18.351 INFO:tasks.rados.rados.0.smithi059.stdout:304: writing smithi05955964-25 from 2293760 to 2826240 tid 2 2022-04-28T19:45:18.354 INFO:tasks.rados.rados.0.smithi059.stdout:304: writing smithi05955964-25 from 2826240 to 3465216 tid 3 2022-04-28T19:45:18.355 INFO:tasks.rados.rados.0.smithi059.stdout:304: writing smithi05955964-25 from 3465216 to 3497984 tid 4 2022-04-28T19:45:18.355 INFO:tasks.rados.rados.0.smithi059.stdout:305: delete oid 42 current snap is 29 2022-04-28T19:45:18.356 INFO:tasks.rados.rados.0.smithi059.stdout:305: done (1 left) 2022-04-28T19:45:18.356 INFO:tasks.rados.rados.0.smithi059.stdout:306: snap_remove snap 25 2022-04-28T19:45:19.280 INFO:tasks.rados.rados.0.smithi059.stdout:306: done (1 left) 2022-04-28T19:45:19.280 INFO:tasks.rados.rados.0.smithi059.stdout:307: delete oid 48 current snap is 29 2022-04-28T19:45:19.280 INFO:tasks.rados.rados.0.smithi059.stdout:304: finishing write tid 1 to smithi05955964-25 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:304: finishing write tid 2 to smithi05955964-25 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:304: finishing write tid 3 to smithi05955964-25 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:304: finishing write tid 4 to smithi05955964-25 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:304: finishing write tid 6 to smithi05955964-25 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:304: finishing write tid 7 to smithi05955964-25 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 91 (ObjNum 116 snap 29 seq_num 116) dirty exists 2022-04-28T19:45:19.281 INFO:tasks.rados.rados.0.smithi059.stdout:304: left oid 25 (ObjNum 116 snap 29 seq_num 116) 2022-04-28T19:45:19.285 INFO:tasks.rados.rados.0.smithi059.stdout:304: done (1 left) 2022-04-28T19:45:19.285 INFO:tasks.rados.rados.0.smithi059.stdout:307: done (0 left) 2022-04-28T19:45:19.285 INFO:tasks.rados.rados.0.smithi059.stdout:308: copy_from oid 36 from oid 46 current snap is 29 2022-04-28T19:45:19.285 INFO:tasks.rados.rados.0.smithi059.stdout:309: rollback oid 15 current snap is 29 2022-04-28T19:45:19.285 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 23 2022-04-28T19:45:19.285 INFO:tasks.rados.rados.0.smithi059.stdout:310: setattr oid 14 current snap is 29 2022-04-28T19:45:19.286 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 29 2022-04-28T19:45:19.286 INFO:tasks.rados.rados.0.smithi059.stdout:311: seq_num 119 ranges {827392=802816,1630208=589824,2220032=409600,2629632=204800} 2022-04-28T19:45:19.295 INFO:tasks.rados.rados.0.smithi059.stdout:311: writing smithi05955964-50 from 827392 to 1630208 tid 1 2022-04-28T19:45:19.299 INFO:tasks.rados.rados.0.smithi059.stdout:311: writing smithi05955964-50 from 1630208 to 2220032 tid 2 2022-04-28T19:45:19.302 INFO:tasks.rados.rados.0.smithi059.stdout:311: writing smithi05955964-50 from 2220032 to 2629632 tid 3 2022-04-28T19:45:19.304 INFO:tasks.rados.rados.0.smithi059.stdout:311: writing smithi05955964-50 from 2629632 to 2834432 tid 4 2022-04-28T19:45:19.304 INFO:tasks.rados.rados.0.smithi059.stdout:309: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:309: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:309: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 128 (ObjNum 4 snap 0 seq_num 4) dirty exists 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:312: read oid 12 snap 20 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:312: expect (ObjNum 73 snap 6 seq_num 73) 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:309: done (4 left) 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 29 2022-04-28T19:45:19.305 INFO:tasks.rados.rados.0.smithi059.stdout:313: seq_num 120 ranges {573440=417792,991232=475136,1466368=622592,2088960=507904} 2022-04-28T19:45:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:313: writing smithi05955964-35 from 573440 to 991232 tid 1 2022-04-28T19:45:19.313 INFO:tasks.rados.rados.0.smithi059.stdout:313: writing smithi05955964-35 from 991232 to 1466368 tid 2 2022-04-28T19:45:19.317 INFO:tasks.rados.rados.0.smithi059.stdout:313: writing smithi05955964-35 from 1466368 to 2088960 tid 3 2022-04-28T19:45:19.320 INFO:tasks.rados.rados.0.smithi059.stdout:313: writing smithi05955964-35 from 2088960 to 2596864 tid 4 2022-04-28T19:45:19.320 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 29 2022-04-28T19:45:19.320 INFO:tasks.rados.rados.0.smithi059.stdout:314: seq_num 121 ranges {2244608=434176,2678784=753664,3432448=425984,3858432=425984} 2022-04-28T19:45:19.335 INFO:tasks.rados.rados.0.smithi059.stdout:314: writing smithi05955964-32 from 2244608 to 2678784 tid 1 2022-04-28T19:45:19.339 INFO:tasks.rados.rados.0.smithi059.stdout:314: writing smithi05955964-32 from 2678784 to 3432448 tid 2 2022-04-28T19:45:19.342 INFO:tasks.rados.rados.0.smithi059.stdout:314: writing smithi05955964-32 from 3432448 to 3858432 tid 3 2022-04-28T19:45:19.345 INFO:tasks.rados.rados.0.smithi059.stdout:314: writing smithi05955964-32 from 3858432 to 4284416 tid 4 2022-04-28T19:45:19.345 INFO:tasks.rados.rados.0.smithi059.stdout:315: rollback oid 11 current snap is 29 2022-04-28T19:45:19.345 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 24 2022-04-28T19:45:19.345 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 101 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:45:19.345 INFO:tasks.rados.rados.0.smithi059.stdout:311: finishing write tid 1 to smithi05955964-50 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:311: finishing write tid 2 to smithi05955964-50 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:311: finishing write tid 3 to smithi05955964-50 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:311: finishing write tid 4 to smithi05955964-50 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:311: finishing write tid 6 to smithi05955964-50 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:311: finishing write tid 7 to smithi05955964-50 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 91 (ObjNum 119 snap 29 seq_num 119) dirty exists 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:311: left oid 50 (ObjNum 119 snap 29 seq_num 119) 2022-04-28T19:45:19.346 INFO:tasks.rados.rados.0.smithi059.stdout:312: expect (ObjNum 73 snap 6 seq_num 73) 2022-04-28T19:45:19.374 INFO:tasks.rados.rados.0.smithi059.stdout:313: finishing write tid 1 to smithi05955964-35 2022-04-28T19:45:19.374 INFO:tasks.rados.rados.0.smithi059.stdout:313: finishing write tid 2 to smithi05955964-35 2022-04-28T19:45:19.374 INFO:tasks.rados.rados.0.smithi059.stdout:308: finishing copy_from to smithi05955964-36 2022-04-28T19:45:19.374 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 37 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:45:19.374 INFO:tasks.rados.rados.0.smithi059.stdout:308: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:313: finishing write tid 3 to smithi05955964-35 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:313: finishing write tid 4 to smithi05955964-35 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:308: done (6 left) 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:310: done (5 left) 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:311: done (4 left) 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:312: done (3 left) 2022-04-28T19:45:19.375 INFO:tasks.rados.rados.0.smithi059.stdout:316: snap_remove snap 28 2022-04-28T19:45:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:19 smithi059 bash[29114]: cluster 2022-04-28T19:45:18.089716+0000 mgr.y (mgr.14140) 397 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 71 MiB data, 117 MiB used, 707 GiB / 715 GiB avail; 572 KiB/s rd, 4.6 MiB/s wr, 27 op/s 2022-04-28T19:45:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:19 smithi059 bash[29114]: cluster 2022-04-28T19:45:18.165780+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2022-04-28T19:45:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:19 smithi059 bash[23786]: cluster 2022-04-28T19:45:18.089716+0000 mgr.y (mgr.14140) 397 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 71 MiB data, 117 MiB used, 707 GiB / 715 GiB avail; 572 KiB/s rd, 4.6 MiB/s wr, 27 op/s 2022-04-28T19:45:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:19 smithi059 bash[23786]: cluster 2022-04-28T19:45:18.165780+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2022-04-28T19:45:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:19 smithi139 bash[23439]: cluster 2022-04-28T19:45:18.089716+0000 mgr.y (mgr.14140) 397 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 71 MiB data, 117 MiB used, 707 GiB / 715 GiB avail; 572 KiB/s rd, 4.6 MiB/s wr, 27 op/s 2022-04-28T19:45:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:19 smithi139 bash[23439]: cluster 2022-04-28T19:45:18.165780+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2022-04-28T19:45:20.284 INFO:tasks.rados.rados.0.smithi059.stdout:316: done (3 left) 2022-04-28T19:45:20.284 INFO:tasks.rados.rados.0.smithi059.stdout:317: rollback oid 18 current snap is 29 2022-04-28T19:45:20.284 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 23 2022-04-28T19:45:20.284 INFO:tasks.rados.rados.0.smithi059.stdout:313: finishing write tid 6 to smithi05955964-35 2022-04-28T19:45:20.284 INFO:tasks.rados.rados.0.smithi059.stdout:313: finishing write tid 7 to smithi05955964-35 2022-04-28T19:45:20.284 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 22 (ObjNum 120 snap 29 seq_num 120) dirty exists 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:313: left oid 35 (ObjNum 120 snap 29 seq_num 120) 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:314: finishing write tid 1 to smithi05955964-32 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:314: finishing write tid 2 to smithi05955964-32 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:314: finishing write tid 3 to smithi05955964-32 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:314: finishing write tid 4 to smithi05955964-32 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:315: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:315: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:314: finishing write tid 6 to smithi05955964-32 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:315: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 58 (ObjNum 10 snap 0 seq_num 10) dirty exists 2022-04-28T19:45:20.285 INFO:tasks.rados.rados.0.smithi059.stdout:314: finishing write tid 7 to smithi05955964-32 2022-04-28T19:45:20.286 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 97 (ObjNum 121 snap 29 seq_num 121) dirty exists 2022-04-28T19:45:20.286 INFO:tasks.rados.rados.0.smithi059.stdout:314: left oid 32 (ObjNum 121 snap 29 seq_num 121) 2022-04-28T19:45:20.286 INFO:tasks.rados.rados.0.smithi059.stdout:313: done (3 left) 2022-04-28T19:45:20.286 INFO:tasks.rados.rados.0.smithi059.stdout:314: done (2 left) 2022-04-28T19:45:20.286 INFO:tasks.rados.rados.0.smithi059.stdout:315: done (1 left) 2022-04-28T19:45:20.286 INFO:tasks.rados.rados.0.smithi059.stdout:318: snap_create 2022-04-28T19:45:20.292 INFO:tasks.rados.rados.0.smithi059.stdout:317: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:45:20.293 INFO:tasks.rados.rados.0.smithi059.stdout:317: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:45:20.293 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 97 (ObjNum -133420032 snap 21893 seq_num 4161562816) dirty dne 2022-04-28T19:45:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:20 smithi059 bash[29114]: cluster 2022-04-28T19:45:19.277858+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2022-04-28T19:45:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:20 smithi059 bash[23786]: cluster 2022-04-28T19:45:19.277858+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2022-04-28T19:45:20.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:20 smithi139 bash[23439]: cluster 2022-04-28T19:45:19.277858+0000 mon.a (mon.0) 722 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2022-04-28T19:45:21.296 INFO:tasks.rados.rados.0.smithi059.stdout:317: done (1 left) 2022-04-28T19:45:21.296 INFO:tasks.rados.rados.0.smithi059.stdout:318: done (0 left) 2022-04-28T19:45:21.296 INFO:tasks.rados.rados.0.smithi059.stdout:319: read oid 23 snap -1 2022-04-28T19:45:21.296 INFO:tasks.rados.rados.0.smithi059.stdout:319: expect (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:45:21.296 INFO:tasks.rados.rados.0.smithi059.stdout:320: delete oid 34 current snap is 30 2022-04-28T19:45:21.303 INFO:tasks.rados.rados.0.smithi059.stdout:320: done (1 left) 2022-04-28T19:45:21.303 INFO:tasks.rados.rados.0.smithi059.stdout:321: delete oid 10 current snap is 30 2022-04-28T19:45:21.312 INFO:tasks.rados.rados.0.smithi059.stdout:319: expect (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:45:21.361 INFO:tasks.rados.rados.0.smithi059.stdout:319: done (1 left) 2022-04-28T19:45:21.366 INFO:tasks.rados.rados.0.smithi059.stdout:321: done (0 left) 2022-04-28T19:45:21.366 INFO:tasks.rados.rados.0.smithi059.stdout:322: read oid 29 snap -1 2022-04-28T19:45:21.366 INFO:tasks.rados.rados.0.smithi059.stdout:322: expect deleted 2022-04-28T19:45:21.366 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 30 2022-04-28T19:45:21.366 INFO:tasks.rados.rados.0.smithi059.stdout:323: seq_num 124 ranges {0=483328,483328=458752,942080=598016,1540096=548864} 2022-04-28T19:45:21.367 INFO:tasks.rados.rados.0.smithi059.stdout:323: writing smithi05955964-48 from 0 to 483328 tid 1 2022-04-28T19:45:21.372 INFO:tasks.rados.rados.0.smithi059.stdout:323: writing smithi05955964-48 from 483328 to 942080 tid 2 2022-04-28T19:45:21.377 INFO:tasks.rados.rados.0.smithi059.stdout:323: writing smithi05955964-48 from 942080 to 1540096 tid 3 2022-04-28T19:45:21.380 INFO:tasks.rados.rados.0.smithi059.stdout:323: writing smithi05955964-48 from 1540096 to 2088960 tid 4 2022-04-28T19:45:21.382 INFO:tasks.rados.rados.0.smithi059.stdout:324: snap_remove snap 29 2022-04-28T19:45:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:21 smithi059 bash[29114]: cluster 2022-04-28T19:45:20.090573+0000 mgr.y (mgr.14140) 398 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 79 MiB data, 127 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2022-04-28T19:45:21.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:21 smithi059 bash[29114]: cluster 2022-04-28T19:45:20.281546+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2022-04-28T19:45:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:21 smithi059 bash[23786]: cluster 2022-04-28T19:45:20.090573+0000 mgr.y (mgr.14140) 398 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 79 MiB data, 127 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2022-04-28T19:45:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:21 smithi059 bash[23786]: cluster 2022-04-28T19:45:20.281546+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2022-04-28T19:45:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:21 smithi139 bash[23439]: cluster 2022-04-28T19:45:20.090573+0000 mgr.y (mgr.14140) 398 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 79 MiB data, 127 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.2 MiB/s wr, 15 op/s 2022-04-28T19:45:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:21 smithi139 bash[23439]: cluster 2022-04-28T19:45:20.281546+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2022-04-28T19:45:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:324: done (2 left) 2022-04-28T19:45:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:325: read oid 7 snap -1 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:325: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:326: read oid 19 snap -1 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:326: expect (ObjNum 18 snap 0 seq_num 18) 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:327: copy_from oid 5 from oid 43 current snap is 30 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:323: finishing write tid 1 to smithi05955964-48 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:323: finishing write tid 2 to smithi05955964-48 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:323: finishing write tid 3 to smithi05955964-48 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:323: finishing write tid 4 to smithi05955964-48 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:323: finishing write tid 6 to smithi05955964-48 2022-04-28T19:45:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:323: finishing write tid 7 to smithi05955964-48 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 110 (ObjNum 124 snap 30 seq_num 124) dirty exists 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:323: left oid 48 (ObjNum 124 snap 30 seq_num 124) 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:322: done (4 left) 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:323: done (3 left) 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:328: rmattr oid 38 current snap is 30 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:328: done (3 left) 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:329: read oid 37 snap -1 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:329: expect (ObjNum 108 snap 25 seq_num 108) 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 30 2022-04-28T19:45:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:330: seq_num 125 ranges {1228800=499712,1728512=745472,2473984=409600,2883584=450560} 2022-04-28T19:45:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:330: writing smithi05955964-12 from 1228800 to 1728512 tid 1 2022-04-28T19:45:22.177 INFO:tasks.rados.rados.0.smithi059.stdout:330: writing smithi05955964-12 from 1728512 to 2473984 tid 2 2022-04-28T19:45:22.179 INFO:tasks.rados.rados.0.smithi059.stdout:330: writing smithi05955964-12 from 2473984 to 2883584 tid 3 2022-04-28T19:45:22.182 INFO:tasks.rados.rados.0.smithi059.stdout:330: writing smithi05955964-12 from 2883584 to 3334144 tid 4 2022-04-28T19:45:22.182 INFO:tasks.rados.rados.0.smithi059.stdout:331: delete oid 11 current snap is 30 2022-04-28T19:45:22.182 INFO:tasks.rados.rados.0.smithi059.stdout:326: expect (ObjNum 18 snap 0 seq_num 18) 2022-04-28T19:45:22.190 INFO:tasks.rados.rados.0.smithi059.stdout:325: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:45:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:329: expect (ObjNum 108 snap 25 seq_num 108) 2022-04-28T19:45:22.265 INFO:tasks.rados.rados.0.smithi059.stdout:330: finishing write tid 1 to smithi05955964-12 2022-04-28T19:45:22.265 INFO:tasks.rados.rados.0.smithi059.stdout:330: finishing write tid 2 to smithi05955964-12 2022-04-28T19:45:22.265 INFO:tasks.rados.rados.0.smithi059.stdout:330: finishing write tid 3 to smithi05955964-12 2022-04-28T19:45:22.265 INFO:tasks.rados.rados.0.smithi059.stdout:330: finishing write tid 4 to smithi05955964-12 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:330: finishing write tid 6 to smithi05955964-12 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:330: finishing write tid 7 to smithi05955964-12 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 136 (ObjNum 125 snap 30 seq_num 125) dirty exists 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:330: left oid 12 (ObjNum 125 snap 30 seq_num 125) 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:327: finishing copy_from to smithi05955964-5 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 33 (ObjNum 85 snap 15 seq_num 85) dirty exists 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:327: finishing copy_from racing read to smithi05955964-5 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:325: done (5 left) 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:326: done (4 left) 2022-04-28T19:45:22.266 INFO:tasks.rados.rados.0.smithi059.stdout:327: done (3 left) 2022-04-28T19:45:22.267 INFO:tasks.rados.rados.0.smithi059.stdout:329: done (2 left) 2022-04-28T19:45:22.267 INFO:tasks.rados.rados.0.smithi059.stdout:330: done (1 left) 2022-04-28T19:45:22.267 INFO:tasks.rados.rados.0.smithi059.stdout:331: done (0 left) 2022-04-28T19:45:22.267 INFO:tasks.rados.rados.0.smithi059.stdout:332: delete oid 49 current snap is 30 2022-04-28T19:45:22.268 INFO:tasks.rados.rados.0.smithi059.stdout:332: done (0 left) 2022-04-28T19:45:22.268 INFO:tasks.rados.rados.0.smithi059.stdout:333: copy_from oid 33 from oid 9 current snap is 30 2022-04-28T19:45:22.268 INFO:tasks.rados.rados.0.smithi059.stdout:334: read oid 27 snap -1 2022-04-28T19:45:22.268 INFO:tasks.rados.rados.0.smithi059.stdout:334: expect (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:335: rollback oid 42 current snap is 30 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 20 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:336: read oid 29 snap -1 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:336: expect deleted 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:337: delete oid 19 current snap is 30 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:335: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:45:22.269 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 0 (ObjNum 1 snap 32762 seq_num 4161839568) dirty dne 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:335: done (4 left) 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:336: done (3 left) 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:337: done (2 left) 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:338: rollback oid 31 current snap is 30 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 17 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:339: rollback oid 40 current snap is 30 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 17 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 30 2022-04-28T19:45:22.271 INFO:tasks.rados.rados.0.smithi059.stdout:340: seq_num 129 ranges {638976=565248,1204224=679936,1884160=786432,2670592=139264} 2022-04-28T19:45:22.285 INFO:tasks.rados.rados.0.smithi059.stdout:340: writing smithi05955964-39 from 638976 to 1204224 tid 1 2022-04-28T19:45:22.289 INFO:tasks.rados.rados.0.smithi059.stdout:340: writing smithi05955964-39 from 1204224 to 1884160 tid 2 2022-04-28T19:45:22.294 INFO:tasks.rados.rados.0.smithi059.stdout:340: writing smithi05955964-39 from 1884160 to 2670592 tid 3 2022-04-28T19:45:22.295 INFO:tasks.rados.rados.0.smithi059.stdout:340: writing smithi05955964-39 from 2670592 to 2809856 tid 4 2022-04-28T19:45:22.295 INFO:tasks.rados.rados.0.smithi059.stdout:334: expect (ObjNum 26 snap 0 seq_num 26) 2022-04-28T19:45:22.306 INFO:tasks.rados.rados.0.smithi059.stdout:339: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:45:22.306 INFO:tasks.rados.rados.0.smithi059.stdout:338: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:338: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:338: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 140 (ObjNum 87 snap 17 seq_num 87) dirty exists 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:339: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:339: finishing rollback tid 2 to smithi05955964-40 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 92 (ObjNum 39 snap 0 seq_num 39) dirty exists 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:333: finishing copy_from to smithi05955964-33 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 42 (ObjNum 89 snap 18 seq_num 89) dirty exists 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:333: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:45:22.307 INFO:tasks.rados.rados.0.smithi059.stdout:340: finishing write tid 1 to smithi05955964-39 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:340: finishing write tid 2 to smithi05955964-39 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:340: finishing write tid 3 to smithi05955964-39 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:340: finishing write tid 4 to smithi05955964-39 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:341: read oid 34 snap -1 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:341: expect deleted 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:333: done (5 left) 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:334: done (4 left) 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:338: done (3 left) 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:339: done (2 left) 2022-04-28T19:45:22.308 INFO:tasks.rados.rados.0.smithi059.stdout:342: rollback oid 29 current snap is 30 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 17 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:340: finishing write tid 6 to smithi05955964-39 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:343: snap_create 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:340: finishing write tid 7 to smithi05955964-39 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 109 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:340: left oid 39 (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:22.309 INFO:tasks.rados.rados.0.smithi059.stdout:342: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:45:22.310 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 65 (ObjNum 0 snap 0 seq_num 0) dirty dne 2022-04-28T19:45:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:22 smithi059 bash[29114]: cluster 2022-04-28T19:45:21.290687+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2022-04-28T19:45:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:22 smithi059 bash[29114]: cluster 2022-04-28T19:45:22.156790+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2022-04-28T19:45:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:22 smithi059 bash[23786]: cluster 2022-04-28T19:45:21.290687+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2022-04-28T19:45:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:22 smithi059 bash[23786]: cluster 2022-04-28T19:45:22.156790+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2022-04-28T19:45:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:22 smithi139 bash[23439]: cluster 2022-04-28T19:45:21.290687+0000 mon.a (mon.0) 724 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2022-04-28T19:45:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:22 smithi139 bash[23439]: cluster 2022-04-28T19:45:22.156790+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2022-04-28T19:45:23.108 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:45:23.304 INFO:tasks.rados.rados.0.smithi059.stdout:340: done (3 left) 2022-04-28T19:45:23.304 INFO:tasks.rados.rados.0.smithi059.stdout:341: done (2 left) 2022-04-28T19:45:23.304 INFO:tasks.rados.rados.0.smithi059.stdout:342: done (1 left) 2022-04-28T19:45:23.304 INFO:tasks.rados.rados.0.smithi059.stdout:343: done (0 left) 2022-04-28T19:45:23.304 INFO:tasks.rados.rados.0.smithi059.stdout:344: rollback oid 50 current snap is 31 2022-04-28T19:45:23.305 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 17 2022-04-28T19:45:23.305 INFO:tasks.rados.rados.0.smithi059.stdout:345: rmattr oid 38 current snap is 31 2022-04-28T19:45:23.305 INFO:tasks.rados.rados.0.smithi059.stdout:345: done (1 left) 2022-04-28T19:45:23.305 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 31 2022-04-28T19:45:23.305 INFO:tasks.rados.rados.0.smithi059.stdout:346: seq_num 130 ranges {393216=581632,974848=614400,1589248=598016,2187264=393216} 2022-04-28T19:45:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:346: writing smithi05955964-21 from 393216 to 974848 tid 1 2022-04-28T19:45:23.315 INFO:tasks.rados.rados.0.smithi059.stdout:346: writing smithi05955964-21 from 974848 to 1589248 tid 2 2022-04-28T19:45:23.319 INFO:tasks.rados.rados.0.smithi059.stdout:346: writing smithi05955964-21 from 1589248 to 2187264 tid 3 2022-04-28T19:45:23.323 INFO:tasks.rados.rados.0.smithi059.stdout:346: writing smithi05955964-21 from 2187264 to 2580480 tid 4 2022-04-28T19:45:23.323 INFO:tasks.rados.rados.0.smithi059.stdout:347: copy_from oid 37 from oid 14 current snap is 31 2022-04-28T19:45:23.323 INFO:tasks.rados.rados.0.smithi059.stdout:344: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:344: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:344: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 113 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:346: finishing write tid 1 to smithi05955964-21 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:346: finishing write tid 2 to smithi05955964-21 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:346: finishing write tid 3 to smithi05955964-21 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:344: done (2 left) 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:348: rmattr oid 44 current snap is 31 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:348: done (2 left) 2022-04-28T19:45:23.324 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 31 2022-04-28T19:45:23.325 INFO:tasks.rados.rados.0.smithi059.stdout:349: seq_num 131 ranges {73728=598016,671744=499712,1171456=753664,1925120=352256} 2022-04-28T19:45:23.328 INFO:tasks.rados.rados.0.smithi059.stdout:349: writing smithi05955964-15 from 73728 to 671744 tid 1 2022-04-28T19:45:23.331 INFO:tasks.rados.rados.0.smithi059.stdout:349: writing smithi05955964-15 from 671744 to 1171456 tid 2 2022-04-28T19:45:23.337 INFO:tasks.rados.rados.0.smithi059.stdout:349: writing smithi05955964-15 from 1171456 to 1925120 tid 3 2022-04-28T19:45:23.449 INFO:tasks.rados.rados.0.smithi059.stdout:349: writing smithi05955964-15 from 1925120 to 2277376 tid 4 2022-04-28T19:45:23.449 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 31 2022-04-28T19:45:23.449 INFO:tasks.rados.rados.0.smithi059.stdout:350: seq_num 132 ranges {0=614400,614400=786432,1400832=565248,1966080=253952} 2022-04-28T19:45:23.449 INFO:tasks.rados.rados.0.smithi059.stdout:350: writing smithi05955964-22 from 0 to 614400 tid 1 2022-04-28T19:45:23.449 INFO:tasks.rados.rados.0.smithi059.stdout:350: writing smithi05955964-22 from 614400 to 1400832 tid 2 2022-04-28T19:45:23.449 INFO:tasks.rados.rados.0.smithi059.stdout:350: writing smithi05955964-22 from 1400832 to 1966080 tid 3 2022-04-28T19:45:23.450 INFO:tasks.rados.rados.0.smithi059.stdout:350: writing smithi05955964-22 from 1966080 to 2220032 tid 4 2022-04-28T19:45:23.450 INFO:tasks.rados.rados.0.smithi059.stdout:351: copy_from oid 23 from oid 11 current snap is 31 2022-04-28T19:45:23.450 INFO:tasks.rados.rados.0.smithi059.stdout:346: finishing write tid 4 to smithi05955964-21 2022-04-28T19:45:23.450 INFO:tasks.rados.rados.0.smithi059.stdout:346: finishing write tid 6 to smithi05955964-21 2022-04-28T19:45:23.450 INFO:tasks.rados.rados.0.smithi059.stdout:346: finishing write tid 7 to smithi05955964-21 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 32 (ObjNum 130 snap 31 seq_num 130) dirty exists 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:346: left oid 21 (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:347: finishing copy_from to smithi05955964-37 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 101 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:347: finishing copy_from racing read to smithi05955964-37 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:349: finishing write tid 1 to smithi05955964-15 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:349: finishing write tid 2 to smithi05955964-15 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:349: finishing write tid 3 to smithi05955964-15 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:346: done (4 left) 2022-04-28T19:45:23.452 INFO:tasks.rados.rados.0.smithi059.stdout:347: done (3 left) 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 31 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:352: seq_num 133 ranges {4284416=630784,4915200=663552,5578752=770048,6348800=172032} 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:352: writing smithi05955964-32 from 4284416 to 4915200 tid 1 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:352: writing smithi05955964-32 from 4915200 to 5578752 tid 2 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:352: writing smithi05955964-32 from 5578752 to 6348800 tid 3 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:352: writing smithi05955964-32 from 6348800 to 6520832 tid 4 2022-04-28T19:45:23.453 INFO:tasks.rados.rados.0.smithi059.stdout:353: snap_remove snap 24 2022-04-28T19:45:23.459 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST STATUS REFRESHED AGE VERSION IMAGE NAME IMAGE ID CONTAINER ID 2022-04-28T19:45:23.459 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (4m) 2m ago 4m 0.20.0 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:45:23.459 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (4m) 2m ago 4m 6.7.4 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:45:23.459 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (2m) 2m ago 2m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 44062639a78b 2022-04-28T19:45:23.459 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (2m) 2m ago 2m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 3516bf4894c9 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (8m) 2m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b8c424c2e210 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (10m) 2m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 320b27bb479e 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (10m) 2m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 239497cf6769 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (8m) 2m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0e12092e0bc3 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (9m) 2m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (4m) 2m ago 4m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (4m) 2m ago 4m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (8m) 2m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 547b5580e073 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (7m) 2m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (7m) 2m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 35cdc0c6604a 2022-04-28T19:45:23.460 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (6m) 2m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 2344bb9dfbae 2022-04-28T19:45:23.461 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (6m) 2m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 5976654882b1 2022-04-28T19:45:23.461 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (6m) 2m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 e68cffb4766c 2022-04-28T19:45:23.461 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (5m) 2m ago 5m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 c36b8738a469 2022-04-28T19:45:23.461 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (5m) 2m ago 5m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:45:23.461 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (4m) 2m ago 5m 2.18.1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 c763409947e4 2022-04-28T19:45:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:23 smithi059 bash[29114]: cluster 2022-04-28T19:45:22.091290+0000 mgr.y (mgr.14140) 399 : cluster [DBG] pgmap v408: 89 pgs: 89 active+clean; 86 MiB data, 132 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2022-04-28T19:45:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:23 smithi059 bash[23786]: cluster 2022-04-28T19:45:22.091290+0000 mgr.y (mgr.14140) 399 : cluster [DBG] pgmap v408: 89 pgs: 89 active+clean; 86 MiB data, 132 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2022-04-28T19:45:23.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:23.515Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:45:23.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:23.515Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:45:23.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:23.515Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:45:23.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:23 smithi139 bash[23439]: cluster 2022-04-28T19:45:22.091290+0000 mgr.y (mgr.14140) 399 : cluster [DBG] pgmap v408: 89 pgs: 89 active+clean; 86 MiB data, 132 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2022-04-28T19:45:23.806 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:45:23.806 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:45:23.806 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:45:23.807 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:23.808 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:45:23.808 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 15 2022-04-28T19:45:23.808 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:45:23.808 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:45:24.324 INFO:tasks.rados.rados.0.smithi059.stdout:353: done (4 left) 2022-04-28T19:45:24.416 INFO:tasks.rados.rados.0.smithi059.stdout:354: copy_from oid 43 from oid 12 current snap is 31 2022-04-28T19:45:24.416 INFO:tasks.rados.rados.0.smithi059.stdout:355: copy_from oid 18 from oid 4 current snap is 31 2022-04-28T19:45:24.416 INFO:tasks.rados.rados.0.smithi059.stdout:349: finishing write tid 4 to smithi05955964-15 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:349: finishing write tid 6 to smithi05955964-15 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:349: finishing write tid 7 to smithi05955964-15 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 146 (ObjNum 131 snap 31 seq_num 131) dirty exists 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:349: left oid 15 (ObjNum 131 snap 31 seq_num 131) 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:350: finishing write tid 1 to smithi05955964-22 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:350: finishing write tid 2 to smithi05955964-22 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:349: done (5 left) 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:356: snap_create 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:350: finishing write tid 3 to smithi05955964-22 2022-04-28T19:45:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:351: finishing copy_from to smithi05955964-23 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:351: got expected ENOENT (src dne) 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:351: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:350: finishing write tid 4 to smithi05955964-22 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:350: finishing write tid 6 to smithi05955964-22 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:350: finishing write tid 7 to smithi05955964-22 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 117 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:350: left oid 22 (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:352: finishing write tid 1 to smithi05955964-32 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:352: finishing write tid 2 to smithi05955964-32 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:352: finishing write tid 3 to smithi05955964-32 2022-04-28T19:45:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:352: finishing write tid 4 to smithi05955964-32 2022-04-28T19:45:24.419 INFO:tasks.rados.rados.0.smithi059.stdout:352: finishing write tid 6 to smithi05955964-32 2022-04-28T19:45:24.419 INFO:tasks.rados.rados.0.smithi059.stdout:352: finishing write tid 7 to smithi05955964-32 2022-04-28T19:45:24.419 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 119 (ObjNum 133 snap 31 seq_num 133) dirty exists 2022-04-28T19:45:24.419 INFO:tasks.rados.rados.0.smithi059.stdout:352: left oid 32 (ObjNum 133 snap 31 seq_num 133) 2022-04-28T19:45:24.420 INFO:tasks.rados.rados.0.smithi059.stdout:355: finishing copy_from to smithi05955964-18 2022-04-28T19:45:24.420 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2022-04-28T19:45:24.420 INFO:tasks.rados.rados.0.smithi059.stdout:355: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:45:24.420 INFO:tasks.rados.rados.0.smithi059.stdout:354: finishing copy_from to smithi05955964-43 2022-04-28T19:45:24.421 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 136 (ObjNum 125 snap 30 seq_num 125) dirty exists 2022-04-28T19:45:24.421 INFO:tasks.rados.rados.0.smithi059.stdout:354: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:45:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:24 smithi059 bash[29114]: audit 2022-04-28T19:45:23.093465+0000 mgr.y (mgr.14140) 400 : audit [DBG] from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:24 smithi059 bash[29114]: cluster 2022-04-28T19:45:23.297797+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2022-04-28T19:45:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:24 smithi059 bash[29114]: audit 2022-04-28T19:45:23.804556+0000 mon.c (mon.1) 34 : audit [DBG] from='client.? 172.21.15.59:0/28737731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:45:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:24 smithi059 bash[23786]: audit 2022-04-28T19:45:23.093465+0000 mgr.y (mgr.14140) 400 : audit [DBG] from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:24.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:24 smithi059 bash[23786]: cluster 2022-04-28T19:45:23.297797+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2022-04-28T19:45:24.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:24 smithi059 bash[23786]: audit 2022-04-28T19:45:23.804556+0000 mon.c (mon.1) 34 : audit [DBG] from='client.? 172.21.15.59:0/28737731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:45:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:24 smithi139 bash[23439]: audit 2022-04-28T19:45:23.093465+0000 mgr.y (mgr.14140) 400 : audit [DBG] from='client.14847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:24 smithi139 bash[23439]: cluster 2022-04-28T19:45:23.297797+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2022-04-28T19:45:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:24 smithi139 bash[23439]: audit 2022-04-28T19:45:23.804556+0000 mon.c (mon.1) 34 : audit [DBG] from='client.? 172.21.15.59:0/28737731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:45:25.326 INFO:tasks.rados.rados.0.smithi059.stdout:350: done (5 left) 2022-04-28T19:45:25.326 INFO:tasks.rados.rados.0.smithi059.stdout:351: done (4 left) 2022-04-28T19:45:25.326 INFO:tasks.rados.rados.0.smithi059.stdout:352: done (3 left) 2022-04-28T19:45:25.326 INFO:tasks.rados.rados.0.smithi059.stdout:354: done (2 left) 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:355: done (1 left) 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:356: done (0 left) 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:357: read oid 15 snap -1 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:357: expect (ObjNum 131 snap 31 seq_num 131) 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:358: copy_from oid 3 from oid 47 current snap is 32 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 32 2022-04-28T19:45:25.327 INFO:tasks.rados.rados.0.smithi059.stdout:359: seq_num 134 ranges {622592=647168,1269760=548864,1818624=573440,2392064=483328} 2022-04-28T19:45:25.333 INFO:tasks.rados.rados.0.smithi059.stdout:359: writing smithi05955964-38 from 622592 to 1269760 tid 1 2022-04-28T19:45:25.337 INFO:tasks.rados.rados.0.smithi059.stdout:359: writing smithi05955964-38 from 1269760 to 1818624 tid 2 2022-04-28T19:45:25.341 INFO:tasks.rados.rados.0.smithi059.stdout:359: writing smithi05955964-38 from 1818624 to 2392064 tid 3 2022-04-28T19:45:25.344 INFO:tasks.rados.rados.0.smithi059.stdout:359: writing smithi05955964-38 from 2392064 to 2875392 tid 4 2022-04-28T19:45:25.345 INFO:tasks.rados.rados.0.smithi059.stdout:360: read oid 39 snap -1 2022-04-28T19:45:25.345 INFO:tasks.rados.rados.0.smithi059.stdout:360: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:25.345 INFO:tasks.rados.rados.0.smithi059.stdout:361: snap_create 2022-04-28T19:45:25.345 INFO:tasks.rados.rados.0.smithi059.stdout:359: finishing write tid 1 to smithi05955964-38 2022-04-28T19:45:25.345 INFO:tasks.rados.rados.0.smithi059.stdout:359: finishing write tid 2 to smithi05955964-38 2022-04-28T19:45:25.345 INFO:tasks.rados.rados.0.smithi059.stdout:357: expect (ObjNum 131 snap 31 seq_num 131) 2022-04-28T19:45:25.400 INFO:tasks.rados.rados.0.smithi059.stdout:358: finishing copy_from to smithi05955964-3 2022-04-28T19:45:25.401 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 8 (ObjNum 46 snap 0 seq_num 46) dirty exists 2022-04-28T19:45:25.401 INFO:tasks.rados.rados.0.smithi059.stdout:358: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:45:25.401 INFO:tasks.rados.rados.0.smithi059.stdout:359: finishing write tid 3 to smithi05955964-38 2022-04-28T19:45:25.401 INFO:tasks.rados.rados.0.smithi059.stdout:359: finishing write tid 4 to smithi05955964-38 2022-04-28T19:45:25.401 INFO:tasks.rados.rados.0.smithi059.stdout:359: finishing write tid 6 to smithi05955964-38 2022-04-28T19:45:25.401 INFO:tasks.rados.rados.0.smithi059.stdout:359: finishing write tid 7 to smithi05955964-38 2022-04-28T19:45:25.402 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:45:25.402 INFO:tasks.rados.rados.0.smithi059.stdout:359: left oid 38 (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:45:25.402 INFO:tasks.rados.rados.0.smithi059.stdout:360: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:25 smithi059 bash[29114]: audit 2022-04-28T19:45:23.445104+0000 mgr.y (mgr.14140) 401 : audit [DBG] from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:25 smithi059 bash[29114]: cluster 2022-04-28T19:45:24.091982+0000 mgr.y (mgr.14140) 402 : cluster [DBG] pgmap v411: 89 pgs: 89 active+clean; 88 MiB data, 138 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.4 MiB/s wr, 13 op/s 2022-04-28T19:45:25.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:25 smithi059 bash[29114]: cluster 2022-04-28T19:45:24.319459+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2022-04-28T19:45:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:25 smithi059 bash[23786]: audit 2022-04-28T19:45:23.445104+0000 mgr.y (mgr.14140) 401 : audit [DBG] from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:25 smithi059 bash[23786]: cluster 2022-04-28T19:45:24.091982+0000 mgr.y (mgr.14140) 402 : cluster [DBG] pgmap v411: 89 pgs: 89 active+clean; 88 MiB data, 138 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.4 MiB/s wr, 13 op/s 2022-04-28T19:45:25.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:25 smithi059 bash[23786]: cluster 2022-04-28T19:45:24.319459+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2022-04-28T19:45:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:25 smithi139 bash[23439]: audit 2022-04-28T19:45:23.445104+0000 mgr.y (mgr.14140) 401 : audit [DBG] from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:25 smithi139 bash[23439]: cluster 2022-04-28T19:45:24.091982+0000 mgr.y (mgr.14140) 402 : cluster [DBG] pgmap v411: 89 pgs: 89 active+clean; 88 MiB data, 138 MiB used, 707 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.4 MiB/s wr, 13 op/s 2022-04-28T19:45:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:25 smithi139 bash[23439]: cluster 2022-04-28T19:45:24.319459+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2022-04-28T19:45:26.344 INFO:tasks.rados.rados.0.smithi059.stdout:357: done (4 left) 2022-04-28T19:45:26.344 INFO:tasks.rados.rados.0.smithi059.stdout:358: done (3 left) 2022-04-28T19:45:26.344 INFO:tasks.rados.rados.0.smithi059.stdout:359: done (2 left) 2022-04-28T19:45:26.345 INFO:tasks.rados.rados.0.smithi059.stdout:360: done (1 left) 2022-04-28T19:45:26.345 INFO:tasks.rados.rados.0.smithi059.stdout:361: done (0 left) 2022-04-28T19:45:26.345 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 33 2022-04-28T19:45:26.345 INFO:tasks.rados.rados.0.smithi059.stdout:362: seq_num 135 ranges {2695168=663552,3358720=434176,3792896=786432,4579328=385024} 2022-04-28T19:45:26.363 INFO:tasks.rados.rados.0.smithi059.stdout:362: writing smithi05955964-5 from 2695168 to 3358720 tid 1 2022-04-28T19:45:26.366 INFO:tasks.rados.rados.0.smithi059.stdout:362: writing smithi05955964-5 from 3358720 to 3792896 tid 2 2022-04-28T19:45:26.370 INFO:tasks.rados.rados.0.smithi059.stdout:362: writing smithi05955964-5 from 3792896 to 4579328 tid 3 2022-04-28T19:45:26.373 INFO:tasks.rados.rados.0.smithi059.stdout:362: writing smithi05955964-5 from 4579328 to 4964352 tid 4 2022-04-28T19:45:26.373 INFO:tasks.rados.rados.0.smithi059.stdout:363: rollback oid 31 current snap is 33 2022-04-28T19:45:26.373 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 31 2022-04-28T19:45:26.373 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 33 2022-04-28T19:45:26.373 INFO:tasks.rados.rados.0.smithi059.stdout:364: seq_num 136 ranges {0=688128,688128=720896,1409024=540672,1949696=344064} 2022-04-28T19:45:26.377 INFO:tasks.rados.rados.0.smithi059.stdout:364: writing smithi05955964-49 from 0 to 688128 tid 1 2022-04-28T19:45:26.382 INFO:tasks.rados.rados.0.smithi059.stdout:364: writing smithi05955964-49 from 688128 to 1409024 tid 2 2022-04-28T19:45:26.386 INFO:tasks.rados.rados.0.smithi059.stdout:364: writing smithi05955964-49 from 1409024 to 1949696 tid 3 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:364: writing smithi05955964-49 from 1949696 to 2293760 tid 4 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:365: rmattr oid 41 current snap is 33 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:365: done (3 left) 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:366: read oid 7 snap -1 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:366: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:367: snap_create 2022-04-28T19:45:26.389 INFO:tasks.rados.rados.0.smithi059.stdout:363: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:363: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:362: finishing write tid 1 to smithi05955964-5 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:362: finishing write tid 2 to smithi05955964-5 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:363: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 156 (ObjNum 87 snap 17 seq_num 87) dirty exists 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:364: finishing write tid 1 to smithi05955964-49 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:362: finishing write tid 3 to smithi05955964-5 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:362: finishing write tid 4 to smithi05955964-5 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:362: finishing write tid 6 to smithi05955964-5 2022-04-28T19:45:26.390 INFO:tasks.rados.rados.0.smithi059.stdout:362: finishing write tid 7 to smithi05955964-5 2022-04-28T19:45:26.391 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 110 (ObjNum 135 snap 33 seq_num 135) dirty exists 2022-04-28T19:45:26.391 INFO:tasks.rados.rados.0.smithi059.stdout:362: left oid 5 (ObjNum 135 snap 33 seq_num 135) 2022-04-28T19:45:26.391 INFO:tasks.rados.rados.0.smithi059.stdout:364: finishing write tid 2 to smithi05955964-49 2022-04-28T19:45:26.392 INFO:tasks.rados.rados.0.smithi059.stdout:364: finishing write tid 3 to smithi05955964-49 2022-04-28T19:45:26.392 INFO:tasks.rados.rados.0.smithi059.stdout:364: finishing write tid 4 to smithi05955964-49 2022-04-28T19:45:26.392 INFO:tasks.rados.rados.0.smithi059.stdout:364: finishing write tid 6 to smithi05955964-49 2022-04-28T19:45:26.392 INFO:tasks.rados.rados.0.smithi059.stdout:366: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:45:26.401 INFO:tasks.rados.rados.0.smithi059.stdout:364: finishing write tid 7 to smithi05955964-49 2022-04-28T19:45:26.401 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 103 (ObjNum 136 snap 33 seq_num 136) dirty exists 2022-04-28T19:45:26.401 INFO:tasks.rados.rados.0.smithi059.stdout:364: left oid 49 (ObjNum 136 snap 33 seq_num 136) 2022-04-28T19:45:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:26 smithi059 bash[29114]: cluster 2022-04-28T19:45:25.323272+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2022-04-28T19:45:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:26 smithi059 bash[23786]: cluster 2022-04-28T19:45:25.323272+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2022-04-28T19:45:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:26 smithi139 bash[23439]: cluster 2022-04-28T19:45:25.323272+0000 mon.a (mon.0) 728 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2022-04-28T19:45:27.166 INFO:tasks.rados.rados.0.smithi059.stdout:362: done (4 left) 2022-04-28T19:45:27.166 INFO:tasks.rados.rados.0.smithi059.stdout:363: done (3 left) 2022-04-28T19:45:27.166 INFO:tasks.rados.rados.0.smithi059.stdout:364: done (2 left) 2022-04-28T19:45:27.166 INFO:tasks.rados.rados.0.smithi059.stdout:366: done (1 left) 2022-04-28T19:45:27.166 INFO:tasks.rados.rados.0.smithi059.stdout:367: done (0 left) 2022-04-28T19:45:27.166 INFO:tasks.rados.rados.0.smithi059.stdout:368: delete oid 49 current snap is 34 2022-04-28T19:45:27.171 INFO:tasks.rados.rados.0.smithi059.stdout:368: done (0 left) 2022-04-28T19:45:27.172 INFO:tasks.rados.rados.0.smithi059.stdout:369: rollback oid 18 current snap is 34 2022-04-28T19:45:27.172 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 20 2022-04-28T19:45:27.172 INFO:tasks.rados.rados.0.smithi059.stdout:370: snap_create 2022-04-28T19:45:27.174 INFO:tasks.rados.rados.0.smithi059.stdout:369: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:45:27.174 INFO:tasks.rados.rados.0.smithi059.stdout:369: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:45:27.174 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 111 (ObjNum -133534832 snap 21893 seq_num 0) dirty dne 2022-04-28T19:45:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:27 smithi059 bash[23786]: cluster 2022-04-28T19:45:26.092766+0000 mgr.y (mgr.14140) 403 : cluster [DBG] pgmap v414: 89 pgs: 89 active+clean; 95 MiB data, 156 MiB used, 707 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2022-04-28T19:45:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:27 smithi059 bash[23786]: cluster 2022-04-28T19:45:26.339813+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2022-04-28T19:45:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:27 smithi059 bash[23786]: cluster 2022-04-28T19:45:27.161671+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2022-04-28T19:45:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:27 smithi059 bash[29114]: cluster 2022-04-28T19:45:26.092766+0000 mgr.y (mgr.14140) 403 : cluster [DBG] pgmap v414: 89 pgs: 89 active+clean; 95 MiB data, 156 MiB used, 707 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2022-04-28T19:45:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:27 smithi059 bash[29114]: cluster 2022-04-28T19:45:26.339813+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2022-04-28T19:45:27.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:27 smithi059 bash[29114]: cluster 2022-04-28T19:45:27.161671+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2022-04-28T19:45:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:27 smithi139 bash[23439]: cluster 2022-04-28T19:45:26.092766+0000 mgr.y (mgr.14140) 403 : cluster [DBG] pgmap v414: 89 pgs: 89 active+clean; 95 MiB data, 156 MiB used, 707 GiB / 715 GiB avail; 4.7 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2022-04-28T19:45:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:27 smithi139 bash[23439]: cluster 2022-04-28T19:45:26.339813+0000 mon.a (mon.0) 729 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2022-04-28T19:45:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:27 smithi139 bash[23439]: cluster 2022-04-28T19:45:27.161671+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2022-04-28T19:45:28.177 INFO:tasks.rados.rados.0.smithi059.stdout:369: done (1 left) 2022-04-28T19:45:28.177 INFO:tasks.rados.rados.0.smithi059.stdout:370: done (0 left) 2022-04-28T19:45:28.178 INFO:tasks.rados.rados.0.smithi059.stdout:371: rollback oid 41 current snap is 35 2022-04-28T19:45:28.178 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 30 2022-04-28T19:45:28.178 INFO:tasks.rados.rados.0.smithi059.stdout:372: rollback oid 18 current snap is 35 2022-04-28T19:45:28.178 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 33 2022-04-28T19:45:28.178 INFO:tasks.rados.rados.0.smithi059.stdout:373: rmattr oid 23 current snap is 35 2022-04-28T19:45:28.179 INFO:tasks.rados.rados.0.smithi059.stdout:374: copy_from oid 13 from oid 37 current snap is 35 2022-04-28T19:45:28.179 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 35 2022-04-28T19:45:28.179 INFO:tasks.rados.rados.0.smithi059.stdout:375: seq_num 138 ranges {3039232=720896,3760128=540672,4300800=557056,4857856=507904} 2022-04-28T19:45:28.199 INFO:tasks.rados.rados.0.smithi059.stdout:375: writing smithi05955964-36 from 3039232 to 3760128 tid 1 2022-04-28T19:45:28.202 INFO:tasks.rados.rados.0.smithi059.stdout:375: writing smithi05955964-36 from 3760128 to 4300800 tid 2 2022-04-28T19:45:28.207 INFO:tasks.rados.rados.0.smithi059.stdout:375: writing smithi05955964-36 from 4300800 to 4857856 tid 3 2022-04-28T19:45:28.209 INFO:tasks.rados.rados.0.smithi059.stdout:375: writing smithi05955964-36 from 4857856 to 5365760 tid 4 2022-04-28T19:45:28.210 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 35 2022-04-28T19:45:28.210 INFO:tasks.rados.rados.0.smithi059.stdout:376: seq_num 139 ranges {4505600=737280,5242880=425984,5668864=761856,6430720=417792} 2022-04-28T19:45:28.239 INFO:tasks.rados.rados.0.smithi059.stdout:376: writing smithi05955964-31 from 4505600 to 5242880 tid 1 2022-04-28T19:45:28.241 INFO:tasks.rados.rados.0.smithi059.stdout:376: writing smithi05955964-31 from 5242880 to 5668864 tid 2 2022-04-28T19:45:28.246 INFO:tasks.rados.rados.0.smithi059.stdout:376: writing smithi05955964-31 from 5668864 to 6430720 tid 3 2022-04-28T19:45:28.249 INFO:tasks.rados.rados.0.smithi059.stdout:376: writing smithi05955964-31 from 6430720 to 6848512 tid 4 2022-04-28T19:45:28.249 INFO:tasks.rados.rados.0.smithi059.stdout:377: setattr oid 39 current snap is 35 2022-04-28T19:45:28.249 INFO:tasks.rados.rados.0.smithi059.stdout:371: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:45:28.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 21 (ObjNum 1 snap 32762 seq_num 4161421856) dirty dne 2022-04-28T19:45:28.249 INFO:tasks.rados.rados.0.smithi059.stdout:372: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:45:28.249 INFO:tasks.rados.rados.0.smithi059.stdout:372: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:45:28.250 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 113 (ObjNum 3 snap 0 seq_num 3) dirty exists 2022-04-28T19:45:28.250 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 81 (ObjNum 53 snap 1 seq_num 53) dirty exists 2022-04-28T19:45:28.250 INFO:tasks.rados.rados.0.smithi059.stdout:374: finishing copy_from to smithi05955964-13 2022-04-28T19:45:28.250 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 101 (ObjNum 13 snap 0 seq_num 13) dirty exists 2022-04-28T19:45:28.250 INFO:tasks.rados.rados.0.smithi059.stdout:374: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:45:28.250 INFO:tasks.rados.rados.0.smithi059.stdout:375: finishing write tid 1 to smithi05955964-36 2022-04-28T19:45:28.251 INFO:tasks.rados.rados.0.smithi059.stdout:375: finishing write tid 2 to smithi05955964-36 2022-04-28T19:45:28.251 INFO:tasks.rados.rados.0.smithi059.stdout:375: finishing write tid 3 to smithi05955964-36 2022-04-28T19:45:28.251 INFO:tasks.rados.rados.0.smithi059.stdout:375: finishing write tid 4 to smithi05955964-36 2022-04-28T19:45:28.251 INFO:tasks.rados.rados.0.smithi059.stdout:375: finishing write tid 6 to smithi05955964-36 2022-04-28T19:45:28.251 INFO:tasks.rados.rados.0.smithi059.stdout:375: finishing write tid 7 to smithi05955964-36 2022-04-28T19:45:28.251 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 111 (ObjNum 138 snap 35 seq_num 138) dirty exists 2022-04-28T19:45:28.252 INFO:tasks.rados.rados.0.smithi059.stdout:375: left oid 36 (ObjNum 138 snap 35 seq_num 138) 2022-04-28T19:45:28.252 INFO:tasks.rados.rados.0.smithi059.stdout:376: finishing write tid 1 to smithi05955964-31 2022-04-28T19:45:28.252 INFO:tasks.rados.rados.0.smithi059.stdout:376: finishing write tid 2 to smithi05955964-31 2022-04-28T19:45:28.253 INFO:tasks.rados.rados.0.smithi059.stdout:376: finishing write tid 3 to smithi05955964-31 2022-04-28T19:45:28.254 INFO:tasks.rados.rados.0.smithi059.stdout:371: done (6 left) 2022-04-28T19:45:28.254 INFO:tasks.rados.rados.0.smithi059.stdout:372: done (5 left) 2022-04-28T19:45:28.254 INFO:tasks.rados.rados.0.smithi059.stdout:373: done (4 left) 2022-04-28T19:45:28.254 INFO:tasks.rados.rados.0.smithi059.stdout:374: done (3 left) 2022-04-28T19:45:28.254 INFO:tasks.rados.rados.0.smithi059.stdout:375: done (2 left) 2022-04-28T19:45:28.255 INFO:tasks.rados.rados.0.smithi059.stdout:378: snap_create 2022-04-28T19:45:28.255 INFO:tasks.rados.rados.0.smithi059.stdout:376: finishing write tid 4 to smithi05955964-31 2022-04-28T19:45:28.255 INFO:tasks.rados.rados.0.smithi059.stdout:376: finishing write tid 6 to smithi05955964-31 2022-04-28T19:45:28.256 INFO:tasks.rados.rados.0.smithi059.stdout:376: finishing write tid 7 to smithi05955964-31 2022-04-28T19:45:28.256 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 162 (ObjNum 139 snap 35 seq_num 139) dirty exists 2022-04-28T19:45:28.256 INFO:tasks.rados.rados.0.smithi059.stdout:376: left oid 31 (ObjNum 139 snap 35 seq_num 139) 2022-04-28T19:45:28.256 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 137 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:45:29.185 INFO:tasks.rados.rados.0.smithi059.stdout:376: done (2 left) 2022-04-28T19:45:29.186 INFO:tasks.rados.rados.0.smithi059.stdout:377: done (1 left) 2022-04-28T19:45:29.186 INFO:tasks.rados.rados.0.smithi059.stdout:378: done (0 left) 2022-04-28T19:45:29.186 INFO:tasks.rados.rados.0.smithi059.stdout:379: copy_from oid 34 from oid 46 current snap is 36 2022-04-28T19:45:29.186 INFO:tasks.rados.rados.0.smithi059.stdout:380: snap_remove snap 33 2022-04-28T19:45:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:29 smithi059 bash[29114]: cluster 2022-04-28T19:45:28.093650+0000 mgr.y (mgr.14140) 404 : cluster [DBG] pgmap v417: 89 pgs: 89 active+clean; 112 MiB data, 174 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.3 MiB/s wr, 24 op/s 2022-04-28T19:45:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:29 smithi059 bash[29114]: cluster 2022-04-28T19:45:28.174002+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2022-04-28T19:45:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:29 smithi059 bash[23786]: cluster 2022-04-28T19:45:28.093650+0000 mgr.y (mgr.14140) 404 : cluster [DBG] pgmap v417: 89 pgs: 89 active+clean; 112 MiB data, 174 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.3 MiB/s wr, 24 op/s 2022-04-28T19:45:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:29 smithi059 bash[23786]: cluster 2022-04-28T19:45:28.174002+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2022-04-28T19:45:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:29 smithi139 bash[23439]: cluster 2022-04-28T19:45:28.093650+0000 mgr.y (mgr.14140) 404 : cluster [DBG] pgmap v417: 89 pgs: 89 active+clean; 112 MiB data, 174 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.3 MiB/s wr, 24 op/s 2022-04-28T19:45:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:29 smithi139 bash[23439]: cluster 2022-04-28T19:45:28.174002+0000 mon.a (mon.0) 731 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2022-04-28T19:45:30.190 INFO:tasks.rados.rados.0.smithi059.stdout:380: done (1 left) 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:381: rollback oid 45 current snap is 36 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 17 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:379: finishing copy_from to smithi05955964-34 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 37 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:379: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:379: done (1 left) 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:382: read oid 28 snap -1 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:382: expect (ObjNum 102 snap 24 seq_num 102) 2022-04-28T19:45:30.191 INFO:tasks.rados.rados.0.smithi059.stdout:383: snap_remove snap 30 2022-04-28T19:45:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:30 smithi059 bash[23786]: cluster 2022-04-28T19:45:29.181500+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2022-04-28T19:45:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:30 smithi059 bash[29114]: cluster 2022-04-28T19:45:29.181500+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2022-04-28T19:45:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:30 smithi139 bash[23439]: cluster 2022-04-28T19:45:29.181500+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2022-04-28T19:45:31.208 INFO:tasks.rados.rados.0.smithi059.stdout:383: done (2 left) 2022-04-28T19:45:31.209 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 36 2022-04-28T19:45:31.209 INFO:tasks.rados.rados.0.smithi059.stdout:384: seq_num 140 ranges {2580480=753664,3334144=704512,4038656=524288,4562944=376832} 2022-04-28T19:45:31.228 INFO:tasks.rados.rados.0.smithi059.stdout:384: writing smithi05955964-21 from 2580480 to 3334144 tid 1 2022-04-28T19:45:31.232 INFO:tasks.rados.rados.0.smithi059.stdout:384: writing smithi05955964-21 from 3334144 to 4038656 tid 2 2022-04-28T19:45:31.235 INFO:tasks.rados.rados.0.smithi059.stdout:384: writing smithi05955964-21 from 4038656 to 4562944 tid 3 2022-04-28T19:45:31.237 INFO:tasks.rados.rados.0.smithi059.stdout:384: writing smithi05955964-21 from 4562944 to 4939776 tid 4 2022-04-28T19:45:31.238 INFO:tasks.rados.rados.0.smithi059.stdout:381: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:45:31.238 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 36 2022-04-28T19:45:31.238 INFO:tasks.rados.rados.0.smithi059.stdout:385: seq_num 141 ranges {221184=770048,991232=589824,1581056=729088,2310144=286720} 2022-04-28T19:45:31.243 INFO:tasks.rados.rados.0.smithi059.stdout:385: writing smithi05955964-37 from 221184 to 991232 tid 1 2022-04-28T19:45:31.247 INFO:tasks.rados.rados.0.smithi059.stdout:385: writing smithi05955964-37 from 991232 to 1581056 tid 2 2022-04-28T19:45:31.251 INFO:tasks.rados.rados.0.smithi059.stdout:385: writing smithi05955964-37 from 1581056 to 2310144 tid 3 2022-04-28T19:45:31.289 INFO:tasks.rados.rados.0.smithi059.stdout:385: writing smithi05955964-37 from 2310144 to 2596864 tid 4 2022-04-28T19:45:31.289 INFO:tasks.rados.rados.0.smithi059.stdout:386: snap_remove snap 27 2022-04-28T19:45:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:31 smithi059 bash[23786]: cluster 2022-04-28T19:45:30.094635+0000 mgr.y (mgr.14140) 405 : cluster [DBG] pgmap v420: 89 pgs: 89 active+clean; 122 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 3.9 MiB/s rd, 6.8 MiB/s wr, 20 op/s 2022-04-28T19:45:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:31 smithi059 bash[23786]: cluster 2022-04-28T19:45:30.186439+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2022-04-28T19:45:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:31 smithi059 bash[29114]: cluster 2022-04-28T19:45:30.094635+0000 mgr.y (mgr.14140) 405 : cluster [DBG] pgmap v420: 89 pgs: 89 active+clean; 122 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 3.9 MiB/s rd, 6.8 MiB/s wr, 20 op/s 2022-04-28T19:45:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:31 smithi059 bash[29114]: cluster 2022-04-28T19:45:30.186439+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2022-04-28T19:45:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:31 smithi139 bash[23439]: cluster 2022-04-28T19:45:30.094635+0000 mgr.y (mgr.14140) 405 : cluster [DBG] pgmap v420: 89 pgs: 89 active+clean; 122 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 3.9 MiB/s rd, 6.8 MiB/s wr, 20 op/s 2022-04-28T19:45:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:31 smithi139 bash[23439]: cluster 2022-04-28T19:45:30.186439+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2022-04-28T19:45:32.165 INFO:tasks.rados.rados.0.smithi059.stdout:386: done (4 left) 2022-04-28T19:45:32.165 INFO:tasks.rados.rados.0.smithi059.stdout:387: snap_create 2022-04-28T19:45:32.165 INFO:tasks.rados.rados.0.smithi059.stdout:381: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:45:32.165 INFO:tasks.rados.rados.0.smithi059.stdout:381: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:45:32.165 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 93 (ObjNum 33 snap 0 seq_num 33) dirty exists 2022-04-28T19:45:32.165 INFO:tasks.rados.rados.0.smithi059.stdout:382: expect (ObjNum 102 snap 24 seq_num 102) 2022-04-28T19:45:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:384: finishing write tid 1 to smithi05955964-21 2022-04-28T19:45:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:384: finishing write tid 2 to smithi05955964-21 2022-04-28T19:45:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:384: finishing write tid 3 to smithi05955964-21 2022-04-28T19:45:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:384: finishing write tid 4 to smithi05955964-21 2022-04-28T19:45:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:384: finishing write tid 6 to smithi05955964-21 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:384: finishing write tid 7 to smithi05955964-21 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 42 (ObjNum 140 snap 36 seq_num 140) dirty exists 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:384: left oid 21 (ObjNum 140 snap 36 seq_num 140) 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:385: finishing write tid 1 to smithi05955964-37 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:385: finishing write tid 2 to smithi05955964-37 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:385: finishing write tid 3 to smithi05955964-37 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:385: finishing write tid 4 to smithi05955964-37 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:385: finishing write tid 6 to smithi05955964-37 2022-04-28T19:45:32.220 INFO:tasks.rados.rados.0.smithi059.stdout:385: finishing write tid 7 to smithi05955964-37 2022-04-28T19:45:32.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 124 (ObjNum 141 snap 36 seq_num 141) dirty exists 2022-04-28T19:45:32.221 INFO:tasks.rados.rados.0.smithi059.stdout:385: left oid 37 (ObjNum 141 snap 36 seq_num 141) 2022-04-28T19:45:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:32 smithi059 bash[29114]: cluster 2022-04-28T19:45:31.204884+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2022-04-28T19:45:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:32 smithi059 bash[29114]: cluster 2022-04-28T19:45:32.162532+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2022-04-28T19:45:32.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:32 smithi059 bash[23786]: cluster 2022-04-28T19:45:31.204884+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2022-04-28T19:45:32.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:32 smithi059 bash[23786]: cluster 2022-04-28T19:45:32.162532+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2022-04-28T19:45:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:32 smithi139 bash[23439]: cluster 2022-04-28T19:45:31.204884+0000 mon.a (mon.0) 734 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2022-04-28T19:45:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:32 smithi139 bash[23439]: cluster 2022-04-28T19:45:32.162532+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2022-04-28T19:45:33.176 INFO:tasks.rados.rados.0.smithi059.stdout:381: done (4 left) 2022-04-28T19:45:33.285 INFO:tasks.rados.rados.0.smithi059.stdout:382: done (3 left) 2022-04-28T19:45:33.286 INFO:tasks.rados.rados.0.smithi059.stdout:384: done (2 left) 2022-04-28T19:45:33.286 INFO:tasks.rados.rados.0.smithi059.stdout:385: done (1 left) 2022-04-28T19:45:33.286 INFO:tasks.rados.rados.0.smithi059.stdout:387: done (0 left) 2022-04-28T19:45:33.286 INFO:tasks.rados.rados.0.smithi059.stdout:388: delete oid 47 current snap is 37 2022-04-28T19:45:33.287 INFO:tasks.rados.rados.0.smithi059.stdout:388: done (0 left) 2022-04-28T19:45:33.287 INFO:tasks.rados.rados.0.smithi059.stdout:389: rollback oid 29 current snap is 37 2022-04-28T19:45:33.287 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 34 2022-04-28T19:45:33.288 INFO:tasks.rados.rados.0.smithi059.stdout:390: rmattr oid 34 current snap is 37 2022-04-28T19:45:33.288 INFO:tasks.rados.rados.0.smithi059.stdout:391: snap_remove snap 31 2022-04-28T19:45:33.517 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[29114]: cluster 2022-04-28T19:45:32.095379+0000 mgr.y (mgr.14140) 406 : cluster [DBG] pgmap v423: 89 pgs: 89 active+clean; 124 MiB data, 188 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.1 MiB/s wr, 17 op/s 2022-04-28T19:45:33.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[29114]: cluster 2022-04-28T19:45:33.173840+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2022-04-28T19:45:33.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[23786]: cluster 2022-04-28T19:45:32.095379+0000 mgr.y (mgr.14140) 406 : cluster [DBG] pgmap v423: 89 pgs: 89 active+clean; 124 MiB data, 188 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.1 MiB/s wr, 17 op/s 2022-04-28T19:45:33.518 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[23786]: cluster 2022-04-28T19:45:33.173840+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2022-04-28T19:45:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:33 smithi139 bash[23439]: cluster 2022-04-28T19:45:32.095379+0000 mgr.y (mgr.14140) 406 : cluster [DBG] pgmap v423: 89 pgs: 89 active+clean; 124 MiB data, 188 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.1 MiB/s wr, 17 op/s 2022-04-28T19:45:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:33 smithi139 bash[23439]: cluster 2022-04-28T19:45:33.173840+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2022-04-28T19:45:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:33.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:45:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:33.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:45:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:33.516Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:45:34.179 INFO:tasks.rados.rados.0.smithi059.stdout:391: done (2 left) 2022-04-28T19:45:34.179 INFO:tasks.rados.rados.0.smithi059.stdout:392: copy_from oid 24 from oid 1 current snap is 37 2022-04-28T19:45:34.180 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 0 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:45:34.180 INFO:tasks.rados.rados.0.smithi059.stdout:389: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:45:34.180 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 65 (ObjNum -133694112 snap 21893 seq_num 4161798960) dirty dne 2022-04-28T19:45:34.180 INFO:tasks.rados.rados.0.smithi059.stdout:389: done (2 left) 2022-04-28T19:45:34.180 INFO:tasks.rados.rados.0.smithi059.stdout:390: done (1 left) 2022-04-28T19:45:34.180 INFO:tasks.rados.rados.0.smithi059.stdout:393: snap_create 2022-04-28T19:45:34.198 INFO:tasks.rados.rados.0.smithi059.stdout:392: finishing copy_from to smithi05955964-24 2022-04-28T19:45:34.199 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 52 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:45:34.199 INFO:tasks.rados.rados.0.smithi059.stdout:392: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:45:35.183 INFO:tasks.rados.rados.0.smithi059.stdout:392: done (1 left) 2022-04-28T19:45:35.183 INFO:tasks.rados.rados.0.smithi059.stdout:393: done (0 left) 2022-04-28T19:45:35.184 INFO:tasks.rados.rados.0.smithi059.stdout:394: copy_from oid 27 from oid 39 current snap is 38 2022-04-28T19:45:35.184 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 38 2022-04-28T19:45:35.184 INFO:tasks.rados.rados.0.smithi059.stdout:395: seq_num 143 ranges {0=802816,802816=761856,1564672=745472,2310144=98304} 2022-04-28T19:45:35.188 INFO:tasks.rados.rados.0.smithi059.stdout:395: writing smithi05955964-8 from 0 to 802816 tid 1 2022-04-28T19:45:35.193 INFO:tasks.rados.rados.0.smithi059.stdout:395: writing smithi05955964-8 from 802816 to 1564672 tid 2 2022-04-28T19:45:35.198 INFO:tasks.rados.rados.0.smithi059.stdout:395: writing smithi05955964-8 from 1564672 to 2310144 tid 3 2022-04-28T19:45:35.199 INFO:tasks.rados.rados.0.smithi059.stdout:395: writing smithi05955964-8 from 2310144 to 2408448 tid 4 2022-04-28T19:45:35.199 INFO:tasks.rados.rados.0.smithi059.stdout:396: copy_from oid 45 from oid 50 current snap is 38 2022-04-28T19:45:35.199 INFO:tasks.rados.rados.0.smithi059.stdout:395: finishing write tid 1 to smithi05955964-8 2022-04-28T19:45:35.199 INFO:tasks.rados.rados.0.smithi059.stdout:395: finishing write tid 2 to smithi05955964-8 2022-04-28T19:45:35.199 INFO:tasks.rados.rados.0.smithi059.stdout:397: snap_create 2022-04-28T19:45:35.203 INFO:tasks.rados.rados.0.smithi059.stdout:395: finishing write tid 3 to smithi05955964-8 2022-04-28T19:45:35.203 INFO:tasks.rados.rados.0.smithi059.stdout:395: finishing write tid 4 to smithi05955964-8 2022-04-28T19:45:35.203 INFO:tasks.rados.rados.0.smithi059.stdout:395: finishing write tid 6 to smithi05955964-8 2022-04-28T19:45:35.207 INFO:tasks.rados.rados.0.smithi059.stdout:395: finishing write tid 7 to smithi05955964-8 2022-04-28T19:45:35.207 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 124 (ObjNum 143 snap 38 seq_num 143) dirty exists 2022-04-28T19:45:35.207 INFO:tasks.rados.rados.0.smithi059.stdout:395: left oid 8 (ObjNum 143 snap 38 seq_num 143) 2022-04-28T19:45:35.212 INFO:tasks.rados.rados.0.smithi059.stdout:394: finishing copy_from to smithi05955964-27 2022-04-28T19:45:35.212 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 137 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:45:35.212 INFO:tasks.rados.rados.0.smithi059.stdout:394: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:45:35.213 INFO:tasks.rados.rados.0.smithi059.stdout:396: finishing copy_from to smithi05955964-45 2022-04-28T19:45:35.213 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 113 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:45:35.213 INFO:tasks.rados.rados.0.smithi059.stdout:396: finishing copy_from racing read to smithi05955964-45 2022-04-28T19:45:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:35 smithi059 bash[23786]: cluster 2022-04-28T19:45:34.096097+0000 mgr.y (mgr.14140) 407 : cluster [DBG] pgmap v426: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 127 MiB data, 197 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 1.2 MiB/s wr, 8 op/s 2022-04-28T19:45:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:35 smithi059 bash[23786]: cluster 2022-04-28T19:45:34.177272+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2022-04-28T19:45:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:35 smithi059 bash[29114]: cluster 2022-04-28T19:45:34.096097+0000 mgr.y (mgr.14140) 407 : cluster [DBG] pgmap v426: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 127 MiB data, 197 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 1.2 MiB/s wr, 8 op/s 2022-04-28T19:45:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:35 smithi059 bash[29114]: cluster 2022-04-28T19:45:34.177272+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2022-04-28T19:45:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:35 smithi139 bash[23439]: cluster 2022-04-28T19:45:34.096097+0000 mgr.y (mgr.14140) 407 : cluster [DBG] pgmap v426: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 127 MiB data, 197 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 1.2 MiB/s wr, 8 op/s 2022-04-28T19:45:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:35 smithi139 bash[23439]: cluster 2022-04-28T19:45:34.177272+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2022-04-28T19:45:36.202 INFO:tasks.rados.rados.0.smithi059.stdout:394: done (3 left) 2022-04-28T19:45:36.234 INFO:tasks.rados.rados.0.smithi059.stdout:395: done (2 left) 2022-04-28T19:45:36.234 INFO:tasks.rados.rados.0.smithi059.stdout:396: done (1 left) 2022-04-28T19:45:36.234 INFO:tasks.rados.rados.0.smithi059.stdout:397: done (0 left) 2022-04-28T19:45:36.234 INFO:tasks.rados.rados.0.smithi059.stdout:398: read oid 36 snap -1 2022-04-28T19:45:36.235 INFO:tasks.rados.rados.0.smithi059.stdout:398: expect (ObjNum 138 snap 35 seq_num 138) 2022-04-28T19:45:36.235 INFO:tasks.rados.rados.0.smithi059.stdout:399: delete oid 21 current snap is 39 2022-04-28T19:45:36.236 INFO:tasks.rados.rados.0.smithi059.stdout:399: done (1 left) 2022-04-28T19:45:36.236 INFO:tasks.rados.rados.0.smithi059.stdout:400: rmattr oid 9 current snap is 39 2022-04-28T19:45:36.236 INFO:tasks.rados.rados.0.smithi059.stdout:400: done (1 left) 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:401: rmattr oid 32 current snap is 39 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:401: done (1 left) 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:402: rmattr oid 6 current snap is 39 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:402: done (1 left) 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:403: copy_from oid 15 from oid 19 current snap is 39 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:404: delete oid 3 current snap is 39 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:404: done (2 left) 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:405: rollback oid 49 current snap is 39 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 17 2022-04-28T19:45:36.237 INFO:tasks.rados.rados.0.smithi059.stdout:406: read oid 45 snap -1 2022-04-28T19:45:36.238 INFO:tasks.rados.rados.0.smithi059.stdout:406: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:45:36.238 INFO:tasks.rados.rados.0.smithi059.stdout:407: snap_remove snap 35 2022-04-28T19:45:36.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:36 smithi059 bash[29114]: cluster 2022-04-28T19:45:35.180104+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2022-04-28T19:45:36.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:36 smithi059 bash[23786]: cluster 2022-04-28T19:45:35.180104+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2022-04-28T19:45:36.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:36 smithi139 bash[23439]: cluster 2022-04-28T19:45:35.180104+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2022-04-28T19:45:37.168 INFO:tasks.rados.rados.0.smithi059.stdout:407: done (4 left) 2022-04-28T19:45:37.169 INFO:tasks.rados.rados.0.smithi059.stdout:408: delete oid 5 current snap is 39 2022-04-28T19:45:37.169 INFO:tasks.rados.rados.0.smithi059.stdout:403: finishing copy_from to smithi05955964-15 2022-04-28T19:45:37.169 INFO:tasks.rados.rados.0.smithi059.stdout:403: got expected ENOENT (src dne) 2022-04-28T19:45:37.169 INFO:tasks.rados.rados.0.smithi059.stdout:403: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:45:37.169 INFO:tasks.rados.rados.0.smithi059.stdout:406: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:45:37.190 INFO:tasks.rados.rados.0.smithi059.stdout:398: expect (ObjNum 138 snap 35 seq_num 138) 2022-04-28T19:45:37.397 INFO:tasks.rados.rados.0.smithi059.stdout:405: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:45:37.397 INFO:tasks.rados.rados.0.smithi059.stdout:405: finishing rollback tid 2 to smithi05955964-49 2022-04-28T19:45:37.397 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 127 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:45:37.397 INFO:tasks.rados.rados.0.smithi059.stdout:398: done (4 left) 2022-04-28T19:45:37.398 INFO:tasks.rados.rados.0.smithi059.stdout:403: done (3 left) 2022-04-28T19:45:37.398 INFO:tasks.rados.rados.0.smithi059.stdout:405: done (2 left) 2022-04-28T19:45:37.398 INFO:tasks.rados.rados.0.smithi059.stdout:406: done (1 left) 2022-04-28T19:45:37.398 INFO:tasks.rados.rados.0.smithi059.stdout:408: done (0 left) 2022-04-28T19:45:37.398 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 39 2022-04-28T19:45:37.399 INFO:tasks.rados.rados.0.smithi059.stdout:409: seq_num 147 ranges {3334144=65536} 2022-04-28T19:45:37.425 INFO:tasks.rados.rados.0.smithi059.stdout:409: writing smithi05955964-12 from 3334144 to 3399680 tid 1 2022-04-28T19:45:37.425 INFO:tasks.rados.rados.0.smithi059.stdout:410: snap_remove snap 38 2022-04-28T19:45:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:37 smithi059 bash[23786]: cluster 2022-04-28T19:45:36.096955+0000 mgr.y (mgr.14140) 408 : cluster [DBG] pgmap v429: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 126 MiB data, 194 MiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2022-04-28T19:45:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:37 smithi059 bash[23786]: cluster 2022-04-28T19:45:36.200703+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2022-04-28T19:45:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:37 smithi059 bash[23786]: cluster 2022-04-28T19:45:37.163809+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2022-04-28T19:45:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:37 smithi059 bash[29114]: cluster 2022-04-28T19:45:36.096955+0000 mgr.y (mgr.14140) 408 : cluster [DBG] pgmap v429: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 126 MiB data, 194 MiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2022-04-28T19:45:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:37 smithi059 bash[29114]: cluster 2022-04-28T19:45:36.200703+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2022-04-28T19:45:37.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:37 smithi059 bash[29114]: cluster 2022-04-28T19:45:37.163809+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2022-04-28T19:45:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:37 smithi139 bash[23439]: cluster 2022-04-28T19:45:36.096955+0000 mgr.y (mgr.14140) 408 : cluster [DBG] pgmap v429: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 126 MiB data, 194 MiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2022-04-28T19:45:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:37 smithi139 bash[23439]: cluster 2022-04-28T19:45:36.200703+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2022-04-28T19:45:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:37 smithi139 bash[23439]: cluster 2022-04-28T19:45:37.163809+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2022-04-28T19:45:38.212 INFO:tasks.rados.rados.0.smithi059.stdout:410: done (1 left) 2022-04-28T19:45:38.212 INFO:tasks.rados.rados.0.smithi059.stdout:411: snap_remove snap 36 2022-04-28T19:45:39.222 INFO:tasks.rados.rados.0.smithi059.stdout:411: done (1 left) 2022-04-28T19:45:39.223 INFO:tasks.rados.rados.0.smithi059.stdout:412: snap_remove snap 37 2022-04-28T19:45:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:39 smithi059 bash[29114]: cluster 2022-04-28T19:45:38.097873+0000 mgr.y (mgr.14140) 409 : cluster [DBG] pgmap v432: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 126 MiB data, 198 MiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.5 MiB/s wr, 5 op/s 2022-04-28T19:45:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:39 smithi059 bash[29114]: cluster 2022-04-28T19:45:38.210442+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2022-04-28T19:45:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:39 smithi059 bash[23786]: cluster 2022-04-28T19:45:38.097873+0000 mgr.y (mgr.14140) 409 : cluster [DBG] pgmap v432: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 126 MiB data, 198 MiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.5 MiB/s wr, 5 op/s 2022-04-28T19:45:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:39 smithi059 bash[23786]: cluster 2022-04-28T19:45:38.210442+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2022-04-28T19:45:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:39 smithi139 bash[23439]: cluster 2022-04-28T19:45:38.097873+0000 mgr.y (mgr.14140) 409 : cluster [DBG] pgmap v432: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 126 MiB data, 198 MiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.5 MiB/s wr, 5 op/s 2022-04-28T19:45:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:39 smithi139 bash[23439]: cluster 2022-04-28T19:45:38.210442+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2022-04-28T19:45:40.233 INFO:tasks.rados.rados.0.smithi059.stdout:412: done (1 left) 2022-04-28T19:45:40.233 INFO:tasks.rados.rados.0.smithi059.stdout:413: snap_remove snap 26 2022-04-28T19:45:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:40 smithi059 bash[23786]: cluster 2022-04-28T19:45:39.219221+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2022-04-28T19:45:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:40 smithi059 bash[29114]: cluster 2022-04-28T19:45:39.219221+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2022-04-28T19:45:40.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:40 smithi139 bash[23439]: cluster 2022-04-28T19:45:39.219221+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2022-04-28T19:45:41.236 INFO:tasks.rados.rados.0.smithi059.stdout:413: done (1 left) 2022-04-28T19:45:41.236 INFO:tasks.rados.rados.0.smithi059.stdout:414: setattr oid 21 current snap is 39 2022-04-28T19:45:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:409: finishing write tid 1 to smithi05955964-12 2022-04-28T19:45:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:409: finishing write tid 3 to smithi05955964-12 2022-04-28T19:45:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:409: finishing write tid 4 to smithi05955964-12 2022-04-28T19:45:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 179 (ObjNum 147 snap 39 seq_num 147) dirty exists 2022-04-28T19:45:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:409: left oid 12 (ObjNum 147 snap 39 seq_num 147) 2022-04-28T19:45:41.241 INFO:tasks.rados.rados.0.smithi059.stdout:409: done (1 left) 2022-04-28T19:45:41.241 INFO:tasks.rados.rados.0.smithi059.stdout:415: read oid 27 snap -1 2022-04-28T19:45:41.241 INFO:tasks.rados.rados.0.smithi059.stdout:415: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:41.241 INFO:tasks.rados.rados.0.smithi059.stdout:416: read oid 37 snap -1 2022-04-28T19:45:41.241 INFO:tasks.rados.rados.0.smithi059.stdout:416: expect (ObjNum 141 snap 36 seq_num 141) 2022-04-28T19:45:41.241 INFO:tasks.rados.rados.0.smithi059.stdout:417: snap_create 2022-04-28T19:45:41.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 57 (ObjNum 1584228979 snap 1633381973 seq_num 1769099368) dirty exists 2022-04-28T19:45:41.268 INFO:tasks.rados.rados.0.smithi059.stdout:415: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:41.351 INFO:tasks.rados.rados.0.smithi059.stdout:416: expect (ObjNum 141 snap 36 seq_num 141) 2022-04-28T19:45:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:41 smithi059 bash[23786]: cluster 2022-04-28T19:45:40.098726+0000 mgr.y (mgr.14140) 410 : cluster [DBG] pgmap v435: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 130 MiB data, 202 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2022-04-28T19:45:41.679 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:41 smithi059 bash[23786]: cluster 2022-04-28T19:45:40.228068+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2022-04-28T19:45:41.680 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:41 smithi059 bash[29114]: cluster 2022-04-28T19:45:40.098726+0000 mgr.y (mgr.14140) 410 : cluster [DBG] pgmap v435: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 130 MiB data, 202 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2022-04-28T19:45:41.680 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:41 smithi059 bash[29114]: cluster 2022-04-28T19:45:40.228068+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2022-04-28T19:45:41.681 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:41 smithi139 bash[23439]: cluster 2022-04-28T19:45:40.098726+0000 mgr.y (mgr.14140) 410 : cluster [DBG] pgmap v435: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 130 MiB data, 202 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2022-04-28T19:45:41.681 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:41 smithi139 bash[23439]: cluster 2022-04-28T19:45:40.228068+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2022-04-28T19:45:42.169 INFO:tasks.rados.rados.0.smithi059.stdout:414: done (3 left) 2022-04-28T19:45:42.169 INFO:tasks.rados.rados.0.smithi059.stdout:415: done (2 left) 2022-04-28T19:45:42.169 INFO:tasks.rados.rados.0.smithi059.stdout:416: done (1 left) 2022-04-28T19:45:42.169 INFO:tasks.rados.rados.0.smithi059.stdout:417: done (0 left) 2022-04-28T19:45:42.169 INFO:tasks.rados.rados.0.smithi059.stdout:418: delete oid 32 current snap is 40 2022-04-28T19:45:42.173 INFO:tasks.rados.rados.0.smithi059.stdout:418: done (0 left) 2022-04-28T19:45:42.174 INFO:tasks.rados.rados.0.smithi059.stdout:419: read oid 29 snap 22 2022-04-28T19:45:42.174 INFO:tasks.rados.rados.0.smithi059.stdout:419: expect deleted 2022-04-28T19:45:42.174 INFO:tasks.rados.rados.0.smithi059.stdout:420: delete oid 1 current snap is 40 2022-04-28T19:45:42.176 INFO:tasks.rados.rados.0.smithi059.stdout:419: done (1 left) 2022-04-28T19:45:42.177 INFO:tasks.rados.rados.0.smithi059.stdout:420: done (0 left) 2022-04-28T19:45:42.177 INFO:tasks.rados.rados.0.smithi059.stdout:421: copy_from oid 11 from oid 50 current snap is 40 2022-04-28T19:45:42.177 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 40 2022-04-28T19:45:42.177 INFO:tasks.rados.rados.0.smithi059.stdout:422: seq_num 150 ranges {4907008=114688} 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:422: writing smithi05955964-16 from 4907008 to 5021696 tid 1 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:423: rmattr oid 2 current snap is 40 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:423: done (2 left) 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:424: delete oid 35 current snap is 40 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:421: finishing copy_from to smithi05955964-11 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 113 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:45:42.206 INFO:tasks.rados.rados.0.smithi059.stdout:421: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:45:42.209 INFO:tasks.rados.rados.0.smithi059.stdout:422: finishing write tid 1 to smithi05955964-16 2022-04-28T19:45:42.211 INFO:tasks.rados.rados.0.smithi059.stdout:422: finishing write tid 3 to smithi05955964-16 2022-04-28T19:45:42.212 INFO:tasks.rados.rados.0.smithi059.stdout:421: done (2 left) 2022-04-28T19:45:42.212 INFO:tasks.rados.rados.0.smithi059.stdout:424: done (1 left) 2022-04-28T19:45:42.212 INFO:tasks.rados.rados.0.smithi059.stdout:425: read oid 3 snap -1 2022-04-28T19:45:42.212 INFO:tasks.rados.rados.0.smithi059.stdout:425: expect deleted 2022-04-28T19:45:42.212 INFO:tasks.rados.rados.0.smithi059.stdout:426: snap_create 2022-04-28T19:45:42.214 INFO:tasks.rados.rados.0.smithi059.stdout:422: finishing write tid 4 to smithi05955964-16 2022-04-28T19:45:42.214 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 82 (ObjNum 150 snap 40 seq_num 150) dirty exists 2022-04-28T19:45:42.214 INFO:tasks.rados.rados.0.smithi059.stdout:422: left oid 16 (ObjNum 150 snap 40 seq_num 150) 2022-04-28T19:45:42.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:42 smithi059 bash[29114]: cluster 2022-04-28T19:45:41.234471+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2022-04-28T19:45:42.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:42 smithi059 bash[29114]: cluster 2022-04-28T19:45:42.166733+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2022-04-28T19:45:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:42 smithi059 bash[23786]: cluster 2022-04-28T19:45:41.234471+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2022-04-28T19:45:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:42 smithi059 bash[23786]: cluster 2022-04-28T19:45:42.166733+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2022-04-28T19:45:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:42 smithi139 bash[23439]: cluster 2022-04-28T19:45:41.234471+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2022-04-28T19:45:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:42 smithi139 bash[23439]: cluster 2022-04-28T19:45:42.166733+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2022-04-28T19:45:43.182 INFO:tasks.rados.rados.0.smithi059.stdout:422: done (2 left) 2022-04-28T19:45:43.182 INFO:tasks.rados.rados.0.smithi059.stdout:425: done (1 left) 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:426: done (0 left) 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:427: rmattr oid 6 current snap is 41 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:428: rollback oid 3 current snap is 41 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 34 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:429: rollback oid 16 current snap is 41 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 32 2022-04-28T19:45:43.183 INFO:tasks.rados.rados.0.smithi059.stdout:430: snap_create 2022-04-28T19:45:43.184 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 0 (ObjNum 1 snap 32762 seq_num 4161709616) dirty dne 2022-04-28T19:45:43.186 INFO:tasks.rados.rados.0.smithi059.stdout:428: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:45:43.187 INFO:tasks.rados.rados.0.smithi059.stdout:428: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:45:43.187 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 129 (ObjNum 46 snap 0 seq_num 46) dirty exists 2022-04-28T19:45:43.188 INFO:tasks.rados.rados.0.smithi059.stdout:429: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:45:43.188 INFO:tasks.rados.rados.0.smithi059.stdout:429: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:45:43.188 INFO:tasks.rados.rados.0.smithi059.stdout:429: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:45:43.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 86 (ObjNum 111 snap 26 seq_num 111) dirty exists 2022-04-28T19:45:43.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[29114]: cluster 2022-04-28T19:45:42.099463+0000 mgr.y (mgr.14140) 411 : cluster [DBG] pgmap v438: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 130 MiB data, 203 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 904 KiB/s wr, 4 op/s 2022-04-28T19:45:43.523 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[29114]: cluster 2022-04-28T19:45:43.179230+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2022-04-28T19:45:43.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[23786]: cluster 2022-04-28T19:45:42.099463+0000 mgr.y (mgr.14140) 411 : cluster [DBG] pgmap v438: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 130 MiB data, 203 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 904 KiB/s wr, 4 op/s 2022-04-28T19:45:43.524 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[23786]: cluster 2022-04-28T19:45:43.179230+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2022-04-28T19:45:43.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:43 smithi139 bash[23439]: cluster 2022-04-28T19:45:42.099463+0000 mgr.y (mgr.14140) 411 : cluster [DBG] pgmap v438: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 130 MiB data, 203 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 904 KiB/s wr, 4 op/s 2022-04-28T19:45:43.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:43 smithi139 bash[23439]: cluster 2022-04-28T19:45:43.179230+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2022-04-28T19:45:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:43.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:45:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:43.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:45:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:43.516Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:427: done (3 left) 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:428: done (2 left) 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:429: done (1 left) 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:430: done (0 left) 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:431: rollback oid 39 current snap is 42 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 39 to 34 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:432: rollback oid 21 current snap is 42 2022-04-28T19:45:44.186 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 32 2022-04-28T19:45:44.187 INFO:tasks.rados.rados.0.smithi059.stdout:433: delete oid 24 current snap is 42 2022-04-28T19:45:44.189 INFO:tasks.rados.rados.0.smithi059.stdout:433: done (2 left) 2022-04-28T19:45:44.189 INFO:tasks.rados.rados.0.smithi059.stdout:434: delete oid 3 current snap is 42 2022-04-28T19:45:44.193 INFO:tasks.rados.rados.0.smithi059.stdout:431: finishing rollback tid 0 to smithi05955964-39 2022-04-28T19:45:44.194 INFO:tasks.rados.rados.0.smithi059.stdout:431: finishing rollback tid 1 to smithi05955964-39 2022-04-28T19:45:44.195 INFO:tasks.rados.rados.0.smithi059.stdout:431: finishing rollback tid 2 to smithi05955964-39 2022-04-28T19:45:44.195 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 169 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:45:44.195 INFO:tasks.rados.rados.0.smithi059.stdout:432: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:45:44.196 INFO:tasks.rados.rados.0.smithi059.stdout:432: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:45:44.196 INFO:tasks.rados.rados.0.smithi059.stdout:431: done (2 left) 2022-04-28T19:45:44.196 INFO:tasks.rados.rados.0.smithi059.stdout:434: done (1 left) 2022-04-28T19:45:44.196 INFO:tasks.rados.rados.0.smithi059.stdout:435: read oid 27 snap 41 2022-04-28T19:45:44.196 INFO:tasks.rados.rados.0.smithi059.stdout:435: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:44.197 INFO:tasks.rados.rados.0.smithi059.stdout:436: read oid 23 snap 17 2022-04-28T19:45:44.197 INFO:tasks.rados.rados.0.smithi059.stdout:436: expect (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:45:44.197 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 42 2022-04-28T19:45:44.197 INFO:tasks.rados.rados.0.smithi059.stdout:437: seq_num 154 ranges {57344=180224} 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:437: writing smithi05955964-18 from 57344 to 237568 tid 1 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:438: copy_from oid 42 from oid 45 current snap is 42 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:432: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 63 (ObjNum 130 snap 31 seq_num 130) dirty exists 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:432: done (4 left) 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:439: read oid 21 snap -1 2022-04-28T19:45:44.198 INFO:tasks.rados.rados.0.smithi059.stdout:439: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:45:44.199 INFO:tasks.rados.rados.0.smithi059.stdout:440: copy_from oid 9 from oid 30 current snap is 42 2022-04-28T19:45:44.199 INFO:tasks.rados.rados.0.smithi059.stdout:441: rmattr oid 4 current snap is 42 2022-04-28T19:45:44.199 INFO:tasks.rados.rados.0.smithi059.stdout:441: done (6 left) 2022-04-28T19:45:44.199 INFO:tasks.rados.rados.0.smithi059.stdout:442: rmattr oid 26 current snap is 42 2022-04-28T19:45:44.199 INFO:tasks.rados.rados.0.smithi059.stdout:442: done (6 left) 2022-04-28T19:45:44.199 INFO:tasks.rados.rados.0.smithi059.stdout:443: setattr oid 1 current snap is 42 2022-04-28T19:45:44.200 INFO:tasks.rados.rados.0.smithi059.stdout:444: rollback oid 29 current snap is 42 2022-04-28T19:45:44.200 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 39 2022-04-28T19:45:44.200 INFO:tasks.rados.rados.0.smithi059.stdout:445: delete oid 41 current snap is 42 2022-04-28T19:45:44.200 INFO:tasks.rados.rados.0.smithi059.stdout:444: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:45:44.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 65 (ObjNum 0 snap 0 seq_num 0) dirty dne 2022-04-28T19:45:44.201 INFO:tasks.rados.rados.0.smithi059.stdout:444: done (8 left) 2022-04-28T19:45:44.202 INFO:tasks.rados.rados.0.smithi059.stdout:445: done (7 left) 2022-04-28T19:45:44.202 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 42 2022-04-28T19:45:44.202 INFO:tasks.rados.rados.0.smithi059.stdout:446: seq_num 156 ranges {0=221184} 2022-04-28T19:45:44.203 INFO:tasks.rados.rados.0.smithi059.stdout:446: writing smithi05955964-5 from 0 to 221184 tid 1 2022-04-28T19:45:44.203 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 42 2022-04-28T19:45:44.203 INFO:tasks.rados.rados.0.smithi059.stdout:447: seq_num 157 ranges {2596864=237568} 2022-04-28T19:45:44.221 INFO:tasks.rados.rados.0.smithi059.stdout:447: writing smithi05955964-37 from 2596864 to 2834432 tid 1 2022-04-28T19:45:44.222 INFO:tasks.rados.rados.0.smithi059.stdout:448: read oid 33 snap -1 2022-04-28T19:45:44.222 INFO:tasks.rados.rados.0.smithi059.stdout:448: expect (ObjNum 89 snap 18 seq_num 89) 2022-04-28T19:45:44.222 INFO:tasks.rados.rados.0.smithi059.stdout:449: read oid 11 snap -1 2022-04-28T19:45:44.222 INFO:tasks.rados.rados.0.smithi059.stdout:449: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:45:44.222 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 42 2022-04-28T19:45:44.222 INFO:tasks.rados.rados.0.smithi059.stdout:450: seq_num 158 ranges {0=253952} 2022-04-28T19:45:44.223 INFO:tasks.rados.rados.0.smithi059.stdout:450: writing smithi05955964-24 from 0 to 253952 tid 1 2022-04-28T19:45:44.223 INFO:tasks.rados.rados.0.smithi059.stdout:451: read oid 31 snap -1 2022-04-28T19:45:44.224 INFO:tasks.rados.rados.0.smithi059.stdout:451: expect (ObjNum 139 snap 35 seq_num 139) 2022-04-28T19:45:44.224 INFO:tasks.rados.rados.0.smithi059.stdout:452: read oid 14 snap -1 2022-04-28T19:45:44.224 INFO:tasks.rados.rados.0.smithi059.stdout:452: expect (ObjNum 13 snap 0 seq_num 13) 2022-04-28T19:45:44.224 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 42 2022-04-28T19:45:44.224 INFO:tasks.rados.rados.0.smithi059.stdout:453: seq_num 159 ranges {2088960=270336} 2022-04-28T19:45:44.238 INFO:tasks.rados.rados.0.smithi059.stdout:453: writing smithi05955964-48 from 2088960 to 2359296 tid 1 2022-04-28T19:45:44.238 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 42 2022-04-28T19:45:44.238 INFO:tasks.rados.rados.0.smithi059.stdout:454: seq_num 160 ranges {0=286720} 2022-04-28T19:45:44.240 INFO:tasks.rados.rados.0.smithi059.stdout:454: writing smithi05955964-44 from 0 to 286720 tid 1 2022-04-28T19:45:44.240 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 16 2022-04-28T19:45:44.240 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 136 (ObjNum 1 snap 32762 seq_num 4161669920) dirty exists 2022-04-28T19:45:44.241 INFO:tasks.rados.rados.0.smithi059.stdout:437: finishing write tid 1 to smithi05955964-18 2022-04-28T19:45:44.241 INFO:tasks.rados.rados.0.smithi059.stdout:437: finishing write tid 3 to smithi05955964-18 2022-04-28T19:45:44.241 INFO:tasks.rados.rados.0.smithi059.stdout:437: finishing write tid 4 to smithi05955964-18 2022-04-28T19:45:44.241 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 145 (ObjNum 154 snap 42 seq_num 154) dirty exists 2022-04-28T19:45:44.242 INFO:tasks.rados.rados.0.smithi059.stdout:437: left oid 18 (ObjNum 154 snap 42 seq_num 154) 2022-04-28T19:45:44.242 INFO:tasks.rados.rados.0.smithi059.stdout:435: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:44.327 INFO:tasks.rados.rados.0.smithi059.stdout:446: finishing write tid 1 to smithi05955964-5 2022-04-28T19:45:44.328 INFO:tasks.rados.rados.0.smithi059.stdout:446: finishing write tid 3 to smithi05955964-5 2022-04-28T19:45:44.328 INFO:tasks.rados.rados.0.smithi059.stdout:439: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:45:44.393 INFO:tasks.rados.rados.0.smithi059.stdout:446: finishing write tid 4 to smithi05955964-5 2022-04-28T19:45:44.393 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 139 (ObjNum 156 snap 42 seq_num 156) dirty exists 2022-04-28T19:45:44.393 INFO:tasks.rados.rados.0.smithi059.stdout:446: left oid 5 (ObjNum 156 snap 42 seq_num 156) 2022-04-28T19:45:44.393 INFO:tasks.rados.rados.0.smithi059.stdout:438: finishing copy_from to smithi05955964-42 2022-04-28T19:45:44.394 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 113 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:45:44.394 INFO:tasks.rados.rados.0.smithi059.stdout:436: expect (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:45:44.448 INFO:tasks.rados.rados.0.smithi059.stdout:438: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:45:44.448 INFO:tasks.rados.rados.0.smithi059.stdout:440: finishing copy_from to smithi05955964-9 2022-04-28T19:45:44.448 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 54 (ObjNum 100 snap 24 seq_num 100) dirty exists 2022-04-28T19:45:44.448 INFO:tasks.rados.rados.0.smithi059.stdout:440: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:435: done (15 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:436: done (14 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:437: done (13 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:438: done (12 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:439: done (11 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:440: done (10 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:443: done (9 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:446: done (8 left) 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 42 2022-04-28T19:45:44.449 INFO:tasks.rados.rados.0.smithi059.stdout:455: seq_num 161 ranges {221184=303104} 2022-04-28T19:45:44.456 INFO:tasks.rados.rados.0.smithi059.stdout:455: writing smithi05955964-13 from 221184 to 524288 tid 1 2022-04-28T19:45:44.456 INFO:tasks.rados.rados.0.smithi059.stdout:456: read oid 10 snap -1 2022-04-28T19:45:44.457 INFO:tasks.rados.rados.0.smithi059.stdout:456: expect deleted 2022-04-28T19:45:44.457 INFO:tasks.rados.rados.0.smithi059.stdout:457: snap_remove snap 34 2022-04-28T19:45:45.265 INFO:tasks.rados.rados.0.smithi059.stdout:457: done (10 left) 2022-04-28T19:45:45.265 INFO:tasks.rados.rados.0.smithi059.stdout:458: rollback oid 45 current snap is 42 2022-04-28T19:45:45.265 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 39 2022-04-28T19:45:45.265 INFO:tasks.rados.rados.0.smithi059.stdout:452: expect (ObjNum 13 snap 0 seq_num 13) 2022-04-28T19:45:45.274 INFO:tasks.rados.rados.0.smithi059.stdout:449: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:45:45.297 INFO:tasks.rados.rados.0.smithi059.stdout:450: finishing write tid 1 to smithi05955964-24 2022-04-28T19:45:45.297 INFO:tasks.rados.rados.0.smithi059.stdout:447: finishing write tid 1 to smithi05955964-37 2022-04-28T19:45:45.297 INFO:tasks.rados.rados.0.smithi059.stdout:450: finishing write tid 3 to smithi05955964-24 2022-04-28T19:45:45.297 INFO:tasks.rados.rados.0.smithi059.stdout:450: finishing write tid 4 to smithi05955964-24 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 116 (ObjNum 158 snap 42 seq_num 158) dirty exists 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:450: left oid 24 (ObjNum 158 snap 42 seq_num 158) 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:447: finishing write tid 3 to smithi05955964-37 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:449: done (10 left) 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:450: done (9 left) 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:452: done (8 left) 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:459: delete oid 42 current snap is 42 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:447: finishing write tid 4 to smithi05955964-37 2022-04-28T19:45:45.298 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 149 (ObjNum 157 snap 42 seq_num 157) dirty exists 2022-04-28T19:45:45.299 INFO:tasks.rados.rados.0.smithi059.stdout:447: left oid 37 (ObjNum 157 snap 42 seq_num 157) 2022-04-28T19:45:45.299 INFO:tasks.rados.rados.0.smithi059.stdout:448: expect (ObjNum 89 snap 18 seq_num 89) 2022-04-28T19:45:45.388 INFO:tasks.rados.rados.0.smithi059.stdout:453: finishing write tid 1 to smithi05955964-48 2022-04-28T19:45:45.388 INFO:tasks.rados.rados.0.smithi059.stdout:453: finishing write tid 3 to smithi05955964-48 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:453: finishing write tid 4 to smithi05955964-48 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 140 (ObjNum 159 snap 42 seq_num 159) dirty exists 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:453: left oid 48 (ObjNum 159 snap 42 seq_num 159) 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:454: finishing write tid 1 to smithi05955964-44 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:447: done (8 left) 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:448: done (7 left) 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:453: done (6 left) 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:459: done (5 left) 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 42 2022-04-28T19:45:45.389 INFO:tasks.rados.rados.0.smithi059.stdout:460: seq_num 163 ranges {1720320=335872} 2022-04-28T19:45:45.406 INFO:tasks.rados.rados.0.smithi059.stdout:460: writing smithi05955964-28 from 1720320 to 2056192 tid 1 2022-04-28T19:45:45.406 INFO:tasks.rados.rados.0.smithi059.stdout:461: snap_remove snap 40 2022-04-28T19:45:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:45 smithi059 bash[29114]: cluster 2022-04-28T19:45:44.100183+0000 mgr.y (mgr.14140) 412 : cluster [DBG] pgmap v441: 89 pgs: 89 active+clean; 128 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 0 B/s rd, 202 KiB/s wr, 5 op/s 2022-04-28T19:45:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:45 smithi059 bash[29114]: cluster 2022-04-28T19:45:44.181306+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2022-04-28T19:45:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:45 smithi059 bash[23786]: cluster 2022-04-28T19:45:44.100183+0000 mgr.y (mgr.14140) 412 : cluster [DBG] pgmap v441: 89 pgs: 89 active+clean; 128 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 0 B/s rd, 202 KiB/s wr, 5 op/s 2022-04-28T19:45:45.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:45 smithi059 bash[23786]: cluster 2022-04-28T19:45:44.181306+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2022-04-28T19:45:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:45 smithi139 bash[23439]: cluster 2022-04-28T19:45:44.100183+0000 mgr.y (mgr.14140) 412 : cluster [DBG] pgmap v441: 89 pgs: 89 active+clean; 128 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 0 B/s rd, 202 KiB/s wr, 5 op/s 2022-04-28T19:45:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:45 smithi139 bash[23439]: cluster 2022-04-28T19:45:44.181306+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2022-04-28T19:45:46.264 INFO:tasks.rados.rados.0.smithi059.stdout:461: done (6 left) 2022-04-28T19:45:46.378 INFO:tasks.rados.rados.0.smithi059.stdout:462: copy_from oid 1 from oid 3 current snap is 42 2022-04-28T19:45:46.378 INFO:tasks.rados.rados.0.smithi059.stdout:463: setattr oid 18 current snap is 42 2022-04-28T19:45:46.383 INFO:tasks.rados.rados.0.smithi059.stdout:464: rollback oid 39 current snap is 42 2022-04-28T19:45:46.383 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 39 to 23 2022-04-28T19:45:46.384 INFO:tasks.rados.rados.0.smithi059.stdout:465: setattr oid 4 current snap is 42 2022-04-28T19:45:46.384 INFO:tasks.rados.rados.0.smithi059.stdout:454: finishing write tid 3 to smithi05955964-44 2022-04-28T19:45:46.384 INFO:tasks.rados.rados.0.smithi059.stdout:454: finishing write tid 4 to smithi05955964-44 2022-04-28T19:45:46.384 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 84 (ObjNum 160 snap 42 seq_num 160) dirty exists 2022-04-28T19:45:46.384 INFO:tasks.rados.rados.0.smithi059.stdout:454: left oid 44 (ObjNum 160 snap 42 seq_num 160) 2022-04-28T19:45:46.384 INFO:tasks.rados.rados.0.smithi059.stdout:451: expect (ObjNum 139 snap 35 seq_num 139) 2022-04-28T19:45:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:46 smithi059 bash[29114]: cluster 2022-04-28T19:45:45.260138+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2022-04-28T19:45:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:46 smithi059 bash[23786]: cluster 2022-04-28T19:45:45.260138+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:455: finishing write tid 1 to smithi05955964-13 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:455: finishing write tid 3 to smithi05955964-13 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:455: finishing write tid 4 to smithi05955964-13 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 103 (ObjNum 161 snap 42 seq_num 161) dirty exists 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:455: left oid 13 (ObjNum 161 snap 42 seq_num 161) 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:458: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:458: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:45:46.593 INFO:tasks.rados.rados.0.smithi059.stdout:458: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 135 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:460: finishing write tid 1 to smithi05955964-28 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:460: finishing write tid 3 to smithi05955964-28 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:460: finishing write tid 4 to smithi05955964-28 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 91 (ObjNum 163 snap 42 seq_num 163) dirty exists 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:460: left oid 28 (ObjNum 163 snap 42 seq_num 163) 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 157 (ObjNum 154 snap 42 seq_num 154) dirty exists 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:462: finishing copy_from to smithi05955964-1 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:462: got expected ENOENT (src dne) 2022-04-28T19:45:46.594 INFO:tasks.rados.rados.0.smithi059.stdout:462: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:45:46.595 INFO:tasks.rados.rados.0.smithi059.stdout:464: finishing rollback tid 0 to smithi05955964-39 2022-04-28T19:45:46.595 INFO:tasks.rados.rados.0.smithi059.stdout:464: finishing rollback tid 1 to smithi05955964-39 2022-04-28T19:45:46.595 INFO:tasks.rados.rados.0.smithi059.stdout:464: finishing rollback tid 2 to smithi05955964-39 2022-04-28T19:45:46.595 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 176 (ObjNum 38 snap 0 seq_num 38) dirty exists 2022-04-28T19:45:46.595 INFO:tasks.rados.rados.0.smithi059.stdout:451: done (9 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:454: done (8 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:455: done (7 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:456: done (6 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:458: done (5 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:460: done (4 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:462: done (3 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:463: done (2 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:464: done (1 left) 2022-04-28T19:45:46.596 INFO:tasks.rados.rados.0.smithi059.stdout:466: snap_create 2022-04-28T19:45:46.597 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 117 (ObjNum 3 snap 0 seq_num 3) dirty exists 2022-04-28T19:45:46.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:46 smithi139 bash[23439]: cluster 2022-04-28T19:45:45.260138+0000 mon.a (mon.0) 748 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2022-04-28T19:45:47.172 INFO:tasks.rados.rados.0.smithi059.stdout:465: done (1 left) 2022-04-28T19:45:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:466: done (0 left) 2022-04-28T19:45:47.288 INFO:tasks.rados.rados.0.smithi059.stdout:467: snap_create 2022-04-28T19:45:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:47 smithi059 bash[29114]: cluster 2022-04-28T19:45:46.100915+0000 mgr.y (mgr.14140) 413 : cluster [DBG] pgmap v444: 89 pgs: 89 active+clean; 121 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 22 op/s 2022-04-28T19:45:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:47 smithi059 bash[29114]: cluster 2022-04-28T19:45:46.262218+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2022-04-28T19:45:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:47 smithi059 bash[29114]: cluster 2022-04-28T19:45:47.170616+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2022-04-28T19:45:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:47 smithi059 bash[23786]: cluster 2022-04-28T19:45:46.100915+0000 mgr.y (mgr.14140) 413 : cluster [DBG] pgmap v444: 89 pgs: 89 active+clean; 121 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 22 op/s 2022-04-28T19:45:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:47 smithi059 bash[23786]: cluster 2022-04-28T19:45:46.262218+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2022-04-28T19:45:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:47 smithi059 bash[23786]: cluster 2022-04-28T19:45:47.170616+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2022-04-28T19:45:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:47 smithi139 bash[23439]: cluster 2022-04-28T19:45:46.100915+0000 mgr.y (mgr.14140) 413 : cluster [DBG] pgmap v444: 89 pgs: 89 active+clean; 121 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 22 op/s 2022-04-28T19:45:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:47 smithi139 bash[23439]: cluster 2022-04-28T19:45:46.262218+0000 mon.a (mon.0) 749 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2022-04-28T19:45:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:47 smithi139 bash[23439]: cluster 2022-04-28T19:45:47.170616+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2022-04-28T19:45:48.184 INFO:tasks.rados.rados.0.smithi059.stdout:467: done (0 left) 2022-04-28T19:45:48.184 INFO:tasks.rados.rados.0.smithi059.stdout:468: setattr oid 4 current snap is 44 2022-04-28T19:45:48.186 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 44 2022-04-28T19:45:48.186 INFO:tasks.rados.rados.0.smithi059.stdout:469: seq_num 164 ranges {221184=352256} 2022-04-28T19:45:48.190 INFO:tasks.rados.rados.0.smithi059.stdout:469: writing smithi05955964-14 from 221184 to 573440 tid 1 2022-04-28T19:45:48.190 INFO:tasks.rados.rados.0.smithi059.stdout:470: read oid 16 snap 42 2022-04-28T19:45:48.190 INFO:tasks.rados.rados.0.smithi059.stdout:470: expect (ObjNum 111 snap 26 seq_num 111) 2022-04-28T19:45:48.191 INFO:tasks.rados.rados.0.smithi059.stdout:471: rmattr oid 10 current snap is 44 2022-04-28T19:45:48.191 INFO:tasks.rados.rados.0.smithi059.stdout:471: done (3 left) 2022-04-28T19:45:48.191 INFO:tasks.rados.rados.0.smithi059.stdout:472: copy_from oid 41 from oid 17 current snap is 44 2022-04-28T19:45:48.191 INFO:tasks.rados.rados.0.smithi059.stdout:473: delete oid 32 current snap is 44 2022-04-28T19:45:48.196 INFO:tasks.rados.rados.0.smithi059.stdout:469: finishing write tid 1 to smithi05955964-14 2022-04-28T19:45:48.196 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 118 (ObjNum 3 snap 0 seq_num 3) dirty exists 2022-04-28T19:45:48.196 INFO:tasks.rados.rados.0.smithi059.stdout:469: finishing write tid 3 to smithi05955964-14 2022-04-28T19:45:48.197 INFO:tasks.rados.rados.0.smithi059.stdout:468: done (4 left) 2022-04-28T19:45:48.198 INFO:tasks.rados.rados.0.smithi059.stdout:473: done (3 left) 2022-04-28T19:45:48.198 INFO:tasks.rados.rados.0.smithi059.stdout:474: rmattr oid 28 current snap is 44 2022-04-28T19:45:48.198 INFO:tasks.rados.rados.0.smithi059.stdout:475: rollback oid 31 current snap is 44 2022-04-28T19:45:48.198 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 43 2022-04-28T19:45:48.198 INFO:tasks.rados.rados.0.smithi059.stdout:476: copy_from oid 19 from oid 46 current snap is 44 2022-04-28T19:45:48.199 INFO:tasks.rados.rados.0.smithi059.stdout:477: read oid 45 snap -1 2022-04-28T19:45:48.199 INFO:tasks.rados.rados.0.smithi059.stdout:477: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:45:48.199 INFO:tasks.rados.rados.0.smithi059.stdout:478: rollback oid 29 current snap is 44 2022-04-28T19:45:48.199 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 17 2022-04-28T19:45:48.199 INFO:tasks.rados.rados.0.smithi059.stdout:479: setattr oid 40 current snap is 44 2022-04-28T19:45:48.200 INFO:tasks.rados.rados.0.smithi059.stdout:469: finishing write tid 4 to smithi05955964-14 2022-04-28T19:45:48.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 147 (ObjNum 164 snap 44 seq_num 164) dirty exists 2022-04-28T19:45:48.200 INFO:tasks.rados.rados.0.smithi059.stdout:469: left oid 14 (ObjNum 164 snap 44 seq_num 164) 2022-04-28T19:45:48.200 INFO:tasks.rados.rados.0.smithi059.stdout:478: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:45:48.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 65 (ObjNum 0 snap 0 seq_num 188341408) dirty dne 2022-04-28T19:45:48.200 INFO:tasks.rados.rados.0.smithi059.stdout:469: done (8 left) 2022-04-28T19:45:48.201 INFO:tasks.rados.rados.0.smithi059.stdout:478: done (7 left) 2022-04-28T19:45:48.201 INFO:tasks.rados.rados.0.smithi059.stdout:480: snap_create 2022-04-28T19:45:48.202 INFO:tasks.rados.rados.0.smithi059.stdout:477: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:45:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:475: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:45:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:475: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:475: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 199 (ObjNum 139 snap 35 seq_num 139) dirty exists 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 142 (ObjNum 39 snap 0 seq_num 39) dirty exists 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:472: finishing copy_from to smithi05955964-41 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 26 (ObjNum 97 snap 20 seq_num 97) dirty exists 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:472: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:476: finishing copy_from to smithi05955964-19 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 37 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:476: finishing copy_from racing read to smithi05955964-19 2022-04-28T19:45:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:470: expect (ObjNum 111 snap 26 seq_num 111) 2022-04-28T19:45:48.423 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 97 (ObjNum 163 snap 42 seq_num 163) dirty exists 2022-04-28T19:45:49.190 INFO:tasks.rados.rados.0.smithi059.stdout:470: done (7 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:472: done (6 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:474: done (5 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:475: done (4 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:476: done (3 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:477: done (2 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:479: done (1 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:480: done (0 left) 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:481: rmattr oid 4 current snap is 45 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 45 2022-04-28T19:45:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:482: seq_num 166 ranges {0=385024} 2022-04-28T19:45:49.193 INFO:tasks.rados.rados.0.smithi059.stdout:482: writing smithi05955964-47 from 0 to 385024 tid 1 2022-04-28T19:45:49.193 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 45 2022-04-28T19:45:49.193 INFO:tasks.rados.rados.0.smithi059.stdout:483: seq_num 167 ranges {663552=401408} 2022-04-28T19:45:49.201 INFO:tasks.rados.rados.0.smithi059.stdout:483: writing smithi05955964-40 from 663552 to 1064960 tid 1 2022-04-28T19:45:49.201 INFO:tasks.rados.rados.0.smithi059.stdout:484: read oid 38 snap 42 2022-04-28T19:45:49.201 INFO:tasks.rados.rados.0.smithi059.stdout:484: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:45:49.201 INFO:tasks.rados.rados.0.smithi059.stdout:485: delete oid 16 current snap is 45 2022-04-28T19:45:49.201 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 119 (ObjNum 3 snap 0 seq_num 3) dirty exists 2022-04-28T19:45:49.202 INFO:tasks.rados.rados.0.smithi059.stdout:482: finishing write tid 1 to smithi05955964-47 2022-04-28T19:45:49.202 INFO:tasks.rados.rados.0.smithi059.stdout:482: finishing write tid 3 to smithi05955964-47 2022-04-28T19:45:49.204 INFO:tasks.rados.rados.0.smithi059.stdout:482: finishing write tid 4 to smithi05955964-47 2022-04-28T19:45:49.204 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 139 (ObjNum 166 snap 45 seq_num 166) dirty exists 2022-04-28T19:45:49.204 INFO:tasks.rados.rados.0.smithi059.stdout:482: left oid 47 (ObjNum 166 snap 45 seq_num 166) 2022-04-28T19:45:49.205 INFO:tasks.rados.rados.0.smithi059.stdout:481: done (4 left) 2022-04-28T19:45:49.205 INFO:tasks.rados.rados.0.smithi059.stdout:482: done (3 left) 2022-04-28T19:45:49.205 INFO:tasks.rados.rados.0.smithi059.stdout:485: done (2 left) 2022-04-28T19:45:49.205 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 45 2022-04-28T19:45:49.205 INFO:tasks.rados.rados.0.smithi059.stdout:486: seq_num 169 ranges {0=434176} 2022-04-28T19:45:49.208 INFO:tasks.rados.rados.0.smithi059.stdout:486: writing smithi05955964-6 from 0 to 434176 tid 1 2022-04-28T19:45:49.208 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 45 2022-04-28T19:45:49.208 INFO:tasks.rados.rados.0.smithi059.stdout:487: seq_num 170 ranges {573440=450560} 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:487: writing smithi05955964-14 from 573440 to 1024000 tid 1 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:488: rollback oid 17 current snap is 45 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 41 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:483: finishing write tid 1 to smithi05955964-40 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:483: finishing write tid 3 to smithi05955964-40 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:483: finishing write tid 4 to smithi05955964-40 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 145 (ObjNum 167 snap 45 seq_num 167) dirty exists 2022-04-28T19:45:49.215 INFO:tasks.rados.rados.0.smithi059.stdout:483: left oid 40 (ObjNum 167 snap 45 seq_num 167) 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:483: done (4 left) 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:489: read oid 35 snap -1 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:489: expect deleted 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:490: rollback oid 18 current snap is 45 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 17 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 45 2022-04-28T19:45:49.216 INFO:tasks.rados.rados.0.smithi059.stdout:491: seq_num 171 ranges {385024=466944} 2022-04-28T19:45:49.220 INFO:tasks.rados.rados.0.smithi059.stdout:491: writing smithi05955964-47 from 385024 to 851968 tid 1 2022-04-28T19:45:49.220 INFO:tasks.rados.rados.0.smithi059.stdout:492: rollback oid 37 current snap is 45 2022-04-28T19:45:49.220 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 37 to 17 2022-04-28T19:45:49.220 INFO:tasks.rados.rados.0.smithi059.stdout:484: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:45:49.298 INFO:tasks.rados.rados.0.smithi059.stdout:486: finishing write tid 1 to smithi05955964-6 2022-04-28T19:45:49.298 INFO:tasks.rados.rados.0.smithi059.stdout:486: finishing write tid 3 to smithi05955964-6 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:488: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:488: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:488: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 106 (ObjNum 97 snap 20 seq_num 97) dirty exists 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:487: finishing write tid 1 to smithi05955964-14 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:490: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:487: finishing write tid 3 to smithi05955964-14 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:490: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 160 (ObjNum -133027232 snap 21893 seq_num 4161546032) dirty dne 2022-04-28T19:45:49.299 INFO:tasks.rados.rados.0.smithi059.stdout:486: finishing write tid 4 to smithi05955964-6 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 143 (ObjNum 169 snap 45 seq_num 169) dirty exists 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:486: left oid 6 (ObjNum 169 snap 45 seq_num 169) 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:491: finishing write tid 1 to smithi05955964-47 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:491: finishing write tid 3 to smithi05955964-47 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:487: finishing write tid 4 to smithi05955964-14 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 151 (ObjNum 170 snap 45 seq_num 170) dirty exists 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:487: left oid 14 (ObjNum 170 snap 45 seq_num 170) 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:491: finishing write tid 4 to smithi05955964-47 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 141 (ObjNum 171 snap 45 seq_num 171) dirty exists 2022-04-28T19:45:49.300 INFO:tasks.rados.rados.0.smithi059.stdout:491: left oid 47 (ObjNum 171 snap 45 seq_num 171) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:492: finishing rollback tid 0 to smithi05955964-37 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:492: finishing rollback tid 1 to smithi05955964-37 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:492: finishing rollback tid 2 to smithi05955964-37 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 164 (ObjNum 36 snap 0 seq_num 36) dirty exists 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:484: done (7 left) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:486: done (6 left) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:487: done (5 left) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:488: done (4 left) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:489: done (3 left) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:490: done (2 left) 2022-04-28T19:45:49.301 INFO:tasks.rados.rados.0.smithi059.stdout:491: done (1 left) 2022-04-28T19:45:49.302 INFO:tasks.rados.rados.0.smithi059.stdout:492: done (0 left) 2022-04-28T19:45:49.302 INFO:tasks.rados.rados.0.smithi059.stdout:493: snap_remove snap 41 2022-04-28T19:45:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:49 smithi059 bash[23786]: cluster 2022-04-28T19:45:48.101905+0000 mgr.y (mgr.14140) 414 : cluster [DBG] pgmap v447: 89 pgs: 89 active+clean; 122 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 29 op/s 2022-04-28T19:45:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:49 smithi059 bash[23786]: cluster 2022-04-28T19:45:48.181305+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2022-04-28T19:45:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:49 smithi059 bash[29114]: cluster 2022-04-28T19:45:48.101905+0000 mgr.y (mgr.14140) 414 : cluster [DBG] pgmap v447: 89 pgs: 89 active+clean; 122 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 29 op/s 2022-04-28T19:45:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:49 smithi059 bash[29114]: cluster 2022-04-28T19:45:48.181305+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2022-04-28T19:45:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:49 smithi139 bash[23439]: cluster 2022-04-28T19:45:48.101905+0000 mgr.y (mgr.14140) 414 : cluster [DBG] pgmap v447: 89 pgs: 89 active+clean; 122 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 1.2 MiB/s wr, 29 op/s 2022-04-28T19:45:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:49 smithi139 bash[23439]: cluster 2022-04-28T19:45:48.181305+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2022-04-28T19:45:50.204 INFO:tasks.rados.rados.0.smithi059.stdout:493: done (0 left) 2022-04-28T19:45:50.205 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 45 2022-04-28T19:45:50.205 INFO:tasks.rados.rados.0.smithi059.stdout:494: seq_num 172 ranges {1024000=483328} 2022-04-28T19:45:50.213 INFO:tasks.rados.rados.0.smithi059.stdout:494: writing smithi05955964-14 from 1024000 to 1507328 tid 1 2022-04-28T19:45:50.213 INFO:tasks.rados.rados.0.smithi059.stdout:495: delete oid 1 current snap is 45 2022-04-28T19:45:50.218 INFO:tasks.rados.rados.0.smithi059.stdout:494: finishing write tid 1 to smithi05955964-14 2022-04-28T19:45:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:494: finishing write tid 3 to smithi05955964-14 2022-04-28T19:45:50.221 INFO:tasks.rados.rados.0.smithi059.stdout:494: finishing write tid 4 to smithi05955964-14 2022-04-28T19:45:50.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 157 (ObjNum 172 snap 45 seq_num 172) dirty exists 2022-04-28T19:45:50.221 INFO:tasks.rados.rados.0.smithi059.stdout:494: left oid 14 (ObjNum 172 snap 45 seq_num 172) 2022-04-28T19:45:50.221 INFO:tasks.rados.rados.0.smithi059.stdout:494: done (1 left) 2022-04-28T19:45:50.221 INFO:tasks.rados.rados.0.smithi059.stdout:495: done (0 left) 2022-04-28T19:45:50.221 INFO:tasks.rados.rados.0.smithi059.stdout:496: snap_create 2022-04-28T19:45:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:50 smithi059 bash[23786]: cluster 2022-04-28T19:45:49.185750+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2022-04-28T19:45:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:50 smithi059 bash[29114]: cluster 2022-04-28T19:45:49.185750+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2022-04-28T19:45:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:50 smithi139 bash[23439]: cluster 2022-04-28T19:45:49.185750+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2022-04-28T19:45:51.220 INFO:tasks.rados.rados.0.smithi059.stdout:496: done (0 left) 2022-04-28T19:45:51.220 INFO:tasks.rados.rados.0.smithi059.stdout:497: snap_create 2022-04-28T19:45:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:51 smithi059 bash[29114]: cluster 2022-04-28T19:45:50.102934+0000 mgr.y (mgr.14140) 415 : cluster [DBG] pgmap v450: 89 pgs: 89 active+clean; 125 MiB data, 189 MiB used, 707 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.1 MiB/s wr, 34 op/s 2022-04-28T19:45:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:51 smithi059 bash[29114]: cluster 2022-04-28T19:45:50.198008+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2022-04-28T19:45:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:51 smithi059 bash[23786]: cluster 2022-04-28T19:45:50.102934+0000 mgr.y (mgr.14140) 415 : cluster [DBG] pgmap v450: 89 pgs: 89 active+clean; 125 MiB data, 189 MiB used, 707 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.1 MiB/s wr, 34 op/s 2022-04-28T19:45:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:51 smithi059 bash[23786]: cluster 2022-04-28T19:45:50.198008+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2022-04-28T19:45:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:51 smithi139 bash[23439]: cluster 2022-04-28T19:45:50.102934+0000 mgr.y (mgr.14140) 415 : cluster [DBG] pgmap v450: 89 pgs: 89 active+clean; 125 MiB data, 189 MiB used, 707 GiB / 715 GiB avail; 8.6 MiB/s rd, 1.1 MiB/s wr, 34 op/s 2022-04-28T19:45:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:51 smithi139 bash[23439]: cluster 2022-04-28T19:45:50.198008+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2022-04-28T19:45:52.174 INFO:tasks.rados.rados.0.smithi059.stdout:497: done (0 left) 2022-04-28T19:45:52.175 INFO:tasks.rados.rados.0.smithi059.stdout:498: snap_remove snap 20 2022-04-28T19:45:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:52 smithi059 bash[23786]: cluster 2022-04-28T19:45:51.218117+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2022-04-28T19:45:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:52 smithi059 bash[23786]: cluster 2022-04-28T19:45:52.171453+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2022-04-28T19:45:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:52 smithi059 bash[29114]: cluster 2022-04-28T19:45:51.218117+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2022-04-28T19:45:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:52 smithi059 bash[29114]: cluster 2022-04-28T19:45:52.171453+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2022-04-28T19:45:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:52 smithi139 bash[23439]: cluster 2022-04-28T19:45:51.218117+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2022-04-28T19:45:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:52 smithi139 bash[23439]: cluster 2022-04-28T19:45:52.171453+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2022-04-28T19:45:53.187 INFO:tasks.rados.rados.0.smithi059.stdout:498: done (0 left) 2022-04-28T19:45:53.187 INFO:tasks.rados.rados.0.smithi059.stdout:499: read oid 21 snap -1 2022-04-28T19:45:53.187 INFO:tasks.rados.rados.0.smithi059.stdout:499: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:45:53.187 INFO:tasks.rados.rados.0.smithi059.stdout:500: snap_create 2022-04-28T19:45:53.204 INFO:tasks.rados.rados.0.smithi059.stdout:499: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:45:53.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[29114]: cluster 2022-04-28T19:45:52.103728+0000 mgr.y (mgr.14140) 416 : cluster [DBG] pgmap v453: 89 pgs: 89 active+clean; 130 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 25 op/s 2022-04-28T19:45:53.519 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[29114]: cluster 2022-04-28T19:45:53.183292+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2022-04-28T19:45:53.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[23786]: cluster 2022-04-28T19:45:52.103728+0000 mgr.y (mgr.14140) 416 : cluster [DBG] pgmap v453: 89 pgs: 89 active+clean; 130 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 25 op/s 2022-04-28T19:45:53.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[23786]: cluster 2022-04-28T19:45:53.183292+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2022-04-28T19:45:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:53 smithi139 bash[23439]: cluster 2022-04-28T19:45:52.103728+0000 mgr.y (mgr.14140) 416 : cluster [DBG] pgmap v453: 89 pgs: 89 active+clean; 130 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 25 op/s 2022-04-28T19:45:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:53 smithi139 bash[23439]: cluster 2022-04-28T19:45:53.183292+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2022-04-28T19:45:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:53.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:45:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:53.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:45:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:45:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:45:53.516Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:45:54.143 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:45:54.187 INFO:tasks.rados.rados.0.smithi059.stdout:499: done (1 left) 2022-04-28T19:45:54.187 INFO:tasks.rados.rados.0.smithi059.stdout:500: done (0 left) 2022-04-28T19:45:54.187 INFO:tasks.rados.rados.0.smithi059.stdout:501: copy_from oid 24 from oid 25 current snap is 48 2022-04-28T19:45:54.187 INFO:tasks.rados.rados.0.smithi059.stdout:502: snap_remove snap 46 2022-04-28T19:45:54.450 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST STATUS REFRESHED AGE VERSION IMAGE NAME IMAGE ID CONTAINER ID 2022-04-28T19:45:54.450 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (4m) 3m ago 5m 0.20.0 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (5m) 3m ago 5m 6.7.4 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (3m) 3m ago 3m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 44062639a78b 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (3m) 3m ago 3m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 3516bf4894c9 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (9m) 3m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b8c424c2e210 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (11m) 3m ago 11m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 320b27bb479e 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (11m) 3m ago 11m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 239497cf6769 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (9m) 3m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0e12092e0bc3 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (9m) 3m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:45:54.451 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (5m) 3m ago 5m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (5m) 3m ago 5m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (8m) 3m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 547b5580e073 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (8m) 3m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (7m) 3m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 35cdc0c6604a 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (7m) 3m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 2344bb9dfbae 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (7m) 3m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 5976654882b1 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (6m) 3m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 e68cffb4766c 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (6m) 3m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 c36b8738a469 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (5m) 3m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:45:54.452 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (4m) 3m ago 5m 2.18.1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 c763409947e4 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:54.794 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 15 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:45:54.795 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:45:55.194 INFO:tasks.rados.rados.0.smithi059.stdout:502: done (1 left) 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:503: rollback oid 16 current snap is 48 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 42 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:501: finishing copy_from to smithi05955964-24 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 91 (ObjNum 116 snap 29 seq_num 116) dirty exists 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:501: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:501: done (1 left) 2022-04-28T19:45:55.312 INFO:tasks.rados.rados.0.smithi059.stdout:504: read oid 47 snap 23 2022-04-28T19:45:55.313 INFO:tasks.rados.rados.0.smithi059.stdout:504: expect (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:45:55.313 INFO:tasks.rados.rados.0.smithi059.stdout:505: rollback oid 17 current snap is 48 2022-04-28T19:45:55.313 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 45 2022-04-28T19:45:55.313 INFO:tasks.rados.rados.0.smithi059.stdout:506: snap_create 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:504: expect (ObjNum 46 snap 0 seq_num 46) 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:505: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:505: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:503: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:503: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 109 (ObjNum 111 snap 26 seq_num 111) dirty exists 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:505: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:45:55.314 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 109 (ObjNum 97 snap 20 seq_num 97) dirty exists 2022-04-28T19:45:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[23786]: cluster 2022-04-28T19:45:54.104400+0000 mgr.y (mgr.14140) 417 : cluster [DBG] pgmap v456: 89 pgs: 89 active+clean; 130 MiB data, 198 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2022-04-28T19:45:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[23786]: audit 2022-04-28T19:45:54.127682+0000 mgr.y (mgr.14140) 418 : audit [DBG] from='client.24775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:55.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[23786]: cluster 2022-04-28T19:45:54.185847+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2022-04-28T19:45:55.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[23786]: audit 2022-04-28T19:45:54.792149+0000 mon.c (mon.1) 35 : audit [DBG] from='client.? 172.21.15.59:0/2418989420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:45:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[29114]: cluster 2022-04-28T19:45:54.104400+0000 mgr.y (mgr.14140) 417 : cluster [DBG] pgmap v456: 89 pgs: 89 active+clean; 130 MiB data, 198 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2022-04-28T19:45:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[29114]: audit 2022-04-28T19:45:54.127682+0000 mgr.y (mgr.14140) 418 : audit [DBG] from='client.24775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[29114]: cluster 2022-04-28T19:45:54.185847+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2022-04-28T19:45:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:55 smithi059 bash[29114]: audit 2022-04-28T19:45:54.792149+0000 mon.c (mon.1) 35 : audit [DBG] from='client.? 172.21.15.59:0/2418989420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:45:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:55 smithi139 bash[23439]: cluster 2022-04-28T19:45:54.104400+0000 mgr.y (mgr.14140) 417 : cluster [DBG] pgmap v456: 89 pgs: 89 active+clean; 130 MiB data, 198 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.0 MiB/s wr, 11 op/s 2022-04-28T19:45:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:55 smithi139 bash[23439]: audit 2022-04-28T19:45:54.127682+0000 mgr.y (mgr.14140) 418 : audit [DBG] from='client.24775 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:55 smithi139 bash[23439]: cluster 2022-04-28T19:45:54.185847+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2022-04-28T19:45:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:55 smithi139 bash[23439]: audit 2022-04-28T19:45:54.792149+0000 mon.c (mon.1) 35 : audit [DBG] from='client.? 172.21.15.59:0/2418989420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:45:56.206 INFO:tasks.rados.rados.0.smithi059.stdout:503: done (3 left) 2022-04-28T19:45:56.206 INFO:tasks.rados.rados.0.smithi059.stdout:504: done (2 left) 2022-04-28T19:45:56.207 INFO:tasks.rados.rados.0.smithi059.stdout:505: done (1 left) 2022-04-28T19:45:56.207 INFO:tasks.rados.rados.0.smithi059.stdout:506: done (0 left) 2022-04-28T19:45:56.207 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 49 2022-04-28T19:45:56.207 INFO:tasks.rados.rados.0.smithi059.stdout:507: seq_num 174 ranges {2850816=516096} 2022-04-28T19:45:56.226 INFO:tasks.rados.rados.0.smithi059.stdout:507: writing smithi05955964-33 from 2850816 to 3366912 tid 1 2022-04-28T19:45:56.226 INFO:tasks.rados.rados.0.smithi059.stdout:508: read oid 27 snap -1 2022-04-28T19:45:56.226 INFO:tasks.rados.rados.0.smithi059.stdout:508: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:56.226 INFO:tasks.rados.rados.0.smithi059.stdout:509: read oid 34 snap -1 2022-04-28T19:45:56.227 INFO:tasks.rados.rados.0.smithi059.stdout:509: expect (ObjNum 84 snap 14 seq_num 84) 2022-04-28T19:45:56.227 INFO:tasks.rados.rados.0.smithi059.stdout:510: copy_from oid 9 from oid 5 current snap is 49 2022-04-28T19:45:56.227 INFO:tasks.rados.rados.0.smithi059.stdout:511: copy_from oid 3 from oid 39 current snap is 49 2022-04-28T19:45:56.227 INFO:tasks.rados.rados.0.smithi059.stdout:512: delete oid 37 current snap is 49 2022-04-28T19:45:56.237 INFO:tasks.rados.rados.0.smithi059.stdout:507: finishing write tid 1 to smithi05955964-33 2022-04-28T19:45:56.237 INFO:tasks.rados.rados.0.smithi059.stdout:507: finishing write tid 3 to smithi05955964-33 2022-04-28T19:45:56.239 INFO:tasks.rados.rados.0.smithi059.stdout:507: finishing write tid 4 to smithi05955964-33 2022-04-28T19:45:56.239 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 194 (ObjNum 174 snap 49 seq_num 174) dirty exists 2022-04-28T19:45:56.239 INFO:tasks.rados.rados.0.smithi059.stdout:507: left oid 33 (ObjNum 174 snap 49 seq_num 174) 2022-04-28T19:45:56.243 INFO:tasks.rados.rados.0.smithi059.stdout:510: finishing copy_from to smithi05955964-9 2022-04-28T19:45:56.243 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 139 (ObjNum 156 snap 42 seq_num 156) dirty exists 2022-04-28T19:45:56.244 INFO:tasks.rados.rados.0.smithi059.stdout:508: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:45:56.327 INFO:tasks.rados.rados.0.smithi059.stdout:510: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:45:56.327 INFO:tasks.rados.rados.0.smithi059.stdout:511: finishing copy_from to smithi05955964-3 2022-04-28T19:45:56.328 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 176 (ObjNum 38 snap 0 seq_num 38) dirty exists 2022-04-28T19:45:56.328 INFO:tasks.rados.rados.0.smithi059.stdout:511: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:45:56.328 INFO:tasks.rados.rados.0.smithi059.stdout:509: expect (ObjNum 84 snap 14 seq_num 84) 2022-04-28T19:45:56.429 INFO:tasks.rados.rados.0.smithi059.stdout:507: done (5 left) 2022-04-28T19:45:56.475 INFO:tasks.rados.rados.0.smithi059.stdout:508: done (4 left) 2022-04-28T19:45:56.475 INFO:tasks.rados.rados.0.smithi059.stdout:509: done (3 left) 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:510: done (2 left) 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:511: done (1 left) 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:512: done (0 left) 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:513: copy_from oid 17 from oid 14 current snap is 49 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:514: read oid 30 snap -1 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:514: expect (ObjNum 100 snap 24 seq_num 100) 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 49 2022-04-28T19:45:56.478 INFO:tasks.rados.rados.0.smithi059.stdout:515: seq_num 176 ranges {3399680=557056} 2022-04-28T19:45:56.479 INFO:tasks.rados.rados.0.smithi059.stdout:515: writing smithi05955964-12 from 3399680 to 3956736 tid 1 2022-04-28T19:45:56.479 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 49 2022-04-28T19:45:56.480 INFO:tasks.rados.rados.0.smithi059.stdout:516: seq_num 177 ranges {2408448=573440} 2022-04-28T19:45:56.480 INFO:tasks.rados.rados.0.smithi059.stdout:516: writing smithi05955964-8 from 2408448 to 2981888 tid 1 2022-04-28T19:45:56.480 INFO:tasks.rados.rados.0.smithi059.stdout:517: snap_remove snap 43 2022-04-28T19:45:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:56 smithi059 bash[29114]: audit 2022-04-28T19:45:54.427718+0000 mgr.y (mgr.14140) 419 : audit [DBG] from='client.14871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:56 smithi059 bash[29114]: cluster 2022-04-28T19:45:55.189340+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2022-04-28T19:45:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:56 smithi059 bash[23786]: audit 2022-04-28T19:45:54.427718+0000 mgr.y (mgr.14140) 419 : audit [DBG] from='client.14871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:56 smithi059 bash[23786]: cluster 2022-04-28T19:45:55.189340+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2022-04-28T19:45:56.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:56 smithi139 bash[23439]: audit 2022-04-28T19:45:54.427718+0000 mgr.y (mgr.14140) 419 : audit [DBG] from='client.14871 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:45:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:56 smithi139 bash[23439]: cluster 2022-04-28T19:45:55.189340+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2022-04-28T19:45:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:517: done (4 left) 2022-04-28T19:45:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:518: snap_create 2022-04-28T19:45:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:514: expect (ObjNum 100 snap 24 seq_num 100) 2022-04-28T19:45:57.241 INFO:tasks.rados.rados.0.smithi059.stdout:513: finishing copy_from to smithi05955964-17 2022-04-28T19:45:57.241 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 157 (ObjNum 172 snap 45 seq_num 172) dirty exists 2022-04-28T19:45:57.241 INFO:tasks.rados.rados.0.smithi059.stdout:513: finishing copy_from racing read to smithi05955964-17 2022-04-28T19:45:57.241 INFO:tasks.rados.rados.0.smithi059.stdout:515: finishing write tid 1 to smithi05955964-12 2022-04-28T19:45:57.241 INFO:tasks.rados.rados.0.smithi059.stdout:515: finishing write tid 3 to smithi05955964-12 2022-04-28T19:45:57.241 INFO:tasks.rados.rados.0.smithi059.stdout:515: finishing write tid 4 to smithi05955964-12 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 210 (ObjNum 176 snap 49 seq_num 176) dirty exists 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:515: left oid 12 (ObjNum 176 snap 49 seq_num 176) 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:516: finishing write tid 1 to smithi05955964-8 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:516: finishing write tid 3 to smithi05955964-8 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:516: finishing write tid 4 to smithi05955964-8 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 153 (ObjNum 177 snap 49 seq_num 177) dirty exists 2022-04-28T19:45:57.242 INFO:tasks.rados.rados.0.smithi059.stdout:516: left oid 8 (ObjNum 177 snap 49 seq_num 177) 2022-04-28T19:45:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:57 smithi059 bash[29114]: cluster 2022-04-28T19:45:56.105200+0000 mgr.y (mgr.14140) 420 : cluster [DBG] pgmap v459: 89 pgs: 89 active+clean; 129 MiB data, 200 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 8 op/s 2022-04-28T19:45:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:57 smithi059 bash[29114]: cluster 2022-04-28T19:45:56.204056+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2022-04-28T19:45:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:57 smithi059 bash[29114]: cluster 2022-04-28T19:45:57.174247+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2022-04-28T19:45:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:57 smithi059 bash[23786]: cluster 2022-04-28T19:45:56.105200+0000 mgr.y (mgr.14140) 420 : cluster [DBG] pgmap v459: 89 pgs: 89 active+clean; 129 MiB data, 200 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 8 op/s 2022-04-28T19:45:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:57 smithi059 bash[23786]: cluster 2022-04-28T19:45:56.204056+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2022-04-28T19:45:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:57 smithi059 bash[23786]: cluster 2022-04-28T19:45:57.174247+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2022-04-28T19:45:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:57 smithi139 bash[23439]: cluster 2022-04-28T19:45:56.105200+0000 mgr.y (mgr.14140) 420 : cluster [DBG] pgmap v459: 89 pgs: 89 active+clean; 129 MiB data, 200 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 0 B/s wr, 8 op/s 2022-04-28T19:45:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:57 smithi139 bash[23439]: cluster 2022-04-28T19:45:56.204056+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2022-04-28T19:45:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:57 smithi139 bash[23439]: cluster 2022-04-28T19:45:57.174247+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2022-04-28T19:45:58.193 INFO:tasks.rados.rados.0.smithi059.stdout:513: done (4 left) 2022-04-28T19:45:58.193 INFO:tasks.rados.rados.0.smithi059.stdout:514: done (3 left) 2022-04-28T19:45:58.193 INFO:tasks.rados.rados.0.smithi059.stdout:515: done (2 left) 2022-04-28T19:45:58.193 INFO:tasks.rados.rados.0.smithi059.stdout:516: done (1 left) 2022-04-28T19:45:58.193 INFO:tasks.rados.rados.0.smithi059.stdout:518: done (0 left) 2022-04-28T19:45:58.194 INFO:tasks.rados.rados.0.smithi059.stdout:519: snap_remove snap 42 2022-04-28T19:45:59.195 INFO:tasks.rados.rados.0.smithi059.stdout:519: done (0 left) 2022-04-28T19:45:59.195 INFO:tasks.rados.rados.0.smithi059.stdout:520: read oid 16 snap 48 2022-04-28T19:45:59.195 INFO:tasks.rados.rados.0.smithi059.stdout:520: expect (ObjNum 111 snap 26 seq_num 111) 2022-04-28T19:45:59.195 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 50 2022-04-28T19:45:59.196 INFO:tasks.rados.rados.0.smithi059.stdout:521: seq_num 178 ranges {0=589824} 2022-04-28T19:45:59.199 INFO:tasks.rados.rados.0.smithi059.stdout:521: writing smithi05955964-2 from 0 to 589824 tid 1 2022-04-28T19:45:59.199 INFO:tasks.rados.rados.0.smithi059.stdout:522: setattr oid 49 current snap is 50 2022-04-28T19:45:59.200 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 50 2022-04-28T19:45:59.201 INFO:tasks.rados.rados.0.smithi059.stdout:523: seq_num 179 ranges {827392=606208} 2022-04-28T19:45:59.210 INFO:tasks.rados.rados.0.smithi059.stdout:523: writing smithi05955964-45 from 827392 to 1433600 tid 1 2022-04-28T19:45:59.210 INFO:tasks.rados.rados.0.smithi059.stdout:524: read oid 35 snap -1 2022-04-28T19:45:59.210 INFO:tasks.rados.rados.0.smithi059.stdout:524: expect deleted 2022-04-28T19:45:59.210 INFO:tasks.rados.rados.0.smithi059.stdout:525: read oid 14 snap -1 2022-04-28T19:45:59.210 INFO:tasks.rados.rados.0.smithi059.stdout:525: expect (ObjNum 172 snap 45 seq_num 172) 2022-04-28T19:45:59.211 INFO:tasks.rados.rados.0.smithi059.stdout:526: delete oid 39 current snap is 50 2022-04-28T19:45:59.211 INFO:tasks.rados.rados.0.smithi059.stdout:521: finishing write tid 1 to smithi05955964-2 2022-04-28T19:45:59.211 INFO:tasks.rados.rados.0.smithi059.stdout:521: finishing write tid 3 to smithi05955964-2 2022-04-28T19:45:59.211 INFO:tasks.rados.rados.0.smithi059.stdout:521: finishing write tid 4 to smithi05955964-2 2022-04-28T19:45:59.211 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 34 (ObjNum 178 snap 50 seq_num 178) dirty exists 2022-04-28T19:45:59.211 INFO:tasks.rados.rados.0.smithi059.stdout:521: left oid 2 (ObjNum 178 snap 50 seq_num 178) 2022-04-28T19:45:59.217 INFO:tasks.rados.rados.0.smithi059.stdout:525: expect (ObjNum 172 snap 45 seq_num 172) 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 163 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:523: finishing write tid 1 to smithi05955964-45 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:523: finishing write tid 3 to smithi05955964-45 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:523: finishing write tid 4 to smithi05955964-45 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 178 (ObjNum 179 snap 50 seq_num 179) dirty exists 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:523: left oid 45 (ObjNum 179 snap 50 seq_num 179) 2022-04-28T19:45:59.287 INFO:tasks.rados.rados.0.smithi059.stdout:520: expect (ObjNum 111 snap 26 seq_num 111) 2022-04-28T19:45:59.479 INFO:tasks.rados.rados.0.smithi059.stdout:520: done (6 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:521: done (5 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:522: done (4 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:523: done (3 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:524: done (2 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:525: done (1 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:526: done (0 left) 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 50 2022-04-28T19:45:59.556 INFO:tasks.rados.rados.0.smithi059.stdout:527: seq_num 181 ranges {1064960=638976} 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:527: writing smithi05955964-40 from 1064960 to 1703936 tid 1 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:528: snap_create 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:527: finishing write tid 1 to smithi05955964-40 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:527: finishing write tid 3 to smithi05955964-40 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:527: finishing write tid 4 to smithi05955964-40 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 170 (ObjNum 181 snap 50 seq_num 181) dirty exists 2022-04-28T19:45:59.558 INFO:tasks.rados.rados.0.smithi059.stdout:527: left oid 40 (ObjNum 181 snap 50 seq_num 181) 2022-04-28T19:45:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:59 smithi059 bash[29114]: cluster 2022-04-28T19:45:58.106107+0000 mgr.y (mgr.14140) 421 : cluster [DBG] pgmap v462: 89 pgs: 89 active+clean; 134 MiB data, 205 MiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.2 MiB/s wr, 2 op/s 2022-04-28T19:45:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:45:59 smithi059 bash[29114]: cluster 2022-04-28T19:45:58.191223+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2022-04-28T19:45:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:59 smithi059 bash[23786]: cluster 2022-04-28T19:45:58.106107+0000 mgr.y (mgr.14140) 421 : cluster [DBG] pgmap v462: 89 pgs: 89 active+clean; 134 MiB data, 205 MiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.2 MiB/s wr, 2 op/s 2022-04-28T19:45:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:45:59 smithi059 bash[23786]: cluster 2022-04-28T19:45:58.191223+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2022-04-28T19:45:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:59 smithi139 bash[23439]: cluster 2022-04-28T19:45:58.106107+0000 mgr.y (mgr.14140) 421 : cluster [DBG] pgmap v462: 89 pgs: 89 active+clean; 134 MiB data, 205 MiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.2 MiB/s wr, 2 op/s 2022-04-28T19:45:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:45:59 smithi139 bash[23439]: cluster 2022-04-28T19:45:58.191223+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2022-04-28T19:46:00.221 INFO:tasks.rados.rados.0.smithi059.stdout:527: done (1 left) 2022-04-28T19:46:00.221 INFO:tasks.rados.rados.0.smithi059.stdout:528: done (0 left) 2022-04-28T19:46:00.222 INFO:tasks.rados.rados.0.smithi059.stdout:529: setattr oid 5 current snap is 51 2022-04-28T19:46:00.222 INFO:tasks.rados.rados.0.smithi059.stdout:530: rollback oid 50 current snap is 51 2022-04-28T19:46:00.222 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 17 2022-04-28T19:46:00.223 INFO:tasks.rados.rados.0.smithi059.stdout:531: setattr oid 47 current snap is 51 2022-04-28T19:46:00.223 INFO:tasks.rados.rados.0.smithi059.stdout:532: rollback oid 10 current snap is 51 2022-04-28T19:46:00.223 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 50 2022-04-28T19:46:00.223 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 51 2022-04-28T19:46:00.223 INFO:tasks.rados.rados.0.smithi059.stdout:533: seq_num 182 ranges {827392=655360} 2022-04-28T19:46:00.232 INFO:tasks.rados.rados.0.smithi059.stdout:533: writing smithi05955964-11 from 827392 to 1482752 tid 1 2022-04-28T19:46:00.232 INFO:tasks.rados.rados.0.smithi059.stdout:534: snap_remove snap 22 2022-04-28T19:46:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:00 smithi059 bash[23786]: cluster 2022-04-28T19:45:59.191745+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2022-04-28T19:46:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:00 smithi059 bash[29114]: cluster 2022-04-28T19:45:59.191745+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2022-04-28T19:46:00.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:00 smithi139 bash[23439]: cluster 2022-04-28T19:45:59.191745+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2022-04-28T19:46:01.228 INFO:tasks.rados.rados.0.smithi059.stdout:534: done (5 left) 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:535: read oid 44 snap -1 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:535: expect (ObjNum 160 snap 42 seq_num 160) 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:536: read oid 45 snap -1 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:536: expect (ObjNum 179 snap 50 seq_num 179) 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:537: read oid 39 snap 50 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:537: expect deleted 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:538: snap_create 2022-04-28T19:46:01.229 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 142 (ObjNum 171 snap 45 seq_num 171) dirty exists 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:532: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 73 (ObjNum 0 snap 0 seq_num 0) dirty dne 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 159 (ObjNum 156 snap 42 seq_num 156) dirty exists 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:530: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:530: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:530: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 204 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:533: finishing write tid 1 to smithi05955964-11 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:533: finishing write tid 3 to smithi05955964-11 2022-04-28T19:46:01.230 INFO:tasks.rados.rados.0.smithi059.stdout:533: finishing write tid 4 to smithi05955964-11 2022-04-28T19:46:01.231 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 131 (ObjNum 182 snap 51 seq_num 182) dirty exists 2022-04-28T19:46:01.231 INFO:tasks.rados.rados.0.smithi059.stdout:533: left oid 11 (ObjNum 182 snap 51 seq_num 182) 2022-04-28T19:46:01.245 INFO:tasks.rados.rados.0.smithi059.stdout:535: expect (ObjNum 160 snap 42 seq_num 160) 2022-04-28T19:46:01.254 INFO:tasks.rados.rados.0.smithi059.stdout:536: expect (ObjNum 179 snap 50 seq_num 179) 2022-04-28T19:46:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:01 smithi059 bash[23786]: cluster 2022-04-28T19:46:00.106988+0000 mgr.y (mgr.14140) 422 : cluster [DBG] pgmap v465: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 141 MiB data, 210 MiB used, 707 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2022-04-28T19:46:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:01 smithi059 bash[23786]: cluster 2022-04-28T19:46:00.220100+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2022-04-28T19:46:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:01 smithi059 bash[29114]: cluster 2022-04-28T19:46:00.106988+0000 mgr.y (mgr.14140) 422 : cluster [DBG] pgmap v465: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 141 MiB data, 210 MiB used, 707 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2022-04-28T19:46:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:01 smithi059 bash[29114]: cluster 2022-04-28T19:46:00.220100+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2022-04-28T19:46:01.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:01 smithi139 bash[23439]: cluster 2022-04-28T19:46:00.106988+0000 mgr.y (mgr.14140) 422 : cluster [DBG] pgmap v465: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 141 MiB data, 210 MiB used, 707 GiB / 715 GiB avail; 9.4 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2022-04-28T19:46:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:01 smithi139 bash[23439]: cluster 2022-04-28T19:46:00.220100+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2022-04-28T19:46:02.181 INFO:tasks.rados.rados.0.smithi059.stdout:529: done (8 left) 2022-04-28T19:46:02.181 INFO:tasks.rados.rados.0.smithi059.stdout:530: done (7 left) 2022-04-28T19:46:02.181 INFO:tasks.rados.rados.0.smithi059.stdout:531: done (6 left) 2022-04-28T19:46:02.182 INFO:tasks.rados.rados.0.smithi059.stdout:532: done (5 left) 2022-04-28T19:46:02.182 INFO:tasks.rados.rados.0.smithi059.stdout:533: done (4 left) 2022-04-28T19:46:02.182 INFO:tasks.rados.rados.0.smithi059.stdout:535: done (3 left) 2022-04-28T19:46:02.182 INFO:tasks.rados.rados.0.smithi059.stdout:536: done (2 left) 2022-04-28T19:46:02.183 INFO:tasks.rados.rados.0.smithi059.stdout:537: done (1 left) 2022-04-28T19:46:02.183 INFO:tasks.rados.rados.0.smithi059.stdout:538: done (0 left) 2022-04-28T19:46:02.183 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 52 2022-04-28T19:46:02.183 INFO:tasks.rados.rados.0.smithi059.stdout:539: seq_num 183 ranges {638976=671744} 2022-04-28T19:46:02.188 INFO:tasks.rados.rados.0.smithi059.stdout:539: writing smithi05955964-3 from 638976 to 1310720 tid 1 2022-04-28T19:46:02.189 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 52 2022-04-28T19:46:02.189 INFO:tasks.rados.rados.0.smithi059.stdout:540: seq_num 184 ranges {811008=688128} 2022-04-28T19:46:02.198 INFO:tasks.rados.rados.0.smithi059.stdout:540: writing smithi05955964-49 from 811008 to 1499136 tid 1 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:541: read oid 32 snap 32 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:541: expect (ObjNum 133 snap 31 seq_num 133) 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:542: rollback oid 50 current snap is 52 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 51 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:539: finishing write tid 1 to smithi05955964-3 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:539: finishing write tid 3 to smithi05955964-3 2022-04-28T19:46:02.199 INFO:tasks.rados.rados.0.smithi059.stdout:539: finishing write tid 4 to smithi05955964-3 2022-04-28T19:46:02.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 180 (ObjNum 183 snap 52 seq_num 183) dirty exists 2022-04-28T19:46:02.200 INFO:tasks.rados.rados.0.smithi059.stdout:539: left oid 3 (ObjNum 183 snap 52 seq_num 183) 2022-04-28T19:46:02.200 INFO:tasks.rados.rados.0.smithi059.stdout:539: done (3 left) 2022-04-28T19:46:02.200 INFO:tasks.rados.rados.0.smithi059.stdout:543: snap_create 2022-04-28T19:46:02.202 INFO:tasks.rados.rados.0.smithi059.stdout:540: finishing write tid 1 to smithi05955964-49 2022-04-28T19:46:02.202 INFO:tasks.rados.rados.0.smithi059.stdout:540: finishing write tid 3 to smithi05955964-49 2022-04-28T19:46:02.204 INFO:tasks.rados.rados.0.smithi059.stdout:540: finishing write tid 4 to smithi05955964-49 2022-04-28T19:46:02.204 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 181 (ObjNum 184 snap 52 seq_num 184) dirty exists 2022-04-28T19:46:02.204 INFO:tasks.rados.rados.0.smithi059.stdout:540: left oid 49 (ObjNum 184 snap 52 seq_num 184) 2022-04-28T19:46:02.227 INFO:tasks.rados.rados.0.smithi059.stdout:542: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:46:02.227 INFO:tasks.rados.rados.0.smithi059.stdout:542: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:46:02.241 INFO:tasks.rados.rados.0.smithi059.stdout:541: expect (ObjNum 133 snap 31 seq_num 133) 2022-04-28T19:46:02.514 INFO:tasks.rados.rados.0.smithi059.stdout:542: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:46:02.514 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 218 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:46:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:02 smithi059 bash[23786]: cluster 2022-04-28T19:46:01.222874+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2022-04-28T19:46:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:02 smithi059 bash[23786]: cluster 2022-04-28T19:46:02.178865+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2022-04-28T19:46:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:02 smithi059 bash[29114]: cluster 2022-04-28T19:46:01.222874+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2022-04-28T19:46:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:02 smithi059 bash[29114]: cluster 2022-04-28T19:46:02.178865+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2022-04-28T19:46:02.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:02 smithi139 bash[23439]: cluster 2022-04-28T19:46:01.222874+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2022-04-28T19:46:02.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:02 smithi139 bash[23439]: cluster 2022-04-28T19:46:02.178865+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2022-04-28T19:46:03.194 INFO:tasks.rados.rados.0.smithi059.stdout:540: done (3 left) 2022-04-28T19:46:03.194 INFO:tasks.rados.rados.0.smithi059.stdout:541: done (2 left) 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:542: done (1 left) 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:543: done (0 left) 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:544: read oid 22 snap -1 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:544: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:545: rollback oid 47 current snap is 53 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 45 2022-04-28T19:46:03.195 INFO:tasks.rados.rados.0.smithi059.stdout:546: snap_remove snap 39 2022-04-28T19:46:03.518 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[29114]: cluster 2022-04-28T19:46:02.107733+0000 mgr.y (mgr.14140) 423 : cluster [DBG] pgmap v468: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 142 MiB data, 210 MiB used, 707 GiB / 715 GiB avail; 8.6 MiB/s rd, 916 KiB/s wr, 21 op/s 2022-04-28T19:46:03.519 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[29114]: cluster 2022-04-28T19:46:03.192350+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2022-04-28T19:46:03.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[23786]: cluster 2022-04-28T19:46:02.107733+0000 mgr.y (mgr.14140) 423 : cluster [DBG] pgmap v468: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 142 MiB data, 210 MiB used, 707 GiB / 715 GiB avail; 8.6 MiB/s rd, 916 KiB/s wr, 21 op/s 2022-04-28T19:46:03.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[23786]: cluster 2022-04-28T19:46:03.192350+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2022-04-28T19:46:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:03 smithi139 bash[23439]: cluster 2022-04-28T19:46:02.107733+0000 mgr.y (mgr.14140) 423 : cluster [DBG] pgmap v468: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 142 MiB data, 210 MiB used, 707 GiB / 715 GiB avail; 8.6 MiB/s rd, 916 KiB/s wr, 21 op/s 2022-04-28T19:46:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:03 smithi139 bash[23439]: cluster 2022-04-28T19:46:03.192350+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2022-04-28T19:46:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:03.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:46:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:03.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:46:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:03.517Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:46:04.200 INFO:tasks.rados.rados.0.smithi059.stdout:546: done (2 left) 2022-04-28T19:46:04.200 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 53 2022-04-28T19:46:04.201 INFO:tasks.rados.rados.0.smithi059.stdout:547: seq_num 185 ranges {434176=704512} 2022-04-28T19:46:04.208 INFO:tasks.rados.rados.0.smithi059.stdout:547: writing smithi05955964-6 from 434176 to 1138688 tid 1 2022-04-28T19:46:04.208 INFO:tasks.rados.rados.0.smithi059.stdout:548: copy_from oid 41 from oid 46 current snap is 53 2022-04-28T19:46:04.208 INFO:tasks.rados.rados.0.smithi059.stdout:549: rollback oid 19 current snap is 53 2022-04-28T19:46:04.208 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 19 to 32 2022-04-28T19:46:04.209 INFO:tasks.rados.rados.0.smithi059.stdout:550: snap_remove snap 44 2022-04-28T19:46:05.200 INFO:tasks.rados.rados.0.smithi059.stdout:550: done (5 left) 2022-04-28T19:46:05.200 INFO:tasks.rados.rados.0.smithi059.stdout:551: read oid 7 snap -1 2022-04-28T19:46:05.201 INFO:tasks.rados.rados.0.smithi059.stdout:551: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:46:05.201 INFO:tasks.rados.rados.0.smithi059.stdout:552: delete oid 45 current snap is 53 2022-04-28T19:46:05.201 INFO:tasks.rados.rados.0.smithi059.stdout:544: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:05.258 INFO:tasks.rados.rados.0.smithi059.stdout:545: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:46:05.258 INFO:tasks.rados.rados.0.smithi059.stdout:545: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:46:05.258 INFO:tasks.rados.rados.0.smithi059.stdout:545: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 148 (ObjNum 171 snap 45 seq_num 171) dirty exists 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:549: finishing rollback tid 0 to smithi05955964-19 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:544: done (6 left) 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:545: done (5 left) 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:552: done (4 left) 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:553: rollback oid 1 current snap is 53 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 32 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:549: finishing rollback tid 1 to smithi05955964-19 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 184 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty dne 2022-04-28T19:46:05.259 INFO:tasks.rados.rados.0.smithi059.stdout:547: finishing write tid 1 to smithi05955964-6 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:547: finishing write tid 3 to smithi05955964-6 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:547: finishing write tid 4 to smithi05955964-6 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 182 (ObjNum 185 snap 53 seq_num 185) dirty exists 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:547: left oid 6 (ObjNum 185 snap 53 seq_num 185) 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:548: finishing copy_from to smithi05955964-41 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 37 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:548: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:46:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:551: expect (ObjNum 23 snap 0 seq_num 23) 2022-04-28T19:46:05.268 INFO:tasks.rados.rados.0.smithi059.stdout:553: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:46:05.268 INFO:tasks.rados.rados.0.smithi059.stdout:553: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:46:05.268 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 195 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:46:05.268 INFO:tasks.rados.rados.0.smithi059.stdout:547: done (4 left) 2022-04-28T19:46:05.269 INFO:tasks.rados.rados.0.smithi059.stdout:548: done (3 left) 2022-04-28T19:46:05.269 INFO:tasks.rados.rados.0.smithi059.stdout:549: done (2 left) 2022-04-28T19:46:05.269 INFO:tasks.rados.rados.0.smithi059.stdout:551: done (1 left) 2022-04-28T19:46:05.269 INFO:tasks.rados.rados.0.smithi059.stdout:553: done (0 left) 2022-04-28T19:46:05.269 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 53 2022-04-28T19:46:05.269 INFO:tasks.rados.rados.0.smithi059.stdout:554: seq_num 187 ranges {393216=737280} 2022-04-28T19:46:05.275 INFO:tasks.rados.rados.0.smithi059.stdout:554: writing smithi05955964-7 from 393216 to 1130496 tid 1 2022-04-28T19:46:05.275 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 53 2022-04-28T19:46:05.276 INFO:tasks.rados.rados.0.smithi059.stdout:555: seq_num 188 ranges {2277376=753664} 2022-04-28T19:46:05.295 INFO:tasks.rados.rados.0.smithi059.stdout:555: writing smithi05955964-15 from 2277376 to 3031040 tid 1 2022-04-28T19:46:05.295 INFO:tasks.rados.rados.0.smithi059.stdout:556: rollback oid 31 current snap is 53 2022-04-28T19:46:05.296 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 51 2022-04-28T19:46:05.296 INFO:tasks.rados.rados.0.smithi059.stdout:554: finishing write tid 1 to smithi05955964-7 2022-04-28T19:46:05.296 INFO:tasks.rados.rados.0.smithi059.stdout:554: finishing write tid 3 to smithi05955964-7 2022-04-28T19:46:05.297 INFO:tasks.rados.rados.0.smithi059.stdout:554: finishing write tid 4 to smithi05955964-7 2022-04-28T19:46:05.297 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 233 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:46:05.298 INFO:tasks.rados.rados.0.smithi059.stdout:554: left oid 7 (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:05.298 INFO:tasks.rados.rados.0.smithi059.stdout:557: read oid 32 snap -1 2022-04-28T19:46:05.298 INFO:tasks.rados.rados.0.smithi059.stdout:557: expect deleted 2022-04-28T19:46:05.298 INFO:tasks.rados.rados.0.smithi059.stdout:554: done (3 left) 2022-04-28T19:46:05.298 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 53 2022-04-28T19:46:05.298 INFO:tasks.rados.rados.0.smithi059.stdout:558: seq_num 189 ranges {3039232=770048} 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:558: writing smithi05955964-34 from 3039232 to 3809280 tid 1 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:559: snap_create 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:555: finishing write tid 1 to smithi05955964-15 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:555: finishing write tid 3 to smithi05955964-15 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:556: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:555: finishing write tid 4 to smithi05955964-15 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 236 (ObjNum 188 snap 53 seq_num 188) dirty exists 2022-04-28T19:46:05.317 INFO:tasks.rados.rados.0.smithi059.stdout:555: left oid 15 (ObjNum 188 snap 53 seq_num 188) 2022-04-28T19:46:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:556: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:46:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:556: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:46:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 240 (ObjNum 139 snap 35 seq_num 139) dirty exists 2022-04-28T19:46:05.321 INFO:tasks.rados.rados.0.smithi059.stdout:558: finishing write tid 1 to smithi05955964-34 2022-04-28T19:46:05.321 INFO:tasks.rados.rados.0.smithi059.stdout:558: finishing write tid 3 to smithi05955964-34 2022-04-28T19:46:05.322 INFO:tasks.rados.rados.0.smithi059.stdout:558: finishing write tid 4 to smithi05955964-34 2022-04-28T19:46:05.322 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 201 (ObjNum 189 snap 53 seq_num 189) dirty exists 2022-04-28T19:46:05.322 INFO:tasks.rados.rados.0.smithi059.stdout:558: left oid 34 (ObjNum 189 snap 53 seq_num 189) 2022-04-28T19:46:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:05 smithi059 bash[23786]: cluster 2022-04-28T19:46:04.108512+0000 mgr.y (mgr.14140) 424 : cluster [DBG] pgmap v471: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 143 MiB data, 213 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 570 KiB/s wr, 9 op/s 2022-04-28T19:46:05.700 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:05 smithi059 bash[23786]: cluster 2022-04-28T19:46:04.195015+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2022-04-28T19:46:05.700 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:05 smithi059 bash[29114]: cluster 2022-04-28T19:46:04.108512+0000 mgr.y (mgr.14140) 424 : cluster [DBG] pgmap v471: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 143 MiB data, 213 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 570 KiB/s wr, 9 op/s 2022-04-28T19:46:05.702 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:05 smithi059 bash[29114]: cluster 2022-04-28T19:46:04.195015+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2022-04-28T19:46:05.703 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:05 smithi139 bash[23439]: cluster 2022-04-28T19:46:04.108512+0000 mgr.y (mgr.14140) 424 : cluster [DBG] pgmap v471: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 143 MiB data, 213 MiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 570 KiB/s wr, 9 op/s 2022-04-28T19:46:05.703 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:05 smithi139 bash[23439]: cluster 2022-04-28T19:46:04.195015+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2022-04-28T19:46:06.218 INFO:tasks.rados.rados.0.smithi059.stdout:555: done (4 left) 2022-04-28T19:46:06.218 INFO:tasks.rados.rados.0.smithi059.stdout:556: done (3 left) 2022-04-28T19:46:06.218 INFO:tasks.rados.rados.0.smithi059.stdout:557: done (2 left) 2022-04-28T19:46:06.218 INFO:tasks.rados.rados.0.smithi059.stdout:558: done (1 left) 2022-04-28T19:46:06.218 INFO:tasks.rados.rados.0.smithi059.stdout:559: done (0 left) 2022-04-28T19:46:06.219 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 54 2022-04-28T19:46:06.219 INFO:tasks.rados.rados.0.smithi059.stdout:560: seq_num 190 ranges {1687552=786432} 2022-04-28T19:46:06.233 INFO:tasks.rados.rados.0.smithi059.stdout:560: writing smithi05955964-23 from 1687552 to 2473984 tid 1 2022-04-28T19:46:06.233 INFO:tasks.rados.rados.0.smithi059.stdout:561: copy_from oid 26 from oid 22 current snap is 54 2022-04-28T19:46:06.233 INFO:tasks.rados.rados.0.smithi059.stdout:562: snap_create 2022-04-28T19:46:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:560: finishing write tid 1 to smithi05955964-23 2022-04-28T19:46:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:560: finishing write tid 3 to smithi05955964-23 2022-04-28T19:46:06.242 INFO:tasks.rados.rados.0.smithi059.stdout:560: finishing write tid 4 to smithi05955964-23 2022-04-28T19:46:06.242 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 151 (ObjNum 190 snap 54 seq_num 190) dirty exists 2022-04-28T19:46:06.242 INFO:tasks.rados.rados.0.smithi059.stdout:560: left oid 23 (ObjNum 190 snap 54 seq_num 190) 2022-04-28T19:46:06.248 INFO:tasks.rados.rados.0.smithi059.stdout:561: finishing copy_from to smithi05955964-26 2022-04-28T19:46:06.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 117 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:46:06.249 INFO:tasks.rados.rados.0.smithi059.stdout:561: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:46:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:06 smithi059 bash[29114]: cluster 2022-04-28T19:46:05.195296+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2022-04-28T19:46:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:06 smithi059 bash[23786]: cluster 2022-04-28T19:46:05.195296+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2022-04-28T19:46:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:06 smithi139 bash[23439]: cluster 2022-04-28T19:46:05.195296+0000 mon.a (mon.0) 768 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2022-04-28T19:46:07.180 INFO:tasks.rados.rados.0.smithi059.stdout:560: done (2 left) 2022-04-28T19:46:07.187 INFO:tasks.rados.rados.0.smithi059.stdout:561: done (1 left) 2022-04-28T19:46:07.187 INFO:tasks.rados.rados.0.smithi059.stdout:562: done (0 left) 2022-04-28T19:46:07.188 INFO:tasks.rados.rados.0.smithi059.stdout:563: snap_create 2022-04-28T19:46:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:07 smithi059 bash[29114]: cluster 2022-04-28T19:46:06.109380+0000 mgr.y (mgr.14140) 425 : cluster [DBG] pgmap v474: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 148 MiB data, 216 MiB used, 707 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2022-04-28T19:46:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:07 smithi059 bash[29114]: cluster 2022-04-28T19:46:06.213975+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2022-04-28T19:46:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:07 smithi059 bash[29114]: cluster 2022-04-28T19:46:07.177996+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2022-04-28T19:46:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:07 smithi059 bash[23786]: cluster 2022-04-28T19:46:06.109380+0000 mgr.y (mgr.14140) 425 : cluster [DBG] pgmap v474: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 148 MiB data, 216 MiB used, 707 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2022-04-28T19:46:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:07 smithi059 bash[23786]: cluster 2022-04-28T19:46:06.213975+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2022-04-28T19:46:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:07 smithi059 bash[23786]: cluster 2022-04-28T19:46:07.177996+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2022-04-28T19:46:07.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:07 smithi139 bash[23439]: cluster 2022-04-28T19:46:06.109380+0000 mgr.y (mgr.14140) 425 : cluster [DBG] pgmap v474: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 148 MiB data, 216 MiB used, 707 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 13 op/s 2022-04-28T19:46:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:07 smithi139 bash[23439]: cluster 2022-04-28T19:46:06.213975+0000 mon.a (mon.0) 769 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2022-04-28T19:46:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:07 smithi139 bash[23439]: cluster 2022-04-28T19:46:07.177996+0000 mon.a (mon.0) 770 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2022-04-28T19:46:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:563: done (0 left) 2022-04-28T19:46:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:564: delete oid 4 current snap is 56 2022-04-28T19:46:08.197 INFO:tasks.rados.rados.0.smithi059.stdout:564: done (0 left) 2022-04-28T19:46:08.197 INFO:tasks.rados.rados.0.smithi059.stdout:565: delete oid 15 current snap is 56 2022-04-28T19:46:08.201 INFO:tasks.rados.rados.0.smithi059.stdout:565: done (0 left) 2022-04-28T19:46:08.202 INFO:tasks.rados.rados.0.smithi059.stdout:566: snap_create 2022-04-28T19:46:09.191 INFO:tasks.rados.rados.0.smithi059.stdout:566: done (0 left) 2022-04-28T19:46:09.207 INFO:tasks.rados.rados.0.smithi059.stdout:567: setattr oid 19 current snap is 57 2022-04-28T19:46:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:568: rmattr oid 42 current snap is 57 2022-04-28T19:46:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:568: done (1 left) 2022-04-28T19:46:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 57 2022-04-28T19:46:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:569: seq_num 193 ranges {0=434176,434176=401408} 2022-04-28T19:46:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:569: writing smithi05955964-39 from 0 to 434176 tid 1 2022-04-28T19:46:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:569: writing smithi05955964-39 from 434176 to 835584 tid 2 2022-04-28T19:46:09.210 INFO:tasks.rados.rados.0.smithi059.stdout:570: read oid 47 snap -1 2022-04-28T19:46:09.210 INFO:tasks.rados.rados.0.smithi059.stdout:570: expect (ObjNum 171 snap 45 seq_num 171) 2022-04-28T19:46:09.210 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 57 2022-04-28T19:46:09.210 INFO:tasks.rados.rados.0.smithi059.stdout:571: seq_num 194 ranges {3039232=458752,3497984=401408} 2022-04-28T19:46:09.218 INFO:tasks.rados.rados.0.smithi059.stdout:571: writing smithi05955964-41 from 3039232 to 3497984 tid 1 2022-04-28T19:46:09.221 INFO:tasks.rados.rados.0.smithi059.stdout:571: writing smithi05955964-41 from 3497984 to 3899392 tid 2 2022-04-28T19:46:09.221 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 57 2022-04-28T19:46:09.221 INFO:tasks.rados.rados.0.smithi059.stdout:572: seq_num 195 ranges {3039232=475136,3514368=401408} 2022-04-28T19:46:09.241 INFO:tasks.rados.rados.0.smithi059.stdout:572: writing smithi05955964-46 from 3039232 to 3514368 tid 1 2022-04-28T19:46:09.243 INFO:tasks.rados.rados.0.smithi059.stdout:572: writing smithi05955964-46 from 3514368 to 3915776 tid 2 2022-04-28T19:46:09.243 INFO:tasks.rados.rados.0.smithi059.stdout:573: read oid 3 snap -1 2022-04-28T19:46:09.244 INFO:tasks.rados.rados.0.smithi059.stdout:573: expect (ObjNum 183 snap 52 seq_num 183) 2022-04-28T19:46:09.244 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 57 2022-04-28T19:46:09.244 INFO:tasks.rados.rados.0.smithi059.stdout:574: seq_num 196 ranges {1499136=491520,1990656=401408} 2022-04-28T19:46:09.255 INFO:tasks.rados.rados.0.smithi059.stdout:574: writing smithi05955964-49 from 1499136 to 1990656 tid 1 2022-04-28T19:46:09.258 INFO:tasks.rados.rados.0.smithi059.stdout:574: writing smithi05955964-49 from 1990656 to 2392064 tid 2 2022-04-28T19:46:09.258 INFO:tasks.rados.rados.0.smithi059.stdout:575: delete oid 20 current snap is 57 2022-04-28T19:46:09.258 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 191 (ObjNum 0 snap 21893 seq_num 4161559984) dirty exists 2022-04-28T19:46:09.258 INFO:tasks.rados.rados.0.smithi059.stdout:569: finishing write tid 1 to smithi05955964-39 2022-04-28T19:46:09.259 INFO:tasks.rados.rados.0.smithi059.stdout:570: expect (ObjNum 171 snap 45 seq_num 171) 2022-04-28T19:46:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:569: finishing write tid 2 to smithi05955964-39 2022-04-28T19:46:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:569: finishing write tid 4 to smithi05955964-39 2022-04-28T19:46:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:569: finishing write tid 5 to smithi05955964-39 2022-04-28T19:46:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 235 (ObjNum 193 snap 57 seq_num 193) dirty exists 2022-04-28T19:46:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:569: left oid 39 (ObjNum 193 snap 57 seq_num 193) 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:571: finishing write tid 1 to smithi05955964-41 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:567: done (7 left) 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:569: done (6 left) 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:570: done (5 left) 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:575: done (4 left) 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 57 2022-04-28T19:46:09.292 INFO:tasks.rados.rados.0.smithi059.stdout:576: seq_num 198 ranges {3497984=524288,4022272=401408} 2022-04-28T19:46:09.320 INFO:tasks.rados.rados.0.smithi059.stdout:576: writing smithi05955964-24 from 3497984 to 4022272 tid 1 2022-04-28T19:46:09.323 INFO:tasks.rados.rados.0.smithi059.stdout:576: writing smithi05955964-24 from 4022272 to 4423680 tid 2 2022-04-28T19:46:09.323 INFO:tasks.rados.rados.0.smithi059.stdout:577: read oid 19 snap -1 2022-04-28T19:46:09.323 INFO:tasks.rados.rados.0.smithi059.stdout:577: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:46:09.323 INFO:tasks.rados.rados.0.smithi059.stdout:578: read oid 8 snap -1 2022-04-28T19:46:09.323 INFO:tasks.rados.rados.0.smithi059.stdout:578: expect (ObjNum 177 snap 49 seq_num 177) 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:579: rollback oid 31 current snap is 57 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 23 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:571: finishing write tid 2 to smithi05955964-41 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:571: finishing write tid 4 to smithi05955964-41 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:571: finishing write tid 5 to smithi05955964-41 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 44 (ObjNum 194 snap 57 seq_num 194) dirty exists 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:571: left oid 41 (ObjNum 194 snap 57 seq_num 194) 2022-04-28T19:46:09.324 INFO:tasks.rados.rados.0.smithi059.stdout:572: finishing write tid 1 to smithi05955964-46 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:572: finishing write tid 2 to smithi05955964-46 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:572: finishing write tid 4 to smithi05955964-46 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:572: finishing write tid 5 to smithi05955964-46 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 175 (ObjNum 195 snap 57 seq_num 195) dirty exists 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:572: left oid 46 (ObjNum 195 snap 57 seq_num 195) 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:571: done (7 left) 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:572: done (6 left) 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 57 2022-04-28T19:46:09.325 INFO:tasks.rados.rados.0.smithi059.stdout:580: seq_num 199 ranges {1703936=540672,2244608=401408} 2022-04-28T19:46:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:580: writing smithi05955964-40 from 1703936 to 2244608 tid 1 2022-04-28T19:46:09.338 INFO:tasks.rados.rados.0.smithi059.stdout:580: writing smithi05955964-40 from 2244608 to 2646016 tid 2 2022-04-28T19:46:09.338 INFO:tasks.rados.rados.0.smithi059.stdout:581: copy_from oid 42 from oid 7 current snap is 57 2022-04-28T19:46:09.338 INFO:tasks.rados.rados.0.smithi059.stdout:573: expect (ObjNum 183 snap 52 seq_num 183) 2022-04-28T19:46:09.382 INFO:tasks.rados.rados.0.smithi059.stdout:574: finishing write tid 1 to smithi05955964-49 2022-04-28T19:46:09.463 INFO:tasks.rados.rados.0.smithi059.stdout:574: finishing write tid 2 to smithi05955964-49 2022-04-28T19:46:09.463 INFO:tasks.rados.rados.0.smithi059.stdout:574: finishing write tid 4 to smithi05955964-49 2022-04-28T19:46:09.463 INFO:tasks.rados.rados.0.smithi059.stdout:574: finishing write tid 5 to smithi05955964-49 2022-04-28T19:46:09.463 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 199 (ObjNum 196 snap 57 seq_num 196) dirty exists 2022-04-28T19:46:09.463 INFO:tasks.rados.rados.0.smithi059.stdout:574: left oid 49 (ObjNum 196 snap 57 seq_num 196) 2022-04-28T19:46:09.465 INFO:tasks.rados.rados.0.smithi059.stdout:577: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:576: finishing write tid 1 to smithi05955964-24 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:576: finishing write tid 2 to smithi05955964-24 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:576: finishing write tid 4 to smithi05955964-24 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:576: finishing write tid 5 to smithi05955964-24 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 155 (ObjNum 198 snap 57 seq_num 198) dirty exists 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:576: left oid 24 (ObjNum 198 snap 57 seq_num 198) 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:579: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:579: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:579: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:46:09.466 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 246 (ObjNum 87 snap 17 seq_num 87) dirty exists 2022-04-28T19:46:09.467 INFO:tasks.rados.rados.0.smithi059.stdout:580: finishing write tid 1 to smithi05955964-40 2022-04-28T19:46:09.467 INFO:tasks.rados.rados.0.smithi059.stdout:580: finishing write tid 2 to smithi05955964-40 2022-04-28T19:46:09.467 INFO:tasks.rados.rados.0.smithi059.stdout:580: finishing write tid 4 to smithi05955964-40 2022-04-28T19:46:09.467 INFO:tasks.rados.rados.0.smithi059.stdout:578: expect (ObjNum 177 snap 49 seq_num 177) 2022-04-28T19:46:09.491 INFO:tasks.rados.rados.0.smithi059.stdout:580: finishing write tid 5 to smithi05955964-40 2022-04-28T19:46:09.491 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 203 (ObjNum 199 snap 57 seq_num 199) dirty exists 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:580: left oid 40 (ObjNum 199 snap 57 seq_num 199) 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:581: finishing copy_from to smithi05955964-42 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 233 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:581: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:573: done (7 left) 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:574: done (6 left) 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:576: done (5 left) 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:577: done (4 left) 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:578: done (3 left) 2022-04-28T19:46:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:579: done (2 left) 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:580: done (1 left) 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:581: done (0 left) 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:582: rollback oid 36 current snap is 57 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 47 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:583: copy_from oid 47 from oid 42 current snap is 57 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:584: rollback oid 28 current snap is 57 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 28 to 48 2022-04-28T19:46:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:585: delete oid 33 current snap is 57 2022-04-28T19:46:09.495 INFO:tasks.rados.rados.0.smithi059.stdout:582: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:46:09.495 INFO:tasks.rados.rados.0.smithi059.stdout:585: done (3 left) 2022-04-28T19:46:09.495 INFO:tasks.rados.rados.0.smithi059.stdout:586: delete oid 40 current snap is 57 2022-04-28T19:46:09.497 INFO:tasks.rados.rados.0.smithi059.stdout:582: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:46:09.497 INFO:tasks.rados.rados.0.smithi059.stdout:582: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:46:09.497 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 207 (ObjNum 138 snap 35 seq_num 138) dirty exists 2022-04-28T19:46:09.499 INFO:tasks.rados.rados.0.smithi059.stdout:584: finishing rollback tid 0 to smithi05955964-28 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:584: finishing rollback tid 1 to smithi05955964-28 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:582: done (3 left) 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:586: done (2 left) 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:587: read oid 9 snap -1 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:587: expect (ObjNum 156 snap 42 seq_num 156) 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:588: rollback oid 49 current snap is 57 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 45 2022-04-28T19:46:09.500 INFO:tasks.rados.rados.0.smithi059.stdout:589: delete oid 45 current snap is 57 2022-04-28T19:46:09.501 INFO:tasks.rados.rados.0.smithi059.stdout:584: finishing rollback tid 2 to smithi05955964-28 2022-04-28T19:46:09.501 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 131 (ObjNum 163 snap 42 seq_num 163) dirty exists 2022-04-28T19:46:09.501 INFO:tasks.rados.rados.0.smithi059.stdout:584: done (4 left) 2022-04-28T19:46:09.501 INFO:tasks.rados.rados.0.smithi059.stdout:589: done (3 left) 2022-04-28T19:46:09.501 INFO:tasks.rados.rados.0.smithi059.stdout:590: read oid 34 snap -1 2022-04-28T19:46:09.502 INFO:tasks.rados.rados.0.smithi059.stdout:590: expect (ObjNum 189 snap 53 seq_num 189) 2022-04-28T19:46:09.502 INFO:tasks.rados.rados.0.smithi059.stdout:591: snap_create 2022-04-28T19:46:09.502 INFO:tasks.rados.rados.0.smithi059.stdout:583: finishing copy_from to smithi05955964-47 2022-04-28T19:46:09.502 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 233 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:46:09.503 INFO:tasks.rados.rados.0.smithi059.stdout:583: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:46:09.503 INFO:tasks.rados.rados.0.smithi059.stdout:588: finishing rollback tid 0 to smithi05955964-49 2022-04-28T19:46:09.503 INFO:tasks.rados.rados.0.smithi059.stdout:588: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:46:09.504 INFO:tasks.rados.rados.0.smithi059.stdout:588: finishing rollback tid 2 to smithi05955964-49 2022-04-28T19:46:09.504 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 211 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:46:09.504 INFO:tasks.rados.rados.0.smithi059.stdout:587: expect (ObjNum 156 snap 42 seq_num 156) 2022-04-28T19:46:09.518 INFO:tasks.rados.rados.0.smithi059.stdout:590: expect (ObjNum 189 snap 53 seq_num 189) 2022-04-28T19:46:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:09 smithi059 bash[29114]: cluster 2022-04-28T19:46:08.110294+0000 mgr.y (mgr.14140) 426 : cluster [DBG] pgmap v477: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 147 MiB data, 218 MiB used, 707 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2022-04-28T19:46:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:09 smithi059 bash[29114]: cluster 2022-04-28T19:46:08.188558+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2022-04-28T19:46:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:09 smithi059 bash[23786]: cluster 2022-04-28T19:46:08.110294+0000 mgr.y (mgr.14140) 426 : cluster [DBG] pgmap v477: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 147 MiB data, 218 MiB used, 707 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2022-04-28T19:46:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:09 smithi059 bash[23786]: cluster 2022-04-28T19:46:08.188558+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2022-04-28T19:46:09.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:09 smithi139 bash[23439]: cluster 2022-04-28T19:46:08.110294+0000 mgr.y (mgr.14140) 426 : cluster [DBG] pgmap v477: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 147 MiB data, 218 MiB used, 707 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2022-04-28T19:46:09.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:09 smithi139 bash[23439]: cluster 2022-04-28T19:46:08.188558+0000 mon.a (mon.0) 771 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2022-04-28T19:46:10.211 INFO:tasks.rados.rados.0.smithi059.stdout:583: done (4 left) 2022-04-28T19:46:10.211 INFO:tasks.rados.rados.0.smithi059.stdout:587: done (3 left) 2022-04-28T19:46:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:588: done (2 left) 2022-04-28T19:46:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:590: done (1 left) 2022-04-28T19:46:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:591: done (0 left) 2022-04-28T19:46:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 58 2022-04-28T19:46:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:592: seq_num 203 ranges {835584=606208,1441792=401408} 2022-04-28T19:46:10.219 INFO:tasks.rados.rados.0.smithi059.stdout:592: writing smithi05955964-39 from 835584 to 1441792 tid 1 2022-04-28T19:46:10.222 INFO:tasks.rados.rados.0.smithi059.stdout:592: writing smithi05955964-39 from 1441792 to 1843200 tid 2 2022-04-28T19:46:10.222 INFO:tasks.rados.rados.0.smithi059.stdout:593: read oid 48 snap -1 2022-04-28T19:46:10.222 INFO:tasks.rados.rados.0.smithi059.stdout:593: expect (ObjNum 159 snap 42 seq_num 159) 2022-04-28T19:46:10.222 INFO:tasks.rados.rados.0.smithi059.stdout:594: rollback oid 18 current snap is 58 2022-04-28T19:46:10.222 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 51 2022-04-28T19:46:10.223 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 58 2022-04-28T19:46:10.223 INFO:tasks.rados.rados.0.smithi059.stdout:595: seq_num 204 ranges {286720=622592,909312=401408} 2022-04-28T19:46:10.227 INFO:tasks.rados.rados.0.smithi059.stdout:595: writing smithi05955964-44 from 286720 to 909312 tid 1 2022-04-28T19:46:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:595: writing smithi05955964-44 from 909312 to 1310720 tid 2 2022-04-28T19:46:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:596: snap_create 2022-04-28T19:46:10.230 INFO:tasks.rados.rados.0.smithi059.stdout:594: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:46:10.231 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 160 (ObjNum 43888 snap 32762 seq_num 41792) dirty dne 2022-04-28T19:46:10.232 INFO:tasks.rados.rados.0.smithi059.stdout:592: finishing write tid 1 to smithi05955964-39 2022-04-28T19:46:10.233 INFO:tasks.rados.rados.0.smithi059.stdout:592: finishing write tid 2 to smithi05955964-39 2022-04-28T19:46:10.233 INFO:tasks.rados.rados.0.smithi059.stdout:592: finishing write tid 4 to smithi05955964-39 2022-04-28T19:46:10.235 INFO:tasks.rados.rados.0.smithi059.stdout:595: finishing write tid 1 to smithi05955964-44 2022-04-28T19:46:10.235 INFO:tasks.rados.rados.0.smithi059.stdout:592: finishing write tid 5 to smithi05955964-39 2022-04-28T19:46:10.236 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 241 (ObjNum 203 snap 58 seq_num 203) dirty exists 2022-04-28T19:46:10.236 INFO:tasks.rados.rados.0.smithi059.stdout:592: left oid 39 (ObjNum 203 snap 58 seq_num 203) 2022-04-28T19:46:10.236 INFO:tasks.rados.rados.0.smithi059.stdout:595: finishing write tid 2 to smithi05955964-44 2022-04-28T19:46:10.236 INFO:tasks.rados.rados.0.smithi059.stdout:595: finishing write tid 4 to smithi05955964-44 2022-04-28T19:46:10.237 INFO:tasks.rados.rados.0.smithi059.stdout:595: finishing write tid 5 to smithi05955964-44 2022-04-28T19:46:10.237 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 142 (ObjNum 204 snap 58 seq_num 204) dirty exists 2022-04-28T19:46:10.237 INFO:tasks.rados.rados.0.smithi059.stdout:595: left oid 44 (ObjNum 204 snap 58 seq_num 204) 2022-04-28T19:46:10.239 INFO:tasks.rados.rados.0.smithi059.stdout:593: expect (ObjNum 159 snap 42 seq_num 159) 2022-04-28T19:46:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:10 smithi059 bash[29114]: cluster 2022-04-28T19:46:09.189854+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2022-04-28T19:46:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:10 smithi059 bash[23786]: cluster 2022-04-28T19:46:09.189854+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2022-04-28T19:46:10.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:10 smithi139 bash[23439]: cluster 2022-04-28T19:46:09.189854+0000 mon.a (mon.0) 772 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2022-04-28T19:46:11.221 INFO:tasks.rados.rados.0.smithi059.stdout:592: done (4 left) 2022-04-28T19:46:11.221 INFO:tasks.rados.rados.0.smithi059.stdout:593: done (3 left) 2022-04-28T19:46:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:594: done (2 left) 2022-04-28T19:46:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:595: done (1 left) 2022-04-28T19:46:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:596: done (0 left) 2022-04-28T19:46:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 59 2022-04-28T19:46:11.222 INFO:tasks.rados.rados.0.smithi059.stdout:597: seq_num 205 ranges {0=638976,638976=401408} 2022-04-28T19:46:11.226 INFO:tasks.rados.rados.0.smithi059.stdout:597: writing smithi05955964-18 from 0 to 638976 tid 1 2022-04-28T19:46:11.229 INFO:tasks.rados.rados.0.smithi059.stdout:597: writing smithi05955964-18 from 638976 to 1040384 tid 2 2022-04-28T19:46:11.229 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 59 2022-04-28T19:46:11.230 INFO:tasks.rados.rados.0.smithi059.stdout:598: seq_num 206 ranges {1843200=655360,2498560=401408} 2022-04-28T19:46:11.245 INFO:tasks.rados.rados.0.smithi059.stdout:598: writing smithi05955964-39 from 1843200 to 2498560 tid 1 2022-04-28T19:46:11.247 INFO:tasks.rados.rados.0.smithi059.stdout:598: writing smithi05955964-39 from 2498560 to 2899968 tid 2 2022-04-28T19:46:11.247 INFO:tasks.rados.rados.0.smithi059.stdout:599: setattr oid 49 current snap is 59 2022-04-28T19:46:11.247 INFO:tasks.rados.rados.0.smithi059.stdout:597: finishing write tid 1 to smithi05955964-18 2022-04-28T19:46:11.247 INFO:tasks.rados.rados.0.smithi059.stdout:597: finishing write tid 2 to smithi05955964-18 2022-04-28T19:46:11.248 INFO:tasks.rados.rados.0.smithi059.stdout:597: finishing write tid 4 to smithi05955964-18 2022-04-28T19:46:11.248 INFO:tasks.rados.rados.0.smithi059.stdout:597: finishing write tid 5 to smithi05955964-18 2022-04-28T19:46:11.248 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 236 (ObjNum 205 snap 59 seq_num 205) dirty exists 2022-04-28T19:46:11.248 INFO:tasks.rados.rados.0.smithi059.stdout:597: left oid 18 (ObjNum 205 snap 59 seq_num 205) 2022-04-28T19:46:11.249 INFO:tasks.rados.rados.0.smithi059.stdout:597: done (2 left) 2022-04-28T19:46:11.249 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 59 2022-04-28T19:46:11.249 INFO:tasks.rados.rados.0.smithi059.stdout:600: seq_num 207 ranges {2981888=671744,3653632=401408} 2022-04-28T19:46:11.270 INFO:tasks.rados.rados.0.smithi059.stdout:600: writing smithi05955964-8 from 2981888 to 3653632 tid 1 2022-04-28T19:46:11.273 INFO:tasks.rados.rados.0.smithi059.stdout:600: writing smithi05955964-8 from 3653632 to 4055040 tid 2 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:601: rollback oid 23 current snap is 59 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 58 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 213 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:598: finishing write tid 1 to smithi05955964-39 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:598: finishing write tid 2 to smithi05955964-39 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:598: finishing write tid 4 to smithi05955964-39 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:598: finishing write tid 5 to smithi05955964-39 2022-04-28T19:46:11.274 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 245 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:46:11.275 INFO:tasks.rados.rados.0.smithi059.stdout:598: left oid 39 (ObjNum 206 snap 59 seq_num 206) 2022-04-28T19:46:11.275 INFO:tasks.rados.rados.0.smithi059.stdout:598: done (3 left) 2022-04-28T19:46:11.275 INFO:tasks.rados.rados.0.smithi059.stdout:599: done (2 left) 2022-04-28T19:46:11.275 INFO:tasks.rados.rados.0.smithi059.stdout:602: copy_from oid 46 from oid 12 current snap is 59 2022-04-28T19:46:11.275 INFO:tasks.rados.rados.0.smithi059.stdout:603: snap_remove snap 52 2022-04-28T19:46:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:11 smithi059 bash[23786]: cluster 2022-04-28T19:46:10.111222+0000 mgr.y (mgr.14140) 427 : cluster [DBG] pgmap v480: 89 pgs: 89 active+clean; 150 MiB data, 221 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2022-04-28T19:46:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:11 smithi059 bash[23786]: cluster 2022-04-28T19:46:10.209515+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2022-04-28T19:46:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:11 smithi059 bash[29114]: cluster 2022-04-28T19:46:10.111222+0000 mgr.y (mgr.14140) 427 : cluster [DBG] pgmap v480: 89 pgs: 89 active+clean; 150 MiB data, 221 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2022-04-28T19:46:11.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:11 smithi059 bash[29114]: cluster 2022-04-28T19:46:10.209515+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2022-04-28T19:46:11.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:11 smithi139 bash[23439]: cluster 2022-04-28T19:46:10.111222+0000 mgr.y (mgr.14140) 427 : cluster [DBG] pgmap v480: 89 pgs: 89 active+clean; 150 MiB data, 221 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2022-04-28T19:46:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:11 smithi139 bash[23439]: cluster 2022-04-28T19:46:10.209515+0000 mon.a (mon.0) 773 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2022-04-28T19:46:12.183 INFO:tasks.rados.rados.0.smithi059.stdout:603: done (3 left) 2022-04-28T19:46:12.183 INFO:tasks.rados.rados.0.smithi059.stdout:604: setattr oid 35 current snap is 59 2022-04-28T19:46:12.187 INFO:tasks.rados.rados.0.smithi059.stdout:605: read oid 28 snap 49 2022-04-28T19:46:12.187 INFO:tasks.rados.rados.0.smithi059.stdout:605: expect (ObjNum 163 snap 42 seq_num 163) 2022-04-28T19:46:12.187 INFO:tasks.rados.rados.0.smithi059.stdout:606: copy_from oid 3 from oid 6 current snap is 59 2022-04-28T19:46:12.187 INFO:tasks.rados.rados.0.smithi059.stdout:601: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:46:12.187 INFO:tasks.rados.rados.0.smithi059.stdout:601: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:601: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 236 (ObjNum 190 snap 54 seq_num 190) dirty exists 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:600: finishing write tid 1 to smithi05955964-8 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:600: finishing write tid 2 to smithi05955964-8 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:600: finishing write tid 4 to smithi05955964-8 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:600: finishing write tid 5 to smithi05955964-8 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 179 (ObjNum 207 snap 59 seq_num 207) dirty exists 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:600: left oid 8 (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:602: finishing copy_from to smithi05955964-46 2022-04-28T19:46:12.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 210 (ObjNum 176 snap 49 seq_num 176) dirty exists 2022-04-28T19:46:12.189 INFO:tasks.rados.rados.0.smithi059.stdout:602: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:46:12.189 INFO:tasks.rados.rados.0.smithi059.stdout:600: done (5 left) 2022-04-28T19:46:12.189 INFO:tasks.rados.rados.0.smithi059.stdout:601: done (4 left) 2022-04-28T19:46:12.189 INFO:tasks.rados.rados.0.smithi059.stdout:602: done (3 left) 2022-04-28T19:46:12.189 INFO:tasks.rados.rados.0.smithi059.stdout:607: delete oid 7 current snap is 59 2022-04-28T19:46:12.195 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 35 (ObjNum 188341584 snap 32762 seq_num 188341584) dirty exists 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:604: done (3 left) 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:607: done (2 left) 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:608: read oid 40 snap -1 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:608: expect deleted 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:609: read oid 23 snap 17 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:609: expect (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:46:12.199 INFO:tasks.rados.rados.0.smithi059.stdout:610: delete oid 46 current snap is 59 2022-04-28T19:46:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:608: done (4 left) 2022-04-28T19:46:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:610: done (3 left) 2022-04-28T19:46:12.202 INFO:tasks.rados.rados.0.smithi059.stdout:611: delete oid 36 current snap is 59 2022-04-28T19:46:12.208 INFO:tasks.rados.rados.0.smithi059.stdout:605: expect (ObjNum 163 snap 42 seq_num 163) 2022-04-28T19:46:12.291 INFO:tasks.rados.rados.0.smithi059.stdout:609: expect (ObjNum 53 snap 1 seq_num 53) 2022-04-28T19:46:12.346 INFO:tasks.rados.rados.0.smithi059.stdout:606: finishing copy_from to smithi05955964-3 2022-04-28T19:46:12.346 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 182 (ObjNum 185 snap 53 seq_num 185) dirty exists 2022-04-28T19:46:12.346 INFO:tasks.rados.rados.0.smithi059.stdout:606: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:46:12.346 INFO:tasks.rados.rados.0.smithi059.stdout:605: done (3 left) 2022-04-28T19:46:12.346 INFO:tasks.rados.rados.0.smithi059.stdout:606: done (2 left) 2022-04-28T19:46:12.346 INFO:tasks.rados.rados.0.smithi059.stdout:609: done (1 left) 2022-04-28T19:46:12.347 INFO:tasks.rados.rados.0.smithi059.stdout:611: done (0 left) 2022-04-28T19:46:12.347 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 59 2022-04-28T19:46:12.347 INFO:tasks.rados.rados.0.smithi059.stdout:612: seq_num 211 ranges {1138688=737280,1875968=401408} 2022-04-28T19:46:12.366 INFO:tasks.rados.rados.0.smithi059.stdout:612: writing smithi05955964-6 from 1138688 to 1875968 tid 1 2022-04-28T19:46:12.369 INFO:tasks.rados.rados.0.smithi059.stdout:612: writing smithi05955964-6 from 1875968 to 2277376 tid 2 2022-04-28T19:46:12.369 INFO:tasks.rados.rados.0.smithi059.stdout:613: rollback oid 43 current snap is 59 2022-04-28T19:46:12.369 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 43 to 47 2022-04-28T19:46:12.369 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 59 2022-04-28T19:46:12.369 INFO:tasks.rados.rados.0.smithi059.stdout:614: seq_num 212 ranges {0=753664,753664=401408} 2022-04-28T19:46:12.373 INFO:tasks.rados.rados.0.smithi059.stdout:614: writing smithi05955964-35 from 0 to 753664 tid 1 2022-04-28T19:46:12.376 INFO:tasks.rados.rados.0.smithi059.stdout:614: writing smithi05955964-35 from 753664 to 1155072 tid 2 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:615: rollback oid 24 current snap is 59 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 54 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:612: finishing write tid 1 to smithi05955964-6 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:612: finishing write tid 2 to smithi05955964-6 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:612: finishing write tid 4 to smithi05955964-6 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:613: finishing rollback tid 0 to smithi05955964-43 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:616: rollback oid 41 current snap is 59 2022-04-28T19:46:12.377 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 55 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:613: finishing rollback tid 1 to smithi05955964-43 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:613: finishing rollback tid 2 to smithi05955964-43 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 251 (ObjNum 125 snap 30 seq_num 125) dirty exists 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:612: finishing write tid 5 to smithi05955964-6 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 187 (ObjNum 211 snap 59 seq_num 211) dirty exists 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:612: left oid 6 (ObjNum 211 snap 59 seq_num 211) 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:612: done (4 left) 2022-04-28T19:46:12.378 INFO:tasks.rados.rados.0.smithi059.stdout:613: done (3 left) 2022-04-28T19:46:12.379 INFO:tasks.rados.rados.0.smithi059.stdout:617: read oid 18 snap 53 2022-04-28T19:46:12.379 INFO:tasks.rados.rados.0.smithi059.stdout:617: expect deleted 2022-04-28T19:46:12.379 INFO:tasks.rados.rados.0.smithi059.stdout:618: copy_from oid 16 from oid 44 current snap is 59 2022-04-28T19:46:12.379 INFO:tasks.rados.rados.0.smithi059.stdout:619: snap_remove snap 48 2022-04-28T19:46:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:12 smithi059 bash[29114]: cluster 2022-04-28T19:46:11.219820+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2022-04-28T19:46:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:12 smithi059 bash[29114]: cluster 2022-04-28T19:46:12.179444+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2022-04-28T19:46:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:12 smithi059 bash[23786]: cluster 2022-04-28T19:46:11.219820+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2022-04-28T19:46:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:12 smithi059 bash[23786]: cluster 2022-04-28T19:46:12.179444+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2022-04-28T19:46:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:12 smithi139 bash[23439]: cluster 2022-04-28T19:46:11.219820+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2022-04-28T19:46:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:12 smithi139 bash[23439]: cluster 2022-04-28T19:46:12.179444+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2022-04-28T19:46:13.237 INFO:tasks.rados.rados.0.smithi059.stdout:619: done (5 left) 2022-04-28T19:46:13.237 INFO:tasks.rados.rados.0.smithi059.stdout:620: snap_remove snap 32 2022-04-28T19:46:13.519 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:13 smithi059 bash[29114]: cluster 2022-04-28T19:46:12.111924+0000 mgr.y (mgr.14140) 428 : cluster [DBG] pgmap v483: 89 pgs: 89 active+clean; 155 MiB data, 225 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2022-04-28T19:46:13.519 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:13 smithi059 bash[23786]: cluster 2022-04-28T19:46:12.111924+0000 mgr.y (mgr.14140) 428 : cluster [DBG] pgmap v483: 89 pgs: 89 active+clean; 155 MiB data, 225 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2022-04-28T19:46:13.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:13 smithi139 bash[23439]: cluster 2022-04-28T19:46:12.111924+0000 mgr.y (mgr.14140) 428 : cluster [DBG] pgmap v483: 89 pgs: 89 active+clean; 155 MiB data, 225 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2022-04-28T19:46:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:13.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:46:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:13.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:46:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:13.517Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:46:14.255 INFO:tasks.rados.rados.0.smithi059.stdout:620: done (5 left) 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:621: copy_from oid 15 from oid 36 current snap is 59 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:622: copy_from oid 11 from oid 46 current snap is 59 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:614: finishing write tid 1 to smithi05955964-35 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:615: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:615: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:614: finishing write tid 2 to smithi05955964-35 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:615: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:46:14.256 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 163 (ObjNum 116 snap 29 seq_num 116) dirty exists 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:616: finishing rollback tid 0 to smithi05955964-41 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:616: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:614: finishing write tid 4 to smithi05955964-35 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:616: finishing rollback tid 2 to smithi05955964-41 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 50 (ObjNum 84 snap 14 seq_num 84) dirty exists 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:614: finishing write tid 5 to smithi05955964-35 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 38 (ObjNum 212 snap 59 seq_num 212) dirty exists 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:614: left oid 35 (ObjNum 212 snap 59 seq_num 212) 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:618: finishing copy_from to smithi05955964-16 2022-04-28T19:46:14.257 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 142 (ObjNum 204 snap 58 seq_num 204) dirty exists 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:618: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:614: done (6 left) 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:615: done (5 left) 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:616: done (4 left) 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:617: done (3 left) 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:618: done (2 left) 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:623: rollback oid 17 current snap is 59 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 23 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:624: read oid 3 snap -1 2022-04-28T19:46:14.258 INFO:tasks.rados.rados.0.smithi059.stdout:624: expect (ObjNum 185 snap 53 seq_num 185) 2022-04-28T19:46:14.259 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 59 2022-04-28T19:46:14.259 INFO:tasks.rados.rados.0.smithi059.stdout:625: seq_num 213 ranges {2179072=770048,2949120=401408} 2022-04-28T19:46:14.273 INFO:tasks.rados.rados.0.smithi059.stdout:625: writing smithi05955964-30 from 2179072 to 2949120 tid 1 2022-04-28T19:46:14.276 INFO:tasks.rados.rados.0.smithi059.stdout:625: writing smithi05955964-30 from 2949120 to 3350528 tid 2 2022-04-28T19:46:14.276 INFO:tasks.rados.rados.0.smithi059.stdout:626: snap_remove snap 54 2022-04-28T19:46:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:14 smithi059 bash[23786]: cluster 2022-04-28T19:46:13.228359+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2022-04-28T19:46:14.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:14 smithi059 bash[29114]: cluster 2022-04-28T19:46:13.228359+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2022-04-28T19:46:14.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:14 smithi139 bash[23439]: cluster 2022-04-28T19:46:13.228359+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2022-04-28T19:46:15.260 INFO:tasks.rados.rados.0.smithi059.stdout:623: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:46:15.260 INFO:tasks.rados.rados.0.smithi059.stdout:623: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:46:15.260 INFO:tasks.rados.rados.0.smithi059.stdout:623: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 160 (ObjNum 97 snap 20 seq_num 97) dirty exists 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:622: finishing copy_from to smithi05955964-11 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:622: got expected ENOENT (src dne) 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:622: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:621: finishing copy_from to smithi05955964-15 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:621: got expected ENOENT (src dne) 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:621: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:46:15.261 INFO:tasks.rados.rados.0.smithi059.stdout:624: expect (ObjNum 185 snap 53 seq_num 185) 2022-04-28T19:46:15.298 INFO:tasks.rados.rados.0.smithi059.stdout:625: finishing write tid 1 to smithi05955964-30 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:625: finishing write tid 2 to smithi05955964-30 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:625: finishing write tid 4 to smithi05955964-30 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:625: finishing write tid 5 to smithi05955964-30 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 239 (ObjNum 213 snap 59 seq_num 213) dirty exists 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:625: left oid 30 (ObjNum 213 snap 59 seq_num 213) 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:621: done (5 left) 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:622: done (4 left) 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:623: done (3 left) 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:624: done (2 left) 2022-04-28T19:46:15.299 INFO:tasks.rados.rados.0.smithi059.stdout:625: done (1 left) 2022-04-28T19:46:15.300 INFO:tasks.rados.rados.0.smithi059.stdout:626: done (0 left) 2022-04-28T19:46:15.300 INFO:tasks.rados.rados.0.smithi059.stdout:627: snap_remove snap 51 2022-04-28T19:46:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:15 smithi059 bash[29114]: cluster 2022-04-28T19:46:14.112549+0000 mgr.y (mgr.14140) 429 : cluster [DBG] pgmap v486: 89 pgs: 89 active+clean; 156 MiB data, 235 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2022-04-28T19:46:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:15 smithi059 bash[29114]: cluster 2022-04-28T19:46:14.252034+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2022-04-28T19:46:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:15 smithi059 bash[23786]: cluster 2022-04-28T19:46:14.112549+0000 mgr.y (mgr.14140) 429 : cluster [DBG] pgmap v486: 89 pgs: 89 active+clean; 156 MiB data, 235 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2022-04-28T19:46:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:15 smithi059 bash[23786]: cluster 2022-04-28T19:46:14.252034+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2022-04-28T19:46:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:15 smithi139 bash[23439]: cluster 2022-04-28T19:46:14.112549+0000 mgr.y (mgr.14140) 429 : cluster [DBG] pgmap v486: 89 pgs: 89 active+clean; 156 MiB data, 235 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 12 op/s 2022-04-28T19:46:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:15 smithi139 bash[23439]: cluster 2022-04-28T19:46:14.252034+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2022-04-28T19:46:16.286 INFO:tasks.rados.rados.0.smithi059.stdout:627: done (0 left) 2022-04-28T19:46:16.286 INFO:tasks.rados.rados.0.smithi059.stdout:628: snap_remove snap 23 2022-04-28T19:46:16.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:16 smithi059 bash[29114]: cluster 2022-04-28T19:46:15.255431+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2022-04-28T19:46:16.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:16 smithi059 bash[29114]: audit 2022-04-28T19:46:16.152709+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:46:16.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:16 smithi059 bash[29114]: audit 2022-04-28T19:46:16.157968+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:46:16.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:16 smithi059 bash[23786]: cluster 2022-04-28T19:46:15.255431+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2022-04-28T19:46:16.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:16 smithi059 bash[23786]: audit 2022-04-28T19:46:16.152709+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:46:16.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:16 smithi059 bash[23786]: audit 2022-04-28T19:46:16.157968+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:46:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:16 smithi139 bash[23439]: cluster 2022-04-28T19:46:15.255431+0000 mon.a (mon.0) 778 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2022-04-28T19:46:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:16 smithi139 bash[23439]: audit 2022-04-28T19:46:16.152709+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:46:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:16 smithi139 bash[23439]: audit 2022-04-28T19:46:16.157968+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:46:17.293 INFO:tasks.rados.rados.0.smithi059.stdout:628: done (0 left) 2022-04-28T19:46:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:629: rollback oid 12 current snap is 59 2022-04-28T19:46:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 12 to 58 2022-04-28T19:46:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:630: read oid 21 snap -1 2022-04-28T19:46:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:630: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:46:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:631: snap_remove snap 47 2022-04-28T19:46:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:17 smithi059 bash[29114]: cluster 2022-04-28T19:46:16.113211+0000 mgr.y (mgr.14140) 430 : cluster [DBG] pgmap v489: 89 pgs: 89 active+clean; 161 MiB data, 239 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2022-04-28T19:46:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:17 smithi059 bash[29114]: cluster 2022-04-28T19:46:16.281298+0000 mon.a (mon.0) 781 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2022-04-28T19:46:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:17 smithi059 bash[23786]: cluster 2022-04-28T19:46:16.113211+0000 mgr.y (mgr.14140) 430 : cluster [DBG] pgmap v489: 89 pgs: 89 active+clean; 161 MiB data, 239 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2022-04-28T19:46:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:17 smithi059 bash[23786]: cluster 2022-04-28T19:46:16.281298+0000 mon.a (mon.0) 781 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2022-04-28T19:46:17.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:17 smithi139 bash[23439]: cluster 2022-04-28T19:46:16.113211+0000 mgr.y (mgr.14140) 430 : cluster [DBG] pgmap v489: 89 pgs: 89 active+clean; 161 MiB data, 239 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2022-04-28T19:46:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:17 smithi139 bash[23439]: cluster 2022-04-28T19:46:16.281298+0000 mon.a (mon.0) 781 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2022-04-28T19:46:18.308 INFO:tasks.rados.rados.0.smithi059.stdout:631: done (2 left) 2022-04-28T19:46:18.316 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 59 2022-04-28T19:46:18.316 INFO:tasks.rados.rados.0.smithi059.stdout:632: seq_num 214 ranges {1310720=794624,2105344=401408} 2022-04-28T19:46:18.322 INFO:tasks.rados.rados.0.smithi059.stdout:632: writing smithi05955964-44 from 1310720 to 2105344 tid 1 2022-04-28T19:46:18.324 INFO:tasks.rados.rados.0.smithi059.stdout:632: writing smithi05955964-44 from 2105344 to 2506752 tid 2 2022-04-28T19:46:18.336 INFO:tasks.rados.rados.0.smithi059.stdout:629: finishing rollback tid 0 to smithi05955964-12 2022-04-28T19:46:18.336 INFO:tasks.rados.rados.0.smithi059.stdout:629: finishing rollback tid 1 to smithi05955964-12 2022-04-28T19:46:18.336 INFO:tasks.rados.rados.0.smithi059.stdout:629: finishing rollback tid 2 to smithi05955964-12 2022-04-28T19:46:18.336 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 294 (ObjNum 176 snap 49 seq_num 176) dirty exists 2022-04-28T19:46:18.336 INFO:tasks.rados.rados.0.smithi059.stdout:630: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:46:18.393 INFO:tasks.rados.rados.0.smithi059.stdout:632: finishing write tid 1 to smithi05955964-44 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:632: finishing write tid 2 to smithi05955964-44 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:632: finishing write tid 4 to smithi05955964-44 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:632: finishing write tid 5 to smithi05955964-44 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 145 (ObjNum 214 snap 59 seq_num 214) dirty exists 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:632: left oid 44 (ObjNum 214 snap 59 seq_num 214) 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:629: done (2 left) 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:630: done (1 left) 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:632: done (0 left) 2022-04-28T19:46:18.394 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 59 2022-04-28T19:46:18.395 INFO:tasks.rados.rados.0.smithi059.stdout:633: seq_num 215 ranges {0=409600,409600=532480,942080=270336} 2022-04-28T19:46:18.398 INFO:tasks.rados.rados.0.smithi059.stdout:633: writing smithi05955964-33 from 0 to 409600 tid 1 2022-04-28T19:46:18.404 INFO:tasks.rados.rados.0.smithi059.stdout:633: writing smithi05955964-33 from 409600 to 942080 tid 2 2022-04-28T19:46:18.406 INFO:tasks.rados.rados.0.smithi059.stdout:633: writing smithi05955964-33 from 942080 to 1212416 tid 3 2022-04-28T19:46:18.406 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 59 2022-04-28T19:46:18.406 INFO:tasks.rados.rados.0.smithi059.stdout:634: seq_num 216 ranges {0=425984,425984=409600,835584=393216} 2022-04-28T19:46:18.408 INFO:tasks.rados.rados.0.smithi059.stdout:634: writing smithi05955964-45 from 0 to 425984 tid 1 2022-04-28T19:46:18.411 INFO:tasks.rados.rados.0.smithi059.stdout:634: writing smithi05955964-45 from 425984 to 835584 tid 2 2022-04-28T19:46:18.413 INFO:tasks.rados.rados.0.smithi059.stdout:634: writing smithi05955964-45 from 835584 to 1228800 tid 3 2022-04-28T19:46:18.414 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 59 2022-04-28T19:46:18.414 INFO:tasks.rados.rados.0.smithi059.stdout:635: seq_num 217 ranges {0=442368,442368=696320,1138688=106496} 2022-04-28T19:46:18.416 INFO:tasks.rados.rados.0.smithi059.stdout:635: writing smithi05955964-19 from 0 to 442368 tid 1 2022-04-28T19:46:18.420 INFO:tasks.rados.rados.0.smithi059.stdout:635: writing smithi05955964-19 from 442368 to 1138688 tid 2 2022-04-28T19:46:18.421 INFO:tasks.rados.rados.0.smithi059.stdout:635: writing smithi05955964-19 from 1138688 to 1245184 tid 3 2022-04-28T19:46:18.421 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 59 2022-04-28T19:46:18.422 INFO:tasks.rados.rados.0.smithi059.stdout:636: seq_num 218 ranges {1507328=458752,1966080=581632,2547712=221184} 2022-04-28T19:46:18.432 INFO:tasks.rados.rados.0.smithi059.stdout:636: writing smithi05955964-14 from 1507328 to 1966080 tid 1 2022-04-28T19:46:18.436 INFO:tasks.rados.rados.0.smithi059.stdout:636: writing smithi05955964-14 from 1966080 to 2547712 tid 2 2022-04-28T19:46:18.439 INFO:tasks.rados.rados.0.smithi059.stdout:636: writing smithi05955964-14 from 2547712 to 2768896 tid 3 2022-04-28T19:46:18.439 INFO:tasks.rados.rados.0.smithi059.stdout:637: setattr oid 29 current snap is 59 2022-04-28T19:46:18.439 INFO:tasks.rados.rados.0.smithi059.stdout:633: finishing write tid 1 to smithi05955964-33 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:633: finishing write tid 2 to smithi05955964-33 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:633: finishing write tid 3 to smithi05955964-33 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:633: finishing write tid 5 to smithi05955964-33 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:633: finishing write tid 6 to smithi05955964-33 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 301 (ObjNum 215 snap 59 seq_num 215) dirty exists 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:633: left oid 33 (ObjNum 215 snap 59 seq_num 215) 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:634: finishing write tid 1 to smithi05955964-45 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:634: finishing write tid 2 to smithi05955964-45 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:634: finishing write tid 3 to smithi05955964-45 2022-04-28T19:46:18.440 INFO:tasks.rados.rados.0.smithi059.stdout:634: finishing write tid 5 to smithi05955964-45 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:634: finishing write tid 6 to smithi05955964-45 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 228 (ObjNum 216 snap 59 seq_num 216) dirty exists 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:634: left oid 45 (ObjNum 216 snap 59 seq_num 216) 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:635: finishing write tid 1 to smithi05955964-19 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:635: finishing write tid 2 to smithi05955964-19 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:635: finishing write tid 3 to smithi05955964-19 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:635: finishing write tid 5 to smithi05955964-19 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:635: finishing write tid 6 to smithi05955964-19 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 226 (ObjNum 217 snap 59 seq_num 217) dirty exists 2022-04-28T19:46:18.441 INFO:tasks.rados.rados.0.smithi059.stdout:635: left oid 19 (ObjNum 217 snap 59 seq_num 217) 2022-04-28T19:46:18.442 INFO:tasks.rados.rados.0.smithi059.stdout:636: finishing write tid 1 to smithi05955964-14 2022-04-28T19:46:18.442 INFO:tasks.rados.rados.0.smithi059.stdout:636: finishing write tid 2 to smithi05955964-14 2022-04-28T19:46:18.442 INFO:tasks.rados.rados.0.smithi059.stdout:633: done (4 left) 2022-04-28T19:46:18.442 INFO:tasks.rados.rados.0.smithi059.stdout:634: done (3 left) 2022-04-28T19:46:18.442 INFO:tasks.rados.rados.0.smithi059.stdout:635: done (2 left) 2022-04-28T19:46:18.442 INFO:tasks.rados.rados.0.smithi059.stdout:638: delete oid 7 current snap is 59 2022-04-28T19:46:18.443 INFO:tasks.rados.rados.0.smithi059.stdout:636: finishing write tid 3 to smithi05955964-14 2022-04-28T19:46:18.443 INFO:tasks.rados.rados.0.smithi059.stdout:636: finishing write tid 5 to smithi05955964-14 2022-04-28T19:46:18.443 INFO:tasks.rados.rados.0.smithi059.stdout:638: done (2 left) 2022-04-28T19:46:18.443 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 59 2022-04-28T19:46:18.443 INFO:tasks.rados.rados.0.smithi059.stdout:639: seq_num 220 ranges {3497984=491520,3989504=753664,4743168=49152} 2022-04-28T19:46:18.464 INFO:tasks.rados.rados.0.smithi059.stdout:639: writing smithi05955964-25 from 3497984 to 3989504 tid 1 2022-04-28T19:46:18.468 INFO:tasks.rados.rados.0.smithi059.stdout:639: writing smithi05955964-25 from 3989504 to 4743168 tid 2 2022-04-28T19:46:18.470 INFO:tasks.rados.rados.0.smithi059.stdout:639: writing smithi05955964-25 from 4743168 to 4792320 tid 3 2022-04-28T19:46:18.470 INFO:tasks.rados.rados.0.smithi059.stdout:640: copy_from oid 45 from oid 6 current snap is 59 2022-04-28T19:46:18.470 INFO:tasks.rados.rados.0.smithi059.stdout:636: finishing write tid 6 to smithi05955964-14 2022-04-28T19:46:18.470 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 231 (ObjNum 218 snap 59 seq_num 218) dirty exists 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:636: left oid 14 (ObjNum 218 snap 59 seq_num 218) 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:636: done (3 left) 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:641: rollback oid 15 current snap is 59 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 57 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 217 (ObjNum 0 snap 0 seq_num 4161406992) dirty exists 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:639: finishing write tid 1 to smithi05955964-25 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:637: done (3 left) 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:642: copy_from oid 33 from oid 19 current snap is 59 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:643: copy_from oid 18 from oid 8 current snap is 59 2022-04-28T19:46:18.471 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 59 2022-04-28T19:46:18.472 INFO:tasks.rados.rados.0.smithi059.stdout:644: seq_num 221 ranges {1908736=507904,2416640=688128,3104768=114688} 2022-04-28T19:46:18.484 INFO:tasks.rados.rados.0.smithi059.stdout:644: writing smithi05955964-17 from 1908736 to 2416640 tid 1 2022-04-28T19:46:18.488 INFO:tasks.rados.rados.0.smithi059.stdout:644: writing smithi05955964-17 from 2416640 to 3104768 tid 2 2022-04-28T19:46:18.488 INFO:tasks.rados.rados.0.smithi059.stdout:644: writing smithi05955964-17 from 3104768 to 3219456 tid 3 2022-04-28T19:46:18.489 INFO:tasks.rados.rados.0.smithi059.stdout:645: snap_remove snap 45 2022-04-28T19:46:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:18 smithi059 bash[29114]: cluster 2022-04-28T19:46:17.291709+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2022-04-28T19:46:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:18 smithi059 bash[23786]: cluster 2022-04-28T19:46:17.291709+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2022-04-28T19:46:18.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:18 smithi139 bash[23439]: cluster 2022-04-28T19:46:17.291709+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2022-04-28T19:46:19.332 INFO:tasks.rados.rados.0.smithi059.stdout:645: done (6 left) 2022-04-28T19:46:19.332 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 59 2022-04-28T19:46:19.332 INFO:tasks.rados.rados.0.smithi059.stdout:646: seq_num 222 ranges {2056192=524288,2580480=565248,3145728=237568} 2022-04-28T19:46:19.347 INFO:tasks.rados.rados.0.smithi059.stdout:646: writing smithi05955964-28 from 2056192 to 2580480 tid 1 2022-04-28T19:46:19.350 INFO:tasks.rados.rados.0.smithi059.stdout:646: writing smithi05955964-28 from 2580480 to 3145728 tid 2 2022-04-28T19:46:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:646: writing smithi05955964-28 from 3145728 to 3383296 tid 3 2022-04-28T19:46:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:647: read oid 35 snap -1 2022-04-28T19:46:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:647: expect (ObjNum 212 snap 59 seq_num 212) 2022-04-28T19:46:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:648: snap_remove snap 55 2022-04-28T19:46:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:19 smithi059 bash[29114]: cluster 2022-04-28T19:46:18.113980+0000 mgr.y (mgr.14140) 431 : cluster [DBG] pgmap v492: 89 pgs: 89 active+clean; 147 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2022-04-28T19:46:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:19 smithi059 bash[29114]: cluster 2022-04-28T19:46:18.304160+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2022-04-28T19:46:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:19 smithi059 bash[23786]: cluster 2022-04-28T19:46:18.113980+0000 mgr.y (mgr.14140) 431 : cluster [DBG] pgmap v492: 89 pgs: 89 active+clean; 147 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2022-04-28T19:46:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:19 smithi059 bash[23786]: cluster 2022-04-28T19:46:18.304160+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2022-04-28T19:46:19.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:19 smithi139 bash[23439]: cluster 2022-04-28T19:46:18.113980+0000 mgr.y (mgr.14140) 431 : cluster [DBG] pgmap v492: 89 pgs: 89 active+clean; 147 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2022-04-28T19:46:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:19 smithi139 bash[23439]: cluster 2022-04-28T19:46:18.304160+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:648: done (8 left) 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:649: read oid 23 snap -1 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:649: expect (ObjNum 190 snap 54 seq_num 190) 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:650: rollback oid 46 current snap is 59 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 58 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:641: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 242 (ObjNum -133514208 snap 21893 seq_num 4161894720) dirty dne 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:641: done (9 left) 2022-04-28T19:46:20.342 INFO:tasks.rados.rados.0.smithi059.stdout:651: setattr oid 39 current snap is 59 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:639: finishing write tid 2 to smithi05955964-25 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:639: finishing write tid 3 to smithi05955964-25 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:640: finishing copy_from to smithi05955964-45 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 187 (ObjNum 211 snap 59 seq_num 211) dirty exists 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:639: finishing write tid 5 to smithi05955964-25 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:642: finishing copy_from to smithi05955964-33 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 226 (ObjNum 217 snap 59 seq_num 217) dirty exists 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:640: finishing copy_from racing read to smithi05955964-45 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:642: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:639: finishing write tid 6 to smithi05955964-25 2022-04-28T19:46:20.343 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 222 (ObjNum 220 snap 59 seq_num 220) dirty exists 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:639: left oid 25 (ObjNum 220 snap 59 seq_num 220) 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:644: finishing write tid 1 to smithi05955964-17 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:644: finishing write tid 2 to smithi05955964-17 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:644: finishing write tid 3 to smithi05955964-17 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:644: finishing write tid 5 to smithi05955964-17 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:644: finishing write tid 6 to smithi05955964-17 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 164 (ObjNum 221 snap 59 seq_num 221) dirty exists 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:644: left oid 17 (ObjNum 221 snap 59 seq_num 221) 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:643: finishing copy_from to smithi05955964-18 2022-04-28T19:46:20.344 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 179 (ObjNum 207 snap 59 seq_num 207) dirty exists 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:643: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:646: finishing write tid 1 to smithi05955964-28 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:646: finishing write tid 2 to smithi05955964-28 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:646: finishing write tid 3 to smithi05955964-28 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:646: finishing write tid 5 to smithi05955964-28 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:646: finishing write tid 6 to smithi05955964-28 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 164 (ObjNum 222 snap 59 seq_num 222) dirty exists 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:646: left oid 28 (ObjNum 222 snap 59 seq_num 222) 2022-04-28T19:46:20.345 INFO:tasks.rados.rados.0.smithi059.stdout:647: expect (ObjNum 212 snap 59 seq_num 212) 2022-04-28T19:46:20.381 INFO:tasks.rados.rados.0.smithi059.stdout:650: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:46:20.381 INFO:tasks.rados.rados.0.smithi059.stdout:650: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:46:20.381 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 232 (ObjNum 195 snap 57 seq_num 195) dirty exists 2022-04-28T19:46:20.381 INFO:tasks.rados.rados.0.smithi059.stdout:649: expect (ObjNum 190 snap 54 seq_num 190) 2022-04-28T19:46:20.482 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 315 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:46:20.482 INFO:tasks.rados.rados.0.smithi059.stdout:639: done (9 left) 2022-04-28T19:46:20.482 INFO:tasks.rados.rados.0.smithi059.stdout:640: done (8 left) 2022-04-28T19:46:20.482 INFO:tasks.rados.rados.0.smithi059.stdout:642: done (7 left) 2022-04-28T19:46:20.482 INFO:tasks.rados.rados.0.smithi059.stdout:643: done (6 left) 2022-04-28T19:46:20.482 INFO:tasks.rados.rados.0.smithi059.stdout:644: done (5 left) 2022-04-28T19:46:20.483 INFO:tasks.rados.rados.0.smithi059.stdout:646: done (4 left) 2022-04-28T19:46:20.483 INFO:tasks.rados.rados.0.smithi059.stdout:647: done (3 left) 2022-04-28T19:46:20.483 INFO:tasks.rados.rados.0.smithi059.stdout:649: done (2 left) 2022-04-28T19:46:20.483 INFO:tasks.rados.rados.0.smithi059.stdout:650: done (1 left) 2022-04-28T19:46:20.483 INFO:tasks.rados.rados.0.smithi059.stdout:651: done (0 left) 2022-04-28T19:46:20.483 INFO:tasks.rados.rados.0.smithi059.stdout:652: delete oid 19 current snap is 59 2022-04-28T19:46:20.486 INFO:tasks.rados.rados.0.smithi059.stdout:652: done (0 left) 2022-04-28T19:46:20.487 INFO:tasks.rados.rados.0.smithi059.stdout:653: rollback oid 15 current snap is 59 2022-04-28T19:46:20.487 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 50 2022-04-28T19:46:20.487 INFO:tasks.rados.rados.0.smithi059.stdout:654: setattr oid 42 current snap is 59 2022-04-28T19:46:20.489 INFO:tasks.rados.rados.0.smithi059.stdout:655: setattr oid 27 current snap is 59 2022-04-28T19:46:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:653: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:46:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:653: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:46:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 322 (ObjNum 131 snap 31 seq_num 131) dirty exists 2022-04-28T19:46:20.492 INFO:tasks.rados.rados.0.smithi059.stdout:653: done (2 left) 2022-04-28T19:46:20.492 INFO:tasks.rados.rados.0.smithi059.stdout:656: snap_remove snap 50 2022-04-28T19:46:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:20 smithi059 bash[23786]: cluster 2022-04-28T19:46:19.329779+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2022-04-28T19:46:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:20 smithi059 bash[29114]: cluster 2022-04-28T19:46:19.329779+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2022-04-28T19:46:20.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:20 smithi139 bash[23439]: cluster 2022-04-28T19:46:19.329779+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2022-04-28T19:46:21.341 INFO:tasks.rados.rados.0.smithi059.stdout:656: done (2 left) 2022-04-28T19:46:21.341 INFO:tasks.rados.rados.0.smithi059.stdout:657: delete oid 25 current snap is 59 2022-04-28T19:46:21.341 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 249 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:46:21.341 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 237 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:46:21.356 INFO:tasks.rados.rados.0.smithi059.stdout:654: done (2 left) 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:655: done (1 left) 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:657: done (0 left) 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:658: read oid 42 snap -1 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:658: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:659: read oid 26 snap -1 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:659: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:21.357 INFO:tasks.rados.rados.0.smithi059.stdout:660: setattr oid 7 current snap is 59 2022-04-28T19:46:21.358 INFO:tasks.rados.rados.0.smithi059.stdout:661: delete oid 43 current snap is 59 2022-04-28T19:46:21.361 INFO:tasks.rados.rados.0.smithi059.stdout:661: done (3 left) 2022-04-28T19:46:21.361 INFO:tasks.rados.rados.0.smithi059.stdout:662: snap_create 2022-04-28T19:46:21.368 INFO:tasks.rados.rados.0.smithi059.stdout:658: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:21.405 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 331 (ObjNum 1683968868 snap 1852665689 seq_num 1951422835) dirty exists 2022-04-28T19:46:21.405 INFO:tasks.rados.rados.0.smithi059.stdout:659: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:21 smithi139 bash[23439]: cluster 2022-04-28T19:46:20.114758+0000 mgr.y (mgr.14140) 432 : cluster [DBG] pgmap v495: 89 pgs: 89 active+clean; 144 MiB data, 221 MiB used, 707 GiB / 715 GiB avail; 1.5 KiB/s rd, 4.1 MiB/s wr, 14 op/s 2022-04-28T19:46:21.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:21 smithi139 bash[23439]: cluster 2022-04-28T19:46:20.336563+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2022-04-28T19:46:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:21 smithi059 bash[23786]: cluster 2022-04-28T19:46:20.114758+0000 mgr.y (mgr.14140) 432 : cluster [DBG] pgmap v495: 89 pgs: 89 active+clean; 144 MiB data, 221 MiB used, 707 GiB / 715 GiB avail; 1.5 KiB/s rd, 4.1 MiB/s wr, 14 op/s 2022-04-28T19:46:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:21 smithi059 bash[23786]: cluster 2022-04-28T19:46:20.336563+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2022-04-28T19:46:21.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:21 smithi059 bash[29114]: cluster 2022-04-28T19:46:20.114758+0000 mgr.y (mgr.14140) 432 : cluster [DBG] pgmap v495: 89 pgs: 89 active+clean; 144 MiB data, 221 MiB used, 707 GiB / 715 GiB avail; 1.5 KiB/s rd, 4.1 MiB/s wr, 14 op/s 2022-04-28T19:46:21.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:21 smithi059 bash[29114]: cluster 2022-04-28T19:46:20.336563+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2022-04-28T19:46:22.187 INFO:tasks.rados.rados.0.smithi059.stdout:658: done (3 left) 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:659: done (2 left) 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:660: done (1 left) 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:662: done (0 left) 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:663: rollback oid 47 current snap is 60 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 49 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:664: read oid 10 snap -1 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:664: expect deleted 2022-04-28T19:46:22.188 INFO:tasks.rados.rados.0.smithi059.stdout:665: read oid 42 snap 57 2022-04-28T19:46:22.189 INFO:tasks.rados.rados.0.smithi059.stdout:665: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:22.189 INFO:tasks.rados.rados.0.smithi059.stdout:666: snap_remove snap 53 2022-04-28T19:46:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:22 smithi139 bash[23439]: cluster 2022-04-28T19:46:21.338607+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2022-04-28T19:46:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:22 smithi139 bash[23439]: cluster 2022-04-28T19:46:22.183654+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2022-04-28T19:46:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:22 smithi059 bash[29114]: cluster 2022-04-28T19:46:21.338607+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2022-04-28T19:46:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:22 smithi059 bash[29114]: cluster 2022-04-28T19:46:22.183654+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2022-04-28T19:46:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:22 smithi059 bash[23786]: cluster 2022-04-28T19:46:21.338607+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2022-04-28T19:46:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:22 smithi059 bash[23786]: cluster 2022-04-28T19:46:22.183654+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2022-04-28T19:46:23.200 INFO:tasks.rados.rados.0.smithi059.stdout:666: done (3 left) 2022-04-28T19:46:23.200 INFO:tasks.rados.rados.0.smithi059.stdout:667: setattr oid 4 current snap is 60 2022-04-28T19:46:23.200 INFO:tasks.rados.rados.0.smithi059.stdout:663: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:46:23.201 INFO:tasks.rados.rados.0.smithi059.stdout:663: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:46:23.201 INFO:tasks.rados.rados.0.smithi059.stdout:663: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:46:23.201 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 240 (ObjNum 171 snap 45 seq_num 171) dirty exists 2022-04-28T19:46:23.201 INFO:tasks.rados.rados.0.smithi059.stdout:665: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:23.235 INFO:tasks.rados.rados.0.smithi059.stdout:663: done (3 left) 2022-04-28T19:46:23.236 INFO:tasks.rados.rados.0.smithi059.stdout:664: done (2 left) 2022-04-28T19:46:23.236 INFO:tasks.rados.rados.0.smithi059.stdout:665: done (1 left) 2022-04-28T19:46:23.236 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 60 2022-04-28T19:46:23.236 INFO:tasks.rados.rados.0.smithi059.stdout:668: seq_num 226 ranges {0=589824,589824=507904,1097728=294912} 2022-04-28T19:46:23.240 INFO:tasks.rados.rados.0.smithi059.stdout:668: writing smithi05955964-20 from 0 to 589824 tid 1 2022-04-28T19:46:23.243 INFO:tasks.rados.rados.0.smithi059.stdout:668: writing smithi05955964-20 from 589824 to 1097728 tid 2 2022-04-28T19:46:23.245 INFO:tasks.rados.rados.0.smithi059.stdout:668: writing smithi05955964-20 from 1097728 to 1392640 tid 3 2022-04-28T19:46:23.245 INFO:tasks.rados.rados.0.smithi059.stdout:669: read oid 1 snap -1 2022-04-28T19:46:23.245 INFO:tasks.rados.rados.0.smithi059.stdout:669: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:46:23.245 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 60 2022-04-28T19:46:23.245 INFO:tasks.rados.rados.0.smithi059.stdout:670: seq_num 227 ranges {0=606208,606208=794624,1400832=8192} 2022-04-28T19:46:23.249 INFO:tasks.rados.rados.0.smithi059.stdout:670: writing smithi05955964-19 from 0 to 606208 tid 1 2022-04-28T19:46:23.255 INFO:tasks.rados.rados.0.smithi059.stdout:670: writing smithi05955964-19 from 606208 to 1400832 tid 2 2022-04-28T19:46:23.255 INFO:tasks.rados.rados.0.smithi059.stdout:670: writing smithi05955964-19 from 1400832 to 1409024 tid 3 2022-04-28T19:46:23.255 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 60 2022-04-28T19:46:23.255 INFO:tasks.rados.rados.0.smithi059.stdout:671: seq_num 228 ranges {0=622592,622592=671744,1294336=131072} 2022-04-28T19:46:23.258 INFO:tasks.rados.rados.0.smithi059.stdout:671: writing smithi05955964-36 from 0 to 622592 tid 1 2022-04-28T19:46:23.263 INFO:tasks.rados.rados.0.smithi059.stdout:671: writing smithi05955964-36 from 622592 to 1294336 tid 2 2022-04-28T19:46:23.264 INFO:tasks.rados.rados.0.smithi059.stdout:671: writing smithi05955964-36 from 1294336 to 1425408 tid 3 2022-04-28T19:46:23.265 INFO:tasks.rados.rados.0.smithi059.stdout:672: read oid 42 snap 57 2022-04-28T19:46:23.265 INFO:tasks.rados.rados.0.smithi059.stdout:672: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:23.265 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 60 2022-04-28T19:46:23.265 INFO:tasks.rados.rados.0.smithi059.stdout:673: seq_num 229 ranges {0=638976,638976=606208,1245184=196608} 2022-04-28T19:46:23.269 INFO:tasks.rados.rados.0.smithi059.stdout:673: writing smithi05955964-32 from 0 to 638976 tid 1 2022-04-28T19:46:23.273 INFO:tasks.rados.rados.0.smithi059.stdout:673: writing smithi05955964-32 from 638976 to 1245184 tid 2 2022-04-28T19:46:23.274 INFO:tasks.rados.rados.0.smithi059.stdout:673: writing smithi05955964-32 from 1245184 to 1441792 tid 3 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:674: rmattr oid 22 current snap is 60 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:674: done (7 left) 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:675: rmattr oid 34 current snap is 60 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:675: done (7 left) 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:676: rmattr oid 49 current snap is 60 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 216 (ObjNum 1449677646 snap 1550143823 seq_num 1499026020) dirty exists 2022-04-28T19:46:23.275 INFO:tasks.rados.rados.0.smithi059.stdout:669: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:46:23.309 INFO:tasks.rados.rados.0.smithi059.stdout:668: finishing write tid 1 to smithi05955964-20 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:668: finishing write tid 2 to smithi05955964-20 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:668: finishing write tid 3 to smithi05955964-20 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:670: finishing write tid 1 to smithi05955964-19 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:667: done (7 left) 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:669: done (6 left) 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 60 2022-04-28T19:46:23.310 INFO:tasks.rados.rados.0.smithi059.stdout:677: seq_num 230 ranges {3809280=655360,4464640=491520,4956160=311296} 2022-04-28T19:46:23.341 INFO:tasks.rados.rados.0.smithi059.stdout:677: writing smithi05955964-34 from 3809280 to 4464640 tid 1 2022-04-28T19:46:23.343 INFO:tasks.rados.rados.0.smithi059.stdout:677: writing smithi05955964-34 from 4464640 to 4956160 tid 2 2022-04-28T19:46:23.345 INFO:tasks.rados.rados.0.smithi059.stdout:677: writing smithi05955964-34 from 4956160 to 5267456 tid 3 2022-04-28T19:46:23.345 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 60 2022-04-28T19:46:23.345 INFO:tasks.rados.rados.0.smithi059.stdout:678: seq_num 231 ranges {3497984=671744,4169728=778240,4947968=24576} 2022-04-28T19:46:23.369 INFO:tasks.rados.rados.0.smithi059.stdout:678: writing smithi05955964-24 from 3497984 to 4169728 tid 1 2022-04-28T19:46:23.374 INFO:tasks.rados.rados.0.smithi059.stdout:678: writing smithi05955964-24 from 4169728 to 4947968 tid 2 2022-04-28T19:46:23.374 INFO:tasks.rados.rados.0.smithi059.stdout:678: writing smithi05955964-24 from 4947968 to 4972544 tid 3 2022-04-28T19:46:23.374 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 60 2022-04-28T19:46:23.374 INFO:tasks.rados.rados.0.smithi059.stdout:679: seq_num 232 ranges {0=688128,688128=663552,1351680=139264} 2022-04-28T19:46:23.378 INFO:tasks.rados.rados.0.smithi059.stdout:679: writing smithi05955964-7 from 0 to 688128 tid 1 2022-04-28T19:46:23.382 INFO:tasks.rados.rados.0.smithi059.stdout:679: writing smithi05955964-7 from 688128 to 1351680 tid 2 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:679: writing smithi05955964-7 from 1351680 to 1490944 tid 3 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:680: read oid 2 snap -1 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:680: expect (ObjNum 178 snap 50 seq_num 178) 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:681: read oid 50 snap -1 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:681: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 60 2022-04-28T19:46:23.384 INFO:tasks.rados.rados.0.smithi059.stdout:682: seq_num 233 ranges {3350528=712704,4063232=548864,4612096=253952} 2022-04-28T19:46:23.407 INFO:tasks.rados.rados.0.smithi059.stdout:682: writing smithi05955964-30 from 3350528 to 4063232 tid 1 2022-04-28T19:46:23.410 INFO:tasks.rados.rados.0.smithi059.stdout:682: writing smithi05955964-30 from 4063232 to 4612096 tid 2 2022-04-28T19:46:23.412 INFO:tasks.rados.rados.0.smithi059.stdout:682: writing smithi05955964-30 from 4612096 to 4866048 tid 3 2022-04-28T19:46:23.412 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 60 2022-04-28T19:46:23.412 INFO:tasks.rados.rados.0.smithi059.stdout:683: seq_num 234 ranges {2277376=729088,3006464=434176,3440640=368640} 2022-04-28T19:46:23.429 INFO:tasks.rados.rados.0.smithi059.stdout:683: writing smithi05955964-15 from 2277376 to 3006464 tid 1 2022-04-28T19:46:23.431 INFO:tasks.rados.rados.0.smithi059.stdout:683: writing smithi05955964-15 from 3006464 to 3440640 tid 2 2022-04-28T19:46:23.434 INFO:tasks.rados.rados.0.smithi059.stdout:683: writing smithi05955964-15 from 3440640 to 3809280 tid 3 2022-04-28T19:46:23.434 INFO:tasks.rados.rados.0.smithi059.stdout:684: snap_remove snap 58 2022-04-28T19:46:23.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:23 smithi139 bash[23439]: cluster 2022-04-28T19:46:22.115443+0000 mgr.y (mgr.14140) 433 : cluster [DBG] pgmap v498: 89 pgs: 89 active+clean; 152 MiB data, 222 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.9 MiB/s wr, 20 op/s 2022-04-28T19:46:23.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:23 smithi139 bash[23439]: cluster 2022-04-28T19:46:23.196330+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2022-04-28T19:46:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[29114]: cluster 2022-04-28T19:46:22.115443+0000 mgr.y (mgr.14140) 433 : cluster [DBG] pgmap v498: 89 pgs: 89 active+clean; 152 MiB data, 222 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.9 MiB/s wr, 20 op/s 2022-04-28T19:46:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[29114]: cluster 2022-04-28T19:46:23.196330+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2022-04-28T19:46:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[23786]: cluster 2022-04-28T19:46:22.115443+0000 mgr.y (mgr.14140) 433 : cluster [DBG] pgmap v498: 89 pgs: 89 active+clean; 152 MiB data, 222 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s rd, 3.9 MiB/s wr, 20 op/s 2022-04-28T19:46:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[23786]: cluster 2022-04-28T19:46:23.196330+0000 mon.a (mon.0) 788 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2022-04-28T19:46:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:23.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:46:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:23.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:46:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:23.517Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:46:24.367 INFO:tasks.rados.rados.0.smithi059.stdout:684: done (13 left) 2022-04-28T19:46:24.367 INFO:tasks.rados.rados.0.smithi059.stdout:685: snap_remove snap 56 2022-04-28T19:46:25.126 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:46:25.378 INFO:tasks.rados.rados.0.smithi059.stdout:685: done (13 left) 2022-04-28T19:46:25.378 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 60 2022-04-28T19:46:25.378 INFO:tasks.rados.rados.0.smithi059.stdout:686: seq_num 235 ranges {3219456=745472,3964928=712704,4677632=90112} 2022-04-28T19:46:25.400 INFO:tasks.rados.rados.0.smithi059.stdout:686: writing smithi05955964-17 from 3219456 to 3964928 tid 1 2022-04-28T19:46:25.404 INFO:tasks.rados.rados.0.smithi059.stdout:686: writing smithi05955964-17 from 3964928 to 4677632 tid 2 2022-04-28T19:46:25.405 INFO:tasks.rados.rados.0.smithi059.stdout:686: writing smithi05955964-17 from 4677632 to 4767744 tid 3 2022-04-28T19:46:25.405 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 60 2022-04-28T19:46:25.405 INFO:tasks.rados.rados.0.smithi059.stdout:687: seq_num 236 ranges {2473984=761856,3235840=647168,3883008=155648} 2022-04-28T19:46:25.423 INFO:tasks.rados.rados.0.smithi059.stdout:687: writing smithi05955964-23 from 2473984 to 3235840 tid 1 2022-04-28T19:46:25.428 INFO:tasks.rados.rados.0.smithi059.stdout:687: writing smithi05955964-23 from 3235840 to 3883008 tid 2 2022-04-28T19:46:25.429 INFO:tasks.rados.rados.0.smithi059.stdout:687: writing smithi05955964-23 from 3883008 to 4038656 tid 3 2022-04-28T19:46:25.429 INFO:tasks.rados.rados.0.smithi059.stdout:688: snap_create 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:668: finishing write tid 5 to smithi05955964-20 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:670: finishing write tid 2 to smithi05955964-19 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:670: finishing write tid 3 to smithi05955964-19 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:670: finishing write tid 5 to smithi05955964-19 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:668: finishing write tid 6 to smithi05955964-20 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 257 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:668: left oid 20 (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:670: finishing write tid 6 to smithi05955964-19 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 254 (ObjNum 227 snap 60 seq_num 227) dirty exists 2022-04-28T19:46:25.430 INFO:tasks.rados.rados.0.smithi059.stdout:670: left oid 19 (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:46:25.431 INFO:tasks.rados.rados.0.smithi059.stdout:671: finishing write tid 1 to smithi05955964-36 2022-04-28T19:46:25.431 INFO:tasks.rados.rados.0.smithi059.stdout:672: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST STATUS REFRESHED AGE VERSION IMAGE NAME IMAGE ID CONTAINER ID 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (5m) 3m ago 5m 0.20.0 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (5m) 3m ago 5m 6.7.4 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (3m) 3m ago 3m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 44062639a78b 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (3m) 3m ago 3m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 3516bf4894c9 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (9m) 3m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b8c424c2e210 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (11m) 3m ago 11m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 320b27bb479e 2022-04-28T19:46:25.444 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (11m) 3m ago 11m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 239497cf6769 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (9m) 3m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0e12092e0bc3 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (10m) 3m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (5m) 3m ago 5m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (5m) 3m ago 5m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (9m) 3m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 547b5580e073 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (8m) 3m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (8m) 3m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 35cdc0c6604a 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (8m) 3m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 2344bb9dfbae 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (7m) 3m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 5976654882b1 2022-04-28T19:46:25.445 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (7m) 3m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 e68cffb4766c 2022-04-28T19:46:25.446 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (6m) 3m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 c36b8738a469 2022-04-28T19:46:25.446 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (6m) 3m ago 6m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:46:25.446 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (5m) 3m ago 6m 2.18.1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 c763409947e4 2022-04-28T19:46:25.471 INFO:tasks.rados.rados.0.smithi059.stdout:671: finishing write tid 2 to smithi05955964-36 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:671: finishing write tid 3 to smithi05955964-36 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:671: finishing write tid 5 to smithi05955964-36 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:673: finishing write tid 1 to smithi05955964-32 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:671: finishing write tid 6 to smithi05955964-36 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 285 (ObjNum 228 snap 60 seq_num 228) dirty exists 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:671: left oid 36 (ObjNum 228 snap 60 seq_num 228) 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:673: finishing write tid 2 to smithi05955964-32 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:673: finishing write tid 3 to smithi05955964-32 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:673: finishing write tid 5 to smithi05955964-32 2022-04-28T19:46:25.472 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 289 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:673: finishing write tid 6 to smithi05955964-32 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 330 (ObjNum 229 snap 60 seq_num 229) dirty exists 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:673: left oid 32 (ObjNum 229 snap 60 seq_num 229) 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:677: finishing write tid 1 to smithi05955964-34 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:677: finishing write tid 2 to smithi05955964-34 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:677: finishing write tid 3 to smithi05955964-34 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:677: finishing write tid 5 to smithi05955964-34 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:677: finishing write tid 6 to smithi05955964-34 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 260 (ObjNum 230 snap 60 seq_num 230) dirty exists 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:677: left oid 34 (ObjNum 230 snap 60 seq_num 230) 2022-04-28T19:46:25.473 INFO:tasks.rados.rados.0.smithi059.stdout:678: finishing write tid 1 to smithi05955964-24 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:678: finishing write tid 2 to smithi05955964-24 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:678: finishing write tid 3 to smithi05955964-24 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:678: finishing write tid 5 to smithi05955964-24 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:678: finishing write tid 6 to smithi05955964-24 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 221 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:678: left oid 24 (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:679: finishing write tid 1 to smithi05955964-7 2022-04-28T19:46:25.474 INFO:tasks.rados.rados.0.smithi059.stdout:680: expect (ObjNum 178 snap 50 seq_num 178) 2022-04-28T19:46:25.485 INFO:tasks.rados.rados.0.smithi059.stdout:679: finishing write tid 2 to smithi05955964-7 2022-04-28T19:46:25.485 INFO:tasks.rados.rados.0.smithi059.stdout:681: expect (ObjNum 49 snap 0 seq_num 49) 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:679: finishing write tid 3 to smithi05955964-7 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:679: finishing write tid 5 to smithi05955964-7 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:679: finishing write tid 6 to smithi05955964-7 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 344 (ObjNum 232 snap 60 seq_num 232) dirty exists 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:679: left oid 7 (ObjNum 232 snap 60 seq_num 232) 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:682: finishing write tid 1 to smithi05955964-30 2022-04-28T19:46:25.504 INFO:tasks.rados.rados.0.smithi059.stdout:682: finishing write tid 2 to smithi05955964-30 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:682: finishing write tid 3 to smithi05955964-30 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:682: finishing write tid 5 to smithi05955964-30 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:682: finishing write tid 6 to smithi05955964-30 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 265 (ObjNum 233 snap 60 seq_num 233) dirty exists 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:682: left oid 30 (ObjNum 233 snap 60 seq_num 233) 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:683: finishing write tid 1 to smithi05955964-15 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:683: finishing write tid 2 to smithi05955964-15 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:683: finishing write tid 3 to smithi05955964-15 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:683: finishing write tid 5 to smithi05955964-15 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:683: finishing write tid 6 to smithi05955964-15 2022-04-28T19:46:25.505 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 349 (ObjNum 234 snap 60 seq_num 234) dirty exists 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:683: left oid 15 (ObjNum 234 snap 60 seq_num 234) 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:686: finishing write tid 1 to smithi05955964-17 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:686: finishing write tid 2 to smithi05955964-17 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:686: finishing write tid 3 to smithi05955964-17 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:686: finishing write tid 5 to smithi05955964-17 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:686: finishing write tid 6 to smithi05955964-17 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 168 (ObjNum 235 snap 60 seq_num 235) dirty exists 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:686: left oid 17 (ObjNum 235 snap 60 seq_num 235) 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:687: finishing write tid 1 to smithi05955964-23 2022-04-28T19:46:25.506 INFO:tasks.rados.rados.0.smithi059.stdout:687: finishing write tid 2 to smithi05955964-23 2022-04-28T19:46:25.507 INFO:tasks.rados.rados.0.smithi059.stdout:687: finishing write tid 3 to smithi05955964-23 2022-04-28T19:46:25.507 INFO:tasks.rados.rados.0.smithi059.stdout:687: finishing write tid 5 to smithi05955964-23 2022-04-28T19:46:25.507 INFO:tasks.rados.rados.0.smithi059.stdout:687: finishing write tid 6 to smithi05955964-23 2022-04-28T19:46:25.507 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 244 (ObjNum 236 snap 60 seq_num 236) dirty exists 2022-04-28T19:46:25.507 INFO:tasks.rados.rados.0.smithi059.stdout:687: left oid 23 (ObjNum 236 snap 60 seq_num 236) 2022-04-28T19:46:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:25 smithi139 bash[23439]: cluster 2022-04-28T19:46:24.116287+0000 mgr.y (mgr.14140) 434 : cluster [DBG] pgmap v501: 89 pgs: 89 active+clean; 152 MiB data, 226 MiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2022-04-28T19:46:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:25 smithi139 bash[23439]: cluster 2022-04-28T19:46:24.364544+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:46:25.776 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 15 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:46:25.777 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:46:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:25 smithi059 bash[29114]: cluster 2022-04-28T19:46:24.116287+0000 mgr.y (mgr.14140) 434 : cluster [DBG] pgmap v501: 89 pgs: 89 active+clean; 152 MiB data, 226 MiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2022-04-28T19:46:25.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:25 smithi059 bash[29114]: cluster 2022-04-28T19:46:24.364544+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2022-04-28T19:46:25.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:25 smithi059 bash[23786]: cluster 2022-04-28T19:46:24.116287+0000 mgr.y (mgr.14140) 434 : cluster [DBG] pgmap v501: 89 pgs: 89 active+clean; 152 MiB data, 226 MiB used, 707 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.8 MiB/s wr, 18 op/s 2022-04-28T19:46:25.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:25 smithi059 bash[23786]: cluster 2022-04-28T19:46:24.364544+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2022-04-28T19:46:26.383 INFO:tasks.rados.rados.0.smithi059.stdout:668: done (15 left) 2022-04-28T19:46:26.383 INFO:tasks.rados.rados.0.smithi059.stdout:670: done (14 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:671: done (13 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:672: done (12 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:673: done (11 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:676: done (10 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:677: done (9 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:678: done (8 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:679: done (7 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:680: done (6 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:681: done (5 left) 2022-04-28T19:46:26.384 INFO:tasks.rados.rados.0.smithi059.stdout:682: done (4 left) 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:683: done (3 left) 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:686: done (2 left) 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:687: done (1 left) 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:688: done (0 left) 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:689: rollback oid 13 current snap is 61 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 13 to 60 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 61 2022-04-28T19:46:26.385 INFO:tasks.rados.rados.0.smithi059.stdout:690: seq_num 237 ranges {1155072=778240,1933312=532480,2465792=270336} 2022-04-28T19:46:26.395 INFO:tasks.rados.rados.0.smithi059.stdout:690: writing smithi05955964-35 from 1155072 to 1933312 tid 1 2022-04-28T19:46:26.398 INFO:tasks.rados.rados.0.smithi059.stdout:690: writing smithi05955964-35 from 1933312 to 2465792 tid 2 2022-04-28T19:46:26.400 INFO:tasks.rados.rados.0.smithi059.stdout:690: writing smithi05955964-35 from 2465792 to 2736128 tid 3 2022-04-28T19:46:26.400 INFO:tasks.rados.rados.0.smithi059.stdout:691: read oid 1 snap -1 2022-04-28T19:46:26.400 INFO:tasks.rados.rados.0.smithi059.stdout:691: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:46:26.401 INFO:tasks.rados.rados.0.smithi059.stdout:692: copy_from oid 17 from oid 38 current snap is 61 2022-04-28T19:46:26.401 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 61 2022-04-28T19:46:26.401 INFO:tasks.rados.rados.0.smithi059.stdout:693: seq_num 238 ranges {589824=794624,1384448=417792,1802240=385024} 2022-04-28T19:46:26.408 INFO:tasks.rados.rados.0.smithi059.stdout:693: writing smithi05955964-2 from 589824 to 1384448 tid 1 2022-04-28T19:46:26.411 INFO:tasks.rados.rados.0.smithi059.stdout:693: writing smithi05955964-2 from 1384448 to 1802240 tid 2 2022-04-28T19:46:26.414 INFO:tasks.rados.rados.0.smithi059.stdout:693: writing smithi05955964-2 from 1802240 to 2187264 tid 3 2022-04-28T19:46:26.414 INFO:tasks.rados.rados.0.smithi059.stdout:694: setattr oid 36 current snap is 61 2022-04-28T19:46:26.414 INFO:tasks.rados.rados.0.smithi059.stdout:689: finishing rollback tid 0 to smithi05955964-13 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:689: finishing rollback tid 1 to smithi05955964-13 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:689: finishing rollback tid 2 to smithi05955964-13 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 171 (ObjNum 161 snap 42 seq_num 161) dirty exists 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:690: finishing write tid 1 to smithi05955964-35 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:690: finishing write tid 2 to smithi05955964-35 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:690: finishing write tid 3 to smithi05955964-35 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:690: finishing write tid 5 to smithi05955964-35 2022-04-28T19:46:26.415 INFO:tasks.rados.rados.0.smithi059.stdout:690: finishing write tid 6 to smithi05955964-35 2022-04-28T19:46:26.416 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 45 (ObjNum 237 snap 61 seq_num 237) dirty exists 2022-04-28T19:46:26.416 INFO:tasks.rados.rados.0.smithi059.stdout:690: left oid 35 (ObjNum 237 snap 61 seq_num 237) 2022-04-28T19:46:26.416 INFO:tasks.rados.rados.0.smithi059.stdout:691: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:693: finishing write tid 1 to smithi05955964-2 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:693: finishing write tid 2 to smithi05955964-2 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:693: finishing write tid 3 to smithi05955964-2 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:693: finishing write tid 5 to smithi05955964-2 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:693: finishing write tid 6 to smithi05955964-2 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 39 (ObjNum 238 snap 61 seq_num 238) dirty exists 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:693: left oid 2 (ObjNum 238 snap 61 seq_num 238) 2022-04-28T19:46:26.453 INFO:tasks.rados.rados.0.smithi059.stdout:692: finishing copy_from to smithi05955964-17 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:692: finishing copy_from racing read to smithi05955964-17 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:689: done (5 left) 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:690: done (4 left) 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:691: done (3 left) 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:692: done (2 left) 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:693: done (1 left) 2022-04-28T19:46:26.454 INFO:tasks.rados.rados.0.smithi059.stdout:695: setattr oid 33 current snap is 61 2022-04-28T19:46:26.455 INFO:tasks.rados.rados.0.smithi059.stdout:696: snap_create 2022-04-28T19:46:26.455 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 301 (ObjNum 228 snap 60 seq_num 228) dirty exists 2022-04-28T19:46:26.458 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 340 (ObjNum 217 snap 59 seq_num 217) dirty exists 2022-04-28T19:46:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:26 smithi059 bash[23786]: audit 2022-04-28T19:46:25.112126+0000 mgr.y (mgr.14140) 435 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:26 smithi059 bash[23786]: cluster 2022-04-28T19:46:25.376419+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2022-04-28T19:46:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:26 smithi059 bash[23786]: audit 2022-04-28T19:46:25.773988+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.59:0/3227498073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:46:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:26 smithi059 bash[29114]: audit 2022-04-28T19:46:25.112126+0000 mgr.y (mgr.14140) 435 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:26 smithi059 bash[29114]: cluster 2022-04-28T19:46:25.376419+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2022-04-28T19:46:26.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:26 smithi059 bash[29114]: audit 2022-04-28T19:46:25.773988+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.59:0/3227498073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:46:26.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:26 smithi139 bash[23439]: audit 2022-04-28T19:46:25.112126+0000 mgr.y (mgr.14140) 435 : audit [DBG] from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:26.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:26 smithi139 bash[23439]: cluster 2022-04-28T19:46:25.376419+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2022-04-28T19:46:26.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:26 smithi139 bash[23439]: audit 2022-04-28T19:46:25.773988+0000 mon.a (mon.0) 791 : audit [DBG] from='client.? 172.21.15.59:0/3227498073' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:46:27.188 INFO:tasks.rados.rados.0.smithi059.stdout:694: done (2 left) 2022-04-28T19:46:27.287 INFO:tasks.rados.rados.0.smithi059.stdout:695: done (1 left) 2022-04-28T19:46:27.287 INFO:tasks.rados.rados.0.smithi059.stdout:696: done (0 left) 2022-04-28T19:46:27.287 INFO:tasks.rados.rados.0.smithi059.stdout:697: rollback oid 32 current snap is 62 2022-04-28T19:46:27.288 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 49 2022-04-28T19:46:27.288 INFO:tasks.rados.rados.0.smithi059.stdout:698: snap_remove snap 59 2022-04-28T19:46:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[29114]: audit 2022-04-28T19:46:25.423362+0000 mgr.y (mgr.14140) 436 : audit [DBG] from='client.24799 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[29114]: cluster 2022-04-28T19:46:26.117022+0000 mgr.y (mgr.14140) 437 : cluster [DBG] pgmap v504: 89 pgs: 89 active+clean; 163 MiB data, 234 MiB used, 707 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.1 MiB/s wr, 36 op/s 2022-04-28T19:46:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[29114]: cluster 2022-04-28T19:46:26.381619+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2022-04-28T19:46:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[29114]: cluster 2022-04-28T19:46:27.186124+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2022-04-28T19:46:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[23786]: audit 2022-04-28T19:46:25.423362+0000 mgr.y (mgr.14140) 436 : audit [DBG] from='client.24799 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[23786]: cluster 2022-04-28T19:46:26.117022+0000 mgr.y (mgr.14140) 437 : cluster [DBG] pgmap v504: 89 pgs: 89 active+clean; 163 MiB data, 234 MiB used, 707 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.1 MiB/s wr, 36 op/s 2022-04-28T19:46:27.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[23786]: cluster 2022-04-28T19:46:26.381619+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2022-04-28T19:46:27.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:27 smithi059 bash[23786]: cluster 2022-04-28T19:46:27.186124+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2022-04-28T19:46:27.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:27 smithi139 bash[23439]: audit 2022-04-28T19:46:25.423362+0000 mgr.y (mgr.14140) 436 : audit [DBG] from='client.24799 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:27.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:27 smithi139 bash[23439]: cluster 2022-04-28T19:46:26.117022+0000 mgr.y (mgr.14140) 437 : cluster [DBG] pgmap v504: 89 pgs: 89 active+clean; 163 MiB data, 234 MiB used, 707 GiB / 715 GiB avail; 9.6 MiB/s rd, 3.1 MiB/s wr, 36 op/s 2022-04-28T19:46:27.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:27 smithi139 bash[23439]: cluster 2022-04-28T19:46:26.381619+0000 mon.a (mon.0) 792 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2022-04-28T19:46:27.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:27 smithi139 bash[23439]: cluster 2022-04-28T19:46:27.186124+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2022-04-28T19:46:28.200 INFO:tasks.rados.rados.0.smithi059.stdout:698: done (1 left) 2022-04-28T19:46:28.259 INFO:tasks.rados.rados.0.smithi059.stdout:699: rollback oid 46 current snap is 62 2022-04-28T19:46:28.259 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 61 2022-04-28T19:46:28.259 INFO:tasks.rados.rados.0.smithi059.stdout:697: finishing rollback tid 0 to smithi05955964-32 2022-04-28T19:46:28.260 INFO:tasks.rados.rados.0.smithi059.stdout:697: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:46:28.260 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 343 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty dne 2022-04-28T19:46:28.260 INFO:tasks.rados.rados.0.smithi059.stdout:697: done (1 left) 2022-04-28T19:46:28.260 INFO:tasks.rados.rados.0.smithi059.stdout:700: snap_remove snap 17 2022-04-28T19:46:29.201 INFO:tasks.rados.rados.0.smithi059.stdout:700: done (1 left) 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:701: read oid 21 snap -1 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:701: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:702: read oid 27 snap 60 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:702: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:703: read oid 2 snap -1 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:703: expect (ObjNum 238 snap 61 seq_num 238) 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:704: delete oid 32 current snap is 62 2022-04-28T19:46:29.202 INFO:tasks.rados.rados.0.smithi059.stdout:699: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:46:29.203 INFO:tasks.rados.rados.0.smithi059.stdout:699: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:46:29.203 INFO:tasks.rados.rados.0.smithi059.stdout:699: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:46:29.203 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 269 (ObjNum 195 snap 57 seq_num 195) dirty exists 2022-04-28T19:46:29.217 INFO:tasks.rados.rados.0.smithi059.stdout:699: done (4 left) 2022-04-28T19:46:29.217 INFO:tasks.rados.rados.0.smithi059.stdout:704: done (3 left) 2022-04-28T19:46:29.217 INFO:tasks.rados.rados.0.smithi059.stdout:705: delete oid 36 current snap is 62 2022-04-28T19:46:29.233 INFO:tasks.rados.rados.0.smithi059.stdout:703: expect (ObjNum 238 snap 61 seq_num 238) 2022-04-28T19:46:29.309 INFO:tasks.rados.rados.0.smithi059.stdout:703: done (3 left) 2022-04-28T19:46:29.310 INFO:tasks.rados.rados.0.smithi059.stdout:705: done (2 left) 2022-04-28T19:46:29.310 INFO:tasks.rados.rados.0.smithi059.stdout:706: snap_create 2022-04-28T19:46:29.310 INFO:tasks.rados.rados.0.smithi059.stdout:702: expect (ObjNum 129 snap 30 seq_num 129) 2022-04-28T19:46:29.411 INFO:tasks.rados.rados.0.smithi059.stdout:701: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:46:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:29 smithi059 bash[29114]: cluster 2022-04-28T19:46:28.117711+0000 mgr.y (mgr.14140) 438 : cluster [DBG] pgmap v507: 89 pgs: 89 active+clean; 171 MiB data, 247 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 5.3 MiB/s wr, 48 op/s 2022-04-28T19:46:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:29 smithi059 bash[29114]: cluster 2022-04-28T19:46:28.195817+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2022-04-28T19:46:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:29 smithi059 bash[23786]: cluster 2022-04-28T19:46:28.117711+0000 mgr.y (mgr.14140) 438 : cluster [DBG] pgmap v507: 89 pgs: 89 active+clean; 171 MiB data, 247 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 5.3 MiB/s wr, 48 op/s 2022-04-28T19:46:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:29 smithi059 bash[23786]: cluster 2022-04-28T19:46:28.195817+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2022-04-28T19:46:29.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:29 smithi139 bash[23439]: cluster 2022-04-28T19:46:28.117711+0000 mgr.y (mgr.14140) 438 : cluster [DBG] pgmap v507: 89 pgs: 89 active+clean; 171 MiB data, 247 MiB used, 707 GiB / 715 GiB avail; 7.9 MiB/s rd, 5.3 MiB/s wr, 48 op/s 2022-04-28T19:46:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:29 smithi139 bash[23439]: cluster 2022-04-28T19:46:28.195817+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2022-04-28T19:46:30.217 INFO:tasks.rados.rados.0.smithi059.stdout:701: done (2 left) 2022-04-28T19:46:30.218 INFO:tasks.rados.rados.0.smithi059.stdout:702: done (1 left) 2022-04-28T19:46:30.218 INFO:tasks.rados.rados.0.smithi059.stdout:706: done (0 left) 2022-04-28T19:46:30.218 INFO:tasks.rados.rados.0.smithi059.stdout:707: delete oid 35 current snap is 63 2022-04-28T19:46:30.228 INFO:tasks.rados.rados.0.smithi059.stdout:707: done (0 left) 2022-04-28T19:46:30.228 INFO:tasks.rados.rados.0.smithi059.stdout:708: snap_create 2022-04-28T19:46:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:30 smithi059 bash[29114]: cluster 2022-04-28T19:46:29.197852+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2022-04-28T19:46:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:30 smithi059 bash[23786]: cluster 2022-04-28T19:46:29.197852+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2022-04-28T19:46:30.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:30 smithi139 bash[23439]: cluster 2022-04-28T19:46:29.197852+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2022-04-28T19:46:31.232 INFO:tasks.rados.rados.0.smithi059.stdout:708: done (0 left) 2022-04-28T19:46:31.233 INFO:tasks.rados.rados.0.smithi059.stdout:709: snap_create 2022-04-28T19:46:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:31 smithi059 bash[29114]: cluster 2022-04-28T19:46:30.118322+0000 mgr.y (mgr.14140) 439 : cluster [DBG] pgmap v510: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 162 MiB data, 246 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.9 MiB/s wr, 23 op/s 2022-04-28T19:46:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:31 smithi059 bash[29114]: cluster 2022-04-28T19:46:30.213510+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2022-04-28T19:46:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:31 smithi059 bash[23786]: cluster 2022-04-28T19:46:30.118322+0000 mgr.y (mgr.14140) 439 : cluster [DBG] pgmap v510: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 162 MiB data, 246 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.9 MiB/s wr, 23 op/s 2022-04-28T19:46:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:31 smithi059 bash[23786]: cluster 2022-04-28T19:46:30.213510+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2022-04-28T19:46:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:31 smithi139 bash[23439]: cluster 2022-04-28T19:46:30.118322+0000 mgr.y (mgr.14140) 439 : cluster [DBG] pgmap v510: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 162 MiB data, 246 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.9 MiB/s wr, 23 op/s 2022-04-28T19:46:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:31 smithi139 bash[23439]: cluster 2022-04-28T19:46:30.213510+0000 mon.a (mon.0) 796 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2022-04-28T19:46:32.197 INFO:tasks.rados.rados.0.smithi059.stdout:709: done (0 left) 2022-04-28T19:46:32.198 INFO:tasks.rados.rados.0.smithi059.stdout:710: read oid 22 snap -1 2022-04-28T19:46:32.198 INFO:tasks.rados.rados.0.smithi059.stdout:710: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:32.198 INFO:tasks.rados.rados.0.smithi059.stdout:711: snap_remove snap 57 2022-04-28T19:46:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:32 smithi059 bash[29114]: cluster 2022-04-28T19:46:31.228008+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2022-04-28T19:46:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:32 smithi059 bash[29114]: cluster 2022-04-28T19:46:32.190243+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2022-04-28T19:46:32.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:32 smithi059 bash[23786]: cluster 2022-04-28T19:46:31.228008+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2022-04-28T19:46:32.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:32 smithi059 bash[23786]: cluster 2022-04-28T19:46:32.190243+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2022-04-28T19:46:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:32 smithi139 bash[23439]: cluster 2022-04-28T19:46:31.228008+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2022-04-28T19:46:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:32 smithi139 bash[23439]: cluster 2022-04-28T19:46:32.190243+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2022-04-28T19:46:33.208 INFO:tasks.rados.rados.0.smithi059.stdout:711: done (1 left) 2022-04-28T19:46:33.208 INFO:tasks.rados.rados.0.smithi059.stdout:712: read oid 48 snap -1 2022-04-28T19:46:33.208 INFO:tasks.rados.rados.0.smithi059.stdout:712: expect (ObjNum 159 snap 42 seq_num 159) 2022-04-28T19:46:33.209 INFO:tasks.rados.rados.0.smithi059.stdout:713: rmattr oid 49 current snap is 65 2022-04-28T19:46:33.209 INFO:tasks.rados.rados.0.smithi059.stdout:714: delete oid 44 current snap is 65 2022-04-28T19:46:33.209 INFO:tasks.rados.rados.0.smithi059.stdout:710: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:33.280 INFO:tasks.rados.rados.0.smithi059.stdout:710: done (3 left) 2022-04-28T19:46:33.281 INFO:tasks.rados.rados.0.smithi059.stdout:714: done (2 left) 2022-04-28T19:46:33.281 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 65 2022-04-28T19:46:33.281 INFO:tasks.rados.rados.0.smithi059.stdout:715: seq_num 243 ranges {2809856=475136,3284992=638976,3923968=565248} 2022-04-28T19:46:33.304 INFO:tasks.rados.rados.0.smithi059.stdout:715: writing smithi05955964-27 from 2809856 to 3284992 tid 1 2022-04-28T19:46:33.309 INFO:tasks.rados.rados.0.smithi059.stdout:715: writing smithi05955964-27 from 3284992 to 3923968 tid 2 2022-04-28T19:46:33.313 INFO:tasks.rados.rados.0.smithi059.stdout:715: writing smithi05955964-27 from 3923968 to 4489216 tid 3 2022-04-28T19:46:33.313 INFO:tasks.rados.rados.0.smithi059.stdout:716: read oid 45 snap -1 2022-04-28T19:46:33.313 INFO:tasks.rados.rados.0.smithi059.stdout:716: expect (ObjNum 211 snap 59 seq_num 211) 2022-04-28T19:46:33.314 INFO:tasks.rados.rados.0.smithi059.stdout:717: read oid 43 snap 64 2022-04-28T19:46:33.314 INFO:tasks.rados.rados.0.smithi059.stdout:717: expect deleted 2022-04-28T19:46:33.314 INFO:tasks.rados.rados.0.smithi059.stdout:718: setattr oid 47 current snap is 65 2022-04-28T19:46:33.314 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 313 (ObjNum 48 snap 0 seq_num 48) dirty exists 2022-04-28T19:46:33.314 INFO:tasks.rados.rados.0.smithi059.stdout:712: expect (ObjNum 159 snap 42 seq_num 159) 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:715: finishing write tid 1 to smithi05955964-27 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:715: finishing write tid 2 to smithi05955964-27 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:715: finishing write tid 3 to smithi05955964-27 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:715: finishing write tid 5 to smithi05955964-27 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:715: finishing write tid 6 to smithi05955964-27 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 257 (ObjNum 243 snap 65 seq_num 243) dirty exists 2022-04-28T19:46:33.413 INFO:tasks.rados.rados.0.smithi059.stdout:715: left oid 27 (ObjNum 243 snap 65 seq_num 243) 2022-04-28T19:46:33.414 INFO:tasks.rados.rados.0.smithi059.stdout:716: expect (ObjNum 211 snap 59 seq_num 211) 2022-04-28T19:46:33.500 INFO:tasks.rados.rados.0.smithi059.stdout:712: done (5 left) 2022-04-28T19:46:33.500 INFO:tasks.rados.rados.0.smithi059.stdout:713: done (4 left) 2022-04-28T19:46:33.500 INFO:tasks.rados.rados.0.smithi059.stdout:715: done (3 left) 2022-04-28T19:46:33.501 INFO:tasks.rados.rados.0.smithi059.stdout:716: done (2 left) 2022-04-28T19:46:33.501 INFO:tasks.rados.rados.0.smithi059.stdout:717: done (1 left) 2022-04-28T19:46:33.501 INFO:tasks.rados.rados.0.smithi059.stdout:719: snap_remove snap 49 2022-04-28T19:46:33.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[29114]: cluster 2022-04-28T19:46:32.119003+0000 mgr.y (mgr.14140) 440 : cluster [DBG] pgmap v513: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 155 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 382 KiB/s wr, 14 op/s 2022-04-28T19:46:33.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[29114]: cluster 2022-04-28T19:46:33.203389+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2022-04-28T19:46:33.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[23786]: cluster 2022-04-28T19:46:32.119003+0000 mgr.y (mgr.14140) 440 : cluster [DBG] pgmap v513: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 155 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 382 KiB/s wr, 14 op/s 2022-04-28T19:46:33.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[23786]: cluster 2022-04-28T19:46:33.203389+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2022-04-28T19:46:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:33 smithi139 bash[23439]: cluster 2022-04-28T19:46:32.119003+0000 mgr.y (mgr.14140) 440 : cluster [DBG] pgmap v513: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 155 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 382 KiB/s wr, 14 op/s 2022-04-28T19:46:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:33 smithi139 bash[23439]: cluster 2022-04-28T19:46:33.203389+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2022-04-28T19:46:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:33.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:46:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:33.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:46:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:33.518Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:46:34.245 INFO:tasks.rados.rados.0.smithi059.stdout:719: done (1 left) 2022-04-28T19:46:34.246 INFO:tasks.rados.rados.0.smithi059.stdout:720: delete oid 10 current snap is 65 2022-04-28T19:46:34.246 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 259 (ObjNum 171 snap 45 seq_num 171) dirty exists 2022-04-28T19:46:34.255 INFO:tasks.rados.rados.0.smithi059.stdout:718: done (1 left) 2022-04-28T19:46:34.255 INFO:tasks.rados.rados.0.smithi059.stdout:720: done (0 left) 2022-04-28T19:46:34.255 INFO:tasks.rados.rados.0.smithi059.stdout:721: snap_create 2022-04-28T19:46:35.255 INFO:tasks.rados.rados.0.smithi059.stdout:721: done (0 left) 2022-04-28T19:46:35.255 INFO:tasks.rados.rados.0.smithi059.stdout:722: setattr oid 43 current snap is 66 2022-04-28T19:46:35.256 INFO:tasks.rados.rados.0.smithi059.stdout:723: snap_remove snap 64 2022-04-28T19:46:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:35 smithi059 bash[29114]: cluster 2022-04-28T19:46:34.119630+0000 mgr.y (mgr.14140) 441 : cluster [DBG] pgmap v516: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 150 MiB data, 234 MiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 0 B/s wr, 8 op/s 2022-04-28T19:46:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:35 smithi059 bash[29114]: cluster 2022-04-28T19:46:34.244108+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2022-04-28T19:46:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:35 smithi059 bash[23786]: cluster 2022-04-28T19:46:34.119630+0000 mgr.y (mgr.14140) 441 : cluster [DBG] pgmap v516: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 150 MiB data, 234 MiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 0 B/s wr, 8 op/s 2022-04-28T19:46:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:35 smithi059 bash[23786]: cluster 2022-04-28T19:46:34.244108+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2022-04-28T19:46:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:35 smithi139 bash[23439]: cluster 2022-04-28T19:46:34.119630+0000 mgr.y (mgr.14140) 441 : cluster [DBG] pgmap v516: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 150 MiB data, 234 MiB used, 707 GiB / 715 GiB avail; 3.4 MiB/s rd, 0 B/s wr, 8 op/s 2022-04-28T19:46:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:35 smithi139 bash[23439]: cluster 2022-04-28T19:46:34.244108+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2022-04-28T19:46:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:723: done (1 left) 2022-04-28T19:46:36.343 INFO:tasks.rados.rados.0.smithi059.stdout:724: read oid 21 snap -1 2022-04-28T19:46:36.344 INFO:tasks.rados.rados.0.smithi059.stdout:724: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:46:36.344 INFO:tasks.rados.rados.0.smithi059.stdout:725: snap_remove snap 62 2022-04-28T19:46:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:36 smithi059 bash[23786]: cluster 2022-04-28T19:46:35.252544+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2022-04-28T19:46:36.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:36 smithi059 bash[29114]: cluster 2022-04-28T19:46:35.252544+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2022-04-28T19:46:36.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:36 smithi139 bash[23439]: cluster 2022-04-28T19:46:35.252544+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2022-04-28T19:46:37.195 INFO:tasks.rados.rados.0.smithi059.stdout:725: done (2 left) 2022-04-28T19:46:37.195 INFO:tasks.rados.rados.0.smithi059.stdout:726: copy_from oid 46 from oid 26 current snap is 66 2022-04-28T19:46:37.195 INFO:tasks.rados.rados.0.smithi059.stdout:727: copy_from oid 44 from oid 42 current snap is 66 2022-04-28T19:46:37.195 INFO:tasks.rados.rados.0.smithi059.stdout:728: setattr oid 22 current snap is 66 2022-04-28T19:46:37.196 INFO:tasks.rados.rados.0.smithi059.stdout:729: rollback oid 32 current snap is 66 2022-04-28T19:46:37.196 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 65 2022-04-28T19:46:37.196 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 66 2022-04-28T19:46:37.196 INFO:tasks.rados.rados.0.smithi059.stdout:730: seq_num 245 ranges {0=507904,507904=458752,966656=671744,1638400=73728} 2022-04-28T19:46:37.200 INFO:tasks.rados.rados.0.smithi059.stdout:730: writing smithi05955964-10 from 0 to 507904 tid 1 2022-04-28T19:46:37.204 INFO:tasks.rados.rados.0.smithi059.stdout:730: writing smithi05955964-10 from 507904 to 966656 tid 2 2022-04-28T19:46:37.209 INFO:tasks.rados.rados.0.smithi059.stdout:730: writing smithi05955964-10 from 966656 to 1638400 tid 3 2022-04-28T19:46:37.210 INFO:tasks.rados.rados.0.smithi059.stdout:730: writing smithi05955964-10 from 1638400 to 1712128 tid 4 2022-04-28T19:46:37.210 INFO:tasks.rados.rados.0.smithi059.stdout:731: rollback oid 12 current snap is 66 2022-04-28T19:46:37.210 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 12 to 61 2022-04-28T19:46:37.211 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 369 (ObjNum 1951952736 snap 1431655514 seq_num 1819240298) dirty exists 2022-04-28T19:46:37.211 INFO:tasks.rados.rados.0.smithi059.stdout:724: expect (ObjNum 130 snap 31 seq_num 130) 2022-04-28T19:46:37.280 INFO:tasks.rados.rados.0.smithi059.stdout:729: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:46:37.281 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 343 (ObjNum -133503920 snap 21893 seq_num 4161463632) dirty dne 2022-04-28T19:46:37.281 INFO:tasks.rados.rados.0.smithi059.stdout:722: done (7 left) 2022-04-28T19:46:37.282 INFO:tasks.rados.rados.0.smithi059.stdout:724: done (6 left) 2022-04-28T19:46:37.282 INFO:tasks.rados.rados.0.smithi059.stdout:729: done (5 left) 2022-04-28T19:46:37.282 INFO:tasks.rados.rados.0.smithi059.stdout:732: read oid 39 snap -1 2022-04-28T19:46:37.283 INFO:tasks.rados.rados.0.smithi059.stdout:732: expect (ObjNum 206 snap 59 seq_num 206) 2022-04-28T19:46:37.283 INFO:tasks.rados.rados.0.smithi059.stdout:733: snap_create 2022-04-28T19:46:37.283 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 280 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:46:37.284 INFO:tasks.rados.rados.0.smithi059.stdout:730: finishing write tid 1 to smithi05955964-10 2022-04-28T19:46:37.284 INFO:tasks.rados.rados.0.smithi059.stdout:730: finishing write tid 2 to smithi05955964-10 2022-04-28T19:46:37.284 INFO:tasks.rados.rados.0.smithi059.stdout:730: finishing write tid 3 to smithi05955964-10 2022-04-28T19:46:37.285 INFO:tasks.rados.rados.0.smithi059.stdout:731: finishing rollback tid 0 to smithi05955964-12 2022-04-28T19:46:37.285 INFO:tasks.rados.rados.0.smithi059.stdout:730: finishing write tid 4 to smithi05955964-10 2022-04-28T19:46:37.286 INFO:tasks.rados.rados.0.smithi059.stdout:731: finishing rollback tid 1 to smithi05955964-12 2022-04-28T19:46:37.286 INFO:tasks.rados.rados.0.smithi059.stdout:730: finishing write tid 6 to smithi05955964-10 2022-04-28T19:46:37.287 INFO:tasks.rados.rados.0.smithi059.stdout:727: finishing copy_from to smithi05955964-44 2022-04-28T19:46:37.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 249 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:46:37.287 INFO:tasks.rados.rados.0.smithi059.stdout:731: finishing rollback tid 2 to smithi05955964-12 2022-04-28T19:46:37.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 387 (ObjNum 176 snap 49 seq_num 176) dirty exists 2022-04-28T19:46:37.288 INFO:tasks.rados.rados.0.smithi059.stdout:727: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:46:37.288 INFO:tasks.rados.rados.0.smithi059.stdout:730: finishing write tid 7 to smithi05955964-10 2022-04-28T19:46:37.288 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 279 (ObjNum 245 snap 66 seq_num 245) dirty exists 2022-04-28T19:46:37.288 INFO:tasks.rados.rados.0.smithi059.stdout:730: left oid 10 (ObjNum 245 snap 66 seq_num 245) 2022-04-28T19:46:37.288 INFO:tasks.rados.rados.0.smithi059.stdout:726: finishing copy_from to smithi05955964-46 2022-04-28T19:46:37.289 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 117 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:46:37.289 INFO:tasks.rados.rados.0.smithi059.stdout:726: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:46:37.297 INFO:tasks.rados.rados.0.smithi059.stdout:732: expect (ObjNum 206 snap 59 seq_num 206) 2022-04-28T19:46:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:37 smithi059 bash[29114]: cluster 2022-04-28T19:46:36.120216+0000 mgr.y (mgr.14140) 442 : cluster [DBG] pgmap v519: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 129 MiB data, 215 MiB used, 707 GiB / 715 GiB avail; 8.3 MiB/s rd, 410 KiB/s wr, 13 op/s 2022-04-28T19:46:37.626 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:37 smithi059 bash[29114]: cluster 2022-04-28T19:46:36.260782+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2022-04-28T19:46:37.626 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:37 smithi059 bash[29114]: cluster 2022-04-28T19:46:37.193127+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2022-04-28T19:46:37.627 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:37 smithi059 bash[23786]: cluster 2022-04-28T19:46:36.120216+0000 mgr.y (mgr.14140) 442 : cluster [DBG] pgmap v519: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 129 MiB data, 215 MiB used, 707 GiB / 715 GiB avail; 8.3 MiB/s rd, 410 KiB/s wr, 13 op/s 2022-04-28T19:46:37.627 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:37 smithi059 bash[23786]: cluster 2022-04-28T19:46:36.260782+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2022-04-28T19:46:37.627 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:37 smithi059 bash[23786]: cluster 2022-04-28T19:46:37.193127+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2022-04-28T19:46:37.628 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:37 smithi139 bash[23439]: cluster 2022-04-28T19:46:36.120216+0000 mgr.y (mgr.14140) 442 : cluster [DBG] pgmap v519: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 129 MiB data, 215 MiB used, 707 GiB / 715 GiB avail; 8.3 MiB/s rd, 410 KiB/s wr, 13 op/s 2022-04-28T19:46:37.628 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:37 smithi139 bash[23439]: cluster 2022-04-28T19:46:36.260782+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2022-04-28T19:46:37.628 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:37 smithi139 bash[23439]: cluster 2022-04-28T19:46:37.193127+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2022-04-28T19:46:38.278 INFO:tasks.rados.rados.0.smithi059.stdout:726: done (6 left) 2022-04-28T19:46:38.279 INFO:tasks.rados.rados.0.smithi059.stdout:727: done (5 left) 2022-04-28T19:46:38.279 INFO:tasks.rados.rados.0.smithi059.stdout:728: done (4 left) 2022-04-28T19:46:38.279 INFO:tasks.rados.rados.0.smithi059.stdout:730: done (3 left) 2022-04-28T19:46:38.279 INFO:tasks.rados.rados.0.smithi059.stdout:731: done (2 left) 2022-04-28T19:46:38.279 INFO:tasks.rados.rados.0.smithi059.stdout:732: done (1 left) 2022-04-28T19:46:38.279 INFO:tasks.rados.rados.0.smithi059.stdout:733: done (0 left) 2022-04-28T19:46:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:734: read oid 2 snap 61 2022-04-28T19:46:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:734: expect (ObjNum 238 snap 61 seq_num 238) 2022-04-28T19:46:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:735: read oid 33 snap -1 2022-04-28T19:46:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:735: expect (ObjNum 217 snap 59 seq_num 217) 2022-04-28T19:46:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 67 2022-04-28T19:46:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:736: seq_num 246 ranges {0=524288,524288=745472,1269760=425984,1695744=32768} 2022-04-28T19:46:38.281 INFO:tasks.rados.rados.0.smithi059.stdout:736: writing smithi05955964-25 from 0 to 524288 tid 1 2022-04-28T19:46:38.286 INFO:tasks.rados.rados.0.smithi059.stdout:736: writing smithi05955964-25 from 524288 to 1269760 tid 2 2022-04-28T19:46:38.289 INFO:tasks.rados.rados.0.smithi059.stdout:736: writing smithi05955964-25 from 1269760 to 1695744 tid 3 2022-04-28T19:46:38.289 INFO:tasks.rados.rados.0.smithi059.stdout:736: writing smithi05955964-25 from 1695744 to 1728512 tid 4 2022-04-28T19:46:38.289 INFO:tasks.rados.rados.0.smithi059.stdout:737: snap_create 2022-04-28T19:46:38.290 INFO:tasks.rados.rados.0.smithi059.stdout:734: expect (ObjNum 238 snap 61 seq_num 238) 2022-04-28T19:46:38.350 INFO:tasks.rados.rados.0.smithi059.stdout:736: finishing write tid 1 to smithi05955964-25 2022-04-28T19:46:38.351 INFO:tasks.rados.rados.0.smithi059.stdout:735: expect (ObjNum 217 snap 59 seq_num 217) 2022-04-28T19:46:38.382 INFO:tasks.rados.rados.0.smithi059.stdout:736: finishing write tid 2 to smithi05955964-25 2022-04-28T19:46:38.382 INFO:tasks.rados.rados.0.smithi059.stdout:736: finishing write tid 3 to smithi05955964-25 2022-04-28T19:46:38.383 INFO:tasks.rados.rados.0.smithi059.stdout:736: finishing write tid 4 to smithi05955964-25 2022-04-28T19:46:38.383 INFO:tasks.rados.rados.0.smithi059.stdout:736: finishing write tid 6 to smithi05955964-25 2022-04-28T19:46:38.383 INFO:tasks.rados.rados.0.smithi059.stdout:736: finishing write tid 7 to smithi05955964-25 2022-04-28T19:46:38.383 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 296 (ObjNum 246 snap 67 seq_num 246) dirty exists 2022-04-28T19:46:38.384 INFO:tasks.rados.rados.0.smithi059.stdout:736: left oid 25 (ObjNum 246 snap 67 seq_num 246) 2022-04-28T19:46:39.280 INFO:tasks.rados.rados.0.smithi059.stdout:734: done (3 left) 2022-04-28T19:46:39.280 INFO:tasks.rados.rados.0.smithi059.stdout:735: done (2 left) 2022-04-28T19:46:39.280 INFO:tasks.rados.rados.0.smithi059.stdout:736: done (1 left) 2022-04-28T19:46:39.280 INFO:tasks.rados.rados.0.smithi059.stdout:737: done (0 left) 2022-04-28T19:46:39.280 INFO:tasks.rados.rados.0.smithi059.stdout:738: delete oid 9 current snap is 68 2022-04-28T19:46:39.285 INFO:tasks.rados.rados.0.smithi059.stdout:738: done (0 left) 2022-04-28T19:46:39.285 INFO:tasks.rados.rados.0.smithi059.stdout:739: snap_create 2022-04-28T19:46:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:39 smithi059 bash[29114]: cluster 2022-04-28T19:46:38.121002+0000 mgr.y (mgr.14140) 443 : cluster [DBG] pgmap v522: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 112 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 410 KiB/s wr, 17 op/s 2022-04-28T19:46:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:39 smithi059 bash[29114]: cluster 2022-04-28T19:46:38.276284+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2022-04-28T19:46:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:39 smithi059 bash[23786]: cluster 2022-04-28T19:46:38.121002+0000 mgr.y (mgr.14140) 443 : cluster [DBG] pgmap v522: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 112 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 410 KiB/s wr, 17 op/s 2022-04-28T19:46:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:39 smithi059 bash[23786]: cluster 2022-04-28T19:46:38.276284+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2022-04-28T19:46:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:39 smithi139 bash[23439]: cluster 2022-04-28T19:46:38.121002+0000 mgr.y (mgr.14140) 443 : cluster [DBG] pgmap v522: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 112 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 410 KiB/s wr, 17 op/s 2022-04-28T19:46:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:39 smithi139 bash[23439]: cluster 2022-04-28T19:46:38.276284+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2022-04-28T19:46:40.297 INFO:tasks.rados.rados.0.smithi059.stdout:739: done (0 left) 2022-04-28T19:46:40.298 INFO:tasks.rados.rados.0.smithi059.stdout:740: rmattr oid 13 current snap is 69 2022-04-28T19:46:40.298 INFO:tasks.rados.rados.0.smithi059.stdout:741: snap_create 2022-04-28T19:46:40.301 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 172 (ObjNum 161 snap 42 seq_num 161) dirty exists 2022-04-28T19:46:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:40 smithi059 bash[29114]: cluster 2022-04-28T19:46:39.278428+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2022-04-28T19:46:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:40 smithi059 bash[23786]: cluster 2022-04-28T19:46:39.278428+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2022-04-28T19:46:40.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:40 smithi139 bash[23439]: cluster 2022-04-28T19:46:39.278428+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2022-04-28T19:46:41.301 INFO:tasks.rados.rados.0.smithi059.stdout:740: done (1 left) 2022-04-28T19:46:41.301 INFO:tasks.rados.rados.0.smithi059.stdout:741: done (0 left) 2022-04-28T19:46:41.301 INFO:tasks.rados.rados.0.smithi059.stdout:742: copy_from oid 43 from oid 39 current snap is 70 2022-04-28T19:46:41.302 INFO:tasks.rados.rados.0.smithi059.stdout:743: delete oid 49 current snap is 70 2022-04-28T19:46:41.307 INFO:tasks.rados.rados.0.smithi059.stdout:743: done (1 left) 2022-04-28T19:46:41.307 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 70 2022-04-28T19:46:41.307 INFO:tasks.rados.rados.0.smithi059.stdout:744: seq_num 249 ranges {4038656=573440,4612096=794624,5406720=409600} 2022-04-28T19:46:41.333 INFO:tasks.rados.rados.0.smithi059.stdout:744: writing smithi05955964-23 from 4038656 to 4612096 tid 1 2022-04-28T19:46:41.338 INFO:tasks.rados.rados.0.smithi059.stdout:744: writing smithi05955964-23 from 4612096 to 5406720 tid 2 2022-04-28T19:46:41.341 INFO:tasks.rados.rados.0.smithi059.stdout:744: writing smithi05955964-23 from 5406720 to 5816320 tid 3 2022-04-28T19:46:41.341 INFO:tasks.rados.rados.0.smithi059.stdout:745: rollback oid 3 current snap is 70 2022-04-28T19:46:41.341 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 61 2022-04-28T19:46:41.341 INFO:tasks.rados.rados.0.smithi059.stdout:742: finishing copy_from to smithi05955964-43 2022-04-28T19:46:41.342 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 315 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:46:41.342 INFO:tasks.rados.rados.0.smithi059.stdout:742: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:46:41.342 INFO:tasks.rados.rados.0.smithi059.stdout:742: done (2 left) 2022-04-28T19:46:41.342 INFO:tasks.rados.rados.0.smithi059.stdout:746: snap_create 2022-04-28T19:46:41.342 INFO:tasks.rados.rados.0.smithi059.stdout:744: finishing write tid 1 to smithi05955964-23 2022-04-28T19:46:41.345 INFO:tasks.rados.rados.0.smithi059.stdout:745: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:46:41.347 INFO:tasks.rados.rados.0.smithi059.stdout:744: finishing write tid 2 to smithi05955964-23 2022-04-28T19:46:41.348 INFO:tasks.rados.rados.0.smithi059.stdout:744: finishing write tid 3 to smithi05955964-23 2022-04-28T19:46:41.348 INFO:tasks.rados.rados.0.smithi059.stdout:744: finishing write tid 5 to smithi05955964-23 2022-04-28T19:46:41.348 INFO:tasks.rados.rados.0.smithi059.stdout:745: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:46:41.348 INFO:tasks.rados.rados.0.smithi059.stdout:745: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:46:41.348 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 291 (ObjNum 185 snap 53 seq_num 185) dirty exists 2022-04-28T19:46:41.349 INFO:tasks.rados.rados.0.smithi059.stdout:744: finishing write tid 6 to smithi05955964-23 2022-04-28T19:46:41.349 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 284 (ObjNum 249 snap 70 seq_num 249) dirty exists 2022-04-28T19:46:41.350 INFO:tasks.rados.rados.0.smithi059.stdout:744: left oid 23 (ObjNum 249 snap 70 seq_num 249) 2022-04-28T19:46:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:41 smithi059 bash[29114]: cluster 2022-04-28T19:46:40.121763+0000 mgr.y (mgr.14140) 444 : cluster [DBG] pgmap v525: 89 pgs: 89 active+clean; 108 MiB data, 179 MiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 694 KiB/s wr, 8 op/s 2022-04-28T19:46:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:41 smithi059 bash[29114]: cluster 2022-04-28T19:46:40.294037+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2022-04-28T19:46:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:41 smithi059 bash[23786]: cluster 2022-04-28T19:46:40.121763+0000 mgr.y (mgr.14140) 444 : cluster [DBG] pgmap v525: 89 pgs: 89 active+clean; 108 MiB data, 179 MiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 694 KiB/s wr, 8 op/s 2022-04-28T19:46:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:41 smithi059 bash[23786]: cluster 2022-04-28T19:46:40.294037+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2022-04-28T19:46:41.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:41 smithi139 bash[23439]: cluster 2022-04-28T19:46:40.121763+0000 mgr.y (mgr.14140) 444 : cluster [DBG] pgmap v525: 89 pgs: 89 active+clean; 108 MiB data, 179 MiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 694 KiB/s wr, 8 op/s 2022-04-28T19:46:41.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:41 smithi139 bash[23439]: cluster 2022-04-28T19:46:40.294037+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2022-04-28T19:46:42.199 INFO:tasks.rados.rados.0.smithi059.stdout:744: done (2 left) 2022-04-28T19:46:42.199 INFO:tasks.rados.rados.0.smithi059.stdout:745: done (1 left) 2022-04-28T19:46:42.200 INFO:tasks.rados.rados.0.smithi059.stdout:746: done (0 left) 2022-04-28T19:46:42.200 INFO:tasks.rados.rados.0.smithi059.stdout:747: read oid 42 snap -1 2022-04-28T19:46:42.200 INFO:tasks.rados.rados.0.smithi059.stdout:747: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:42.200 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 71 2022-04-28T19:46:42.200 INFO:tasks.rados.rados.0.smithi059.stdout:748: seq_num 250 ranges {0=589824,589824=679936,1269760=409600,1679360=114688} 2022-04-28T19:46:42.202 INFO:tasks.rados.rados.0.smithi059.stdout:748: writing smithi05955964-29 from 0 to 589824 tid 1 2022-04-28T19:46:42.207 INFO:tasks.rados.rados.0.smithi059.stdout:748: writing smithi05955964-29 from 589824 to 1269760 tid 2 2022-04-28T19:46:42.209 INFO:tasks.rados.rados.0.smithi059.stdout:748: writing smithi05955964-29 from 1269760 to 1679360 tid 3 2022-04-28T19:46:42.210 INFO:tasks.rados.rados.0.smithi059.stdout:748: writing smithi05955964-29 from 1679360 to 1794048 tid 4 2022-04-28T19:46:42.210 INFO:tasks.rados.rados.0.smithi059.stdout:749: delete oid 3 current snap is 71 2022-04-28T19:46:42.210 INFO:tasks.rados.rados.0.smithi059.stdout:747: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:42.245 INFO:tasks.rados.rados.0.smithi059.stdout:748: finishing write tid 1 to smithi05955964-29 2022-04-28T19:46:42.245 INFO:tasks.rados.rados.0.smithi059.stdout:748: finishing write tid 2 to smithi05955964-29 2022-04-28T19:46:42.245 INFO:tasks.rados.rados.0.smithi059.stdout:748: finishing write tid 3 to smithi05955964-29 2022-04-28T19:46:42.245 INFO:tasks.rados.rados.0.smithi059.stdout:748: finishing write tid 4 to smithi05955964-29 2022-04-28T19:46:42.245 INFO:tasks.rados.rados.0.smithi059.stdout:748: finishing write tid 6 to smithi05955964-29 2022-04-28T19:46:42.245 INFO:tasks.rados.rados.0.smithi059.stdout:748: finishing write tid 7 to smithi05955964-29 2022-04-28T19:46:42.246 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 302 (ObjNum 250 snap 71 seq_num 250) dirty exists 2022-04-28T19:46:42.246 INFO:tasks.rados.rados.0.smithi059.stdout:748: left oid 29 (ObjNum 250 snap 71 seq_num 250) 2022-04-28T19:46:42.246 INFO:tasks.rados.rados.0.smithi059.stdout:747: done (2 left) 2022-04-28T19:46:42.246 INFO:tasks.rados.rados.0.smithi059.stdout:748: done (1 left) 2022-04-28T19:46:42.246 INFO:tasks.rados.rados.0.smithi059.stdout:749: done (0 left) 2022-04-28T19:46:42.246 INFO:tasks.rados.rados.0.smithi059.stdout:750: snap_create 2022-04-28T19:46:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:42 smithi059 bash[23786]: cluster 2022-04-28T19:46:41.298349+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2022-04-28T19:46:42.676 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:42 smithi059 bash[23786]: cluster 2022-04-28T19:46:42.193309+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2022-04-28T19:46:42.677 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:42 smithi059 bash[29114]: cluster 2022-04-28T19:46:41.298349+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2022-04-28T19:46:42.677 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:42 smithi059 bash[29114]: cluster 2022-04-28T19:46:42.193309+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2022-04-28T19:46:42.678 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:42 smithi139 bash[23439]: cluster 2022-04-28T19:46:41.298349+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2022-04-28T19:46:42.678 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:42 smithi139 bash[23439]: cluster 2022-04-28T19:46:42.193309+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2022-04-28T19:46:43.210 INFO:tasks.rados.rados.0.smithi059.stdout:750: done (0 left) 2022-04-28T19:46:43.210 INFO:tasks.rados.rados.0.smithi059.stdout:751: snap_create 2022-04-28T19:46:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[29114]: cluster 2022-04-28T19:46:42.122505+0000 mgr.y (mgr.14140) 445 : cluster [DBG] pgmap v528: 89 pgs: 89 active+clean; 112 MiB data, 181 MiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2022-04-28T19:46:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[29114]: cluster 2022-04-28T19:46:43.207953+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2022-04-28T19:46:43.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:43.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:46:43.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:43.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:46:43.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:43.518Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:46:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[23786]: cluster 2022-04-28T19:46:42.122505+0000 mgr.y (mgr.14140) 445 : cluster [DBG] pgmap v528: 89 pgs: 89 active+clean; 112 MiB data, 181 MiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2022-04-28T19:46:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:43 smithi059 bash[23786]: cluster 2022-04-28T19:46:43.207953+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2022-04-28T19:46:43.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:43 smithi139 bash[23439]: cluster 2022-04-28T19:46:42.122505+0000 mgr.y (mgr.14140) 445 : cluster [DBG] pgmap v528: 89 pgs: 89 active+clean; 112 MiB data, 181 MiB used, 707 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2022-04-28T19:46:43.687 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:43 smithi139 bash[23439]: cluster 2022-04-28T19:46:43.207953+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2022-04-28T19:46:44.213 INFO:tasks.rados.rados.0.smithi059.stdout:751: done (0 left) 2022-04-28T19:46:44.213 INFO:tasks.rados.rados.0.smithi059.stdout:752: copy_from oid 48 from oid 1 current snap is 73 2022-04-28T19:46:44.214 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 73 2022-04-28T19:46:44.214 INFO:tasks.rados.rados.0.smithi059.stdout:753: seq_num 252 ranges {3039232=630784,3670016=499712,4169728=425984,4595712=278528} 2022-04-28T19:46:44.236 INFO:tasks.rados.rados.0.smithi059.stdout:753: writing smithi05955964-41 from 3039232 to 3670016 tid 1 2022-04-28T19:46:44.239 INFO:tasks.rados.rados.0.smithi059.stdout:753: writing smithi05955964-41 from 3670016 to 4169728 tid 2 2022-04-28T19:46:44.242 INFO:tasks.rados.rados.0.smithi059.stdout:753: writing smithi05955964-41 from 4169728 to 4595712 tid 3 2022-04-28T19:46:44.244 INFO:tasks.rados.rados.0.smithi059.stdout:753: writing smithi05955964-41 from 4595712 to 4874240 tid 4 2022-04-28T19:46:44.244 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 73 2022-04-28T19:46:44.244 INFO:tasks.rados.rados.0.smithi059.stdout:754: seq_num 253 ranges {1728512=647168,2375680=786432,3162112=417792} 2022-04-28T19:46:44.258 INFO:tasks.rados.rados.0.smithi059.stdout:754: writing smithi05955964-25 from 1728512 to 2375680 tid 1 2022-04-28T19:46:44.263 INFO:tasks.rados.rados.0.smithi059.stdout:754: writing smithi05955964-25 from 2375680 to 3162112 tid 2 2022-04-28T19:46:44.266 INFO:tasks.rados.rados.0.smithi059.stdout:754: writing smithi05955964-25 from 3162112 to 3579904 tid 3 2022-04-28T19:46:44.267 INFO:tasks.rados.rados.0.smithi059.stdout:755: rmattr oid 32 current snap is 73 2022-04-28T19:46:44.267 INFO:tasks.rados.rados.0.smithi059.stdout:755: done (3 left) 2022-04-28T19:46:44.267 INFO:tasks.rados.rados.0.smithi059.stdout:756: read oid 36 snap -1 2022-04-28T19:46:44.267 INFO:tasks.rados.rados.0.smithi059.stdout:756: expect deleted 2022-04-28T19:46:44.267 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 73 2022-04-28T19:46:44.267 INFO:tasks.rados.rados.0.smithi059.stdout:757: seq_num 254 ranges {3383296=663552,4046848=663552,4710400=540672} 2022-04-28T19:46:44.288 INFO:tasks.rados.rados.0.smithi059.stdout:757: writing smithi05955964-28 from 3383296 to 4046848 tid 1 2022-04-28T19:46:44.292 INFO:tasks.rados.rados.0.smithi059.stdout:757: writing smithi05955964-28 from 4046848 to 4710400 tid 2 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:757: writing smithi05955964-28 from 4710400 to 5251072 tid 3 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:758: copy_from oid 49 from oid 44 current snap is 73 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:752: finishing copy_from to smithi05955964-48 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 195 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:752: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:753: finishing write tid 1 to smithi05955964-41 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:753: finishing write tid 2 to smithi05955964-41 2022-04-28T19:46:44.297 INFO:tasks.rados.rados.0.smithi059.stdout:753: finishing write tid 3 to smithi05955964-41 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:753: finishing write tid 4 to smithi05955964-41 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:753: finishing write tid 6 to smithi05955964-41 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:753: finishing write tid 7 to smithi05955964-41 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 84 (ObjNum 252 snap 73 seq_num 252) dirty exists 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:753: left oid 41 (ObjNum 252 snap 73 seq_num 252) 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:754: finishing write tid 1 to smithi05955964-25 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:752: done (5 left) 2022-04-28T19:46:44.298 INFO:tasks.rados.rados.0.smithi059.stdout:753: done (4 left) 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:759: snap_create 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:754: finishing write tid 2 to smithi05955964-25 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:754: finishing write tid 3 to smithi05955964-25 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:754: finishing write tid 5 to smithi05955964-25 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:754: finishing write tid 6 to smithi05955964-25 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 307 (ObjNum 253 snap 73 seq_num 253) dirty exists 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:754: left oid 25 (ObjNum 253 snap 73 seq_num 253) 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:757: finishing write tid 1 to smithi05955964-28 2022-04-28T19:46:44.299 INFO:tasks.rados.rados.0.smithi059.stdout:757: finishing write tid 2 to smithi05955964-28 2022-04-28T19:46:44.300 INFO:tasks.rados.rados.0.smithi059.stdout:757: finishing write tid 3 to smithi05955964-28 2022-04-28T19:46:44.300 INFO:tasks.rados.rados.0.smithi059.stdout:757: finishing write tid 5 to smithi05955964-28 2022-04-28T19:46:44.303 INFO:tasks.rados.rados.0.smithi059.stdout:757: finishing write tid 6 to smithi05955964-28 2022-04-28T19:46:44.303 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 201 (ObjNum 254 snap 73 seq_num 254) dirty exists 2022-04-28T19:46:44.303 INFO:tasks.rados.rados.0.smithi059.stdout:757: left oid 28 (ObjNum 254 snap 73 seq_num 254) 2022-04-28T19:46:44.304 INFO:tasks.rados.rados.0.smithi059.stdout:758: finishing copy_from to smithi05955964-49 2022-04-28T19:46:44.304 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 249 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:46:44.304 INFO:tasks.rados.rados.0.smithi059.stdout:758: finishing copy_from racing read to smithi05955964-49 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:754: done (4 left) 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:756: done (3 left) 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:757: done (2 left) 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:758: done (1 left) 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:759: done (0 left) 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:760: rollback oid 46 current snap is 74 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 69 2022-04-28T19:46:45.222 INFO:tasks.rados.rados.0.smithi059.stdout:761: copy_from oid 35 from oid 11 current snap is 74 2022-04-28T19:46:45.223 INFO:tasks.rados.rados.0.smithi059.stdout:762: read oid 30 snap -1 2022-04-28T19:46:45.223 INFO:tasks.rados.rados.0.smithi059.stdout:762: expect (ObjNum 233 snap 60 seq_num 233) 2022-04-28T19:46:45.223 INFO:tasks.rados.rados.0.smithi059.stdout:763: copy_from oid 6 from oid 38 current snap is 74 2022-04-28T19:46:45.223 INFO:tasks.rados.rados.0.smithi059.stdout:764: delete oid 33 current snap is 74 2022-04-28T19:46:45.231 INFO:tasks.rados.rados.0.smithi059.stdout:760: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:760: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:764: done (4 left) 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:765: read oid 32 snap -1 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:765: expect deleted 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:766: read oid 23 snap 73 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:766: expect (ObjNum 249 snap 70 seq_num 249) 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 74 2022-04-28T19:46:45.233 INFO:tasks.rados.rados.0.smithi059.stdout:767: seq_num 256 ranges {2220032=696320,2916352=434176,3350528=770048} 2022-04-28T19:46:45.251 INFO:tasks.rados.rados.0.smithi059.stdout:767: writing smithi05955964-26 from 2220032 to 2916352 tid 1 2022-04-28T19:46:45.254 INFO:tasks.rados.rados.0.smithi059.stdout:767: writing smithi05955964-26 from 2916352 to 3350528 tid 2 2022-04-28T19:46:45.259 INFO:tasks.rados.rados.0.smithi059.stdout:767: writing smithi05955964-26 from 3350528 to 4120576 tid 3 2022-04-28T19:46:45.259 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 74 2022-04-28T19:46:45.259 INFO:tasks.rados.rados.0.smithi059.stdout:768: seq_num 257 ranges {851968=712704,1564672=720896,2285568=483328} 2022-04-28T19:46:45.268 INFO:tasks.rados.rados.0.smithi059.stdout:768: writing smithi05955964-47 from 851968 to 1564672 tid 1 2022-04-28T19:46:45.272 INFO:tasks.rados.rados.0.smithi059.stdout:768: writing smithi05955964-47 from 1564672 to 2285568 tid 2 2022-04-28T19:46:45.276 INFO:tasks.rados.rados.0.smithi059.stdout:768: writing smithi05955964-47 from 2285568 to 2768896 tid 3 2022-04-28T19:46:45.276 INFO:tasks.rados.rados.0.smithi059.stdout:769: read oid 24 snap -1 2022-04-28T19:46:45.276 INFO:tasks.rados.rados.0.smithi059.stdout:769: expect (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:46:45.276 INFO:tasks.rados.rados.0.smithi059.stdout:770: delete oid 37 current snap is 74 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:760: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 311 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:761: finishing copy_from to smithi05955964-35 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 131 (ObjNum 182 snap 51 seq_num 182) dirty exists 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:761: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:763: finishing copy_from to smithi05955964-6 2022-04-28T19:46:45.277 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:46:45.278 INFO:tasks.rados.rados.0.smithi059.stdout:763: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:46:45.278 INFO:tasks.rados.rados.0.smithi059.stdout:767: finishing write tid 1 to smithi05955964-26 2022-04-28T19:46:45.278 INFO:tasks.rados.rados.0.smithi059.stdout:767: finishing write tid 2 to smithi05955964-26 2022-04-28T19:46:45.278 INFO:tasks.rados.rados.0.smithi059.stdout:766: expect (ObjNum 249 snap 70 seq_num 249) 2022-04-28T19:46:45.567 INFO:tasks.rados.rados.0.smithi059.stdout:767: finishing write tid 3 to smithi05955964-26 2022-04-28T19:46:45.567 INFO:tasks.rados.rados.0.smithi059.stdout:767: finishing write tid 5 to smithi05955964-26 2022-04-28T19:46:45.567 INFO:tasks.rados.rados.0.smithi059.stdout:767: finishing write tid 6 to smithi05955964-26 2022-04-28T19:46:45.567 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 287 (ObjNum 256 snap 74 seq_num 256) dirty exists 2022-04-28T19:46:45.567 INFO:tasks.rados.rados.0.smithi059.stdout:767: left oid 26 (ObjNum 256 snap 74 seq_num 256) 2022-04-28T19:46:45.567 INFO:tasks.rados.rados.0.smithi059.stdout:760: done (9 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:761: done (8 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:763: done (7 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:765: done (6 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:766: done (5 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:767: done (4 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:770: done (3 left) 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:771: rmattr oid 39 current snap is 74 2022-04-28T19:46:45.568 INFO:tasks.rados.rados.0.smithi059.stdout:762: expect (ObjNum 233 snap 60 seq_num 233) 2022-04-28T19:46:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:45 smithi059 bash[23786]: cluster 2022-04-28T19:46:44.123211+0000 mgr.y (mgr.14140) 446 : cluster [DBG] pgmap v531: 89 pgs: 89 active+clean; 112 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 4.6 MiB/s rd, 964 KiB/s wr, 11 op/s 2022-04-28T19:46:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:45 smithi059 bash[23786]: cluster 2022-04-28T19:46:44.211630+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2022-04-28T19:46:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:45 smithi059 bash[29114]: cluster 2022-04-28T19:46:44.123211+0000 mgr.y (mgr.14140) 446 : cluster [DBG] pgmap v531: 89 pgs: 89 active+clean; 112 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 4.6 MiB/s rd, 964 KiB/s wr, 11 op/s 2022-04-28T19:46:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:45 smithi059 bash[29114]: cluster 2022-04-28T19:46:44.211630+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2022-04-28T19:46:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:45 smithi139 bash[23439]: cluster 2022-04-28T19:46:44.123211+0000 mgr.y (mgr.14140) 446 : cluster [DBG] pgmap v531: 89 pgs: 89 active+clean; 112 MiB data, 185 MiB used, 707 GiB / 715 GiB avail; 4.6 MiB/s rd, 964 KiB/s wr, 11 op/s 2022-04-28T19:46:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:45 smithi139 bash[23439]: cluster 2022-04-28T19:46:44.211630+0000 mon.a (mon.0) 810 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2022-04-28T19:46:45.774 INFO:tasks.rados.rados.0.smithi059.stdout:768: finishing write tid 1 to smithi05955964-47 2022-04-28T19:46:45.774 INFO:tasks.rados.rados.0.smithi059.stdout:768: finishing write tid 2 to smithi05955964-47 2022-04-28T19:46:45.774 INFO:tasks.rados.rados.0.smithi059.stdout:768: finishing write tid 3 to smithi05955964-47 2022-04-28T19:46:45.774 INFO:tasks.rados.rados.0.smithi059.stdout:768: finishing write tid 5 to smithi05955964-47 2022-04-28T19:46:45.774 INFO:tasks.rados.rados.0.smithi059.stdout:768: finishing write tid 6 to smithi05955964-47 2022-04-28T19:46:45.775 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 289 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:46:45.775 INFO:tasks.rados.rados.0.smithi059.stdout:768: left oid 47 (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:46:45.775 INFO:tasks.rados.rados.0.smithi059.stdout:762: done (3 left) 2022-04-28T19:46:45.775 INFO:tasks.rados.rados.0.smithi059.stdout:768: done (2 left) 2022-04-28T19:46:45.775 INFO:tasks.rados.rados.0.smithi059.stdout:772: copy_from oid 34 from oid 8 current snap is 74 2022-04-28T19:46:45.775 INFO:tasks.rados.rados.0.smithi059.stdout:769: expect (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:46:45.978 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 376 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:46:45.978 INFO:tasks.rados.rados.0.smithi059.stdout:772: finishing copy_from to smithi05955964-34 2022-04-28T19:46:45.978 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 179 (ObjNum 207 snap 59 seq_num 207) dirty exists 2022-04-28T19:46:45.978 INFO:tasks.rados.rados.0.smithi059.stdout:772: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:46:45.978 INFO:tasks.rados.rados.0.smithi059.stdout:769: done (2 left) 2022-04-28T19:46:45.978 INFO:tasks.rados.rados.0.smithi059.stdout:771: done (1 left) 2022-04-28T19:46:45.979 INFO:tasks.rados.rados.0.smithi059.stdout:772: done (0 left) 2022-04-28T19:46:45.979 INFO:tasks.rados.rados.0.smithi059.stdout:773: read oid 48 snap -1 2022-04-28T19:46:45.979 INFO:tasks.rados.rados.0.smithi059.stdout:773: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:46:45.979 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 74 2022-04-28T19:46:45.979 INFO:tasks.rados.rados.0.smithi059.stdout:774: seq_num 259 ranges {4972544=745472,5718016=540672,6258688=581632,6840320=81920} 2022-04-28T19:46:46.017 INFO:tasks.rados.rados.0.smithi059.stdout:774: writing smithi05955964-24 from 4972544 to 5718016 tid 1 2022-04-28T19:46:46.020 INFO:tasks.rados.rados.0.smithi059.stdout:774: writing smithi05955964-24 from 5718016 to 6258688 tid 2 2022-04-28T19:46:46.024 INFO:tasks.rados.rados.0.smithi059.stdout:774: writing smithi05955964-24 from 6258688 to 6840320 tid 3 2022-04-28T19:46:46.024 INFO:tasks.rados.rados.0.smithi059.stdout:774: writing smithi05955964-24 from 6840320 to 6922240 tid 4 2022-04-28T19:46:46.025 INFO:tasks.rados.rados.0.smithi059.stdout:775: setattr oid 31 current snap is 74 2022-04-28T19:46:46.025 INFO:tasks.rados.rados.0.smithi059.stdout:773: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:46:46.059 INFO:tasks.rados.rados.0.smithi059.stdout:774: finishing write tid 1 to smithi05955964-24 2022-04-28T19:46:46.060 INFO:tasks.rados.rados.0.smithi059.stdout:774: finishing write tid 2 to smithi05955964-24 2022-04-28T19:46:46.060 INFO:tasks.rados.rados.0.smithi059.stdout:774: finishing write tid 3 to smithi05955964-24 2022-04-28T19:46:46.060 INFO:tasks.rados.rados.0.smithi059.stdout:774: finishing write tid 4 to smithi05955964-24 2022-04-28T19:46:46.060 INFO:tasks.rados.rados.0.smithi059.stdout:774: finishing write tid 6 to smithi05955964-24 2022-04-28T19:46:46.061 INFO:tasks.rados.rados.0.smithi059.stdout:774: finishing write tid 7 to smithi05955964-24 2022-04-28T19:46:46.061 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 249 (ObjNum 259 snap 74 seq_num 259) dirty exists 2022-04-28T19:46:46.061 INFO:tasks.rados.rados.0.smithi059.stdout:774: left oid 24 (ObjNum 259 snap 74 seq_num 259) 2022-04-28T19:46:46.061 INFO:tasks.rados.rados.0.smithi059.stdout:773: done (2 left) 2022-04-28T19:46:46.061 INFO:tasks.rados.rados.0.smithi059.stdout:774: done (1 left) 2022-04-28T19:46:46.061 INFO:tasks.rados.rados.0.smithi059.stdout:776: snap_create 2022-04-28T19:46:46.063 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 389 (ObjNum 87 snap 17 seq_num 87) dirty exists 2022-04-28T19:46:46.509 INFO:tasks.rados.rados.0.smithi059.stdout:775: done (1 left) 2022-04-28T19:46:46.509 INFO:tasks.rados.rados.0.smithi059.stdout:776: done (0 left) 2022-04-28T19:46:46.509 INFO:tasks.rados.rados.0.smithi059.stdout:777: snap_create 2022-04-28T19:46:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:46 smithi059 bash[29114]: cluster 2022-04-28T19:46:45.217907+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2022-04-28T19:46:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:46 smithi059 bash[29114]: audit 2022-04-28T19:46:45.496024+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:46:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:46 smithi059 bash[29114]: audit 2022-04-28T19:46:45.496898+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:46:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:46 smithi059 bash[23786]: cluster 2022-04-28T19:46:45.217907+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2022-04-28T19:46:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:46 smithi059 bash[23786]: audit 2022-04-28T19:46:45.496024+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:46:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:46 smithi059 bash[23786]: audit 2022-04-28T19:46:45.496898+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:46:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:46 smithi139 bash[23439]: cluster 2022-04-28T19:46:45.217907+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2022-04-28T19:46:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:46 smithi139 bash[23439]: audit 2022-04-28T19:46:45.496024+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:46:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:46 smithi139 bash[23439]: audit 2022-04-28T19:46:45.496898+0000 mon.a (mon.0) 813 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:46:47.198 INFO:tasks.rados.rados.0.smithi059.stdout:777: done (0 left) 2022-04-28T19:46:47.198 INFO:tasks.rados.rados.0.smithi059.stdout:778: copy_from oid 22 from oid 47 current snap is 76 2022-04-28T19:46:47.198 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 76 2022-04-28T19:46:47.199 INFO:tasks.rados.rados.0.smithi059.stdout:779: seq_num 260 ranges {4489216=761856,5251072=417792,5668864=786432} 2022-04-28T19:46:47.230 INFO:tasks.rados.rados.0.smithi059.stdout:779: writing smithi05955964-27 from 4489216 to 5251072 tid 1 2022-04-28T19:46:47.232 INFO:tasks.rados.rados.0.smithi059.stdout:779: writing smithi05955964-27 from 5251072 to 5668864 tid 2 2022-04-28T19:46:47.237 INFO:tasks.rados.rados.0.smithi059.stdout:779: writing smithi05955964-27 from 5668864 to 6455296 tid 3 2022-04-28T19:46:47.238 INFO:tasks.rados.rados.0.smithi059.stdout:780: snap_create 2022-04-28T19:46:47.238 INFO:tasks.rados.rados.0.smithi059.stdout:778: finishing copy_from to smithi05955964-22 2022-04-28T19:46:47.238 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 289 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:46:47.238 INFO:tasks.rados.rados.0.smithi059.stdout:778: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:46:47.238 INFO:tasks.rados.rados.0.smithi059.stdout:779: finishing write tid 1 to smithi05955964-27 2022-04-28T19:46:47.238 INFO:tasks.rados.rados.0.smithi059.stdout:779: finishing write tid 2 to smithi05955964-27 2022-04-28T19:46:47.242 INFO:tasks.rados.rados.0.smithi059.stdout:779: finishing write tid 3 to smithi05955964-27 2022-04-28T19:46:47.242 INFO:tasks.rados.rados.0.smithi059.stdout:779: finishing write tid 5 to smithi05955964-27 2022-04-28T19:46:47.243 INFO:tasks.rados.rados.0.smithi059.stdout:779: finishing write tid 6 to smithi05955964-27 2022-04-28T19:46:47.243 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 294 (ObjNum 260 snap 76 seq_num 260) dirty exists 2022-04-28T19:46:47.243 INFO:tasks.rados.rados.0.smithi059.stdout:779: left oid 27 (ObjNum 260 snap 76 seq_num 260) 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cephadm 2022-04-28T19:46:45.496578+0000 mgr.y (mgr.14140) 447 : cephadm [INF] Upgrade: Target is quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 with id 863db77b5b5ebb9f6537dd0f2bea7edb8cdbedb7e03005109ebbf1f6c68c0b65 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cephadm 2022-04-28T19:46:45.498177+0000 mgr.y (mgr.14140) 448 : cephadm [INF] Upgrade: Checking mgr daemons... 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cephadm 2022-04-28T19:46:45.498886+0000 mgr.y (mgr.14140) 449 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cephadm 2022-04-28T19:46:45.851656+0000 mgr.y (mgr.14140) 450 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 on smithi139 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cluster 2022-04-28T19:46:46.124055+0000 mgr.y (mgr.14140) 451 : cluster [DBG] pgmap v534: 89 pgs: 89 active+clean; 118 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cluster 2022-04-28T19:46:46.505749+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2022-04-28T19:46:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[29114]: cluster 2022-04-28T19:46:47.195532+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cephadm 2022-04-28T19:46:45.496578+0000 mgr.y (mgr.14140) 447 : cephadm [INF] Upgrade: Target is quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 with id 863db77b5b5ebb9f6537dd0f2bea7edb8cdbedb7e03005109ebbf1f6c68c0b65 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cephadm 2022-04-28T19:46:45.498177+0000 mgr.y (mgr.14140) 448 : cephadm [INF] Upgrade: Checking mgr daemons... 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cephadm 2022-04-28T19:46:45.498886+0000 mgr.y (mgr.14140) 449 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cephadm 2022-04-28T19:46:45.851656+0000 mgr.y (mgr.14140) 450 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 on smithi139 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cluster 2022-04-28T19:46:46.124055+0000 mgr.y (mgr.14140) 451 : cluster [DBG] pgmap v534: 89 pgs: 89 active+clean; 118 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cluster 2022-04-28T19:46:46.505749+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2022-04-28T19:46:47.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:47 smithi059 bash[23786]: cluster 2022-04-28T19:46:47.195532+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cephadm 2022-04-28T19:46:45.496578+0000 mgr.y (mgr.14140) 447 : cephadm [INF] Upgrade: Target is quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 with id 863db77b5b5ebb9f6537dd0f2bea7edb8cdbedb7e03005109ebbf1f6c68c0b65 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cephadm 2022-04-28T19:46:45.498177+0000 mgr.y (mgr.14140) 448 : cephadm [INF] Upgrade: Checking mgr daemons... 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cephadm 2022-04-28T19:46:45.498886+0000 mgr.y (mgr.14140) 449 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cephadm 2022-04-28T19:46:45.851656+0000 mgr.y (mgr.14140) 450 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 on smithi139 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cluster 2022-04-28T19:46:46.124055+0000 mgr.y (mgr.14140) 451 : cluster [DBG] pgmap v534: 89 pgs: 89 active+clean; 118 MiB data, 192 MiB used, 707 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.5 MiB/s wr, 8 op/s 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cluster 2022-04-28T19:46:46.505749+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2022-04-28T19:46:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:47 smithi139 bash[23439]: cluster 2022-04-28T19:46:47.195532+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2022-04-28T19:46:48.234 INFO:tasks.rados.rados.0.smithi059.stdout:778: done (2 left) 2022-04-28T19:46:48.234 INFO:tasks.rados.rados.0.smithi059.stdout:779: done (1 left) 2022-04-28T19:46:48.235 INFO:tasks.rados.rados.0.smithi059.stdout:780: done (0 left) 2022-04-28T19:46:48.235 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 77 2022-04-28T19:46:48.235 INFO:tasks.rados.rados.0.smithi059.stdout:781: seq_num 261 ranges {1482752=778240,2260992=704512,2965504=499712} 2022-04-28T19:46:48.249 INFO:tasks.rados.rados.0.smithi059.stdout:781: writing smithi05955964-35 from 1482752 to 2260992 tid 1 2022-04-28T19:46:48.253 INFO:tasks.rados.rados.0.smithi059.stdout:781: writing smithi05955964-35 from 2260992 to 2965504 tid 2 2022-04-28T19:46:48.256 INFO:tasks.rados.rados.0.smithi059.stdout:781: writing smithi05955964-35 from 2965504 to 3465216 tid 3 2022-04-28T19:46:48.256 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 77 2022-04-28T19:46:48.256 INFO:tasks.rados.rados.0.smithi059.stdout:782: seq_num 262 ranges {0=794624,794624=589824,1384448=614400} 2022-04-28T19:46:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:782: writing smithi05955964-40 from 0 to 794624 tid 1 2022-04-28T19:46:48.264 INFO:tasks.rados.rados.0.smithi059.stdout:782: writing smithi05955964-40 from 794624 to 1384448 tid 2 2022-04-28T19:46:48.268 INFO:tasks.rados.rados.0.smithi059.stdout:782: writing smithi05955964-40 from 1384448 to 1998848 tid 3 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:783: read oid 22 snap -1 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:783: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:784: delete oid 31 current snap is 77 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:781: finishing write tid 1 to smithi05955964-35 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:781: finishing write tid 2 to smithi05955964-35 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:781: finishing write tid 3 to smithi05955964-35 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:781: finishing write tid 5 to smithi05955964-35 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:781: finishing write tid 6 to smithi05955964-35 2022-04-28T19:46:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 135 (ObjNum 261 snap 77 seq_num 261) dirty exists 2022-04-28T19:46:48.270 INFO:tasks.rados.rados.0.smithi059.stdout:781: left oid 35 (ObjNum 261 snap 77 seq_num 261) 2022-04-28T19:46:48.270 INFO:tasks.rados.rados.0.smithi059.stdout:782: finishing write tid 1 to smithi05955964-40 2022-04-28T19:46:48.270 INFO:tasks.rados.rados.0.smithi059.stdout:782: finishing write tid 2 to smithi05955964-40 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:782: finishing write tid 3 to smithi05955964-40 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:781: done (3 left) 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:784: done (2 left) 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:785: read oid 8 snap -1 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:785: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:786: rollback oid 39 current snap is 77 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 39 to 70 2022-04-28T19:46:48.273 INFO:tasks.rados.rados.0.smithi059.stdout:782: finishing write tid 5 to smithi05955964-40 2022-04-28T19:46:48.274 INFO:tasks.rados.rados.0.smithi059.stdout:787: read oid 4 snap -1 2022-04-28T19:46:48.274 INFO:tasks.rados.rados.0.smithi059.stdout:787: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:46:48.274 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 77 2022-04-28T19:46:48.274 INFO:tasks.rados.rados.0.smithi059.stdout:788: seq_num 264 ranges {524288=425984,950272=761856,1712128=770048,2482176=73728} 2022-04-28T19:46:48.278 INFO:tasks.rados.rados.0.smithi059.stdout:788: writing smithi05955964-13 from 524288 to 950272 tid 1 2022-04-28T19:46:48.283 INFO:tasks.rados.rados.0.smithi059.stdout:788: writing smithi05955964-13 from 950272 to 1712128 tid 2 2022-04-28T19:46:48.288 INFO:tasks.rados.rados.0.smithi059.stdout:788: writing smithi05955964-13 from 1712128 to 2482176 tid 3 2022-04-28T19:46:48.289 INFO:tasks.rados.rados.0.smithi059.stdout:788: writing smithi05955964-13 from 2482176 to 2555904 tid 4 2022-04-28T19:46:48.289 INFO:tasks.rados.rados.0.smithi059.stdout:789: read oid 11 snap 63 2022-04-28T19:46:48.290 INFO:tasks.rados.rados.0.smithi059.stdout:789: expect (ObjNum 182 snap 51 seq_num 182) 2022-04-28T19:46:48.290 INFO:tasks.rados.rados.0.smithi059.stdout:790: rollback oid 35 current snap is 77 2022-04-28T19:46:48.290 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 35 to 60 2022-04-28T19:46:48.290 INFO:tasks.rados.rados.0.smithi059.stdout:787: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:46:48.295 INFO:tasks.rados.rados.0.smithi059.stdout:782: finishing write tid 6 to smithi05955964-40 2022-04-28T19:46:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 334 (ObjNum 262 snap 77 seq_num 262) dirty exists 2022-04-28T19:46:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:782: left oid 40 (ObjNum 262 snap 77 seq_num 262) 2022-04-28T19:46:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:786: finishing rollback tid 0 to smithi05955964-39 2022-04-28T19:46:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:786: finishing rollback tid 1 to smithi05955964-39 2022-04-28T19:46:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:786: finishing rollback tid 2 to smithi05955964-39 2022-04-28T19:46:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 380 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:46:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:788: finishing write tid 1 to smithi05955964-13 2022-04-28T19:46:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:783: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:46:48.379 INFO:tasks.rados.rados.0.smithi059.stdout:788: finishing write tid 2 to smithi05955964-13 2022-04-28T19:46:48.379 INFO:tasks.rados.rados.0.smithi059.stdout:790: finishing rollback tid 0 to smithi05955964-35 2022-04-28T19:46:48.379 INFO:tasks.rados.rados.0.smithi059.stdout:790: finishing rollback tid 1 to smithi05955964-35 2022-04-28T19:46:48.379 INFO:tasks.rados.rados.0.smithi059.stdout:788: finishing write tid 3 to smithi05955964-13 2022-04-28T19:46:48.380 INFO:tasks.rados.rados.0.smithi059.stdout:789: expect (ObjNum 182 snap 51 seq_num 182) 2022-04-28T19:46:48.427 INFO:tasks.rados.rados.0.smithi059.stdout:788: finishing write tid 4 to smithi05955964-13 2022-04-28T19:46:48.427 INFO:tasks.rados.rados.0.smithi059.stdout:790: finishing rollback tid 2 to smithi05955964-35 2022-04-28T19:46:48.427 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 138 (ObjNum 212 snap 59 seq_num 212) dirty exists 2022-04-28T19:46:48.427 INFO:tasks.rados.rados.0.smithi059.stdout:788: finishing write tid 6 to smithi05955964-13 2022-04-28T19:46:48.427 INFO:tasks.rados.rados.0.smithi059.stdout:785: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:46:48.607 INFO:tasks.rados.rados.0.smithi059.stdout:788: finishing write tid 7 to smithi05955964-13 2022-04-28T19:46:48.607 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 177 (ObjNum 264 snap 77 seq_num 264) dirty exists 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:788: left oid 13 (ObjNum 264 snap 77 seq_num 264) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:782: done (7 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:783: done (6 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:785: done (5 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:786: done (4 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:787: done (3 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:788: done (2 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:789: done (1 left) 2022-04-28T19:46:48.608 INFO:tasks.rados.rados.0.smithi059.stdout:790: done (0 left) 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:791: read oid 24 snap -1 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:791: expect (ObjNum 259 snap 74 seq_num 259) 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:792: rollback oid 21 current snap is 77 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 71 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:793: rollback oid 23 current snap is 77 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 72 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 77 2022-04-28T19:46:48.609 INFO:tasks.rados.rados.0.smithi059.stdout:794: seq_num 265 ranges {1155072=442368,1597440=647168,2244608=573440,2818048=385024} 2022-04-28T19:46:48.622 INFO:tasks.rados.rados.0.smithi059.stdout:794: writing smithi05955964-35 from 1155072 to 1597440 tid 1 2022-04-28T19:46:48.626 INFO:tasks.rados.rados.0.smithi059.stdout:794: writing smithi05955964-35 from 1597440 to 2244608 tid 2 2022-04-28T19:46:48.629 INFO:tasks.rados.rados.0.smithi059.stdout:794: writing smithi05955964-35 from 2244608 to 2818048 tid 3 2022-04-28T19:46:48.632 INFO:tasks.rados.rados.0.smithi059.stdout:794: writing smithi05955964-35 from 2818048 to 3203072 tid 4 2022-04-28T19:46:48.632 INFO:tasks.rados.rados.0.smithi059.stdout:795: copy_from oid 15 from oid 12 current snap is 77 2022-04-28T19:46:48.632 INFO:tasks.rados.rados.0.smithi059.stdout:793: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:46:48.632 INFO:tasks.rados.rados.0.smithi059.stdout:792: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:46:48.632 INFO:tasks.rados.rados.0.smithi059.stdout:796: snap_remove snap 60 2022-04-28T19:46:49.237 INFO:tasks.rados.rados.0.smithi059.stdout:796: done (5 left) 2022-04-28T19:46:49.238 INFO:tasks.rados.rados.0.smithi059.stdout:797: copy_from oid 26 from oid 40 current snap is 77 2022-04-28T19:46:49.238 INFO:tasks.rados.rados.0.smithi059.stdout:798: copy_from oid 9 from oid 19 current snap is 77 2022-04-28T19:46:49.238 INFO:tasks.rados.rados.0.smithi059.stdout:799: setattr oid 16 current snap is 77 2022-04-28T19:46:49.240 INFO:tasks.rados.rados.0.smithi059.stdout:800: setattr oid 38 current snap is 77 2022-04-28T19:46:49.241 INFO:tasks.rados.rados.0.smithi059.stdout:801: snap_create 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:792: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:793: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:793: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 298 (ObjNum 249 snap 70 seq_num 249) dirty exists 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:792: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 85 (ObjNum 130 snap 31 seq_num 130) dirty exists 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:794: finishing write tid 1 to smithi05955964-35 2022-04-28T19:46:49.244 INFO:tasks.rados.rados.0.smithi059.stdout:791: expect (ObjNum 259 snap 74 seq_num 259) 2022-04-28T19:46:49.583 INFO:tasks.rados.rados.0.smithi059.stdout:794: finishing write tid 2 to smithi05955964-35 2022-04-28T19:46:49.583 INFO:tasks.rados.rados.0.smithi059.stdout:794: finishing write tid 3 to smithi05955964-35 2022-04-28T19:46:49.583 INFO:tasks.rados.rados.0.smithi059.stdout:794: finishing write tid 4 to smithi05955964-35 2022-04-28T19:46:49.583 INFO:tasks.rados.rados.0.smithi059.stdout:794: finishing write tid 6 to smithi05955964-35 2022-04-28T19:46:49.583 INFO:tasks.rados.rados.0.smithi059.stdout:794: finishing write tid 7 to smithi05955964-35 2022-04-28T19:46:49.583 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 143 (ObjNum 265 snap 77 seq_num 265) dirty exists 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:794: left oid 35 (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:795: finishing copy_from to smithi05955964-15 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 387 (ObjNum 176 snap 49 seq_num 176) dirty exists 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:795: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 203 (ObjNum 204 snap 58 seq_num 204) dirty exists 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:797: finishing copy_from to smithi05955964-26 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 334 (ObjNum 262 snap 77 seq_num 262) dirty exists 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:797: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 251 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:46:49.584 INFO:tasks.rados.rados.0.smithi059.stdout:798: finishing copy_from to smithi05955964-9 2022-04-28T19:46:49.585 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 254 (ObjNum 227 snap 60 seq_num 227) dirty exists 2022-04-28T19:46:49.585 INFO:tasks.rados.rados.0.smithi059.stdout:798: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:46:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:49 smithi059 bash[29114]: cluster 2022-04-28T19:46:48.124780+0000 mgr.y (mgr.14140) 452 : cluster [DBG] pgmap v537: 89 pgs: 89 active+clean; 127 MiB data, 214 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 3.9 MiB/s wr, 15 op/s 2022-04-28T19:46:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:49 smithi059 bash[29114]: cluster 2022-04-28T19:46:48.229889+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2022-04-28T19:46:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:49 smithi059 bash[23786]: cluster 2022-04-28T19:46:48.124780+0000 mgr.y (mgr.14140) 452 : cluster [DBG] pgmap v537: 89 pgs: 89 active+clean; 127 MiB data, 214 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 3.9 MiB/s wr, 15 op/s 2022-04-28T19:46:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:49 smithi059 bash[23786]: cluster 2022-04-28T19:46:48.229889+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2022-04-28T19:46:49.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:49 smithi139 bash[23439]: cluster 2022-04-28T19:46:48.124780+0000 mgr.y (mgr.14140) 452 : cluster [DBG] pgmap v537: 89 pgs: 89 active+clean; 127 MiB data, 214 MiB used, 707 GiB / 715 GiB avail; 4.4 MiB/s rd, 3.9 MiB/s wr, 15 op/s 2022-04-28T19:46:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:49 smithi139 bash[23439]: cluster 2022-04-28T19:46:48.229889+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:791: done (9 left) 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:792: done (8 left) 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:793: done (7 left) 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:794: done (6 left) 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:795: done (5 left) 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:797: done (4 left) 2022-04-28T19:46:50.255 INFO:tasks.rados.rados.0.smithi059.stdout:798: done (3 left) 2022-04-28T19:46:50.256 INFO:tasks.rados.rados.0.smithi059.stdout:799: done (2 left) 2022-04-28T19:46:50.256 INFO:tasks.rados.rados.0.smithi059.stdout:800: done (1 left) 2022-04-28T19:46:50.256 INFO:tasks.rados.rados.0.smithi059.stdout:801: done (0 left) 2022-04-28T19:46:50.256 INFO:tasks.rados.rados.0.smithi059.stdout:802: rollback oid 27 current snap is 78 2022-04-28T19:46:50.256 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 63 2022-04-28T19:46:50.256 INFO:tasks.rados.rados.0.smithi059.stdout:803: snap_remove snap 76 2022-04-28T19:46:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:50 smithi059 bash[29114]: cluster 2022-04-28T19:46:49.234253+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2022-04-28T19:46:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:50 smithi059 bash[23786]: cluster 2022-04-28T19:46:49.234253+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2022-04-28T19:46:50.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:50 smithi139 bash[23439]: cluster 2022-04-28T19:46:49.234253+0000 mon.a (mon.0) 817 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2022-04-28T19:46:51.258 INFO:tasks.rados.rados.0.smithi059.stdout:803: done (1 left) 2022-04-28T19:46:51.258 INFO:tasks.rados.rados.0.smithi059.stdout:804: copy_from oid 11 from oid 41 current snap is 78 2022-04-28T19:46:51.258 INFO:tasks.rados.rados.0.smithi059.stdout:805: copy_from oid 24 from oid 48 current snap is 78 2022-04-28T19:46:51.259 INFO:tasks.rados.rados.0.smithi059.stdout:806: delete oid 43 current snap is 78 2022-04-28T19:46:51.259 INFO:tasks.rados.rados.0.smithi059.stdout:802: finishing rollback tid 0 to smithi05955964-27 2022-04-28T19:46:51.259 INFO:tasks.rados.rados.0.smithi059.stdout:802: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:46:51.259 INFO:tasks.rados.rados.0.smithi059.stdout:802: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:46:51.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 308 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:46:51.272 INFO:tasks.rados.rados.0.smithi059.stdout:802: done (3 left) 2022-04-28T19:46:51.272 INFO:tasks.rados.rados.0.smithi059.stdout:806: done (2 left) 2022-04-28T19:46:51.272 INFO:tasks.rados.rados.0.smithi059.stdout:807: snap_remove snap 67 2022-04-28T19:46:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:51 smithi059 bash[23786]: cluster 2022-04-28T19:46:50.125617+0000 mgr.y (mgr.14140) 453 : cluster [DBG] pgmap v540: 89 pgs: 89 active+clean; 146 MiB data, 228 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 MiB/s wr, 23 op/s 2022-04-28T19:46:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:51 smithi059 bash[23786]: cluster 2022-04-28T19:46:50.250766+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2022-04-28T19:46:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:51 smithi059 bash[29114]: cluster 2022-04-28T19:46:50.125617+0000 mgr.y (mgr.14140) 453 : cluster [DBG] pgmap v540: 89 pgs: 89 active+clean; 146 MiB data, 228 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 MiB/s wr, 23 op/s 2022-04-28T19:46:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:51 smithi059 bash[29114]: cluster 2022-04-28T19:46:50.250766+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2022-04-28T19:46:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:51 smithi139 bash[23439]: cluster 2022-04-28T19:46:50.125617+0000 mgr.y (mgr.14140) 453 : cluster [DBG] pgmap v540: 89 pgs: 89 active+clean; 146 MiB data, 228 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 7.2 MiB/s wr, 23 op/s 2022-04-28T19:46:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:51 smithi139 bash[23439]: cluster 2022-04-28T19:46:50.250766+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2022-04-28T19:46:52.279 INFO:tasks.rados.rados.0.smithi059.stdout:807: done (2 left) 2022-04-28T19:46:52.279 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 78 2022-04-28T19:46:52.279 INFO:tasks.rados.rados.0.smithi059.stdout:808: seq_num 267 ranges {3956736=475136,4431872=458752,4890624=540672,5431296=606208} 2022-04-28T19:46:52.302 INFO:tasks.rados.rados.0.smithi059.stdout:808: writing smithi05955964-12 from 3956736 to 4431872 tid 1 2022-04-28T19:46:52.304 INFO:tasks.rados.rados.0.smithi059.stdout:808: writing smithi05955964-12 from 4431872 to 4890624 tid 2 2022-04-28T19:46:52.307 INFO:tasks.rados.rados.0.smithi059.stdout:808: writing smithi05955964-12 from 4890624 to 5431296 tid 3 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:808: writing smithi05955964-12 from 5431296 to 6037504 tid 4 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:809: delete oid 21 current snap is 78 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:805: finishing copy_from to smithi05955964-24 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 195 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:805: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:804: finishing copy_from to smithi05955964-11 2022-04-28T19:46:52.311 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 84 (ObjNum 252 snap 73 seq_num 252) dirty exists 2022-04-28T19:46:52.312 INFO:tasks.rados.rados.0.smithi059.stdout:804: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:46:52.312 INFO:tasks.rados.rados.0.smithi059.stdout:808: finishing write tid 1 to smithi05955964-12 2022-04-28T19:46:52.312 INFO:tasks.rados.rados.0.smithi059.stdout:808: finishing write tid 2 to smithi05955964-12 2022-04-28T19:46:52.312 INFO:tasks.rados.rados.0.smithi059.stdout:808: finishing write tid 3 to smithi05955964-12 2022-04-28T19:46:52.315 INFO:tasks.rados.rados.0.smithi059.stdout:804: done (3 left) 2022-04-28T19:46:52.315 INFO:tasks.rados.rados.0.smithi059.stdout:805: done (2 left) 2022-04-28T19:46:52.315 INFO:tasks.rados.rados.0.smithi059.stdout:809: done (1 left) 2022-04-28T19:46:52.315 INFO:tasks.rados.rados.0.smithi059.stdout:810: snap_remove snap 75 2022-04-28T19:46:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:52 smithi059 bash[29114]: cluster 2022-04-28T19:46:51.253499+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2022-04-28T19:46:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:52 smithi059 bash[23786]: cluster 2022-04-28T19:46:51.253499+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2022-04-28T19:46:52.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:52 smithi139 bash[23439]: cluster 2022-04-28T19:46:51.253499+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2022-04-28T19:46:53.280 INFO:tasks.rados.rados.0.smithi059.stdout:810: done (1 left) 2022-04-28T19:46:53.280 INFO:tasks.rados.rados.0.smithi059.stdout:811: read oid 9 snap -1 2022-04-28T19:46:53.280 INFO:tasks.rados.rados.0.smithi059.stdout:811: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:46:53.280 INFO:tasks.rados.rados.0.smithi059.stdout:812: snap_remove snap 61 2022-04-28T19:46:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[29114]: cluster 2022-04-28T19:46:52.126399+0000 mgr.y (mgr.14140) 454 : cluster [DBG] pgmap v543: 89 pgs: 89 active+clean; 153 MiB data, 241 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 7.8 MiB/s wr, 25 op/s 2022-04-28T19:46:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[29114]: cluster 2022-04-28T19:46:52.273639+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2022-04-28T19:46:53.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:53.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:46:53.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:53.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:46:53.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:46:53.518Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:46:53.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[23786]: cluster 2022-04-28T19:46:52.126399+0000 mgr.y (mgr.14140) 454 : cluster [DBG] pgmap v543: 89 pgs: 89 active+clean; 153 MiB data, 241 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 7.8 MiB/s wr, 25 op/s 2022-04-28T19:46:53.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:53 smithi059 bash[23786]: cluster 2022-04-28T19:46:52.273639+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2022-04-28T19:46:53.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:53 smithi139 bash[23439]: cluster 2022-04-28T19:46:52.126399+0000 mgr.y (mgr.14140) 454 : cluster [DBG] pgmap v543: 89 pgs: 89 active+clean; 153 MiB data, 241 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 7.8 MiB/s wr, 25 op/s 2022-04-28T19:46:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:53 smithi139 bash[23439]: cluster 2022-04-28T19:46:52.273639+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2022-04-28T19:46:54.305 INFO:tasks.rados.rados.0.smithi059.stdout:812: done (2 left) 2022-04-28T19:46:54.305 INFO:tasks.rados.rados.0.smithi059.stdout:813: read oid 47 snap -1 2022-04-28T19:46:54.305 INFO:tasks.rados.rados.0.smithi059.stdout:813: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:46:54.305 INFO:tasks.rados.rados.0.smithi059.stdout:814: snap_remove snap 72 2022-04-28T19:46:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:54 smithi059 bash[29114]: cluster 2022-04-28T19:46:53.276767+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2022-04-28T19:46:54.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:54 smithi059 bash[23786]: cluster 2022-04-28T19:46:53.276767+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2022-04-28T19:46:54.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:54 smithi139 bash[23439]: cluster 2022-04-28T19:46:53.276767+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2022-04-28T19:46:55.303 INFO:tasks.rados.rados.0.smithi059.stdout:814: done (3 left) 2022-04-28T19:46:55.303 INFO:tasks.rados.rados.0.smithi059.stdout:815: rollback oid 20 current snap is 78 2022-04-28T19:46:55.304 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 63 2022-04-28T19:46:55.307 INFO:tasks.rados.rados.0.smithi059.stdout:808: finishing write tid 4 to smithi05955964-12 2022-04-28T19:46:55.308 INFO:tasks.rados.rados.0.smithi059.stdout:808: finishing write tid 6 to smithi05955964-12 2022-04-28T19:46:55.308 INFO:tasks.rados.rados.0.smithi059.stdout:808: finishing write tid 7 to smithi05955964-12 2022-04-28T19:46:55.308 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 413 (ObjNum 267 snap 78 seq_num 267) dirty exists 2022-04-28T19:46:55.308 INFO:tasks.rados.rados.0.smithi059.stdout:808: left oid 12 (ObjNum 267 snap 78 seq_num 267) 2022-04-28T19:46:55.308 INFO:tasks.rados.rados.0.smithi059.stdout:811: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:46:55.346 INFO:tasks.rados.rados.0.smithi059.stdout:813: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:46:55.433 INFO:tasks.rados.rados.0.smithi059.stdout:815: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:46:55.433 INFO:tasks.rados.rados.0.smithi059.stdout:815: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:46:55.433 INFO:tasks.rados.rados.0.smithi059.stdout:815: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:46:55.433 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 311 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:46:55.433 INFO:tasks.rados.rados.0.smithi059.stdout:808: done (3 left) 2022-04-28T19:46:55.433 INFO:tasks.rados.rados.0.smithi059.stdout:811: done (2 left) 2022-04-28T19:46:55.434 INFO:tasks.rados.rados.0.smithi059.stdout:813: done (1 left) 2022-04-28T19:46:55.434 INFO:tasks.rados.rados.0.smithi059.stdout:815: done (0 left) 2022-04-28T19:46:55.434 INFO:tasks.rados.rados.0.smithi059.stdout:816: read oid 31 snap -1 2022-04-28T19:46:55.434 INFO:tasks.rados.rados.0.smithi059.stdout:816: expect deleted 2022-04-28T19:46:55.434 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 78 2022-04-28T19:46:55.434 INFO:tasks.rados.rados.0.smithi059.stdout:817: seq_num 269 ranges {5251072=507904,5758976=630784,6389760=557056,6946816=417792} 2022-04-28T19:46:55.475 INFO:tasks.rados.rados.0.smithi059.stdout:817: writing smithi05955964-28 from 5251072 to 5758976 tid 1 2022-04-28T19:46:55.478 INFO:tasks.rados.rados.0.smithi059.stdout:817: writing smithi05955964-28 from 5758976 to 6389760 tid 2 2022-04-28T19:46:55.481 INFO:tasks.rados.rados.0.smithi059.stdout:817: writing smithi05955964-28 from 6389760 to 6946816 tid 3 2022-04-28T19:46:55.484 INFO:tasks.rados.rados.0.smithi059.stdout:817: writing smithi05955964-28 from 6946816 to 7364608 tid 4 2022-04-28T19:46:55.484 INFO:tasks.rados.rados.0.smithi059.stdout:818: setattr oid 16 current snap is 78 2022-04-28T19:46:55.484 INFO:tasks.rados.rados.0.smithi059.stdout:817: finishing write tid 1 to smithi05955964-28 2022-04-28T19:46:55.485 INFO:tasks.rados.rados.0.smithi059.stdout:816: done (2 left) 2022-04-28T19:46:55.486 INFO:tasks.rados.rados.0.smithi059.stdout:819: rmattr oid 20 current snap is 78 2022-04-28T19:46:55.486 INFO:tasks.rados.rados.0.smithi059.stdout:819: done (2 left) 2022-04-28T19:46:55.486 INFO:tasks.rados.rados.0.smithi059.stdout:820: setattr oid 21 current snap is 78 2022-04-28T19:46:55.486 INFO:tasks.rados.rados.0.smithi059.stdout:817: finishing write tid 2 to smithi05955964-28 2022-04-28T19:46:55.487 INFO:tasks.rados.rados.0.smithi059.stdout:821: delete oid 10 current snap is 78 2022-04-28T19:46:55.489 INFO:tasks.rados.rados.0.smithi059.stdout:817: finishing write tid 3 to smithi05955964-28 2022-04-28T19:46:55.489 INFO:tasks.rados.rados.0.smithi059.stdout:817: finishing write tid 4 to smithi05955964-28 2022-04-28T19:46:55.491 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 100 (ObjNum 1 snap 32762 seq_num 4161759872) dirty exists 2022-04-28T19:46:55.491 INFO:tasks.rados.rados.0.smithi059.stdout:820: done (3 left) 2022-04-28T19:46:55.491 INFO:tasks.rados.rados.0.smithi059.stdout:821: done (2 left) 2022-04-28T19:46:55.491 INFO:tasks.rados.rados.0.smithi059.stdout:822: copy_from oid 2 from oid 47 current snap is 78 2022-04-28T19:46:55.491 INFO:tasks.rados.rados.0.smithi059.stdout:823: read oid 46 snap -1 2022-04-28T19:46:55.492 INFO:tasks.rados.rados.0.smithi059.stdout:823: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:55.492 INFO:tasks.rados.rados.0.smithi059.stdout:824: setattr oid 7 current snap is 78 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:817: finishing write tid 6 to smithi05955964-28 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:825: rollback oid 24 current snap is 78 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 71 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:826: copy_from oid 33 from oid 22 current snap is 78 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:827: read oid 15 snap -1 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:827: expect (ObjNum 176 snap 49 seq_num 176) 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 78 2022-04-28T19:46:55.493 INFO:tasks.rados.rados.0.smithi059.stdout:828: seq_num 271 ranges {0=540672,540672=802816,1343488=516096,1859584=286720} 2022-04-28T19:46:55.496 INFO:tasks.rados.rados.0.smithi059.stdout:828: writing smithi05955964-37 from 0 to 540672 tid 1 2022-04-28T19:46:55.501 INFO:tasks.rados.rados.0.smithi059.stdout:828: writing smithi05955964-37 from 540672 to 1343488 tid 2 2022-04-28T19:46:55.504 INFO:tasks.rados.rados.0.smithi059.stdout:828: writing smithi05955964-37 from 1343488 to 1859584 tid 3 2022-04-28T19:46:55.507 INFO:tasks.rados.rados.0.smithi059.stdout:828: writing smithi05955964-37 from 1859584 to 2146304 tid 4 2022-04-28T19:46:55.507 INFO:tasks.rados.rados.0.smithi059.stdout:829: read oid 19 snap -1 2022-04-28T19:46:55.507 INFO:tasks.rados.rados.0.smithi059.stdout:829: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:46:55.507 INFO:tasks.rados.rados.0.smithi059.stdout:830: snap_remove snap 77 2022-04-28T19:46:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:55 smithi059 bash[29114]: cluster 2022-04-28T19:46:54.127096+0000 mgr.y (mgr.14140) 455 : cluster [DBG] pgmap v546: 89 pgs: 89 active+clean; 159 MiB data, 247 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 19 op/s 2022-04-28T19:46:55.778 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:55 smithi059 bash[29114]: cluster 2022-04-28T19:46:54.298321+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2022-04-28T19:46:55.778 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:55 smithi059 bash[23786]: cluster 2022-04-28T19:46:54.127096+0000 mgr.y (mgr.14140) 455 : cluster [DBG] pgmap v546: 89 pgs: 89 active+clean; 159 MiB data, 247 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 19 op/s 2022-04-28T19:46:55.778 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:55 smithi059 bash[23786]: cluster 2022-04-28T19:46:54.298321+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2022-04-28T19:46:55.779 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:55 smithi139 bash[23439]: cluster 2022-04-28T19:46:54.127096+0000 mgr.y (mgr.14140) 455 : cluster [DBG] pgmap v546: 89 pgs: 89 active+clean; 159 MiB data, 247 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 19 op/s 2022-04-28T19:46:55.779 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:55 smithi139 bash[23439]: cluster 2022-04-28T19:46:54.298321+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2022-04-28T19:46:56.116 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:46:56.319 INFO:tasks.rados.rados.0.smithi059.stdout:830: done (10 left) 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:831: snap_create 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 229 (ObjNum 204 snap 58 seq_num 204) dirty exists 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:825: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:817: finishing write tid 7 to smithi05955964-28 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 227 (ObjNum 269 snap 78 seq_num 269) dirty exists 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:817: left oid 28 (ObjNum 269 snap 78 seq_num 269) 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:825: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:825: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:46:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 286 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:46:56.321 INFO:tasks.rados.rados.0.smithi059.stdout:823: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:46:56.374 INFO:tasks.rados.rados.0.smithi059.stdout:829: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:46:56.406 INFO:tasks.rados.rados.0.smithi059.stdout:822: finishing copy_from to smithi05955964-2 2022-04-28T19:46:56.406 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 289 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:46:56.406 INFO:tasks.rados.rados.0.smithi059.stdout:822: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:46:56.407 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 433 (ObjNum 232 snap 60 seq_num 232) dirty exists 2022-04-28T19:46:56.407 INFO:tasks.rados.rados.0.smithi059.stdout:826: finishing copy_from to smithi05955964-33 2022-04-28T19:46:56.407 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 289 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:46:56.407 INFO:tasks.rados.rados.0.smithi059.stdout:826: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:46:56.407 INFO:tasks.rados.rados.0.smithi059.stdout:828: finishing write tid 1 to smithi05955964-37 2022-04-28T19:46:56.407 INFO:tasks.rados.rados.0.smithi059.stdout:827: expect (ObjNum 176 snap 49 seq_num 176) 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST STATUS REFRESHED AGE VERSION IMAGE NAME IMAGE ID CONTAINER ID 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (5m) 4m ago 6m 0.20.0 quay.io/prometheus/alertmanager:v0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (6m) 4m ago 6m 6.7.4 quay.io/ceph/ceph-grafana:6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (4m) 4m ago 4m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 44062639a78b 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (4m) 4m ago 4m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 3516bf4894c9 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (10m) 4m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b8c424c2e210 2022-04-28T19:46:56.431 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (12m) 4m ago 12m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 320b27bb479e 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (12m) 4m ago 12m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 239497cf6769 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (10m) 4m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0e12092e0bc3 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (10m) 4m ago 10m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (6m) 4m ago 6m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (6m) 4m ago 6m 0.18.1 quay.io/prometheus/node-exporter:v0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (9m) 4m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 547b5580e073 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (9m) 4m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (8m) 4m ago 9m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 35cdc0c6604a 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (8m) 4m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 2344bb9dfbae 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (8m) 4m ago 8m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 5976654882b1 2022-04-28T19:46:56.432 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (7m) 4m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 e68cffb4766c 2022-04-28T19:46:56.433 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (7m) 4m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 c36b8738a469 2022-04-28T19:46:56.433 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (6m) 4m ago 7m 15.2.16-79-gb51d8665 quay.ceph.io/ceph-ci/ceph:octopus 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:46:56.433 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (5m) 4m ago 6m 2.18.1 quay.io/prometheus/prometheus:v2.18.1 de242295e225 c763409947e4 2022-04-28T19:46:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:56 smithi059 bash[29114]: cluster 2022-04-28T19:46:55.301877+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2022-04-28T19:46:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:56 smithi059 bash[23786]: cluster 2022-04-28T19:46:55.301877+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2022-04-28T19:46:56.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:56 smithi139 bash[23439]: cluster 2022-04-28T19:46:55.301877+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2022-04-28T19:46:56.634 INFO:tasks.rados.rados.0.smithi059.stdout:828: finishing write tid 2 to smithi05955964-37 2022-04-28T19:46:56.634 INFO:tasks.rados.rados.0.smithi059.stdout:828: finishing write tid 3 to smithi05955964-37 2022-04-28T19:46:56.634 INFO:tasks.rados.rados.0.smithi059.stdout:828: finishing write tid 4 to smithi05955964-37 2022-04-28T19:46:56.634 INFO:tasks.rados.rados.0.smithi059.stdout:828: finishing write tid 6 to smithi05955964-37 2022-04-28T19:46:56.634 INFO:tasks.rados.rados.0.smithi059.stdout:828: finishing write tid 7 to smithi05955964-37 2022-04-28T19:46:56.634 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 311 (ObjNum 271 snap 78 seq_num 271) dirty exists 2022-04-28T19:46:56.635 INFO:tasks.rados.rados.0.smithi059.stdout:828: left oid 37 (ObjNum 271 snap 78 seq_num 271) 2022-04-28T19:46:56.765 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:46:56.766 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 15 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:46:56.767 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:46:57.203 INFO:tasks.rados.rados.0.smithi059.stdout:817: done (10 left) 2022-04-28T19:46:57.203 INFO:tasks.rados.rados.0.smithi059.stdout:818: done (9 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:822: done (8 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:823: done (7 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:824: done (6 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:825: done (5 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:826: done (4 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:827: done (3 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:828: done (2 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:829: done (1 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:831: done (0 left) 2022-04-28T19:46:57.204 INFO:tasks.rados.rados.0.smithi059.stdout:832: rollback oid 21 current snap is 79 2022-04-28T19:46:57.205 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 69 2022-04-28T19:46:57.205 INFO:tasks.rados.rados.0.smithi059.stdout:833: copy_from oid 28 from oid 37 current snap is 79 2022-04-28T19:46:57.205 INFO:tasks.rados.rados.0.smithi059.stdout:834: delete oid 1 current snap is 79 2022-04-28T19:46:57.209 INFO:tasks.rados.rados.0.smithi059.stdout:832: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:46:57.209 INFO:tasks.rados.rados.0.smithi059.stdout:832: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:46:57.211 INFO:tasks.rados.rados.0.smithi059.stdout:832: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:46:57.211 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 106 (ObjNum 130 snap 31 seq_num 130) dirty exists 2022-04-28T19:46:57.211 INFO:tasks.rados.rados.0.smithi059.stdout:832: done (2 left) 2022-04-28T19:46:57.212 INFO:tasks.rados.rados.0.smithi059.stdout:834: done (1 left) 2022-04-28T19:46:57.212 INFO:tasks.rados.rados.0.smithi059.stdout:835: delete oid 15 current snap is 79 2022-04-28T19:46:57.216 INFO:tasks.rados.rados.0.smithi059.stdout:835: done (1 left) 2022-04-28T19:46:57.216 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 79 2022-04-28T19:46:57.216 INFO:tasks.rados.rados.0.smithi059.stdout:836: seq_num 274 ranges {4972544=598016,5570560=499712,6070272=745472,6815744=360448} 2022-04-28T19:46:57.248 INFO:tasks.rados.rados.0.smithi059.stdout:836: writing smithi05955964-24 from 4972544 to 5570560 tid 1 2022-04-28T19:46:57.251 INFO:tasks.rados.rados.0.smithi059.stdout:836: writing smithi05955964-24 from 5570560 to 6070272 tid 2 2022-04-28T19:46:57.256 INFO:tasks.rados.rados.0.smithi059.stdout:836: writing smithi05955964-24 from 6070272 to 6815744 tid 3 2022-04-28T19:46:57.258 INFO:tasks.rados.rados.0.smithi059.stdout:836: writing smithi05955964-24 from 6815744 to 7176192 tid 4 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:837: rmattr oid 2 current snap is 79 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:833: finishing copy_from to smithi05955964-28 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 311 (ObjNum 271 snap 78 seq_num 271) dirty exists 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:833: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:836: finishing write tid 1 to smithi05955964-24 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:836: finishing write tid 2 to smithi05955964-24 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:833: done (2 left) 2022-04-28T19:46:57.259 INFO:tasks.rados.rados.0.smithi059.stdout:838: snap_create 2022-04-28T19:46:57.261 INFO:tasks.rados.rados.0.smithi059.stdout:836: finishing write tid 3 to smithi05955964-24 2022-04-28T19:46:57.261 INFO:tasks.rados.rados.0.smithi059.stdout:836: finishing write tid 4 to smithi05955964-24 2022-04-28T19:46:57.262 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 290 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:46:57.262 INFO:tasks.rados.rados.0.smithi059.stdout:836: finishing write tid 6 to smithi05955964-24 2022-04-28T19:46:57.263 INFO:tasks.rados.rados.0.smithi059.stdout:836: finishing write tid 7 to smithi05955964-24 2022-04-28T19:46:57.263 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 296 (ObjNum 274 snap 79 seq_num 274) dirty exists 2022-04-28T19:46:57.263 INFO:tasks.rados.rados.0.smithi059.stdout:836: left oid 24 (ObjNum 274 snap 79 seq_num 274) 2022-04-28T19:46:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[29114]: audit 2022-04-28T19:46:56.101131+0000 mgr.y (mgr.14140) 456 : audit [DBG] from='client.24811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[29114]: cluster 2022-04-28T19:46:56.127636+0000 mgr.y (mgr.14140) 457 : cluster [DBG] pgmap v549: 89 pgs: 89 active+clean; 160 MiB data, 256 MiB used, 707 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2022-04-28T19:46:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[29114]: cluster 2022-04-28T19:46:56.318210+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2022-04-28T19:46:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[29114]: audit 2022-04-28T19:46:56.764261+0000 mon.c (mon.1) 36 : audit [DBG] from='client.? 172.21.15.59:0/3536953964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:46:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[29114]: cluster 2022-04-28T19:46:57.200309+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2022-04-28T19:46:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[23786]: audit 2022-04-28T19:46:56.101131+0000 mgr.y (mgr.14140) 456 : audit [DBG] from='client.24811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[23786]: cluster 2022-04-28T19:46:56.127636+0000 mgr.y (mgr.14140) 457 : cluster [DBG] pgmap v549: 89 pgs: 89 active+clean; 160 MiB data, 256 MiB used, 707 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2022-04-28T19:46:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[23786]: cluster 2022-04-28T19:46:56.318210+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2022-04-28T19:46:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[23786]: audit 2022-04-28T19:46:56.764261+0000 mon.c (mon.1) 36 : audit [DBG] from='client.? 172.21.15.59:0/3536953964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:46:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:57 smithi059 bash[23786]: cluster 2022-04-28T19:46:57.200309+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2022-04-28T19:46:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:57 smithi139 bash[23439]: audit 2022-04-28T19:46:56.101131+0000 mgr.y (mgr.14140) 456 : audit [DBG] from='client.24811 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:57 smithi139 bash[23439]: cluster 2022-04-28T19:46:56.127636+0000 mgr.y (mgr.14140) 457 : cluster [DBG] pgmap v549: 89 pgs: 89 active+clean; 160 MiB data, 256 MiB used, 707 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2022-04-28T19:46:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:57 smithi139 bash[23439]: cluster 2022-04-28T19:46:56.318210+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2022-04-28T19:46:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:57 smithi139 bash[23439]: audit 2022-04-28T19:46:56.764261+0000 mon.c (mon.1) 36 : audit [DBG] from='client.? 172.21.15.59:0/3536953964' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:46:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:57 smithi139 bash[23439]: cluster 2022-04-28T19:46:57.200309+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2022-04-28T19:46:58.213 INFO:tasks.rados.rados.0.smithi059.stdout:836: done (2 left) 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:837: done (1 left) 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:838: done (0 left) 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:839: rmattr oid 35 current snap is 80 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:840: read oid 16 snap -1 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:840: expect (ObjNum 204 snap 58 seq_num 204) 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:841: read oid 18 snap -1 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:841: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 80 2022-04-28T19:46:58.214 INFO:tasks.rados.rados.0.smithi059.stdout:842: seq_num 275 ranges {2768896=614400,3383296=786432,4169728=499712,4669440=319488} 2022-04-28T19:46:58.233 INFO:tasks.rados.rados.0.smithi059.stdout:842: writing smithi05955964-14 from 2768896 to 3383296 tid 1 2022-04-28T19:46:58.238 INFO:tasks.rados.rados.0.smithi059.stdout:842: writing smithi05955964-14 from 3383296 to 4169728 tid 2 2022-04-28T19:46:58.241 INFO:tasks.rados.rados.0.smithi059.stdout:842: writing smithi05955964-14 from 4169728 to 4669440 tid 3 2022-04-28T19:46:58.243 INFO:tasks.rados.rados.0.smithi059.stdout:842: writing smithi05955964-14 from 4669440 to 4988928 tid 4 2022-04-28T19:46:58.244 INFO:tasks.rados.rados.0.smithi059.stdout:843: read oid 45 snap -1 2022-04-28T19:46:58.244 INFO:tasks.rados.rados.0.smithi059.stdout:843: expect (ObjNum 211 snap 59 seq_num 211) 2022-04-28T19:46:58.244 INFO:tasks.rados.rados.0.smithi059.stdout:844: rollback oid 41 current snap is 80 2022-04-28T19:46:58.244 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 78 2022-04-28T19:46:58.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 144 (ObjNum 265 snap 77 seq_num 265) dirty exists 2022-04-28T19:46:58.244 INFO:tasks.rados.rados.0.smithi059.stdout:840: expect (ObjNum 204 snap 58 seq_num 204) 2022-04-28T19:46:58.286 INFO:tasks.rados.rados.0.smithi059.stdout:842: finishing write tid 1 to smithi05955964-14 2022-04-28T19:46:58.287 INFO:tasks.rados.rados.0.smithi059.stdout:841: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:46:58.470 INFO:tasks.rados.rados.0.smithi059.stdout:842: finishing write tid 2 to smithi05955964-14 2022-04-28T19:46:58.471 INFO:tasks.rados.rados.0.smithi059.stdout:842: finishing write tid 3 to smithi05955964-14 2022-04-28T19:46:58.471 INFO:tasks.rados.rados.0.smithi059.stdout:842: finishing write tid 4 to smithi05955964-14 2022-04-28T19:46:58.471 INFO:tasks.rados.rados.0.smithi059.stdout:842: finishing write tid 6 to smithi05955964-14 2022-04-28T19:46:58.471 INFO:tasks.rados.rados.0.smithi059.stdout:844: finishing rollback tid 0 to smithi05955964-41 2022-04-28T19:46:58.471 INFO:tasks.rados.rados.0.smithi059.stdout:844: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:46:58.471 INFO:tasks.rados.rados.0.smithi059.stdout:843: expect (ObjNum 211 snap 59 seq_num 211) 2022-04-28T19:46:58.549 INFO:tasks.rados.rados.0.smithi059.stdout:844: finishing rollback tid 2 to smithi05955964-41 2022-04-28T19:46:58.549 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 96 (ObjNum 252 snap 73 seq_num 252) dirty exists 2022-04-28T19:46:58.549 INFO:tasks.rados.rados.0.smithi059.stdout:842: finishing write tid 7 to smithi05955964-14 2022-04-28T19:46:58.549 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 339 (ObjNum 275 snap 80 seq_num 275) dirty exists 2022-04-28T19:46:58.549 INFO:tasks.rados.rados.0.smithi059.stdout:842: left oid 14 (ObjNum 275 snap 80 seq_num 275) 2022-04-28T19:46:58.549 INFO:tasks.rados.rados.0.smithi059.stdout:839: done (5 left) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:840: done (4 left) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:841: done (3 left) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:842: done (2 left) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:843: done (1 left) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:844: done (0 left) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:845: read oid 17 snap -1 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:845: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:846: read oid 42 snap -1 2022-04-28T19:46:58.550 INFO:tasks.rados.rados.0.smithi059.stdout:846: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:58.551 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 80 2022-04-28T19:46:58.551 INFO:tasks.rados.rados.0.smithi059.stdout:847: seq_num 276 ranges {2277376=630784,2908160=671744,3579904=704512,4284416=229376} 2022-04-28T19:46:58.573 INFO:tasks.rados.rados.0.smithi059.stdout:847: writing smithi05955964-45 from 2277376 to 2908160 tid 1 2022-04-28T19:46:58.577 INFO:tasks.rados.rados.0.smithi059.stdout:847: writing smithi05955964-45 from 2908160 to 3579904 tid 2 2022-04-28T19:46:58.581 INFO:tasks.rados.rados.0.smithi059.stdout:847: writing smithi05955964-45 from 3579904 to 4284416 tid 3 2022-04-28T19:46:58.583 INFO:tasks.rados.rados.0.smithi059.stdout:847: writing smithi05955964-45 from 4284416 to 4513792 tid 4 2022-04-28T19:46:58.583 INFO:tasks.rados.rados.0.smithi059.stdout:848: read oid 44 snap -1 2022-04-28T19:46:58.583 INFO:tasks.rados.rados.0.smithi059.stdout:848: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:58.583 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 80 2022-04-28T19:46:58.583 INFO:tasks.rados.rados.0.smithi059.stdout:849: seq_num 277 ranges {1998848=647168,2646016=557056,3203072=516096,3719168=532480} 2022-04-28T19:46:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:58 smithi059 bash[29114]: audit 2022-04-28T19:46:56.413190+0000 mgr.y (mgr.14140) 458 : audit [DBG] from='client.24817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:58 smithi059 bash[29114]: cluster 2022-04-28T19:46:58.210154+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2022-04-28T19:46:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:58 smithi059 bash[23786]: audit 2022-04-28T19:46:56.413190+0000 mgr.y (mgr.14140) 458 : audit [DBG] from='client.24817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:58 smithi059 bash[23786]: cluster 2022-04-28T19:46:58.210154+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2022-04-28T19:46:58.598 INFO:tasks.rados.rados.0.smithi059.stdout:849: writing smithi05955964-26 from 1998848 to 2646016 tid 1 2022-04-28T19:46:58.601 INFO:tasks.rados.rados.0.smithi059.stdout:849: writing smithi05955964-26 from 2646016 to 3203072 tid 2 2022-04-28T19:46:58.604 INFO:tasks.rados.rados.0.smithi059.stdout:849: writing smithi05955964-26 from 3203072 to 3719168 tid 3 2022-04-28T19:46:58.608 INFO:tasks.rados.rados.0.smithi059.stdout:849: writing smithi05955964-26 from 3719168 to 4251648 tid 4 2022-04-28T19:46:58.608 INFO:tasks.rados.rados.0.smithi059.stdout:850: copy_from oid 22 from oid 40 current snap is 80 2022-04-28T19:46:58.608 INFO:tasks.rados.rados.0.smithi059.stdout:846: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:58.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:58 smithi139 bash[23439]: audit 2022-04-28T19:46:56.413190+0000 mgr.y (mgr.14140) 458 : audit [DBG] from='client.24817 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:46:58.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:58 smithi139 bash[23439]: cluster 2022-04-28T19:46:58.210154+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2022-04-28T19:46:58.643 INFO:tasks.rados.rados.0.smithi059.stdout:845: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:46:58.717 INFO:tasks.rados.rados.0.smithi059.stdout:847: finishing write tid 1 to smithi05955964-45 2022-04-28T19:46:58.717 INFO:tasks.rados.rados.0.smithi059.stdout:847: finishing write tid 2 to smithi05955964-45 2022-04-28T19:46:58.717 INFO:tasks.rados.rados.0.smithi059.stdout:847: finishing write tid 3 to smithi05955964-45 2022-04-28T19:46:58.717 INFO:tasks.rados.rados.0.smithi059.stdout:848: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:46:58.750 INFO:tasks.rados.rados.0.smithi059.stdout:847: finishing write tid 4 to smithi05955964-45 2022-04-28T19:46:58.750 INFO:tasks.rados.rados.0.smithi059.stdout:847: finishing write tid 6 to smithi05955964-45 2022-04-28T19:46:58.750 INFO:tasks.rados.rados.0.smithi059.stdout:847: finishing write tid 7 to smithi05955964-45 2022-04-28T19:46:58.750 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 321 (ObjNum 276 snap 80 seq_num 276) dirty exists 2022-04-28T19:46:58.750 INFO:tasks.rados.rados.0.smithi059.stdout:847: left oid 45 (ObjNum 276 snap 80 seq_num 276) 2022-04-28T19:46:58.750 INFO:tasks.rados.rados.0.smithi059.stdout:849: finishing write tid 1 to smithi05955964-26 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:849: finishing write tid 2 to smithi05955964-26 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:849: finishing write tid 3 to smithi05955964-26 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:849: finishing write tid 4 to smithi05955964-26 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:849: finishing write tid 6 to smithi05955964-26 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:849: finishing write tid 7 to smithi05955964-26 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 344 (ObjNum 277 snap 80 seq_num 277) dirty exists 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:849: left oid 26 (ObjNum 277 snap 80 seq_num 277) 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:850: finishing copy_from to smithi05955964-22 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 334 (ObjNum 262 snap 77 seq_num 262) dirty exists 2022-04-28T19:46:58.751 INFO:tasks.rados.rados.0.smithi059.stdout:845: done (5 left) 2022-04-28T19:46:58.752 INFO:tasks.rados.rados.0.smithi059.stdout:846: done (4 left) 2022-04-28T19:46:58.752 INFO:tasks.rados.rados.0.smithi059.stdout:847: done (3 left) 2022-04-28T19:46:58.752 INFO:tasks.rados.rados.0.smithi059.stdout:848: done (2 left) 2022-04-28T19:46:58.752 INFO:tasks.rados.rados.0.smithi059.stdout:849: done (1 left) 2022-04-28T19:46:58.752 INFO:tasks.rados.rados.0.smithi059.stdout:851: setattr oid 42 current snap is 80 2022-04-28T19:46:58.752 INFO:tasks.rados.rados.0.smithi059.stdout:850: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:46:58.753 INFO:tasks.rados.rados.0.smithi059.stdout:850: done (1 left) 2022-04-28T19:46:58.753 INFO:tasks.rados.rados.0.smithi059.stdout:852: read oid 40 snap 68 2022-04-28T19:46:58.753 INFO:tasks.rados.rados.0.smithi059.stdout:852: expect deleted 2022-04-28T19:46:58.753 INFO:tasks.rados.rados.0.smithi059.stdout:853: rmattr oid 3 current snap is 80 2022-04-28T19:46:58.753 INFO:tasks.rados.rados.0.smithi059.stdout:853: done (2 left) 2022-04-28T19:46:58.753 INFO:tasks.rados.rados.0.smithi059.stdout:854: snap_remove snap 69 2022-04-28T19:46:59.342 INFO:tasks.rados.rados.0.smithi059.stdout:854: done (2 left) 2022-04-28T19:46:59.358 INFO:tasks.rados.rados.0.smithi059.stdout:855: read oid 15 snap -1 2022-04-28T19:46:59.358 INFO:tasks.rados.rados.0.smithi059.stdout:855: expect deleted 2022-04-28T19:46:59.358 INFO:tasks.rados.rados.0.smithi059.stdout:856: rmattr oid 30 current snap is 80 2022-04-28T19:46:59.359 INFO:tasks.rados.rados.0.smithi059.stdout:856: done (3 left) 2022-04-28T19:46:59.359 INFO:tasks.rados.rados.0.smithi059.stdout:857: snap_remove snap 79 2022-04-28T19:46:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:46:59 smithi139 bash[23439]: cluster 2022-04-28T19:46:58.128398+0000 mgr.y (mgr.14140) 459 : cluster [DBG] pgmap v552: 89 pgs: 89 active+clean; 166 MiB data, 263 MiB used, 707 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2022-04-28T19:46:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:46:59 smithi059 bash[29114]: cluster 2022-04-28T19:46:58.128398+0000 mgr.y (mgr.14140) 459 : cluster [DBG] pgmap v552: 89 pgs: 89 active+clean; 166 MiB data, 263 MiB used, 707 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2022-04-28T19:46:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:46:59 smithi059 bash[23786]: cluster 2022-04-28T19:46:58.128398+0000 mgr.y (mgr.14140) 459 : cluster [DBG] pgmap v552: 89 pgs: 89 active+clean; 166 MiB data, 263 MiB used, 707 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.3 MiB/s wr, 20 op/s 2022-04-28T19:47:00.353 INFO:tasks.rados.rados.0.smithi059.stdout:857: done (3 left) 2022-04-28T19:47:00.353 INFO:tasks.rados.rados.0.smithi059.stdout:858: copy_from oid 1 from oid 32 current snap is 80 2022-04-28T19:47:00.353 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 313 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:851: done (3 left) 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:852: done (2 left) 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:855: done (1 left) 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:859: read oid 10 snap -1 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:859: expect deleted 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:860: copy_from oid 25 from oid 17 current snap is 80 2022-04-28T19:47:00.354 INFO:tasks.rados.rados.0.smithi059.stdout:861: delete oid 14 current snap is 80 2022-04-28T19:47:00.359 INFO:tasks.rados.rados.0.smithi059.stdout:858: finishing copy_from to smithi05955964-1 2022-04-28T19:47:00.359 INFO:tasks.rados.rados.0.smithi059.stdout:858: got expected ENOENT (src dne) 2022-04-28T19:47:00.359 INFO:tasks.rados.rados.0.smithi059.stdout:858: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:47:00.362 INFO:tasks.rados.rados.0.smithi059.stdout:858: done (3 left) 2022-04-28T19:47:00.362 INFO:tasks.rados.rados.0.smithi059.stdout:859: done (2 left) 2022-04-28T19:47:00.363 INFO:tasks.rados.rados.0.smithi059.stdout:861: done (1 left) 2022-04-28T19:47:00.363 INFO:tasks.rados.rados.0.smithi059.stdout:862: rollback oid 38 current snap is 80 2022-04-28T19:47:00.363 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 73 2022-04-28T19:47:00.363 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 80 2022-04-28T19:47:00.363 INFO:tasks.rados.rados.0.smithi059.stdout:863: seq_num 279 ranges {1409024=679936,2088960=720896,2809856=483328,3293184=401408} 2022-04-28T19:47:00.374 INFO:tasks.rados.rados.0.smithi059.stdout:863: writing smithi05955964-9 from 1409024 to 2088960 tid 1 2022-04-28T19:47:00.379 INFO:tasks.rados.rados.0.smithi059.stdout:863: writing smithi05955964-9 from 2088960 to 2809856 tid 2 2022-04-28T19:47:00.382 INFO:tasks.rados.rados.0.smithi059.stdout:863: writing smithi05955964-9 from 2809856 to 3293184 tid 3 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:863: writing smithi05955964-9 from 3293184 to 3694592 tid 4 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:864: copy_from oid 21 from oid 26 current snap is 80 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:860: finishing copy_from to smithi05955964-25 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:862: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:860: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:47:00.385 INFO:tasks.rados.rados.0.smithi059.stdout:862: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:47:00.386 INFO:tasks.rados.rados.0.smithi059.stdout:862: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:47:00.386 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 306 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:00.386 INFO:tasks.rados.rados.0.smithi059.stdout:863: finishing write tid 1 to smithi05955964-9 2022-04-28T19:47:00.386 INFO:tasks.rados.rados.0.smithi059.stdout:860: done (3 left) 2022-04-28T19:47:00.386 INFO:tasks.rados.rados.0.smithi059.stdout:862: done (2 left) 2022-04-28T19:47:00.386 INFO:tasks.rados.rados.0.smithi059.stdout:865: snap_remove snap 65 2022-04-28T19:47:00.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:00 smithi139 bash[23439]: cluster 2022-04-28T19:46:59.337931+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2022-04-28T19:47:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:00 smithi059 bash[29114]: cluster 2022-04-28T19:46:59.337931+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2022-04-28T19:47:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:00 smithi059 bash[23786]: cluster 2022-04-28T19:46:59.337931+0000 mon.a (mon.0) 827 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:865: done (2 left) 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:866: rollback oid 46 current snap is 80 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 63 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:863: finishing write tid 2 to smithi05955964-9 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:863: finishing write tid 3 to smithi05955964-9 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:863: finishing write tid 4 to smithi05955964-9 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:863: finishing write tid 6 to smithi05955964-9 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:863: finishing write tid 7 to smithi05955964-9 2022-04-28T19:47:01.378 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 259 (ObjNum 279 snap 80 seq_num 279) dirty exists 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:863: left oid 9 (ObjNum 279 snap 80 seq_num 279) 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:864: finishing copy_from to smithi05955964-21 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 344 (ObjNum 277 snap 80 seq_num 277) dirty exists 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:864: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:863: done (2 left) 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:864: done (1 left) 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 80 2022-04-28T19:47:01.379 INFO:tasks.rados.rados.0.smithi059.stdout:867: seq_num 280 ranges {4874240=696320,5570560=606208,6176768=688128,6864896=311296} 2022-04-28T19:47:01.411 INFO:tasks.rados.rados.0.smithi059.stdout:867: writing smithi05955964-41 from 4874240 to 5570560 tid 1 2022-04-28T19:47:01.414 INFO:tasks.rados.rados.0.smithi059.stdout:867: writing smithi05955964-41 from 5570560 to 6176768 tid 2 2022-04-28T19:47:01.418 INFO:tasks.rados.rados.0.smithi059.stdout:867: writing smithi05955964-41 from 6176768 to 6864896 tid 3 2022-04-28T19:47:01.421 INFO:tasks.rados.rados.0.smithi059.stdout:867: writing smithi05955964-41 from 6864896 to 7176192 tid 4 2022-04-28T19:47:01.421 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 80 2022-04-28T19:47:01.421 INFO:tasks.rados.rados.0.smithi059.stdout:868: seq_num 281 ranges {5816320=712704,6529024=491520,7020544=491520,7512064=622592} 2022-04-28T19:47:01.458 INFO:tasks.rados.rados.0.smithi059.stdout:868: writing smithi05955964-23 from 5816320 to 6529024 tid 1 2022-04-28T19:47:01.460 INFO:tasks.rados.rados.0.smithi059.stdout:868: writing smithi05955964-23 from 6529024 to 7020544 tid 2 2022-04-28T19:47:01.464 INFO:tasks.rados.rados.0.smithi059.stdout:868: writing smithi05955964-23 from 7020544 to 7512064 tid 3 2022-04-28T19:47:01.468 INFO:tasks.rados.rados.0.smithi059.stdout:868: writing smithi05955964-23 from 7512064 to 8134656 tid 4 2022-04-28T19:47:01.468 INFO:tasks.rados.rados.0.smithi059.stdout:869: snap_create 2022-04-28T19:47:01.468 INFO:tasks.rados.rados.0.smithi059.stdout:866: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:47:01.468 INFO:tasks.rados.rados.0.smithi059.stdout:866: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:47:01.468 INFO:tasks.rados.rados.0.smithi059.stdout:866: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 341 (ObjNum 195 snap 57 seq_num 195) dirty exists 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: finishing write tid 1 to smithi05955964-41 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: finishing write tid 2 to smithi05955964-41 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: finishing write tid 3 to smithi05955964-41 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: finishing write tid 4 to smithi05955964-41 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: finishing write tid 6 to smithi05955964-41 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: finishing write tid 7 to smithi05955964-41 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 107 (ObjNum 280 snap 80 seq_num 280) dirty exists 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:867: left oid 41 (ObjNum 280 snap 80 seq_num 280) 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:868: finishing write tid 1 to smithi05955964-23 2022-04-28T19:47:01.469 INFO:tasks.rados.rados.0.smithi059.stdout:868: finishing write tid 2 to smithi05955964-23 2022-04-28T19:47:01.470 INFO:tasks.rados.rados.0.smithi059.stdout:868: finishing write tid 3 to smithi05955964-23 2022-04-28T19:47:01.470 INFO:tasks.rados.rados.0.smithi059.stdout:868: finishing write tid 4 to smithi05955964-23 2022-04-28T19:47:01.471 INFO:tasks.rados.rados.0.smithi059.stdout:868: finishing write tid 6 to smithi05955964-23 2022-04-28T19:47:01.472 INFO:tasks.rados.rados.0.smithi059.stdout:868: finishing write tid 7 to smithi05955964-23 2022-04-28T19:47:01.472 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 360 (ObjNum 281 snap 80 seq_num 281) dirty exists 2022-04-28T19:47:01.472 INFO:tasks.rados.rados.0.smithi059.stdout:868: left oid 23 (ObjNum 281 snap 80 seq_num 281) 2022-04-28T19:47:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:01 smithi139 bash[23439]: cluster 2022-04-28T19:47:00.129150+0000 mgr.y (mgr.14140) 460 : cluster [DBG] pgmap v555: 89 pgs: 89 active+clean; 173 MiB data, 270 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2022-04-28T19:47:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:01 smithi139 bash[23439]: cluster 2022-04-28T19:47:00.350555+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2022-04-28T19:47:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:01 smithi059 bash[29114]: cluster 2022-04-28T19:47:00.129150+0000 mgr.y (mgr.14140) 460 : cluster [DBG] pgmap v555: 89 pgs: 89 active+clean; 173 MiB data, 270 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2022-04-28T19:47:01.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:01 smithi059 bash[29114]: cluster 2022-04-28T19:47:00.350555+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2022-04-28T19:47:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:01 smithi059 bash[23786]: cluster 2022-04-28T19:47:00.129150+0000 mgr.y (mgr.14140) 460 : cluster [DBG] pgmap v555: 89 pgs: 89 active+clean; 173 MiB data, 270 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 3.9 MiB/s wr, 39 op/s 2022-04-28T19:47:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:01 smithi059 bash[23786]: cluster 2022-04-28T19:47:00.350555+0000 mon.a (mon.0) 828 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2022-04-28T19:47:02.205 INFO:tasks.rados.rados.0.smithi059.stdout:866: done (3 left) 2022-04-28T19:47:02.205 INFO:tasks.rados.rados.0.smithi059.stdout:867: done (2 left) 2022-04-28T19:47:02.205 INFO:tasks.rados.rados.0.smithi059.stdout:868: done (1 left) 2022-04-28T19:47:02.206 INFO:tasks.rados.rados.0.smithi059.stdout:869: done (0 left) 2022-04-28T19:47:02.206 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 81 2022-04-28T19:47:02.206 INFO:tasks.rados.rados.0.smithi059.stdout:870: seq_num 282 ranges {4513792=729088,5242880=425984,5668864=704512,6373376=475136} 2022-04-28T19:47:02.235 INFO:tasks.rados.rados.0.smithi059.stdout:870: writing smithi05955964-45 from 4513792 to 5242880 tid 1 2022-04-28T19:47:02.237 INFO:tasks.rados.rados.0.smithi059.stdout:870: writing smithi05955964-45 from 5242880 to 5668864 tid 2 2022-04-28T19:47:02.241 INFO:tasks.rados.rados.0.smithi059.stdout:870: writing smithi05955964-45 from 5668864 to 6373376 tid 3 2022-04-28T19:47:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:870: writing smithi05955964-45 from 6373376 to 6848512 tid 4 2022-04-28T19:47:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 81 2022-04-28T19:47:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:871: seq_num 283 ranges {0=745472,745472=712704,1458176=458752,1916928=434176} 2022-04-28T19:47:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:871: writing smithi05955964-10 from 0 to 745472 tid 1 2022-04-28T19:47:02.253 INFO:tasks.rados.rados.0.smithi059.stdout:871: writing smithi05955964-10 from 745472 to 1458176 tid 2 2022-04-28T19:47:02.256 INFO:tasks.rados.rados.0.smithi059.stdout:871: writing smithi05955964-10 from 1458176 to 1916928 tid 3 2022-04-28T19:47:02.260 INFO:tasks.rados.rados.0.smithi059.stdout:871: writing smithi05955964-10 from 1916928 to 2351104 tid 4 2022-04-28T19:47:02.260 INFO:tasks.rados.rados.0.smithi059.stdout:872: delete oid 7 current snap is 81 2022-04-28T19:47:02.260 INFO:tasks.rados.rados.0.smithi059.stdout:870: finishing write tid 1 to smithi05955964-45 2022-04-28T19:47:02.260 INFO:tasks.rados.rados.0.smithi059.stdout:870: finishing write tid 2 to smithi05955964-45 2022-04-28T19:47:02.260 INFO:tasks.rados.rados.0.smithi059.stdout:870: finishing write tid 3 to smithi05955964-45 2022-04-28T19:47:02.260 INFO:tasks.rados.rados.0.smithi059.stdout:870: finishing write tid 4 to smithi05955964-45 2022-04-28T19:47:02.261 INFO:tasks.rados.rados.0.smithi059.stdout:870: finishing write tid 6 to smithi05955964-45 2022-04-28T19:47:02.261 INFO:tasks.rados.rados.0.smithi059.stdout:870: finishing write tid 7 to smithi05955964-45 2022-04-28T19:47:02.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 345 (ObjNum 282 snap 81 seq_num 282) dirty exists 2022-04-28T19:47:02.261 INFO:tasks.rados.rados.0.smithi059.stdout:870: left oid 45 (ObjNum 282 snap 81 seq_num 282) 2022-04-28T19:47:02.261 INFO:tasks.rados.rados.0.smithi059.stdout:871: finishing write tid 1 to smithi05955964-10 2022-04-28T19:47:02.261 INFO:tasks.rados.rados.0.smithi059.stdout:871: finishing write tid 2 to smithi05955964-10 2022-04-28T19:47:02.262 INFO:tasks.rados.rados.0.smithi059.stdout:871: finishing write tid 3 to smithi05955964-10 2022-04-28T19:47:02.263 INFO:tasks.rados.rados.0.smithi059.stdout:871: finishing write tid 4 to smithi05955964-10 2022-04-28T19:47:02.263 INFO:tasks.rados.rados.0.smithi059.stdout:871: finishing write tid 6 to smithi05955964-10 2022-04-28T19:47:02.263 INFO:tasks.rados.rados.0.smithi059.stdout:870: done (2 left) 2022-04-28T19:47:02.263 INFO:tasks.rados.rados.0.smithi059.stdout:872: done (1 left) 2022-04-28T19:47:02.263 INFO:tasks.rados.rados.0.smithi059.stdout:873: rmattr oid 20 current snap is 81 2022-04-28T19:47:02.264 INFO:tasks.rados.rados.0.smithi059.stdout:873: done (1 left) 2022-04-28T19:47:02.264 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 81 2022-04-28T19:47:02.264 INFO:tasks.rados.rados.0.smithi059.stdout:874: seq_num 285 ranges {7176192=778240,7954432=475136,8429568=475136,8904704=655360} 2022-04-28T19:47:02.308 INFO:tasks.rados.rados.0.smithi059.stdout:874: writing smithi05955964-24 from 7176192 to 7954432 tid 1 2022-04-28T19:47:02.310 INFO:tasks.rados.rados.0.smithi059.stdout:874: writing smithi05955964-24 from 7954432 to 8429568 tid 2 2022-04-28T19:47:02.314 INFO:tasks.rados.rados.0.smithi059.stdout:874: writing smithi05955964-24 from 8429568 to 8904704 tid 3 2022-04-28T19:47:02.318 INFO:tasks.rados.rados.0.smithi059.stdout:874: writing smithi05955964-24 from 8904704 to 9560064 tid 4 2022-04-28T19:47:02.318 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 81 2022-04-28T19:47:02.318 INFO:tasks.rados.rados.0.smithi059.stdout:875: seq_num 286 ranges {2146304=794624,2940928=761856,3702784=679936,4382720=163840} 2022-04-28T19:47:02.334 INFO:tasks.rados.rados.0.smithi059.stdout:875: writing smithi05955964-37 from 2146304 to 2940928 tid 1 2022-04-28T19:47:02.339 INFO:tasks.rados.rados.0.smithi059.stdout:875: writing smithi05955964-37 from 2940928 to 3702784 tid 2 2022-04-28T19:47:02.343 INFO:tasks.rados.rados.0.smithi059.stdout:875: writing smithi05955964-37 from 3702784 to 4382720 tid 3 2022-04-28T19:47:02.345 INFO:tasks.rados.rados.0.smithi059.stdout:875: writing smithi05955964-37 from 4382720 to 4546560 tid 4 2022-04-28T19:47:02.345 INFO:tasks.rados.rados.0.smithi059.stdout:876: delete oid 43 current snap is 81 2022-04-28T19:47:02.345 INFO:tasks.rados.rados.0.smithi059.stdout:871: finishing write tid 7 to smithi05955964-10 2022-04-28T19:47:02.346 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 365 (ObjNum 283 snap 81 seq_num 283) dirty exists 2022-04-28T19:47:02.346 INFO:tasks.rados.rados.0.smithi059.stdout:871: left oid 10 (ObjNum 283 snap 81 seq_num 283) 2022-04-28T19:47:02.346 INFO:tasks.rados.rados.0.smithi059.stdout:874: finishing write tid 1 to smithi05955964-24 2022-04-28T19:47:02.346 INFO:tasks.rados.rados.0.smithi059.stdout:874: finishing write tid 2 to smithi05955964-24 2022-04-28T19:47:02.346 INFO:tasks.rados.rados.0.smithi059.stdout:874: finishing write tid 3 to smithi05955964-24 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:874: finishing write tid 4 to smithi05955964-24 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:874: finishing write tid 6 to smithi05955964-24 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:874: finishing write tid 7 to smithi05955964-24 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 318 (ObjNum 285 snap 81 seq_num 285) dirty exists 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:874: left oid 24 (ObjNum 285 snap 81 seq_num 285) 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:875: finishing write tid 1 to smithi05955964-37 2022-04-28T19:47:02.347 INFO:tasks.rados.rados.0.smithi059.stdout:875: finishing write tid 2 to smithi05955964-37 2022-04-28T19:47:02.348 INFO:tasks.rados.rados.0.smithi059.stdout:871: done (3 left) 2022-04-28T19:47:02.348 INFO:tasks.rados.rados.0.smithi059.stdout:874: done (2 left) 2022-04-28T19:47:02.348 INFO:tasks.rados.rados.0.smithi059.stdout:876: done (1 left) 2022-04-28T19:47:02.348 INFO:tasks.rados.rados.0.smithi059.stdout:877: snap_create 2022-04-28T19:47:02.348 INFO:tasks.rados.rados.0.smithi059.stdout:875: finishing write tid 3 to smithi05955964-37 2022-04-28T19:47:02.349 INFO:tasks.rados.rados.0.smithi059.stdout:875: finishing write tid 4 to smithi05955964-37 2022-04-28T19:47:02.349 INFO:tasks.rados.rados.0.smithi059.stdout:875: finishing write tid 6 to smithi05955964-37 2022-04-28T19:47:02.349 INFO:tasks.rados.rados.0.smithi059.stdout:875: finishing write tid 7 to smithi05955964-37 2022-04-28T19:47:02.349 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 329 (ObjNum 286 snap 81 seq_num 286) dirty exists 2022-04-28T19:47:02.349 INFO:tasks.rados.rados.0.smithi059.stdout:875: left oid 37 (ObjNum 286 snap 81 seq_num 286) 2022-04-28T19:47:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:02 smithi059 bash[29114]: cluster 2022-04-28T19:47:01.374266+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2022-04-28T19:47:02.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:02 smithi059 bash[29114]: cluster 2022-04-28T19:47:02.201454+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2022-04-28T19:47:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:02 smithi059 bash[23786]: cluster 2022-04-28T19:47:01.374266+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2022-04-28T19:47:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:02 smithi059 bash[23786]: cluster 2022-04-28T19:47:02.201454+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2022-04-28T19:47:02.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:02 smithi139 bash[23439]: cluster 2022-04-28T19:47:01.374266+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2022-04-28T19:47:02.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:02 smithi139 bash[23439]: cluster 2022-04-28T19:47:02.201454+0000 mon.a (mon.0) 830 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2022-04-28T19:47:03.222 INFO:tasks.rados.rados.0.smithi059.stdout:875: done (1 left) 2022-04-28T19:47:03.222 INFO:tasks.rados.rados.0.smithi059.stdout:877: done (0 left) 2022-04-28T19:47:03.223 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 82 2022-04-28T19:47:03.223 INFO:tasks.rados.rados.0.smithi059.stdout:878: seq_num 288 ranges {2555904=24576} 2022-04-28T19:47:03.239 INFO:tasks.rados.rados.0.smithi059.stdout:878: writing smithi05955964-13 from 2555904 to 2580480 tid 1 2022-04-28T19:47:03.239 INFO:tasks.rados.rados.0.smithi059.stdout:879: delete oid 50 current snap is 82 2022-04-28T19:47:03.245 INFO:tasks.rados.rados.0.smithi059.stdout:878: finishing write tid 1 to smithi05955964-13 2022-04-28T19:47:03.246 INFO:tasks.rados.rados.0.smithi059.stdout:878: finishing write tid 3 to smithi05955964-13 2022-04-28T19:47:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:879: done (1 left) 2022-04-28T19:47:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:880: rollback oid 21 current snap is 82 2022-04-28T19:47:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 71 2022-04-28T19:47:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:881: snap_remove snap 81 2022-04-28T19:47:03.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[29114]: cluster 2022-04-28T19:47:02.129935+0000 mgr.y (mgr.14140) 461 : cluster [DBG] pgmap v558: 89 pgs: 89 active+clean; 182 MiB data, 278 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 35 op/s 2022-04-28T19:47:03.888 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[29114]: cluster 2022-04-28T19:47:03.219478+0000 mon.a (mon.0) 831 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2022-04-28T19:47:03.889 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:03.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:47:03.889 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:03.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:47:03.889 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:03.519Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:47:03.890 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[23786]: cluster 2022-04-28T19:47:02.129935+0000 mgr.y (mgr.14140) 461 : cluster [DBG] pgmap v558: 89 pgs: 89 active+clean; 182 MiB data, 278 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 35 op/s 2022-04-28T19:47:03.890 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:03 smithi059 bash[23786]: cluster 2022-04-28T19:47:03.219478+0000 mon.a (mon.0) 831 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2022-04-28T19:47:03.891 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:03 smithi139 bash[23439]: cluster 2022-04-28T19:47:02.129935+0000 mgr.y (mgr.14140) 461 : cluster [DBG] pgmap v558: 89 pgs: 89 active+clean; 182 MiB data, 278 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 35 op/s 2022-04-28T19:47:03.891 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:03 smithi139 bash[23439]: cluster 2022-04-28T19:47:03.219478+0000 mon.a (mon.0) 831 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2022-04-28T19:47:04.225 INFO:tasks.rados.rados.0.smithi059.stdout:878: finishing write tid 4 to smithi05955964-13 2022-04-28T19:47:04.225 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 181 (ObjNum 288 snap 82 seq_num 288) dirty exists 2022-04-28T19:47:04.225 INFO:tasks.rados.rados.0.smithi059.stdout:878: left oid 13 (ObjNum 288 snap 82 seq_num 288) 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:880: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:880: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:880: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 347 (ObjNum 130 snap 31 seq_num 130) dirty exists 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:878: done (2 left) 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:880: done (1 left) 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:881: done (0 left) 2022-04-28T19:47:04.226 INFO:tasks.rados.rados.0.smithi059.stdout:882: snap_create 2022-04-28T19:47:05.231 INFO:tasks.rados.rados.0.smithi059.stdout:882: done (0 left) 2022-04-28T19:47:05.231 INFO:tasks.rados.rados.0.smithi059.stdout:883: read oid 46 snap -1 2022-04-28T19:47:05.231 INFO:tasks.rados.rados.0.smithi059.stdout:883: expect (ObjNum 195 snap 57 seq_num 195) 2022-04-28T19:47:05.232 INFO:tasks.rados.rados.0.smithi059.stdout:884: read oid 12 snap -1 2022-04-28T19:47:05.232 INFO:tasks.rados.rados.0.smithi059.stdout:884: expect (ObjNum 267 snap 78 seq_num 267) 2022-04-28T19:47:05.232 INFO:tasks.rados.rados.0.smithi059.stdout:885: read oid 29 snap -1 2022-04-28T19:47:05.232 INFO:tasks.rados.rados.0.smithi059.stdout:885: expect (ObjNum 250 snap 71 seq_num 250) 2022-04-28T19:47:05.232 INFO:tasks.rados.rados.0.smithi059.stdout:886: delete oid 16 current snap is 83 2022-04-28T19:47:05.242 INFO:tasks.rados.rados.0.smithi059.stdout:886: done (3 left) 2022-04-28T19:47:05.242 INFO:tasks.rados.rados.0.smithi059.stdout:887: read oid 25 snap 63 2022-04-28T19:47:05.242 INFO:tasks.rados.rados.0.smithi059.stdout:887: expect deleted 2022-04-28T19:47:05.243 INFO:tasks.rados.rados.0.smithi059.stdout:888: rollback oid 1 current snap is 83 2022-04-28T19:47:05.243 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 74 2022-04-28T19:47:05.243 INFO:tasks.rados.rados.0.smithi059.stdout:889: rollback oid 24 current snap is 83 2022-04-28T19:47:05.243 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 63 2022-04-28T19:47:05.243 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 83 2022-04-28T19:47:05.243 INFO:tasks.rados.rados.0.smithi059.stdout:890: seq_num 291 ranges {2580480=81920} 2022-04-28T19:47:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:890: writing smithi05955964-13 from 2580480 to 2662400 tid 1 2022-04-28T19:47:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:891: rollback oid 50 current snap is 83 2022-04-28T19:47:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 71 2022-04-28T19:47:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:889: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:47:05.260 INFO:tasks.rados.rados.0.smithi059.stdout:889: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:888: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:888: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 356 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:889: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 322 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:888: done (7 left) 2022-04-28T19:47:05.261 INFO:tasks.rados.rados.0.smithi059.stdout:889: done (6 left) 2022-04-28T19:47:05.262 INFO:tasks.rados.rados.0.smithi059.stdout:892: read oid 3 snap -1 2022-04-28T19:47:05.262 INFO:tasks.rados.rados.0.smithi059.stdout:892: expect deleted 2022-04-28T19:47:05.262 INFO:tasks.rados.rados.0.smithi059.stdout:893: snap_create 2022-04-28T19:47:05.263 INFO:tasks.rados.rados.0.smithi059.stdout:890: finishing write tid 1 to smithi05955964-13 2022-04-28T19:47:05.263 INFO:tasks.rados.rados.0.smithi059.stdout:890: finishing write tid 3 to smithi05955964-13 2022-04-28T19:47:05.264 INFO:tasks.rados.rados.0.smithi059.stdout:890: finishing write tid 4 to smithi05955964-13 2022-04-28T19:47:05.264 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 186 (ObjNum 291 snap 83 seq_num 291) dirty exists 2022-04-28T19:47:05.264 INFO:tasks.rados.rados.0.smithi059.stdout:890: left oid 13 (ObjNum 291 snap 83 seq_num 291) 2022-04-28T19:47:05.267 INFO:tasks.rados.rados.0.smithi059.stdout:883: expect (ObjNum 195 snap 57 seq_num 195) 2022-04-28T19:47:05.445 INFO:tasks.rados.rados.0.smithi059.stdout:885: expect (ObjNum 250 snap 71 seq_num 250) 2022-04-28T19:47:05.488 INFO:tasks.rados.rados.0.smithi059.stdout:891: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:47:05.488 INFO:tasks.rados.rados.0.smithi059.stdout:891: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:47:05.488 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 432 (ObjNum 49 snap 0 seq_num 49) dirty exists 2022-04-28T19:47:05.488 INFO:tasks.rados.rados.0.smithi059.stdout:884: expect (ObjNum 267 snap 78 seq_num 267) 2022-04-28T19:47:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:05 smithi059 bash[29114]: cluster 2022-04-28T19:47:04.130637+0000 mgr.y (mgr.14140) 462 : cluster [DBG] pgmap v561: 89 pgs: 89 active+clean; 188 MiB data, 292 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.8 MiB/s wr, 10 op/s 2022-04-28T19:47:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:05 smithi059 bash[29114]: cluster 2022-04-28T19:47:04.219471+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2022-04-28T19:47:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:05 smithi059 bash[23786]: cluster 2022-04-28T19:47:04.130637+0000 mgr.y (mgr.14140) 462 : cluster [DBG] pgmap v561: 89 pgs: 89 active+clean; 188 MiB data, 292 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.8 MiB/s wr, 10 op/s 2022-04-28T19:47:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:05 smithi059 bash[23786]: cluster 2022-04-28T19:47:04.219471+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2022-04-28T19:47:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:05 smithi139 bash[23439]: cluster 2022-04-28T19:47:04.130637+0000 mgr.y (mgr.14140) 462 : cluster [DBG] pgmap v561: 89 pgs: 89 active+clean; 188 MiB data, 292 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.8 MiB/s wr, 10 op/s 2022-04-28T19:47:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:05 smithi139 bash[23439]: cluster 2022-04-28T19:47:04.219471+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2022-04-28T19:47:06.239 INFO:tasks.rados.rados.0.smithi059.stdout:883: done (7 left) 2022-04-28T19:47:06.239 INFO:tasks.rados.rados.0.smithi059.stdout:884: done (6 left) 2022-04-28T19:47:06.239 INFO:tasks.rados.rados.0.smithi059.stdout:885: done (5 left) 2022-04-28T19:47:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:887: done (4 left) 2022-04-28T19:47:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:890: done (3 left) 2022-04-28T19:47:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:891: done (2 left) 2022-04-28T19:47:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:892: done (1 left) 2022-04-28T19:47:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:893: done (0 left) 2022-04-28T19:47:06.240 INFO:tasks.rados.rados.0.smithi059.stdout:894: snap_remove snap 82 2022-04-28T19:47:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:06 smithi059 bash[29114]: cluster 2022-04-28T19:47:05.229289+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2022-04-28T19:47:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:06 smithi059 bash[23786]: cluster 2022-04-28T19:47:05.229289+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2022-04-28T19:47:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:06 smithi139 bash[23439]: cluster 2022-04-28T19:47:05.229289+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2022-04-28T19:47:07.209 INFO:tasks.rados.rados.0.smithi059.stdout:894: done (0 left) 2022-04-28T19:47:07.231 INFO:tasks.rados.rados.0.smithi059.stdout:895: rmattr oid 41 current snap is 84 2022-04-28T19:47:07.231 INFO:tasks.rados.rados.0.smithi059.stdout:895: done (0 left) 2022-04-28T19:47:07.232 INFO:tasks.rados.rados.0.smithi059.stdout:896: snap_remove snap 66 2022-04-28T19:47:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:07 smithi059 bash[29114]: cluster 2022-04-28T19:47:06.131456+0000 mgr.y (mgr.14140) 463 : cluster [DBG] pgmap v564: 89 pgs: 89 active+clean; 199 MiB data, 306 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s rd, 4.9 MiB/s wr, 12 op/s 2022-04-28T19:47:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:07 smithi059 bash[29114]: cluster 2022-04-28T19:47:06.235603+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2022-04-28T19:47:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:07 smithi059 bash[29114]: cluster 2022-04-28T19:47:07.204274+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2022-04-28T19:47:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:07 smithi059 bash[23786]: cluster 2022-04-28T19:47:06.131456+0000 mgr.y (mgr.14140) 463 : cluster [DBG] pgmap v564: 89 pgs: 89 active+clean; 199 MiB data, 306 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s rd, 4.9 MiB/s wr, 12 op/s 2022-04-28T19:47:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:07 smithi059 bash[23786]: cluster 2022-04-28T19:47:06.235603+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2022-04-28T19:47:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:07 smithi059 bash[23786]: cluster 2022-04-28T19:47:07.204274+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2022-04-28T19:47:07.629 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:07 smithi139 bash[23439]: cluster 2022-04-28T19:47:06.131456+0000 mgr.y (mgr.14140) 463 : cluster [DBG] pgmap v564: 89 pgs: 89 active+clean; 199 MiB data, 306 MiB used, 707 GiB / 715 GiB avail; 1.7 KiB/s rd, 4.9 MiB/s wr, 12 op/s 2022-04-28T19:47:07.629 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:07 smithi139 bash[23439]: cluster 2022-04-28T19:47:06.235603+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2022-04-28T19:47:07.629 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:07 smithi139 bash[23439]: cluster 2022-04-28T19:47:07.204274+0000 mon.a (mon.0) 835 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2022-04-28T19:47:08.224 INFO:tasks.rados.rados.0.smithi059.stdout:896: done (0 left) 2022-04-28T19:47:08.225 INFO:tasks.rados.rados.0.smithi059.stdout:897: setattr oid 11 current snap is 84 2022-04-28T19:47:08.227 INFO:tasks.rados.rados.0.smithi059.stdout:898: rmattr oid 31 current snap is 84 2022-04-28T19:47:08.227 INFO:tasks.rados.rados.0.smithi059.stdout:898: done (1 left) 2022-04-28T19:47:08.227 INFO:tasks.rados.rados.0.smithi059.stdout:899: rollback oid 42 current snap is 84 2022-04-28T19:47:08.227 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 70 2022-04-28T19:47:08.228 INFO:tasks.rados.rados.0.smithi059.stdout:900: rollback oid 38 current snap is 84 2022-04-28T19:47:08.228 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 83 2022-04-28T19:47:08.228 INFO:tasks.rados.rados.0.smithi059.stdout:901: snap_create 2022-04-28T19:47:08.236 INFO:tasks.rados.rados.0.smithi059.stdout:899: finishing rollback tid 0 to smithi05955964-42 2022-04-28T19:47:08.237 INFO:tasks.rados.rados.0.smithi059.stdout:899: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:47:08.237 INFO:tasks.rados.rados.0.smithi059.stdout:899: finishing rollback tid 2 to smithi05955964-42 2022-04-28T19:47:08.237 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 337 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:47:08.237 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 330 (ObjNum 252 snap 73 seq_num 252) dirty exists 2022-04-28T19:47:08.238 INFO:tasks.rados.rados.0.smithi059.stdout:900: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:47:08.243 INFO:tasks.rados.rados.0.smithi059.stdout:900: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:47:08.244 INFO:tasks.rados.rados.0.smithi059.stdout:900: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:47:08.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 334 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:897: done (3 left) 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:899: done (2 left) 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:900: done (1 left) 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:901: done (0 left) 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:902: rollback oid 43 current snap is 85 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 43 to 70 2022-04-28T19:47:09.225 INFO:tasks.rados.rados.0.smithi059.stdout:903: snap_create 2022-04-28T19:47:09.231 INFO:tasks.rados.rados.0.smithi059.stdout:902: finishing rollback tid 1 to smithi05955964-43 2022-04-28T19:47:09.232 INFO:tasks.rados.rados.0.smithi059.stdout:902: finishing rollback tid 2 to smithi05955964-43 2022-04-28T19:47:09.232 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 442 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:47:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:09 smithi139 bash[23439]: cluster 2022-04-28T19:47:08.132263+0000 mgr.y (mgr.14140) 464 : cluster [DBG] pgmap v567: 89 pgs: 89 active+clean; 209 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.4 MiB/s wr, 15 op/s 2022-04-28T19:47:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:09 smithi139 bash[23439]: cluster 2022-04-28T19:47:08.221871+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2022-04-28T19:47:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:09 smithi059 bash[29114]: cluster 2022-04-28T19:47:08.132263+0000 mgr.y (mgr.14140) 464 : cluster [DBG] pgmap v567: 89 pgs: 89 active+clean; 209 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.4 MiB/s wr, 15 op/s 2022-04-28T19:47:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:09 smithi059 bash[29114]: cluster 2022-04-28T19:47:08.221871+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2022-04-28T19:47:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:09 smithi059 bash[23786]: cluster 2022-04-28T19:47:08.132263+0000 mgr.y (mgr.14140) 464 : cluster [DBG] pgmap v567: 89 pgs: 89 active+clean; 209 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.4 MiB/s wr, 15 op/s 2022-04-28T19:47:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:09 smithi059 bash[23786]: cluster 2022-04-28T19:47:08.221871+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2022-04-28T19:47:10.239 INFO:tasks.rados.rados.0.smithi059.stdout:902: done (1 left) 2022-04-28T19:47:10.239 INFO:tasks.rados.rados.0.smithi059.stdout:903: done (0 left) 2022-04-28T19:47:10.240 INFO:tasks.rados.rados.0.smithi059.stdout:904: rmattr oid 29 current snap is 86 2022-04-28T19:47:10.240 INFO:tasks.rados.rados.0.smithi059.stdout:905: snap_create 2022-04-28T19:47:10.245 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 351 (ObjNum 250 snap 71 seq_num 250) dirty exists 2022-04-28T19:47:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[23786]: cluster 2022-04-28T19:47:09.223268+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2022-04-28T19:47:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[23786]: audit 2022-04-28T19:47:09.717595+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[23786]: audit 2022-04-28T19:47:09.719609+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.x"}]: dispatch 2022-04-28T19:47:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[23786]: audit 2022-04-28T19:47:09.721096+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:47:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[23786]: audit 2022-04-28T19:47:09.723023+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.x", "key": "container_image"}]: dispatch 2022-04-28T19:47:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[29114]: cluster 2022-04-28T19:47:09.223268+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2022-04-28T19:47:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[29114]: audit 2022-04-28T19:47:09.717595+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[29114]: audit 2022-04-28T19:47:09.719609+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.x"}]: dispatch 2022-04-28T19:47:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[29114]: audit 2022-04-28T19:47:09.721096+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:47:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:10 smithi059 bash[29114]: audit 2022-04-28T19:47:09.723023+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.x", "key": "container_image"}]: dispatch 2022-04-28T19:47:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:10 smithi139 bash[23439]: cluster 2022-04-28T19:47:09.223268+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2022-04-28T19:47:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:10 smithi139 bash[23439]: audit 2022-04-28T19:47:09.717595+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:10 smithi139 bash[23439]: audit 2022-04-28T19:47:09.719609+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mgr.x"}]: dispatch 2022-04-28T19:47:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:10 smithi139 bash[23439]: audit 2022-04-28T19:47:09.721096+0000 mon.a (mon.0) 840 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:47:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:10 smithi139 bash[23439]: audit 2022-04-28T19:47:09.723023+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mgr.x", "key": "container_image"}]: dispatch 2022-04-28T19:47:11.245 INFO:tasks.rados.rados.0.smithi059.stdout:904: done (1 left) 2022-04-28T19:47:11.246 INFO:tasks.rados.rados.0.smithi059.stdout:905: done (0 left) 2022-04-28T19:47:11.246 INFO:tasks.rados.rados.0.smithi059.stdout:906: delete oid 33 current snap is 87 2022-04-28T19:47:11.253 INFO:tasks.rados.rados.0.smithi059.stdout:906: done (0 left) 2022-04-28T19:47:11.253 INFO:tasks.rados.rados.0.smithi059.stdout:907: setattr oid 48 current snap is 87 2022-04-28T19:47:11.255 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 87 2022-04-28T19:47:11.255 INFO:tasks.rados.rados.0.smithi059.stdout:908: seq_num 293 ranges {2768896=114688} 2022-04-28T19:47:11.272 INFO:tasks.rados.rados.0.smithi059.stdout:908: writing smithi05955964-2 from 2768896 to 2883584 tid 1 2022-04-28T19:47:11.272 INFO:tasks.rados.rados.0.smithi059.stdout:909: read oid 37 snap -1 2022-04-28T19:47:11.272 INFO:tasks.rados.rados.0.smithi059.stdout:909: expect (ObjNum 286 snap 81 seq_num 286) 2022-04-28T19:47:11.272 INFO:tasks.rados.rados.0.smithi059.stdout:910: copy_from oid 40 from oid 33 current snap is 87 2022-04-28T19:47:11.273 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 368 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:47:11.273 INFO:tasks.rados.rados.0.smithi059.stdout:907: done (3 left) 2022-04-28T19:47:11.273 INFO:tasks.rados.rados.0.smithi059.stdout:911: snap_remove snap 63 2022-04-28T19:47:11.324 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:11 smithi139 bash[23439]: cephadm 2022-04-28T19:47:09.709197+0000 mgr.y (mgr.14140) 465 : cephadm [INF] It is presumed safe to stop ['mgr.x'] 2022-04-28T19:47:11.324 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:11 smithi139 bash[23439]: cephadm 2022-04-28T19:47:09.709383+0000 mgr.y (mgr.14140) 466 : cephadm [INF] Upgrade: It is presumed safe to stop ['mgr.x'] 2022-04-28T19:47:11.325 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:11 smithi139 bash[23439]: cephadm 2022-04-28T19:47:09.709537+0000 mgr.y (mgr.14140) 467 : cephadm [INF] Upgrade: Redeploying mgr.x 2022-04-28T19:47:11.325 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:11 smithi139 bash[23439]: cephadm 2022-04-28T19:47:09.722194+0000 mgr.y (mgr.14140) 468 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:47:11.325 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:11 smithi139 bash[23439]: cluster 2022-04-28T19:47:10.133111+0000 mgr.y (mgr.14140) 469 : cluster [DBG] pgmap v570: 89 pgs: 89 active+clean; 210 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 8.4 MiB/s rd, 26 KiB/s wr, 7 op/s 2022-04-28T19:47:11.325 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:11 smithi139 bash[23439]: cluster 2022-04-28T19:47:10.237304+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2022-04-28T19:47:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[29114]: cephadm 2022-04-28T19:47:09.709197+0000 mgr.y (mgr.14140) 465 : cephadm [INF] It is presumed safe to stop ['mgr.x'] 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[29114]: cephadm 2022-04-28T19:47:09.709383+0000 mgr.y (mgr.14140) 466 : cephadm [INF] Upgrade: It is presumed safe to stop ['mgr.x'] 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[29114]: cephadm 2022-04-28T19:47:09.709537+0000 mgr.y (mgr.14140) 467 : cephadm [INF] Upgrade: Redeploying mgr.x 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[29114]: cephadm 2022-04-28T19:47:09.722194+0000 mgr.y (mgr.14140) 468 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[29114]: cluster 2022-04-28T19:47:10.133111+0000 mgr.y (mgr.14140) 469 : cluster [DBG] pgmap v570: 89 pgs: 89 active+clean; 210 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 8.4 MiB/s rd, 26 KiB/s wr, 7 op/s 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[29114]: cluster 2022-04-28T19:47:10.237304+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[23786]: cephadm 2022-04-28T19:47:09.709197+0000 mgr.y (mgr.14140) 465 : cephadm [INF] It is presumed safe to stop ['mgr.x'] 2022-04-28T19:47:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[23786]: cephadm 2022-04-28T19:47:09.709383+0000 mgr.y (mgr.14140) 466 : cephadm [INF] Upgrade: It is presumed safe to stop ['mgr.x'] 2022-04-28T19:47:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[23786]: cephadm 2022-04-28T19:47:09.709537+0000 mgr.y (mgr.14140) 467 : cephadm [INF] Upgrade: Redeploying mgr.x 2022-04-28T19:47:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[23786]: cephadm 2022-04-28T19:47:09.722194+0000 mgr.y (mgr.14140) 468 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:47:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[23786]: cluster 2022-04-28T19:47:10.133111+0000 mgr.y (mgr.14140) 469 : cluster [DBG] pgmap v570: 89 pgs: 89 active+clean; 210 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 8.4 MiB/s rd, 26 KiB/s wr, 7 op/s 2022-04-28T19:47:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:11 smithi059 bash[23786]: cluster 2022-04-28T19:47:10.237304+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2022-04-28T19:47:12.209 INFO:tasks.rados.rados.0.smithi059.stdout:911: done (3 left) 2022-04-28T19:47:12.209 INFO:tasks.rados.rados.0.smithi059.stdout:912: setattr oid 27 current snap is 87 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:910: finishing copy_from to smithi05955964-40 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:910: got expected ENOENT (src dne) 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:910: finishing copy_from racing read to smithi05955964-40 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:908: finishing write tid 1 to smithi05955964-2 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:908: finishing write tid 3 to smithi05955964-2 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:908: finishing write tid 4 to smithi05955964-2 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 292 (ObjNum 293 snap 87 seq_num 293) dirty exists 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:908: left oid 2 (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:47:12.212 INFO:tasks.rados.rados.0.smithi059.stdout:909: expect (ObjNum 286 snap 81 seq_num 286) 2022-04-28T19:47:12.251 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:12 smithi139 systemd[1]: Stopping Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:47:12.360 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 375 (ObjNum 129 snap 30 seq_num 129) dirty exists 2022-04-28T19:47:12.360 INFO:tasks.rados.rados.0.smithi059.stdout:908: done (3 left) 2022-04-28T19:47:12.360 INFO:tasks.rados.rados.0.smithi059.stdout:909: done (2 left) 2022-04-28T19:47:12.360 INFO:tasks.rados.rados.0.smithi059.stdout:910: done (1 left) 2022-04-28T19:47:12.361 INFO:tasks.rados.rados.0.smithi059.stdout:912: done (0 left) 2022-04-28T19:47:12.361 INFO:tasks.rados.rados.0.smithi059.stdout:913: snap_create 2022-04-28T19:47:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:12 smithi059 bash[29114]: cluster 2022-04-28T19:47:11.244350+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2022-04-28T19:47:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:12 smithi059 bash[29114]: cluster 2022-04-28T19:47:12.206384+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2022-04-28T19:47:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:12 smithi059 bash[23786]: cluster 2022-04-28T19:47:11.244350+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2022-04-28T19:47:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:12 smithi059 bash[23786]: cluster 2022-04-28T19:47:12.206384+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2022-04-28T19:47:12.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:12 smithi139 bash[23439]: cluster 2022-04-28T19:47:11.244350+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2022-04-28T19:47:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:12 smithi139 bash[23439]: cluster 2022-04-28T19:47:12.206384+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2022-04-28T19:47:13.271 INFO:tasks.rados.rados.0.smithi059.stdout:913: done (0 left) 2022-04-28T19:47:13.374 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 88 2022-04-28T19:47:13.375 INFO:tasks.rados.rados.0.smithi059.stdout:914: seq_num 294 ranges {4546560=131072} 2022-04-28T19:47:13.376 INFO:tasks.rados.rados.0.smithi059.stdout:914: writing smithi05955964-37 from 4546560 to 4677632 tid 1 2022-04-28T19:47:13.376 INFO:tasks.rados.rados.0.smithi059.stdout:915: snap_create 2022-04-28T19:47:13.376 INFO:tasks.rados.rados.0.smithi059.stdout:914: finishing write tid 1 to smithi05955964-37 2022-04-28T19:47:13.376 INFO:tasks.rados.rados.0.smithi059.stdout:914: finishing write tid 3 to smithi05955964-37 2022-04-28T19:47:13.376 INFO:tasks.rados.rados.0.smithi059.stdout:914: finishing write tid 4 to smithi05955964-37 2022-04-28T19:47:13.377 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 344 (ObjNum 294 snap 88 seq_num 294) dirty exists 2022-04-28T19:47:13.377 INFO:tasks.rados.rados.0.smithi059.stdout:914: left oid 37 (ObjNum 294 snap 88 seq_num 294) 2022-04-28T19:47:13.520 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:13 smithi059 bash[29114]: cluster 2022-04-28T19:47:12.133867+0000 mgr.y (mgr.14140) 470 : cluster [DBG] pgmap v573: 89 pgs: 89 active+clean; 213 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.3 MiB/s rd, 0 B/s wr, 3 op/s 2022-04-28T19:47:13.521 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:13 smithi059 bash[23786]: cluster 2022-04-28T19:47:12.133867+0000 mgr.y (mgr.14140) 470 : cluster [DBG] pgmap v573: 89 pgs: 89 active+clean; 213 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.3 MiB/s rd, 0 B/s wr, 3 op/s 2022-04-28T19:47:13.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:13 smithi139 bash[23439]: cluster 2022-04-28T19:47:12.133867+0000 mgr.y (mgr.14140) 470 : cluster [DBG] pgmap v573: 89 pgs: 89 active+clean; 213 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.3 MiB/s rd, 0 B/s wr, 3 op/s 2022-04-28T19:47:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:13.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:47:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:13.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:47:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:13.519Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:47:14.274 INFO:tasks.rados.rados.0.smithi059.stdout:914: done (1 left) 2022-04-28T19:47:14.274 INFO:tasks.rados.rados.0.smithi059.stdout:915: done (0 left) 2022-04-28T19:47:14.274 INFO:tasks.rados.rados.0.smithi059.stdout:916: read oid 11 snap 83 2022-04-28T19:47:14.274 INFO:tasks.rados.rados.0.smithi059.stdout:916: expect (ObjNum 252 snap 73 seq_num 252) 2022-04-28T19:47:14.274 INFO:tasks.rados.rados.0.smithi059.stdout:917: snap_create 2022-04-28T19:47:14.292 INFO:tasks.rados.rados.0.smithi059.stdout:916: expect (ObjNum 252 snap 73 seq_num 252) 2022-04-28T19:47:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:14 smithi059 bash[23786]: cluster 2022-04-28T19:47:13.269410+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2022-04-28T19:47:14.680 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:14 smithi059 bash[29114]: cluster 2022-04-28T19:47:13.269410+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2022-04-28T19:47:14.680 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:14 smithi139 bash[23439]: cluster 2022-04-28T19:47:13.269410+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2022-04-28T19:47:15.283 INFO:tasks.rados.rados.0.smithi059.stdout:916: done (1 left) 2022-04-28T19:47:15.283 INFO:tasks.rados.rados.0.smithi059.stdout:917: done (0 left) 2022-04-28T19:47:15.283 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 90 2022-04-28T19:47:15.283 INFO:tasks.rados.rados.0.smithi059.stdout:918: seq_num 295 ranges {2351104=147456} 2022-04-28T19:47:15.300 INFO:tasks.rados.rados.0.smithi059.stdout:918: writing smithi05955964-10 from 2351104 to 2498560 tid 1 2022-04-28T19:47:15.300 INFO:tasks.rados.rados.0.smithi059.stdout:919: copy_from oid 5 from oid 16 current snap is 90 2022-04-28T19:47:15.300 INFO:tasks.rados.rados.0.smithi059.stdout:920: copy_from oid 36 from oid 3 current snap is 90 2022-04-28T19:47:15.300 INFO:tasks.rados.rados.0.smithi059.stdout:921: read oid 6 snap -1 2022-04-28T19:47:15.301 INFO:tasks.rados.rados.0.smithi059.stdout:921: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:47:15.301 INFO:tasks.rados.rados.0.smithi059.stdout:922: read oid 47 snap -1 2022-04-28T19:47:15.301 INFO:tasks.rados.rados.0.smithi059.stdout:922: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:47:15.301 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 90 2022-04-28T19:47:15.301 INFO:tasks.rados.rados.0.smithi059.stdout:923: seq_num 296 ranges {1130496=163840} 2022-04-28T19:47:15.307 INFO:tasks.rados.rados.0.smithi059.stdout:923: writing smithi05955964-42 from 1130496 to 1294336 tid 1 2022-04-28T19:47:15.307 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 90 2022-04-28T19:47:15.308 INFO:tasks.rados.rados.0.smithi059.stdout:924: seq_num 297 ranges {0=180224} 2022-04-28T19:47:15.309 INFO:tasks.rados.rados.0.smithi059.stdout:924: writing smithi05955964-7 from 0 to 180224 tid 1 2022-04-28T19:47:15.309 INFO:tasks.rados.rados.0.smithi059.stdout:925: read oid 37 snap -1 2022-04-28T19:47:15.309 INFO:tasks.rados.rados.0.smithi059.stdout:925: expect (ObjNum 294 snap 88 seq_num 294) 2022-04-28T19:47:15.309 INFO:tasks.rados.rados.0.smithi059.stdout:926: rmattr oid 43 current snap is 90 2022-04-28T19:47:15.310 INFO:tasks.rados.rados.0.smithi059.stdout:919: finishing copy_from to smithi05955964-5 2022-04-28T19:47:15.310 INFO:tasks.rados.rados.0.smithi059.stdout:919: got expected ENOENT (src dne) 2022-04-28T19:47:15.310 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 90 2022-04-28T19:47:15.310 INFO:tasks.rados.rados.0.smithi059.stdout:927: seq_num 298 ranges {2875392=196608} 2022-04-28T19:47:15.327 INFO:tasks.rados.rados.0.smithi059.stdout:927: writing smithi05955964-17 from 2875392 to 3072000 tid 1 2022-04-28T19:47:15.327 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 90 2022-04-28T19:47:15.327 INFO:tasks.rados.rados.0.smithi059.stdout:928: seq_num 299 ranges {0=212992} 2022-04-28T19:47:15.328 INFO:tasks.rados.rados.0.smithi059.stdout:928: writing smithi05955964-31 from 0 to 212992 tid 1 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:929: rmattr oid 15 current snap is 90 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:929: done (11 left) 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:930: snap_create 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:919: finishing copy_from racing read to smithi05955964-5 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:920: finishing copy_from to smithi05955964-36 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:920: got expected ENOENT (src dne) 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:920: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:47:15.329 INFO:tasks.rados.rados.0.smithi059.stdout:918: finishing write tid 1 to smithi05955964-10 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:918: finishing write tid 3 to smithi05955964-10 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 452 (ObjNum 206 snap 59 seq_num 206) dirty exists 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:924: finishing write tid 1 to smithi05955964-7 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:924: finishing write tid 3 to smithi05955964-7 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:923: finishing write tid 1 to smithi05955964-42 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:923: finishing write tid 3 to smithi05955964-42 2022-04-28T19:47:15.330 INFO:tasks.rados.rados.0.smithi059.stdout:922: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:47:15.416 INFO:tasks.rados.rados.0.smithi059.stdout:921: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:47:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:918: finishing write tid 4 to smithi05955964-10 2022-04-28T19:47:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 384 (ObjNum 295 snap 90 seq_num 295) dirty exists 2022-04-28T19:47:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:918: left oid 10 (ObjNum 295 snap 90 seq_num 295) 2022-04-28T19:47:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:924: finishing write tid 4 to smithi05955964-7 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 475 (ObjNum 297 snap 90 seq_num 297) dirty exists 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:924: left oid 7 (ObjNum 297 snap 90 seq_num 297) 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:927: finishing write tid 1 to smithi05955964-17 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:927: finishing write tid 3 to smithi05955964-17 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:927: finishing write tid 4 to smithi05955964-17 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 195 (ObjNum 298 snap 90 seq_num 298) dirty exists 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:927: left oid 17 (ObjNum 298 snap 90 seq_num 298) 2022-04-28T19:47:15.493 INFO:tasks.rados.rados.0.smithi059.stdout:925: expect (ObjNum 294 snap 88 seq_num 294) 2022-04-28T19:47:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:15 smithi059 bash[29114]: cluster 2022-04-28T19:47:14.134646+0000 mgr.y (mgr.14140) 471 : cluster [DBG] pgmap v576: 89 pgs: 89 active+clean; 213 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 255 B/s rd, 28 KiB/s wr, 7 op/s 2022-04-28T19:47:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:15 smithi059 bash[29114]: cluster 2022-04-28T19:47:14.271022+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2022-04-28T19:47:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:15 smithi059 bash[23786]: cluster 2022-04-28T19:47:14.134646+0000 mgr.y (mgr.14140) 471 : cluster [DBG] pgmap v576: 89 pgs: 89 active+clean; 213 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 255 B/s rd, 28 KiB/s wr, 7 op/s 2022-04-28T19:47:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:15 smithi059 bash[23786]: cluster 2022-04-28T19:47:14.271022+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2022-04-28T19:47:15.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:15 smithi139 bash[23439]: cluster 2022-04-28T19:47:14.134646+0000 mgr.y (mgr.14140) 471 : cluster [DBG] pgmap v576: 89 pgs: 89 active+clean; 213 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 255 B/s rd, 28 KiB/s wr, 7 op/s 2022-04-28T19:47:15.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:15 smithi139 bash[23439]: cluster 2022-04-28T19:47:14.271022+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2022-04-28T19:47:15.709 INFO:tasks.rados.rados.0.smithi059.stdout:923: finishing write tid 4 to smithi05955964-42 2022-04-28T19:47:15.709 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 347 (ObjNum 296 snap 90 seq_num 296) dirty exists 2022-04-28T19:47:15.709 INFO:tasks.rados.rados.0.smithi059.stdout:923: left oid 42 (ObjNum 296 snap 90 seq_num 296) 2022-04-28T19:47:15.709 INFO:tasks.rados.rados.0.smithi059.stdout:928: finishing write tid 1 to smithi05955964-31 2022-04-28T19:47:15.710 INFO:tasks.rados.rados.0.smithi059.stdout:928: finishing write tid 3 to smithi05955964-31 2022-04-28T19:47:15.710 INFO:tasks.rados.rados.0.smithi059.stdout:928: finishing write tid 4 to smithi05955964-31 2022-04-28T19:47:15.710 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 477 (ObjNum 299 snap 90 seq_num 299) dirty exists 2022-04-28T19:47:15.710 INFO:tasks.rados.rados.0.smithi059.stdout:928: left oid 31 (ObjNum 299 snap 90 seq_num 299) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:918: done (11 left) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:919: done (10 left) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:920: done (9 left) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:921: done (8 left) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:922: done (7 left) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:923: done (6 left) 2022-04-28T19:47:16.296 INFO:tasks.rados.rados.0.smithi059.stdout:924: done (5 left) 2022-04-28T19:47:16.297 INFO:tasks.rados.rados.0.smithi059.stdout:925: done (4 left) 2022-04-28T19:47:16.297 INFO:tasks.rados.rados.0.smithi059.stdout:926: done (3 left) 2022-04-28T19:47:16.297 INFO:tasks.rados.rados.0.smithi059.stdout:927: done (2 left) 2022-04-28T19:47:16.297 INFO:tasks.rados.rados.0.smithi059.stdout:928: done (1 left) 2022-04-28T19:47:16.297 INFO:tasks.rados.rados.0.smithi059.stdout:930: done (0 left) 2022-04-28T19:47:16.297 INFO:tasks.rados.rados.0.smithi059.stdout:931: snap_create 2022-04-28T19:47:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:16 smithi059 bash[29114]: cluster 2022-04-28T19:47:15.280798+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2022-04-28T19:47:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:16 smithi059 bash[29114]: audit 2022-04-28T19:47:16.166662+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:16 smithi059 bash[29114]: audit 2022-04-28T19:47:16.168165+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:16 smithi059 bash[23786]: cluster 2022-04-28T19:47:15.280798+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2022-04-28T19:47:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:16 smithi059 bash[23786]: audit 2022-04-28T19:47:16.166662+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:16 smithi059 bash[23786]: audit 2022-04-28T19:47:16.168165+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:16 smithi139 bash[23439]: cluster 2022-04-28T19:47:15.280798+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2022-04-28T19:47:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:16 smithi139 bash[23439]: audit 2022-04-28T19:47:16.166662+0000 mon.a (mon.0) 848 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:16 smithi139 bash[23439]: audit 2022-04-28T19:47:16.168165+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:17.211 INFO:tasks.rados.rados.0.smithi059.stdout:931: done (0 left) 2022-04-28T19:47:17.211 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 92 2022-04-28T19:47:17.211 INFO:tasks.rados.rados.0.smithi059.stdout:932: seq_num 300 ranges {3072000=229376} 2022-04-28T19:47:17.229 INFO:tasks.rados.rados.0.smithi059.stdout:932: writing smithi05955964-17 from 3072000 to 3301376 tid 1 2022-04-28T19:47:17.230 INFO:tasks.rados.rados.0.smithi059.stdout:933: delete oid 15 current snap is 92 2022-04-28T19:47:17.234 INFO:tasks.rados.rados.0.smithi059.stdout:932: finishing write tid 1 to smithi05955964-17 2022-04-28T19:47:17.234 INFO:tasks.rados.rados.0.smithi059.stdout:932: finishing write tid 3 to smithi05955964-17 2022-04-28T19:47:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:933: done (1 left) 2022-04-28T19:47:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 92 2022-04-28T19:47:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:934: seq_num 302 ranges {2809856=262144} 2022-04-28T19:47:17.254 INFO:tasks.rados.rados.0.smithi059.stdout:934: writing smithi05955964-27 from 2809856 to 3072000 tid 1 2022-04-28T19:47:17.254 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 92 2022-04-28T19:47:17.254 INFO:tasks.rados.rados.0.smithi059.stdout:935: seq_num 303 ranges {2498560=278528} 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:935: writing smithi05955964-10 from 2498560 to 2777088 tid 1 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:936: delete oid 32 current snap is 92 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:932: finishing write tid 4 to smithi05955964-17 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 198 (ObjNum 300 snap 92 seq_num 300) dirty exists 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:932: left oid 17 (ObjNum 300 snap 92 seq_num 300) 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:934: finishing write tid 1 to smithi05955964-27 2022-04-28T19:47:17.271 INFO:tasks.rados.rados.0.smithi059.stdout:934: finishing write tid 3 to smithi05955964-27 2022-04-28T19:47:17.272 INFO:tasks.rados.rados.0.smithi059.stdout:934: finishing write tid 4 to smithi05955964-27 2022-04-28T19:47:17.272 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 387 (ObjNum 302 snap 92 seq_num 302) dirty exists 2022-04-28T19:47:17.272 INFO:tasks.rados.rados.0.smithi059.stdout:934: left oid 27 (ObjNum 302 snap 92 seq_num 302) 2022-04-28T19:47:17.273 INFO:tasks.rados.rados.0.smithi059.stdout:932: done (3 left) 2022-04-28T19:47:17.273 INFO:tasks.rados.rados.0.smithi059.stdout:934: done (2 left) 2022-04-28T19:47:17.273 INFO:tasks.rados.rados.0.smithi059.stdout:936: done (1 left) 2022-04-28T19:47:17.273 INFO:tasks.rados.rados.0.smithi059.stdout:937: read oid 41 snap -1 2022-04-28T19:47:17.274 INFO:tasks.rados.rados.0.smithi059.stdout:937: expect (ObjNum 280 snap 80 seq_num 280) 2022-04-28T19:47:17.274 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 92 2022-04-28T19:47:17.274 INFO:tasks.rados.rados.0.smithi059.stdout:938: seq_num 305 ranges {1794048=311296} 2022-04-28T19:47:17.284 INFO:tasks.rados.rados.0.smithi059.stdout:938: writing smithi05955964-29 from 1794048 to 2105344 tid 1 2022-04-28T19:47:17.284 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 92 2022-04-28T19:47:17.284 INFO:tasks.rados.rados.0.smithi059.stdout:939: seq_num 306 ranges {2899968=327680} 2022-04-28T19:47:17.303 INFO:tasks.rados.rados.0.smithi059.stdout:939: writing smithi05955964-39 from 2899968 to 3227648 tid 1 2022-04-28T19:47:17.303 INFO:tasks.rados.rados.0.smithi059.stdout:940: delete oid 46 current snap is 92 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:935: finishing write tid 1 to smithi05955964-10 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:935: finishing write tid 3 to smithi05955964-10 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:935: finishing write tid 4 to smithi05955964-10 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 390 (ObjNum 303 snap 92 seq_num 303) dirty exists 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:935: left oid 10 (ObjNum 303 snap 92 seq_num 303) 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:938: finishing write tid 1 to smithi05955964-29 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:938: finishing write tid 3 to smithi05955964-29 2022-04-28T19:47:17.304 INFO:tasks.rados.rados.0.smithi059.stdout:938: finishing write tid 4 to smithi05955964-29 2022-04-28T19:47:17.305 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 358 (ObjNum 305 snap 92 seq_num 305) dirty exists 2022-04-28T19:47:17.305 INFO:tasks.rados.rados.0.smithi059.stdout:938: left oid 29 (ObjNum 305 snap 92 seq_num 305) 2022-04-28T19:47:17.305 INFO:tasks.rados.rados.0.smithi059.stdout:937: expect (ObjNum 280 snap 80 seq_num 280) 2022-04-28T19:47:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:17 smithi059 bash[29114]: cluster 2022-04-28T19:47:16.135371+0000 mgr.y (mgr.14140) 472 : cluster [DBG] pgmap v579: 89 pgs: 89 active+clean; 211 MiB data, 307 MiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 60 KiB/s wr, 18 op/s 2022-04-28T19:47:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:17 smithi059 bash[29114]: cluster 2022-04-28T19:47:16.293668+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2022-04-28T19:47:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:17 smithi059 bash[29114]: cluster 2022-04-28T19:47:17.208556+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2022-04-28T19:47:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:17 smithi059 bash[23786]: cluster 2022-04-28T19:47:16.135371+0000 mgr.y (mgr.14140) 472 : cluster [DBG] pgmap v579: 89 pgs: 89 active+clean; 211 MiB data, 307 MiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 60 KiB/s wr, 18 op/s 2022-04-28T19:47:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:17 smithi059 bash[23786]: cluster 2022-04-28T19:47:16.293668+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2022-04-28T19:47:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:17 smithi059 bash[23786]: cluster 2022-04-28T19:47:17.208556+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2022-04-28T19:47:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:17 smithi139 bash[23439]: cluster 2022-04-28T19:47:16.135371+0000 mgr.y (mgr.14140) 472 : cluster [DBG] pgmap v579: 89 pgs: 89 active+clean; 211 MiB data, 307 MiB used, 707 GiB / 715 GiB avail; 3.3 MiB/s rd, 60 KiB/s wr, 18 op/s 2022-04-28T19:47:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:17 smithi139 bash[23439]: cluster 2022-04-28T19:47:16.293668+0000 mon.a (mon.0) 850 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2022-04-28T19:47:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:17 smithi139 bash[23439]: cluster 2022-04-28T19:47:17.208556+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2022-04-28T19:47:17.627 INFO:tasks.rados.rados.0.smithi059.stdout:939: finishing write tid 1 to smithi05955964-39 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:939: finishing write tid 3 to smithi05955964-39 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:939: finishing write tid 4 to smithi05955964-39 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 456 (ObjNum 306 snap 92 seq_num 306) dirty exists 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:939: left oid 39 (ObjNum 306 snap 92 seq_num 306) 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:935: done (4 left) 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:937: done (3 left) 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:938: done (2 left) 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:939: done (1 left) 2022-04-28T19:47:17.628 INFO:tasks.rados.rados.0.smithi059.stdout:940: done (0 left) 2022-04-28T19:47:17.629 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 92 2022-04-28T19:47:17.629 INFO:tasks.rados.rados.0.smithi059.stdout:941: seq_num 308 ranges {827392=360448} 2022-04-28T19:47:17.640 INFO:tasks.rados.rados.0.smithi059.stdout:941: writing smithi05955964-50 from 827392 to 1187840 tid 1 2022-04-28T19:47:17.640 INFO:tasks.rados.rados.0.smithi059.stdout:942: copy_from oid 44 from oid 47 current snap is 92 2022-04-28T19:47:17.640 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 92 2022-04-28T19:47:17.640 INFO:tasks.rados.rados.0.smithi059.stdout:943: seq_num 309 ranges {221184=376832} 2022-04-28T19:47:17.644 INFO:tasks.rados.rados.0.smithi059.stdout:943: writing smithi05955964-5 from 221184 to 598016 tid 1 2022-04-28T19:47:17.644 INFO:tasks.rados.rados.0.smithi059.stdout:944: delete oid 18 current snap is 92 2022-04-28T19:47:17.649 INFO:tasks.rados.rados.0.smithi059.stdout:941: finishing write tid 1 to smithi05955964-50 2022-04-28T19:47:17.649 INFO:tasks.rados.rados.0.smithi059.stdout:941: finishing write tid 3 to smithi05955964-50 2022-04-28T19:47:17.649 INFO:tasks.rados.rados.0.smithi059.stdout:944: done (3 left) 2022-04-28T19:47:17.649 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 92 2022-04-28T19:47:17.649 INFO:tasks.rados.rados.0.smithi059.stdout:945: seq_num 311 ranges {1998848=417792} 2022-04-28T19:47:17.663 INFO:tasks.rados.rados.0.smithi059.stdout:945: writing smithi05955964-22 from 1998848 to 2416640 tid 1 2022-04-28T19:47:17.663 INFO:tasks.rados.rados.0.smithi059.stdout:946: snap_create 2022-04-28T19:47:17.663 INFO:tasks.rados.rados.0.smithi059.stdout:943: finishing write tid 1 to smithi05955964-5 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:943: finishing write tid 3 to smithi05955964-5 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:941: finishing write tid 4 to smithi05955964-50 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 459 (ObjNum 308 snap 92 seq_num 308) dirty exists 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:941: left oid 50 (ObjNum 308 snap 92 seq_num 308) 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:943: finishing write tid 4 to smithi05955964-5 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 363 (ObjNum 309 snap 92 seq_num 309) dirty exists 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:943: left oid 5 (ObjNum 309 snap 92 seq_num 309) 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:942: finishing copy_from to smithi05955964-44 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 289 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:47:17.664 INFO:tasks.rados.rados.0.smithi059.stdout:942: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:47:17.666 INFO:tasks.rados.rados.0.smithi059.stdout:945: finishing write tid 1 to smithi05955964-22 2022-04-28T19:47:17.666 INFO:tasks.rados.rados.0.smithi059.stdout:945: finishing write tid 3 to smithi05955964-22 2022-04-28T19:47:17.670 INFO:tasks.rados.rados.0.smithi059.stdout:945: finishing write tid 4 to smithi05955964-22 2022-04-28T19:47:17.670 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 379 (ObjNum 311 snap 92 seq_num 311) dirty exists 2022-04-28T19:47:17.670 INFO:tasks.rados.rados.0.smithi059.stdout:945: left oid 22 (ObjNum 311 snap 92 seq_num 311) 2022-04-28T19:47:18.320 INFO:tasks.rados.rados.0.smithi059.stdout:941: done (4 left) 2022-04-28T19:47:18.320 INFO:tasks.rados.rados.0.smithi059.stdout:942: done (3 left) 2022-04-28T19:47:18.320 INFO:tasks.rados.rados.0.smithi059.stdout:943: done (2 left) 2022-04-28T19:47:18.320 INFO:tasks.rados.rados.0.smithi059.stdout:945: done (1 left) 2022-04-28T19:47:18.320 INFO:tasks.rados.rados.0.smithi059.stdout:946: done (0 left) 2022-04-28T19:47:18.321 INFO:tasks.rados.rados.0.smithi059.stdout:947: snap_remove snap 80 2022-04-28T19:47:19.322 INFO:tasks.rados.rados.0.smithi059.stdout:947: done (0 left) 2022-04-28T19:47:19.323 INFO:tasks.rados.rados.0.smithi059.stdout:948: rmattr oid 46 current snap is 93 2022-04-28T19:47:19.323 INFO:tasks.rados.rados.0.smithi059.stdout:948: done (0 left) 2022-04-28T19:47:19.323 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 93 2022-04-28T19:47:19.323 INFO:tasks.rados.rados.0.smithi059.stdout:949: seq_num 312 ranges {2580480=434176} 2022-04-28T19:47:19.341 INFO:tasks.rados.rados.0.smithi059.stdout:949: writing smithi05955964-21 from 2580480 to 3014656 tid 1 2022-04-28T19:47:19.341 INFO:tasks.rados.rados.0.smithi059.stdout:950: delete oid 39 current snap is 93 2022-04-28T19:47:19.347 INFO:tasks.rados.rados.0.smithi059.stdout:949: finishing write tid 1 to smithi05955964-21 2022-04-28T19:47:19.347 INFO:tasks.rados.rados.0.smithi059.stdout:949: finishing write tid 3 to smithi05955964-21 2022-04-28T19:47:19.348 INFO:tasks.rados.rados.0.smithi059.stdout:950: done (1 left) 2022-04-28T19:47:19.348 INFO:tasks.rados.rados.0.smithi059.stdout:951: read oid 49 snap -1 2022-04-28T19:47:19.348 INFO:tasks.rados.rados.0.smithi059.stdout:951: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:47:19.348 INFO:tasks.rados.rados.0.smithi059.stdout:952: delete oid 30 current snap is 93 2022-04-28T19:47:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:949: finishing write tid 4 to smithi05955964-21 2022-04-28T19:47:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 349 (ObjNum 312 snap 93 seq_num 312) dirty exists 2022-04-28T19:47:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:949: left oid 21 (ObjNum 312 snap 93 seq_num 312) 2022-04-28T19:47:19.354 INFO:tasks.rados.rados.0.smithi059.stdout:949: done (2 left) 2022-04-28T19:47:19.354 INFO:tasks.rados.rados.0.smithi059.stdout:952: done (1 left) 2022-04-28T19:47:19.354 INFO:tasks.rados.rados.0.smithi059.stdout:953: snap_remove snap 92 2022-04-28T19:47:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:19 smithi139 bash[23439]: cluster 2022-04-28T19:47:18.136250+0000 mgr.y (mgr.14140) 473 : cluster [DBG] pgmap v582: 89 pgs: 89 active+clean; 207 MiB data, 306 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 80 KiB/s wr, 15 op/s 2022-04-28T19:47:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:19 smithi139 bash[23439]: cluster 2022-04-28T19:47:18.317139+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2022-04-28T19:47:19.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:19 smithi059 bash[23786]: cluster 2022-04-28T19:47:18.136250+0000 mgr.y (mgr.14140) 473 : cluster [DBG] pgmap v582: 89 pgs: 89 active+clean; 207 MiB data, 306 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 80 KiB/s wr, 15 op/s 2022-04-28T19:47:19.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:19 smithi059 bash[23786]: cluster 2022-04-28T19:47:18.317139+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2022-04-28T19:47:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:19 smithi059 bash[29114]: cluster 2022-04-28T19:47:18.136250+0000 mgr.y (mgr.14140) 473 : cluster [DBG] pgmap v582: 89 pgs: 89 active+clean; 207 MiB data, 306 MiB used, 707 GiB / 715 GiB avail; 6.7 MiB/s rd, 80 KiB/s wr, 15 op/s 2022-04-28T19:47:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:19 smithi059 bash[29114]: cluster 2022-04-28T19:47:18.317139+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2022-04-28T19:47:20.358 INFO:tasks.rados.rados.0.smithi059.stdout:953: done (1 left) 2022-04-28T19:47:20.358 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 93 2022-04-28T19:47:20.359 INFO:tasks.rados.rados.0.smithi059.stdout:954: seq_num 315 ranges {4055040=483328} 2022-04-28T19:47:20.387 INFO:tasks.rados.rados.0.smithi059.stdout:954: writing smithi05955964-8 from 4055040 to 4538368 tid 1 2022-04-28T19:47:20.388 INFO:tasks.rados.rados.0.smithi059.stdout:955: read oid 32 snap -1 2022-04-28T19:47:20.388 INFO:tasks.rados.rados.0.smithi059.stdout:955: expect deleted 2022-04-28T19:47:20.388 INFO:tasks.rados.rados.0.smithi059.stdout:956: read oid 48 snap -1 2022-04-28T19:47:20.388 INFO:tasks.rados.rados.0.smithi059.stdout:956: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:47:20.388 INFO:tasks.rados.rados.0.smithi059.stdout:957: delete oid 25 current snap is 93 2022-04-28T19:47:20.388 INFO:tasks.rados.rados.0.smithi059.stdout:951: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:47:20.424 INFO:tasks.rados.rados.0.smithi059.stdout:956: expect (ObjNum 88 snap 18 seq_num 88) 2022-04-28T19:47:20.460 INFO:tasks.rados.rados.0.smithi059.stdout:954: finishing write tid 1 to smithi05955964-8 2022-04-28T19:47:20.460 INFO:tasks.rados.rados.0.smithi059.stdout:954: finishing write tid 3 to smithi05955964-8 2022-04-28T19:47:20.460 INFO:tasks.rados.rados.0.smithi059.stdout:954: finishing write tid 4 to smithi05955964-8 2022-04-28T19:47:20.460 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 372 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:47:20.460 INFO:tasks.rados.rados.0.smithi059.stdout:954: left oid 8 (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:47:20.464 INFO:tasks.rados.rados.0.smithi059.stdout:951: done (4 left) 2022-04-28T19:47:20.464 INFO:tasks.rados.rados.0.smithi059.stdout:954: done (3 left) 2022-04-28T19:47:20.464 INFO:tasks.rados.rados.0.smithi059.stdout:955: done (2 left) 2022-04-28T19:47:20.464 INFO:tasks.rados.rados.0.smithi059.stdout:956: done (1 left) 2022-04-28T19:47:20.464 INFO:tasks.rados.rados.0.smithi059.stdout:957: done (0 left) 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:958: rollback oid 10 current snap is 93 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 71 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:959: rollback oid 22 current snap is 93 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 70 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:960: rmattr oid 46 current snap is 93 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:960: done (2 left) 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:961: read oid 3 snap -1 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:961: expect deleted 2022-04-28T19:47:20.465 INFO:tasks.rados.rados.0.smithi059.stdout:962: snap_create 2022-04-28T19:47:20.467 INFO:tasks.rados.rados.0.smithi059.stdout:958: finishing rollback tid 0 to smithi05955964-10 2022-04-28T19:47:20.467 INFO:tasks.rados.rados.0.smithi059.stdout:958: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:47:20.468 INFO:tasks.rados.rados.0.smithi059.stdout:959: finishing rollback tid 0 to smithi05955964-22 2022-04-28T19:47:20.468 INFO:tasks.rados.rados.0.smithi059.stdout:959: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:47:20.469 INFO:tasks.rados.rados.0.smithi059.stdout:958: finishing rollback tid 2 to smithi05955964-10 2022-04-28T19:47:20.469 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 395 (ObjNum 245 snap 66 seq_num 245) dirty exists 2022-04-28T19:47:20.469 INFO:tasks.rados.rados.0.smithi059.stdout:959: finishing rollback tid 2 to smithi05955964-22 2022-04-28T19:47:20.469 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 386 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:47:20.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:20 smithi139 bash[23439]: cluster 2022-04-28T19:47:19.319840+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2022-04-28T19:47:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:20 smithi059 bash[29114]: cluster 2022-04-28T19:47:19.319840+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2022-04-28T19:47:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:20 smithi059 bash[23786]: cluster 2022-04-28T19:47:19.319840+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2022-04-28T19:47:21.361 INFO:tasks.rados.rados.0.smithi059.stdout:958: done (3 left) 2022-04-28T19:47:21.361 INFO:tasks.rados.rados.0.smithi059.stdout:959: done (2 left) 2022-04-28T19:47:21.362 INFO:tasks.rados.rados.0.smithi059.stdout:961: done (1 left) 2022-04-28T19:47:21.362 INFO:tasks.rados.rados.0.smithi059.stdout:962: done (0 left) 2022-04-28T19:47:21.362 INFO:tasks.rados.rados.0.smithi059.stdout:963: snap_remove snap 87 2022-04-28T19:47:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:21 smithi139 bash[23439]: cluster 2022-04-28T19:47:20.137022+0000 mgr.y (mgr.14140) 474 : cluster [DBG] pgmap v585: 89 pgs: 89 active+clean; 211 MiB data, 311 MiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2022-04-28T19:47:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:21 smithi139 bash[23439]: cluster 2022-04-28T19:47:20.356966+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2022-04-28T19:47:21.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:21 smithi059 bash[23786]: cluster 2022-04-28T19:47:20.137022+0000 mgr.y (mgr.14140) 474 : cluster [DBG] pgmap v585: 89 pgs: 89 active+clean; 211 MiB data, 311 MiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2022-04-28T19:47:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:21 smithi059 bash[23786]: cluster 2022-04-28T19:47:20.356966+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2022-04-28T19:47:21.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:21 smithi059 bash[29114]: cluster 2022-04-28T19:47:20.137022+0000 mgr.y (mgr.14140) 474 : cluster [DBG] pgmap v585: 89 pgs: 89 active+clean; 211 MiB data, 311 MiB used, 707 GiB / 715 GiB avail; 17 MiB/s rd, 1.0 MiB/s wr, 14 op/s 2022-04-28T19:47:21.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:21 smithi059 bash[29114]: cluster 2022-04-28T19:47:20.356966+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2022-04-28T19:47:22.218 INFO:tasks.rados.rados.0.smithi059.stdout:963: done (0 left) 2022-04-28T19:47:22.218 INFO:tasks.rados.rados.0.smithi059.stdout:964: read oid 47 snap -1 2022-04-28T19:47:22.218 INFO:tasks.rados.rados.0.smithi059.stdout:964: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:47:22.218 INFO:tasks.rados.rados.0.smithi059.stdout:965: copy_from oid 21 from oid 5 current snap is 94 2022-04-28T19:47:22.219 INFO:tasks.rados.rados.0.smithi059.stdout:966: snap_create 2022-04-28T19:47:22.241 INFO:tasks.rados.rados.0.smithi059.stdout:964: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:47:22.328 INFO:tasks.rados.rados.0.smithi059.stdout:965: finishing copy_from to smithi05955964-21 2022-04-28T19:47:22.328 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 363 (ObjNum 309 snap 92 seq_num 309) dirty exists 2022-04-28T19:47:22.328 INFO:tasks.rados.rados.0.smithi059.stdout:965: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:47:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:22 smithi139 bash[23439]: cluster 2022-04-28T19:47:21.359678+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2022-04-28T19:47:22.722 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:22 smithi139 bash[23439]: cluster 2022-04-28T19:47:22.214604+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2022-04-28T19:47:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:22 smithi059 bash[23786]: cluster 2022-04-28T19:47:21.359678+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2022-04-28T19:47:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:22 smithi059 bash[23786]: cluster 2022-04-28T19:47:22.214604+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2022-04-28T19:47:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:22 smithi059 bash[29114]: cluster 2022-04-28T19:47:21.359678+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2022-04-28T19:47:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:22 smithi059 bash[29114]: cluster 2022-04-28T19:47:22.214604+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2022-04-28T19:47:23.229 INFO:tasks.rados.rados.0.smithi059.stdout:964: done (2 left) 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:965: done (1 left) 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:966: done (0 left) 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:967: rollback oid 44 current snap is 95 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 44 to 85 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:968: copy_from oid 18 from oid 38 current snap is 95 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 95 2022-04-28T19:47:23.230 INFO:tasks.rados.rados.0.smithi059.stdout:969: seq_num 317 ranges {0=516096} 2022-04-28T19:47:23.233 INFO:tasks.rados.rados.0.smithi059.stdout:969: writing smithi05955964-3 from 0 to 516096 tid 1 2022-04-28T19:47:23.233 INFO:tasks.rados.rados.0.smithi059.stdout:970: snap_remove snap 91 2022-04-28T19:47:23.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:23 smithi139 docker[37498]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr.x 2022-04-28T19:47:23.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:23 smithi139 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=137/n/a 2022-04-28T19:47:23.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:23 smithi139 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2022-04-28T19:47:23.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:23 smithi139 systemd[1]: Stopped Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:47:23.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:23 smithi139 systemd[1]: Started Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:47:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[23786]: cluster 2022-04-28T19:47:22.137939+0000 mgr.y (mgr.14140) 475 : cluster [DBG] pgmap v588: 89 pgs: 89 active+clean; 213 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 1.6 MiB/s wr, 16 op/s 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[23786]: audit 2022-04-28T19:47:23.192777+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[23786]: audit 2022-04-28T19:47:23.194125+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[23786]: cluster 2022-04-28T19:47:23.225716+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[29114]: cluster 2022-04-28T19:47:22.137939+0000 mgr.y (mgr.14140) 475 : cluster [DBG] pgmap v588: 89 pgs: 89 active+clean; 213 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 1.6 MiB/s wr, 16 op/s 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[29114]: audit 2022-04-28T19:47:23.192777+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[29114]: audit 2022-04-28T19:47:23.194125+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[29114]: cluster 2022-04-28T19:47:23.225716+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2022-04-28T19:47:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:23.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:47:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:23.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:47:23.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:23.519Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:47:23.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:23 smithi139 bash[23439]: cluster 2022-04-28T19:47:22.137939+0000 mgr.y (mgr.14140) 475 : cluster [DBG] pgmap v588: 89 pgs: 89 active+clean; 213 MiB data, 310 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 1.6 MiB/s wr, 16 op/s 2022-04-28T19:47:23.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:23 smithi139 bash[23439]: audit 2022-04-28T19:47:23.192777+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:23.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:23 smithi139 bash[23439]: audit 2022-04-28T19:47:23.194125+0000 mon.a (mon.0) 858 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:23.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:23 smithi139 bash[23439]: cluster 2022-04-28T19:47:23.225716+0000 mon.a (mon.0) 859 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:970: done (3 left) 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:971: snap_create 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:967: finishing rollback tid 0 to smithi05955964-44 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:967: finishing rollback tid 1 to smithi05955964-44 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:967: finishing rollback tid 2 to smithi05955964-44 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 292 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:47:24.233 INFO:tasks.rados.rados.0.smithi059.stdout:969: finishing write tid 1 to smithi05955964-3 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:969: finishing write tid 3 to smithi05955964-3 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:969: finishing write tid 4 to smithi05955964-3 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 365 (ObjNum 317 snap 95 seq_num 317) dirty exists 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:969: left oid 3 (ObjNum 317 snap 95 seq_num 317) 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:968: finishing copy_from to smithi05955964-18 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 334 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:24.234 INFO:tasks.rados.rados.0.smithi059.stdout:968: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:47:24.582 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[37584]: debug 2022-04-28T19:47:24.517+0000 7fdd8f979700 1 -- 172.21.15.139:0/3050397172 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564fd4d824e0 con 0x564fd4e74c00 2022-04-28T19:47:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[23786]: audit 2022-04-28T19:47:23.577929+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[23786]: audit 2022-04-28T19:47:23.579889+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[23786]: audit 2022-04-28T19:47:23.594908+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[23786]: cephadm 2022-04-28T19:47:23.595644+0000 mgr.y (mgr.14140) 476 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[23786]: cluster 2022-04-28T19:47:24.138815+0000 mgr.y (mgr.14140) 477 : cluster [DBG] pgmap v591: 89 pgs: 89 active+clean; 207 MiB data, 307 MiB used, 707 GiB / 715 GiB avail; 2.0 KiB/s rd, 718 KiB/s wr, 6 op/s 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[23786]: cluster 2022-04-28T19:47:24.231774+0000 mon.a (mon.0) 863 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[29114]: audit 2022-04-28T19:47:23.577929+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[29114]: audit 2022-04-28T19:47:23.579889+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:47:24.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[29114]: audit 2022-04-28T19:47:23.594908+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:24.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[29114]: cephadm 2022-04-28T19:47:23.595644+0000 mgr.y (mgr.14140) 476 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:47:24.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[29114]: cluster 2022-04-28T19:47:24.138815+0000 mgr.y (mgr.14140) 477 : cluster [DBG] pgmap v591: 89 pgs: 89 active+clean; 207 MiB data, 307 MiB used, 707 GiB / 715 GiB avail; 2.0 KiB/s rd, 718 KiB/s wr, 6 op/s 2022-04-28T19:47:24.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:24 smithi059 bash[29114]: cluster 2022-04-28T19:47:24.231774+0000 mon.a (mon.0) 863 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[37584]: debug 2022-04-28T19:47:24.705+0000 7fdd9fd33680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[37584]: debug 2022-04-28T19:47:24.809+0000 7fdd9fd33680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[23439]: audit 2022-04-28T19:47:23.577929+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[23439]: audit 2022-04-28T19:47:23.579889+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "container_image"}]: dispatch 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[23439]: audit 2022-04-28T19:47:23.594908+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[23439]: cephadm 2022-04-28T19:47:23.595644+0000 mgr.y (mgr.14140) 476 : cephadm [INF] refreshing smithi059 facts 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[23439]: cluster 2022-04-28T19:47:24.138815+0000 mgr.y (mgr.14140) 477 : cluster [DBG] pgmap v591: 89 pgs: 89 active+clean; 207 MiB data, 307 MiB used, 707 GiB / 715 GiB avail; 2.0 KiB/s rd, 718 KiB/s wr, 6 op/s 2022-04-28T19:47:24.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:24 smithi139 bash[23439]: cluster 2022-04-28T19:47:24.231774+0000 mon.a (mon.0) 863 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2022-04-28T19:47:25.235 INFO:tasks.rados.rados.0.smithi059.stdout:967: done (3 left) 2022-04-28T19:47:25.235 INFO:tasks.rados.rados.0.smithi059.stdout:968: done (2 left) 2022-04-28T19:47:25.235 INFO:tasks.rados.rados.0.smithi059.stdout:969: done (1 left) 2022-04-28T19:47:25.235 INFO:tasks.rados.rados.0.smithi059.stdout:971: done (0 left) 2022-04-28T19:47:25.235 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 96 2022-04-28T19:47:25.235 INFO:tasks.rados.rados.0.smithi059.stdout:972: seq_num 318 ranges {1482752=532480} 2022-04-28T19:47:25.246 INFO:tasks.rados.rados.0.smithi059.stdout:972: writing smithi05955964-1 from 1482752 to 2015232 tid 1 2022-04-28T19:47:25.246 INFO:tasks.rados.rados.0.smithi059.stdout:973: read oid 35 snap -1 2022-04-28T19:47:25.246 INFO:tasks.rados.rados.0.smithi059.stdout:973: expect (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:47:25.246 INFO:tasks.rados.rados.0.smithi059.stdout:974: read oid 6 snap -1 2022-04-28T19:47:25.246 INFO:tasks.rados.rados.0.smithi059.stdout:974: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:47:25.247 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 96 2022-04-28T19:47:25.247 INFO:tasks.rados.rados.0.smithi059.stdout:975: seq_num 319 ranges {1998848=548864} 2022-04-28T19:47:25.261 INFO:tasks.rados.rados.0.smithi059.stdout:975: writing smithi05955964-40 from 1998848 to 2547712 tid 1 2022-04-28T19:47:25.261 INFO:tasks.rados.rados.0.smithi059.stdout:976: snap_remove snap 70 2022-04-28T19:47:25.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:25 smithi139 bash[37584]: debug 2022-04-28T19:47:25.293+0000 7fdd9fd33680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-04-28T19:47:26.073 INFO:tasks.rados.rados.0.smithi059.stdout:976: done (4 left) 2022-04-28T19:47:26.073 INFO:tasks.rados.rados.0.smithi059.stdout:977: rollback oid 45 current snap is 96 2022-04-28T19:47:26.074 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 84 2022-04-28T19:47:26.074 INFO:tasks.rados.rados.0.smithi059.stdout:972: finishing write tid 1 to smithi05955964-1 2022-04-28T19:47:26.074 INFO:tasks.rados.rados.0.smithi059.stdout:972: finishing write tid 3 to smithi05955964-1 2022-04-28T19:47:26.074 INFO:tasks.rados.rados.0.smithi059.stdout:974: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:47:26.159 INFO:tasks.rados.rados.0.smithi059.stdout:973: expect (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:972: finishing write tid 4 to smithi05955964-1 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 367 (ObjNum 318 snap 96 seq_num 318) dirty exists 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:972: left oid 1 (ObjNum 318 snap 96 seq_num 318) 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:975: finishing write tid 1 to smithi05955964-40 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:975: finishing write tid 3 to smithi05955964-40 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:975: finishing write tid 4 to smithi05955964-40 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 370 (ObjNum 319 snap 96 seq_num 319) dirty exists 2022-04-28T19:47:26.256 INFO:tasks.rados.rados.0.smithi059.stdout:975: left oid 40 (ObjNum 319 snap 96 seq_num 319) 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:977: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:977: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:977: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 369 (ObjNum 282 snap 81 seq_num 282) dirty exists 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:972: done (4 left) 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:973: done (3 left) 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:974: done (2 left) 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:975: done (1 left) 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:977: done (0 left) 2022-04-28T19:47:26.257 INFO:tasks.rados.rados.0.smithi059.stdout:978: rmattr oid 18 current snap is 96 2022-04-28T19:47:26.258 INFO:tasks.rados.rados.0.smithi059.stdout:979: snap_create 2022-04-28T19:47:26.258 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 0 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: cluster 2022-04-28T19:47:25.233546+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2022-04-28T19:47:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: audit 2022-04-28T19:47:25.672057+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: audit 2022-04-28T19:47:26.045904+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: audit 2022-04-28T19:47:26.056168+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: audit 2022-04-28T19:47:26.061231+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2022-04-28T19:47:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: cluster 2022-04-28T19:47:26.068221+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2022-04-28T19:47:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[29114]: cluster 2022-04-28T19:47:26.168835+0000 mon.a (mon.0) 870 : cluster [DBG] Standby manager daemon y started 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: cluster 2022-04-28T19:47:25.233546+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: audit 2022-04-28T19:47:25.672057+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: audit 2022-04-28T19:47:26.045904+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: audit 2022-04-28T19:47:26.056168+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: audit 2022-04-28T19:47:26.061231+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: cluster 2022-04-28T19:47:26.068221+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2022-04-28T19:47:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:26 smithi059 bash[23786]: cluster 2022-04-28T19:47:26.168835+0000 mon.a (mon.0) 870 : cluster [DBG] Standby manager daemon y started 2022-04-28T19:47:26.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[37584]: debug 2022-04-28T19:47:26.293+0000 7fdd9fd33680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-04-28T19:47:26.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[37584]: debug 2022-04-28T19:47:26.385+0000 7fdd9fd33680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-04-28T19:47:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: cluster 2022-04-28T19:47:25.233546+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2022-04-28T19:47:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: audit 2022-04-28T19:47:25.672057+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: audit 2022-04-28T19:47:26.045904+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' 2022-04-28T19:47:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: audit 2022-04-28T19:47:26.056168+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: audit 2022-04-28T19:47:26.061231+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2022-04-28T19:47:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: cluster 2022-04-28T19:47:26.068221+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2022-04-28T19:47:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[23439]: cluster 2022-04-28T19:47:26.168835+0000 mon.a (mon.0) 870 : cluster [DBG] Standby manager daemon y started 2022-04-28T19:47:27.083 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:26 smithi139 bash[37584]: debug 2022-04-28T19:47:26.625+0000 7fdd9fd33680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-04-28T19:47:27.216 INFO:tasks.rados.rados.0.smithi059.stdout:978: done (1 left) 2022-04-28T19:47:27.217 INFO:tasks.rados.rados.0.smithi059.stdout:979: done (0 left) 2022-04-28T19:47:27.217 INFO:tasks.rados.rados.0.smithi059.stdout:980: copy_from oid 33 from oid 24 current snap is 97 2022-04-28T19:47:27.217 INFO:tasks.rados.rados.0.smithi059.stdout:981: snap_remove snap 68 2022-04-28T19:47:27.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[24088]: ignoring --setuser ceph since I am not root 2022-04-28T19:47:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[24088]: ignoring --setgroup ceph since I am not root 2022-04-28T19:47:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[24088]: debug 2022-04-28T19:47:27.090+0000 7facd81d7700 1 -- 172.21.15.59:0/803612965 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561e100ac4e0 con 0x561e10195000 2022-04-28T19:47:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cephadm 2022-04-28T19:47:25.673358+0000 mgr.y (mgr.14140) 478 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:47:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cephadm 2022-04-28T19:47:26.055726+0000 mgr.y (mgr.14140) 479 : cephadm [INF] Upgrade: Target is quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 with id 863db77b5b5ebb9f6537dd0f2bea7edb8cdbedb7e03005109ebbf1f6c68c0b65 2022-04-28T19:47:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cephadm 2022-04-28T19:47:26.057437+0000 mgr.y (mgr.14140) 480 : cephadm [INF] Upgrade: Checking mgr daemons... 2022-04-28T19:47:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cephadm 2022-04-28T19:47:26.057587+0000 mgr.y (mgr.14140) 481 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cephadm 2022-04-28T19:47:26.060651+0000 mgr.y (mgr.14140) 482 : cephadm [INF] Failing over to other MGR 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cluster 2022-04-28T19:47:26.139391+0000 mgr.y (mgr.14140) 483 : cluster [DBG] pgmap v595: 89 pgs: 89 active+clean; 215 MiB data, 308 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 843 KiB/s wr, 5 op/s 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: audit 2022-04-28T19:47:27.054102+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cluster 2022-04-28T19:47:27.054221+0000 mon.a (mon.0) 872 : cluster [DBG] mgrmap e15: x(active, starting, since 0.99181s), standbys: y 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[29114]: cluster 2022-04-28T19:47:27.214876+0000 mon.a (mon.0) 873 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cephadm 2022-04-28T19:47:25.673358+0000 mgr.y (mgr.14140) 478 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cephadm 2022-04-28T19:47:26.055726+0000 mgr.y (mgr.14140) 479 : cephadm [INF] Upgrade: Target is quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 with id 863db77b5b5ebb9f6537dd0f2bea7edb8cdbedb7e03005109ebbf1f6c68c0b65 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cephadm 2022-04-28T19:47:26.057437+0000 mgr.y (mgr.14140) 480 : cephadm [INF] Upgrade: Checking mgr daemons... 2022-04-28T19:47:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cephadm 2022-04-28T19:47:26.057587+0000 mgr.y (mgr.14140) 481 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-04-28T19:47:27.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cephadm 2022-04-28T19:47:26.060651+0000 mgr.y (mgr.14140) 482 : cephadm [INF] Failing over to other MGR 2022-04-28T19:47:27.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cluster 2022-04-28T19:47:26.139391+0000 mgr.y (mgr.14140) 483 : cluster [DBG] pgmap v595: 89 pgs: 89 active+clean; 215 MiB data, 308 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 843 KiB/s wr, 5 op/s 2022-04-28T19:47:27.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: audit 2022-04-28T19:47:27.054102+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2022-04-28T19:47:27.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cluster 2022-04-28T19:47:27.054221+0000 mon.a (mon.0) 872 : cluster [DBG] mgrmap e15: x(active, starting, since 0.99181s), standbys: y 2022-04-28T19:47:27.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:27 smithi059 bash[23786]: cluster 2022-04-28T19:47:27.214876+0000 mon.a (mon.0) 873 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cephadm 2022-04-28T19:47:25.673358+0000 mgr.y (mgr.14140) 478 : cephadm [INF] refreshing smithi139 facts 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cephadm 2022-04-28T19:47:26.055726+0000 mgr.y (mgr.14140) 479 : cephadm [INF] Upgrade: Target is quay.ceph.io/ceph-ci/ceph@sha256:cec3bce4b338ff961c71c7146d545e7e00f2441397d5365a335b160d854f7a49 with id 863db77b5b5ebb9f6537dd0f2bea7edb8cdbedb7e03005109ebbf1f6c68c0b65 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cephadm 2022-04-28T19:47:26.057437+0000 mgr.y (mgr.14140) 480 : cephadm [INF] Upgrade: Checking mgr daemons... 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cephadm 2022-04-28T19:47:26.057587+0000 mgr.y (mgr.14140) 481 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cephadm 2022-04-28T19:47:26.060651+0000 mgr.y (mgr.14140) 482 : cephadm [INF] Failing over to other MGR 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cluster 2022-04-28T19:47:26.139391+0000 mgr.y (mgr.14140) 483 : cluster [DBG] pgmap v595: 89 pgs: 89 active+clean; 215 MiB data, 308 MiB used, 707 GiB / 715 GiB avail; 3.1 MiB/s rd, 843 KiB/s wr, 5 op/s 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: audit 2022-04-28T19:47:27.054102+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.14140 172.21.15.59:0/1201833208' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2022-04-28T19:47:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cluster 2022-04-28T19:47:27.054221+0000 mon.a (mon.0) 872 : cluster [DBG] mgrmap e15: x(active, starting, since 0.99181s), standbys: y 2022-04-28T19:47:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[23439]: cluster 2022-04-28T19:47:27.214876+0000 mon.a (mon.0) 873 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2022-04-28T19:47:28.039 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[37584]: debug 2022-04-28T19:47:27.781+0000 7fdd9fd33680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-04-28T19:47:28.040 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:27 smithi139 bash[37584]: debug 2022-04-28T19:47:27.937+0000 7fdd9fd33680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:47:28.230 INFO:tasks.rados.rados.0.smithi059.stdout:981: done (1 left) 2022-04-28T19:47:28.320 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 97 2022-04-28T19:47:28.320 INFO:tasks.rados.rados.0.smithi059.stdout:982: seq_num 320 ranges {0=565248} 2022-04-28T19:47:28.326 INFO:tasks.rados.rados.0.smithi059.stdout:982: writing smithi05955964-36 from 0 to 565248 tid 1 2022-04-28T19:47:28.326 INFO:tasks.rados.rados.0.smithi059.stdout:983: delete oid 25 current snap is 97 2022-04-28T19:47:28.326 INFO:tasks.rados.rados.0.smithi059.stdout:980: finishing copy_from to smithi05955964-33 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 322 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:980: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:980: done (2 left) 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:983: done (1 left) 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 97 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:984: seq_num 322 ranges {1409024=598016} 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:984: writing smithi05955964-19 from 1409024 to 2007040 tid 1 2022-04-28T19:47:28.327 INFO:tasks.rados.rados.0.smithi059.stdout:985: rollback oid 35 current snap is 97 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 35 to 93 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:986: snap_create 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:982: finishing write tid 1 to smithi05955964-36 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:982: finishing write tid 3 to smithi05955964-36 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:982: finishing write tid 4 to smithi05955964-36 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 378 (ObjNum 320 snap 97 seq_num 320) dirty exists 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:982: left oid 36 (ObjNum 320 snap 97 seq_num 320) 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:985: finishing rollback tid 0 to smithi05955964-35 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:985: finishing rollback tid 1 to smithi05955964-35 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:984: finishing write tid 1 to smithi05955964-19 2022-04-28T19:47:28.328 INFO:tasks.rados.rados.0.smithi059.stdout:984: finishing write tid 3 to smithi05955964-19 2022-04-28T19:47:28.329 INFO:tasks.rados.rados.0.smithi059.stdout:985: finishing rollback tid 2 to smithi05955964-35 2022-04-28T19:47:28.329 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 147 (ObjNum 265 snap 77 seq_num 265) dirty exists 2022-04-28T19:47:28.329 INFO:tasks.rados.rados.0.smithi059.stdout:984: finishing write tid 4 to smithi05955964-19 2022-04-28T19:47:28.329 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 409 (ObjNum 322 snap 97 seq_num 322) dirty exists 2022-04-28T19:47:28.329 INFO:tasks.rados.rados.0.smithi059.stdout:984: left oid 19 (ObjNum 322 snap 97 seq_num 322) 2022-04-28T19:47:28.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:28 smithi139 bash[37584]: debug 2022-04-28T19:47:28.033+0000 7fdd9fd33680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-04-28T19:47:28.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:28 smithi139 bash[37584]: debug 2022-04-28T19:47:28.137+0000 7fdd9fd33680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-04-28T19:47:28.735 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:28 smithi139 bash[37584]: debug 2022-04-28T19:47:28.365+0000 7fdd9fd33680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-04-28T19:47:28.735 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:28 smithi139 bash[37584]: debug 2022-04-28T19:47:28.729+0000 7fdd9fd33680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-04-28T19:47:29.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:28 smithi139 bash[37584]: debug 2022-04-28T19:47:28.845+0000 7fdd9fd33680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:982: done (3 left) 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:984: done (2 left) 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:985: done (1 left) 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:986: done (0 left) 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:987: rmattr oid 32 current snap is 98 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:987: done (0 left) 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:988: read oid 50 snap -1 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:988: expect (ObjNum 308 snap 92 seq_num 308) 2022-04-28T19:47:29.235 INFO:tasks.rados.rados.0.smithi059.stdout:989: rollback oid 48 current snap is 98 2022-04-28T19:47:29.236 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 73 2022-04-28T19:47:29.236 INFO:tasks.rados.rados.0.smithi059.stdout:990: delete oid 5 current snap is 98 2022-04-28T19:47:29.241 INFO:tasks.rados.rados.0.smithi059.stdout:989: finishing rollback tid 0 to smithi05955964-48 2022-04-28T19:47:29.241 INFO:tasks.rados.rados.0.smithi059.stdout:989: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:47:29.241 INFO:tasks.rados.rados.0.smithi059.stdout:990: done (2 left) 2022-04-28T19:47:29.242 INFO:tasks.rados.rados.0.smithi059.stdout:991: delete oid 36 current snap is 98 2022-04-28T19:47:29.242 INFO:tasks.rados.rados.0.smithi059.stdout:989: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:47:29.242 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 413 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:47:29.244 INFO:tasks.rados.rados.0.smithi059.stdout:989: done (2 left) 2022-04-28T19:47:29.244 INFO:tasks.rados.rados.0.smithi059.stdout:991: done (1 left) 2022-04-28T19:47:29.245 INFO:tasks.rados.rados.0.smithi059.stdout:992: copy_from oid 29 from oid 16 current snap is 98 2022-04-28T19:47:29.245 INFO:tasks.rados.rados.0.smithi059.stdout:993: read oid 19 snap 86 2022-04-28T19:47:29.245 INFO:tasks.rados.rados.0.smithi059.stdout:993: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:47:29.245 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 98 2022-04-28T19:47:29.246 INFO:tasks.rados.rados.0.smithi059.stdout:994: seq_num 325 ranges {4874240=647168} 2022-04-28T19:47:29.275 INFO:tasks.rados.rados.0.smithi059.stdout:994: writing smithi05955964-11 from 4874240 to 5521408 tid 1 2022-04-28T19:47:29.276 INFO:tasks.rados.rados.0.smithi059.stdout:995: copy_from oid 26 from oid 18 current snap is 98 2022-04-28T19:47:29.276 INFO:tasks.rados.rados.0.smithi059.stdout:988: expect (ObjNum 308 snap 92 seq_num 308) 2022-04-28T19:47:29.320 INFO:tasks.rados.rados.0.smithi059.stdout:992: finishing copy_from to smithi05955964-29 2022-04-28T19:47:29.320 INFO:tasks.rados.rados.0.smithi059.stdout:992: got expected ENOENT (src dne) 2022-04-28T19:47:29.320 INFO:tasks.rados.rados.0.smithi059.stdout:992: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:47:29.321 INFO:tasks.rados.rados.0.smithi059.stdout:993: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:47:29.353 INFO:tasks.rados.rados.0.smithi059.stdout:994: finishing write tid 1 to smithi05955964-11 2022-04-28T19:47:29.353 INFO:tasks.rados.rados.0.smithi059.stdout:994: finishing write tid 3 to smithi05955964-11 2022-04-28T19:47:29.353 INFO:tasks.rados.rados.0.smithi059.stdout:994: finishing write tid 4 to smithi05955964-11 2022-04-28T19:47:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 363 (ObjNum 325 snap 98 seq_num 325) dirty exists 2022-04-28T19:47:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:994: left oid 11 (ObjNum 325 snap 98 seq_num 325) 2022-04-28T19:47:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:995: finishing copy_from to smithi05955964-26 2022-04-28T19:47:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 334 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:995: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:47:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:988: done (4 left) 2022-04-28T19:47:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:992: done (3 left) 2022-04-28T19:47:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:993: done (2 left) 2022-04-28T19:47:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:994: done (1 left) 2022-04-28T19:47:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:995: done (0 left) 2022-04-28T19:47:29.356 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 98 2022-04-28T19:47:29.356 INFO:tasks.rados.rados.0.smithi059.stdout:996: seq_num 326 ranges {1187840=663552} 2022-04-28T19:47:29.371 INFO:tasks.rados.rados.0.smithi059.stdout:996: writing smithi05955964-50 from 1187840 to 1851392 tid 1 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:997: read oid 19 snap 95 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:997: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:998: read oid 41 snap -1 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:998: expect (ObjNum 280 snap 80 seq_num 280) 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:999: read oid 7 snap 93 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:999: expect (ObjNum 297 snap 90 seq_num 297) 2022-04-28T19:47:29.372 INFO:tasks.rados.rados.0.smithi059.stdout:1000: snap_create 2022-04-28T19:47:29.377 INFO:tasks.rados.rados.0.smithi059.stdout:999: expect (ObjNum 297 snap 90 seq_num 297) 2022-04-28T19:47:29.393 INFO:tasks.rados.rados.0.smithi059.stdout:997: expect (ObjNum 227 snap 60 seq_num 227) 2022-04-28T19:47:29.448 INFO:tasks.rados.rados.0.smithi059.stdout:996: finishing write tid 1 to smithi05955964-50 2022-04-28T19:47:29.448 INFO:tasks.rados.rados.0.smithi059.stdout:996: finishing write tid 3 to smithi05955964-50 2022-04-28T19:47:29.448 INFO:tasks.rados.rados.0.smithi059.stdout:996: finishing write tid 4 to smithi05955964-50 2022-04-28T19:47:29.448 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 499 (ObjNum 326 snap 98 seq_num 326) dirty exists 2022-04-28T19:47:29.448 INFO:tasks.rados.rados.0.smithi059.stdout:996: left oid 50 (ObjNum 326 snap 98 seq_num 326) 2022-04-28T19:47:29.449 INFO:tasks.rados.rados.0.smithi059.stdout:998: expect (ObjNum 280 snap 80 seq_num 280) 2022-04-28T19:47:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:29 smithi059 bash[29114]: cluster 2022-04-28T19:47:28.226734+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2022-04-28T19:47:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:29 smithi059 bash[23786]: cluster 2022-04-28T19:47:28.226734+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2022-04-28T19:47:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:29 smithi139 bash[23439]: cluster 2022-04-28T19:47:28.226734+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2022-04-28T19:47:30.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:29 smithi139 bash[37584]: debug 2022-04-28T19:47:29.701+0000 7fdd9fd33680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-04-28T19:47:30.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:29 smithi139 bash[37584]: debug 2022-04-28T19:47:29.801+0000 7fdd9fd33680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-04-28T19:47:30.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:29 smithi139 bash[37584]: debug 2022-04-28T19:47:29.913+0000 7fdd9fd33680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-04-28T19:47:30.244 INFO:tasks.rados.rados.0.smithi059.stdout:996: done (4 left) 2022-04-28T19:47:30.245 INFO:tasks.rados.rados.0.smithi059.stdout:997: done (3 left) 2022-04-28T19:47:30.245 INFO:tasks.rados.rados.0.smithi059.stdout:998: done (2 left) 2022-04-28T19:47:30.246 INFO:tasks.rados.rados.0.smithi059.stdout:999: done (1 left) 2022-04-28T19:47:30.246 INFO:tasks.rados.rados.0.smithi059.stdout:1000: done (0 left) 2022-04-28T19:47:30.246 INFO:tasks.rados.rados.0.smithi059.stdout:1001: read oid 3 snap -1 2022-04-28T19:47:30.246 INFO:tasks.rados.rados.0.smithi059.stdout:1001: expect (ObjNum 317 snap 95 seq_num 317) 2022-04-28T19:47:30.246 INFO:tasks.rados.rados.0.smithi059.stdout:1002: copy_from oid 50 from oid 8 current snap is 99 2022-04-28T19:47:30.246 INFO:tasks.rados.rados.0.smithi059.stdout:1003: snap_remove snap 74 2022-04-28T19:47:30.427 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:30 smithi139 bash[23439]: cluster 2022-04-28T19:47:29.232274+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2022-04-28T19:47:30.427 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:30 smithi139 bash[37584]: debug 2022-04-28T19:47:30.129+0000 7fdd9fd33680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-04-28T19:47:30.427 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:30 smithi139 bash[37584]: debug 2022-04-28T19:47:30.421+0000 7fdd9fd33680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-04-28T19:47:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:30 smithi059 bash[23786]: cluster 2022-04-28T19:47:29.232274+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2022-04-28T19:47:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:30 smithi059 bash[29114]: cluster 2022-04-28T19:47:29.232274+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2022-04-28T19:47:30.749 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:30 smithi139 bash[37584]: debug 2022-04-28T19:47:30.577+0000 7fdd9fd33680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-04-28T19:47:30.749 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:30 smithi139 bash[37584]: debug 2022-04-28T19:47:30.741+0000 7fdd9fd33680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:47:31.056 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:30 smithi139 bash[37584]: debug 2022-04-28T19:47:30.945+0000 7fdd9fd33680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-04-28T19:47:31.056 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[37584]: debug 2022-04-28T19:47:31.049+0000 7fdd9fd33680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-04-28T19:47:31.084 INFO:tasks.rados.rados.0.smithi059.stdout:1003: done (2 left) 2022-04-28T19:47:31.084 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 99 2022-04-28T19:47:31.085 INFO:tasks.rados.rados.0.smithi059.stdout:1004: seq_num 327 ranges {2875392=679936} 2022-04-28T19:47:31.106 INFO:tasks.rados.rados.0.smithi059.stdout:1004: writing smithi05955964-38 from 2875392 to 3555328 tid 1 2022-04-28T19:47:31.106 INFO:tasks.rados.rados.0.smithi059.stdout:1001: expect (ObjNum 317 snap 95 seq_num 317) 2022-04-28T19:47:31.136 INFO:tasks.rados.rados.0.smithi059.stdout:1002: finishing copy_from to smithi05955964-50 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 372 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:1002: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:1005: read oid 20 snap -1 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:1005: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:1001: done (3 left) 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:1002: done (2 left) 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 99 2022-04-28T19:47:31.137 INFO:tasks.rados.rados.0.smithi059.stdout:1006: seq_num 328 ranges {4055040=696320} 2022-04-28T19:47:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:1006: writing smithi05955964-34 from 4055040 to 4751360 tid 1 2022-04-28T19:47:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:1007: delete oid 1 current snap is 99 2022-04-28T19:47:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:1004: finishing write tid 1 to smithi05955964-38 2022-04-28T19:47:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:1004: finishing write tid 3 to smithi05955964-38 2022-04-28T19:47:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:1004: finishing write tid 4 to smithi05955964-38 2022-04-28T19:47:31.167 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 370 (ObjNum 327 snap 99 seq_num 327) dirty exists 2022-04-28T19:47:31.167 INFO:tasks.rados.rados.0.smithi059.stdout:1004: left oid 38 (ObjNum 327 snap 99 seq_num 327) 2022-04-28T19:47:31.167 INFO:tasks.rados.rados.0.smithi059.stdout:1005: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:47:31.199 INFO:tasks.rados.rados.0.smithi059.stdout:1006: finishing write tid 1 to smithi05955964-34 2022-04-28T19:47:31.199 INFO:tasks.rados.rados.0.smithi059.stdout:1006: finishing write tid 3 to smithi05955964-34 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1006: finishing write tid 4 to smithi05955964-34 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 397 (ObjNum 328 snap 99 seq_num 328) dirty exists 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1006: left oid 34 (ObjNum 328 snap 99 seq_num 328) 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1004: done (3 left) 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1005: done (2 left) 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1006: done (1 left) 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1007: done (0 left) 2022-04-28T19:47:31.200 INFO:tasks.rados.rados.0.smithi059.stdout:1008: snap_remove snap 84 2022-04-28T19:47:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: cluster 2022-04-28T19:47:30.243617+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2022-04-28T19:47:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: audit 2022-04-28T19:47:31.076925+0000 mon.b (mon.2) 23 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:47:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: cluster 2022-04-28T19:47:31.076947+0000 mon.a (mon.0) 877 : cluster [INF] Active manager daemon x restarted 2022-04-28T19:47:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: cluster 2022-04-28T19:47:31.078303+0000 mon.a (mon.0) 878 : cluster [INF] Activating manager daemon x 2022-04-28T19:47:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: audit 2022-04-28T19:47:31.078316+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: audit 2022-04-28T19:47:31.082513+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:47:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: cluster 2022-04-28T19:47:31.083553+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2022-04-28T19:47:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:31 smithi139 bash[23439]: audit 2022-04-28T19:47:31.084551+0000 mon.b (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: cluster 2022-04-28T19:47:30.243617+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: audit 2022-04-28T19:47:31.076925+0000 mon.b (mon.2) 23 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: cluster 2022-04-28T19:47:31.076947+0000 mon.a (mon.0) 877 : cluster [INF] Active manager daemon x restarted 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: cluster 2022-04-28T19:47:31.078303+0000 mon.a (mon.0) 878 : cluster [INF] Activating manager daemon x 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: audit 2022-04-28T19:47:31.078316+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: audit 2022-04-28T19:47:31.082513+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:47:31.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: cluster 2022-04-28T19:47:31.083553+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[29114]: audit 2022-04-28T19:47:31.084551+0000 mon.b (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: cluster 2022-04-28T19:47:30.243617+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: audit 2022-04-28T19:47:31.076925+0000 mon.b (mon.2) 23 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: cluster 2022-04-28T19:47:31.076947+0000 mon.a (mon.0) 877 : cluster [INF] Active manager daemon x restarted 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: cluster 2022-04-28T19:47:31.078303+0000 mon.a (mon.0) 878 : cluster [INF] Activating manager daemon x 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: audit 2022-04-28T19:47:31.078316+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:31.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: audit 2022-04-28T19:47:31.082513+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:47:31.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: cluster 2022-04-28T19:47:31.083553+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2022-04-28T19:47:31.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:31 smithi059 bash[23786]: audit 2022-04-28T19:47:31.084551+0000 mon.b (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:32.090 INFO:tasks.rados.rados.0.smithi059.stdout:1008: done (0 left) 2022-04-28T19:47:32.090 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 99 2022-04-28T19:47:32.090 INFO:tasks.rados.rados.0.smithi059.stdout:1009: seq_num 330 ranges {6848512=737280} 2022-04-28T19:47:32.133 INFO:tasks.rados.rados.0.smithi059.stdout:1009: writing smithi05955964-45 from 6848512 to 7585792 tid 1 2022-04-28T19:47:32.133 INFO:tasks.rados.rados.0.smithi059.stdout:1010: copy_from oid 41 from oid 49 current snap is 99 2022-04-28T19:47:32.133 INFO:tasks.rados.rados.0.smithi059.stdout:1011: snap_remove snap 85 2022-04-28T19:47:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: cluster 2022-04-28T19:47:31.251943+0000 mon.a (mon.0) 880 : cluster [DBG] mgrmap e16: x(active, starting, since 0.173953s), standbys: y 2022-04-28T19:47:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: cluster 2022-04-28T19:47:31.718855+0000 mon.a (mon.0) 881 : cluster [DBG] Standby manager daemon y restarted 2022-04-28T19:47:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: cluster 2022-04-28T19:47:31.719041+0000 mon.a (mon.0) 882 : cluster [DBG] Standby manager daemon y started 2022-04-28T19:47:32.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: audit 2022-04-28T19:47:31.719745+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-04-28T19:47:32.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: audit 2022-04-28T19:47:31.721713+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:32.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: audit 2022-04-28T19:47:31.723693+0000 mon.c (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-04-28T19:47:32.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: audit 2022-04-28T19:47:31.724459+0000 mon.c (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:32.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[29114]: cluster 2022-04-28T19:47:32.088754+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2022-04-28T19:47:32.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: cluster 2022-04-28T19:47:31.251943+0000 mon.a (mon.0) 880 : cluster [DBG] mgrmap e16: x(active, starting, since 0.173953s), standbys: y 2022-04-28T19:47:32.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: cluster 2022-04-28T19:47:31.718855+0000 mon.a (mon.0) 881 : cluster [DBG] Standby manager daemon y restarted 2022-04-28T19:47:32.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: cluster 2022-04-28T19:47:31.719041+0000 mon.a (mon.0) 882 : cluster [DBG] Standby manager daemon y started 2022-04-28T19:47:32.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: audit 2022-04-28T19:47:31.719745+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-04-28T19:47:32.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: audit 2022-04-28T19:47:31.721713+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:32.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: audit 2022-04-28T19:47:31.723693+0000 mon.c (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-04-28T19:47:32.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: audit 2022-04-28T19:47:31.724459+0000 mon.c (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:32.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:32 smithi059 bash[23786]: cluster 2022-04-28T19:47:32.088754+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2022-04-28T19:47:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: cluster 2022-04-28T19:47:31.251943+0000 mon.a (mon.0) 880 : cluster [DBG] mgrmap e16: x(active, starting, since 0.173953s), standbys: y 2022-04-28T19:47:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: cluster 2022-04-28T19:47:31.718855+0000 mon.a (mon.0) 881 : cluster [DBG] Standby manager daemon y restarted 2022-04-28T19:47:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: cluster 2022-04-28T19:47:31.719041+0000 mon.a (mon.0) 882 : cluster [DBG] Standby manager daemon y started 2022-04-28T19:47:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: audit 2022-04-28T19:47:31.719745+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-04-28T19:47:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: audit 2022-04-28T19:47:31.721713+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: audit 2022-04-28T19:47:31.723693+0000 mon.c (mon.1) 39 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-04-28T19:47:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: audit 2022-04-28T19:47:31.724459+0000 mon.c (mon.1) 40 : audit [DBG] from='mgr.? 172.21.15.59:0/3780551439' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:32 smithi139 bash[23439]: cluster 2022-04-28T19:47:32.088754+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2022-04-28T19:47:33.093 INFO:tasks.rados.rados.0.smithi059.stdout:1011: done (2 left) 2022-04-28T19:47:33.093 INFO:tasks.rados.rados.0.smithi059.stdout:1012: snap_remove snap 90 2022-04-28T19:47:33.521 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[29114]: cluster 2022-04-28T19:47:32.266781+0000 mon.a (mon.0) 884 : cluster [DBG] mgrmap e17: x(active, starting, since 1.18879s), standbys: y 2022-04-28T19:47:33.521 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[29114]: audit 2022-04-28T19:47:32.270824+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:47:33.521 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[29114]: audit 2022-04-28T19:47:32.271205+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:47:33.522 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[29114]: cluster 2022-04-28T19:47:33.090876+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2022-04-28T19:47:33.522 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:33.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:47:33.522 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:33.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:47:33.522 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[23786]: cluster 2022-04-28T19:47:32.266781+0000 mon.a (mon.0) 884 : cluster [DBG] mgrmap e17: x(active, starting, since 1.18879s), standbys: y 2022-04-28T19:47:33.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[23786]: audit 2022-04-28T19:47:32.270824+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:47:33.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[23786]: audit 2022-04-28T19:47:32.271205+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:47:33.523 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[23786]: cluster 2022-04-28T19:47:33.090876+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2022-04-28T19:47:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:33 smithi139 bash[23439]: cluster 2022-04-28T19:47:32.266781+0000 mon.a (mon.0) 884 : cluster [DBG] mgrmap e17: x(active, starting, since 1.18879s), standbys: y 2022-04-28T19:47:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:33 smithi139 bash[23439]: audit 2022-04-28T19:47:32.270824+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:47:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:33 smithi139 bash[23439]: audit 2022-04-28T19:47:32.271205+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.? 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:47:33.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:33 smithi139 bash[23439]: cluster 2022-04-28T19:47:33.090876+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2022-04-28T19:47:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:33.520Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:47:34.101 INFO:tasks.rados.rados.0.smithi059.stdout:1012: done (2 left) 2022-04-28T19:47:34.102 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 99 2022-04-28T19:47:34.102 INFO:tasks.rados.rados.0.smithi059.stdout:1013: seq_num 331 ranges {0=753664} 2022-04-28T19:47:34.103 INFO:tasks.rados.rados.0.smithi059.stdout:1013: writing smithi05955964-5 from 0 to 753664 tid 1 2022-04-28T19:47:34.103 INFO:tasks.rados.rados.0.smithi059.stdout:1014: read oid 4 snap -1 2022-04-28T19:47:34.103 INFO:tasks.rados.rados.0.smithi059.stdout:1014: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1015: setattr oid 16 current snap is 99 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1009: finishing write tid 1 to smithi05955964-45 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1009: finishing write tid 3 to smithi05955964-45 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1009: finishing write tid 4 to smithi05955964-45 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 394 (ObjNum 330 snap 99 seq_num 330) dirty exists 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1009: left oid 45 (ObjNum 330 snap 99 seq_num 330) 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1010: finishing copy_from to smithi05955964-41 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 249 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:47:34.104 INFO:tasks.rados.rados.0.smithi059.stdout:1010: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:47:34.105 INFO:tasks.rados.rados.0.smithi059.stdout:1014: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:47:34.111 INFO:tasks.rados.rados.0.smithi059.stdout:1009: done (4 left) 2022-04-28T19:47:34.112 INFO:tasks.rados.rados.0.smithi059.stdout:1010: done (3 left) 2022-04-28T19:47:34.112 INFO:tasks.rados.rados.0.smithi059.stdout:1014: done (2 left) 2022-04-28T19:47:34.112 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 99 2022-04-28T19:47:34.112 INFO:tasks.rados.rados.0.smithi059.stdout:1016: seq_num 332 ranges {2875392=770048} 2022-04-28T19:47:34.130 INFO:tasks.rados.rados.0.smithi059.stdout:1016: writing smithi05955964-26 from 2875392 to 3645440 tid 1 2022-04-28T19:47:34.130 INFO:tasks.rados.rados.0.smithi059.stdout:1017: delete oid 30 current snap is 99 2022-04-28T19:47:34.130 INFO:tasks.rados.rados.0.smithi059.stdout:1013: finishing write tid 1 to smithi05955964-5 2022-04-28T19:47:34.130 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 313 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:47:34.130 INFO:tasks.rados.rados.0.smithi059.stdout:1013: finishing write tid 3 to smithi05955964-5 2022-04-28T19:47:34.131 INFO:tasks.rados.rados.0.smithi059.stdout:1013: finishing write tid 4 to smithi05955964-5 2022-04-28T19:47:34.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 453 (ObjNum 331 snap 99 seq_num 331) dirty exists 2022-04-28T19:47:34.131 INFO:tasks.rados.rados.0.smithi059.stdout:1013: left oid 5 (ObjNum 331 snap 99 seq_num 331) 2022-04-28T19:47:34.132 INFO:tasks.rados.rados.0.smithi059.stdout:1013: done (3 left) 2022-04-28T19:47:34.132 INFO:tasks.rados.rados.0.smithi059.stdout:1015: done (2 left) 2022-04-28T19:47:34.132 INFO:tasks.rados.rados.0.smithi059.stdout:1017: done (1 left) 2022-04-28T19:47:34.132 INFO:tasks.rados.rados.0.smithi059.stdout:1018: read oid 5 snap -1 2022-04-28T19:47:34.132 INFO:tasks.rados.rados.0.smithi059.stdout:1018: expect (ObjNum 331 snap 99 seq_num 331) 2022-04-28T19:47:34.132 INFO:tasks.rados.rados.0.smithi059.stdout:1019: rmattr oid 32 current snap is 99 2022-04-28T19:47:34.133 INFO:tasks.rados.rados.0.smithi059.stdout:1019: done (2 left) 2022-04-28T19:47:34.133 INFO:tasks.rados.rados.0.smithi059.stdout:1020: rmattr oid 47 current snap is 99 2022-04-28T19:47:34.133 INFO:tasks.rados.rados.0.smithi059.stdout:1021: setattr oid 20 current snap is 99 2022-04-28T19:47:34.134 INFO:tasks.rados.rados.0.smithi059.stdout:1016: finishing write tid 1 to smithi05955964-26 2022-04-28T19:47:34.135 INFO:tasks.rados.rados.0.smithi059.stdout:1016: finishing write tid 3 to smithi05955964-26 2022-04-28T19:47:34.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 439 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:47:34.136 INFO:tasks.rados.rados.0.smithi059.stdout:1020: done (3 left) 2022-04-28T19:47:34.136 INFO:tasks.rados.rados.0.smithi059.stdout:1022: read oid 34 snap 96 2022-04-28T19:47:34.137 INFO:tasks.rados.rados.0.smithi059.stdout:1022: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:47:34.137 INFO:tasks.rados.rados.0.smithi059.stdout:1023: rollback oid 9 current snap is 99 2022-04-28T19:47:34.137 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 73 2022-04-28T19:47:34.137 INFO:tasks.rados.rados.0.smithi059.stdout:1024: setattr oid 4 current snap is 99 2022-04-28T19:47:34.137 INFO:tasks.rados.rados.0.smithi059.stdout:1016: finishing write tid 4 to smithi05955964-26 2022-04-28T19:47:34.138 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 452 (ObjNum 332 snap 99 seq_num 332) dirty exists 2022-04-28T19:47:34.138 INFO:tasks.rados.rados.0.smithi059.stdout:1016: left oid 26 (ObjNum 332 snap 99 seq_num 332) 2022-04-28T19:47:34.144 INFO:tasks.rados.rados.0.smithi059.stdout:1018: expect (ObjNum 331 snap 99 seq_num 331) 2022-04-28T19:47:34.157 INFO:tasks.rados.rados.0.smithi059.stdout:1023: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:47:34.157 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 400 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:47:34.157 INFO:tasks.rados.rados.0.smithi059.stdout:1023: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:47:34.157 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 294 (ObjNum -133516784 snap 21893 seq_num 4161984128) dirty dne 2022-04-28T19:47:34.157 INFO:tasks.rados.rados.0.smithi059.stdout:1022: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:47:34.341 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 386 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty exists 2022-04-28T19:47:34.341 INFO:tasks.rados.rados.0.smithi059.stdout:1016: done (5 left) 2022-04-28T19:47:34.341 INFO:tasks.rados.rados.0.smithi059.stdout:1018: done (4 left) 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1021: done (3 left) 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1022: done (2 left) 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1023: done (1 left) 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1024: done (0 left) 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1025: rollback oid 7 current snap is 99 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 7 to 83 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1026: rollback oid 9 current snap is 99 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 95 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1027: read oid 43 snap -1 2022-04-28T19:47:34.342 INFO:tasks.rados.rados.0.smithi059.stdout:1027: expect (ObjNum 206 snap 59 seq_num 206) 2022-04-28T19:47:34.343 INFO:tasks.rados.rados.0.smithi059.stdout:1028: copy_from oid 28 from oid 8 current snap is 99 2022-04-28T19:47:34.343 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 99 2022-04-28T19:47:34.343 INFO:tasks.rados.rados.0.smithi059.stdout:1029: seq_num 334 ranges {2662400=802816} 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:1029: writing smithi05955964-13 from 2662400 to 3465216 tid 1 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:1030: setattr oid 19 current snap is 99 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:1026: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:1026: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 296 (ObjNum 279 snap 80 seq_num 279) dirty exists 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:1025: finishing rollback tid 0 to smithi05955964-7 2022-04-28T19:47:34.369 INFO:tasks.rados.rados.0.smithi059.stdout:1025: finishing rollback tid 1 to smithi05955964-7 2022-04-28T19:47:34.370 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 507 (ObjNum 0 snap 32762 seq_num 4162160768) dirty dne 2022-04-28T19:47:34.370 INFO:tasks.rados.rados.0.smithi059.stdout:1027: expect (ObjNum 206 snap 59 seq_num 206) 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:1028: finishing copy_from to smithi05955964-28 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 372 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:1028: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:1029: finishing write tid 1 to smithi05955964-13 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:1029: finishing write tid 3 to smithi05955964-13 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:1029: finishing write tid 4 to smithi05955964-13 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 227 (ObjNum 334 snap 99 seq_num 334) dirty exists 2022-04-28T19:47:34.487 INFO:tasks.rados.rados.0.smithi059.stdout:1029: left oid 13 (ObjNum 334 snap 99 seq_num 334) 2022-04-28T19:47:34.488 INFO:tasks.rados.rados.0.smithi059.stdout:1025: done (5 left) 2022-04-28T19:47:34.488 INFO:tasks.rados.rados.0.smithi059.stdout:1026: done (4 left) 2022-04-28T19:47:34.488 INFO:tasks.rados.rados.0.smithi059.stdout:1027: done (3 left) 2022-04-28T19:47:34.488 INFO:tasks.rados.rados.0.smithi059.stdout:1028: done (2 left) 2022-04-28T19:47:34.488 INFO:tasks.rados.rados.0.smithi059.stdout:1029: done (1 left) 2022-04-28T19:47:34.488 INFO:tasks.rados.rados.0.smithi059.stdout:1031: snap_create 2022-04-28T19:47:34.490 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 454 (ObjNum 322 snap 97 seq_num 322) dirty exists 2022-04-28T19:47:35.103 INFO:tasks.rados.rados.0.smithi059.stdout:1030: done (1 left) 2022-04-28T19:47:35.103 INFO:tasks.rados.rados.0.smithi059.stdout:1031: done (0 left) 2022-04-28T19:47:35.103 INFO:tasks.rados.rados.0.smithi059.stdout:1032: snap_create 2022-04-28T19:47:35.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:35 smithi139 bash[23439]: cluster 2022-04-28T19:47:34.090372+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2022-04-28T19:47:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:35 smithi059 bash[29114]: cluster 2022-04-28T19:47:34.090372+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2022-04-28T19:47:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:35 smithi059 bash[23786]: cluster 2022-04-28T19:47:34.090372+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2022-04-28T19:47:36.114 INFO:tasks.rados.rados.0.smithi059.stdout:1032: done (0 left) 2022-04-28T19:47:36.114 INFO:tasks.rados.rados.0.smithi059.stdout:1033: copy_from oid 10 from oid 16 current snap is 101 2022-04-28T19:47:36.114 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 101 2022-04-28T19:47:36.114 INFO:tasks.rados.rados.0.smithi059.stdout:1034: seq_num 335 ranges {753664=417792,1171456=401408} 2022-04-28T19:47:36.125 INFO:tasks.rados.rados.0.smithi059.stdout:1034: writing smithi05955964-5 from 753664 to 1171456 tid 1 2022-04-28T19:47:36.125 INFO:tasks.rados.rados.0.smithi059.stdout:1034: writing smithi05955964-5 from 1171456 to 1572864 tid 2 2022-04-28T19:47:36.125 INFO:tasks.rados.rados.0.smithi059.stdout:1035: read oid 23 snap 88 2022-04-28T19:47:36.125 INFO:tasks.rados.rados.0.smithi059.stdout:1035: expect (ObjNum 281 snap 80 seq_num 281) 2022-04-28T19:47:36.125 INFO:tasks.rados.rados.0.smithi059.stdout:1036: delete oid 21 current snap is 101 2022-04-28T19:47:36.126 INFO:tasks.rados.rados.0.smithi059.stdout:1033: finishing copy_from to smithi05955964-10 2022-04-28T19:47:36.126 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 313 (ObjNum -133313928 snap 21893 seq_num 4161653368) dirty exists 2022-04-28T19:47:36.126 INFO:tasks.rados.rados.0.smithi059.stdout:1033: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:47:36.127 INFO:tasks.rados.rados.0.smithi059.stdout:1033: done (3 left) 2022-04-28T19:47:36.128 INFO:tasks.rados.rados.0.smithi059.stdout:1036: done (2 left) 2022-04-28T19:47:36.128 INFO:tasks.rados.rados.0.smithi059.stdout:1037: snap_remove snap 89 2022-04-28T19:47:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:36 smithi139 bash[23439]: cluster 2022-04-28T19:47:35.101489+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2022-04-28T19:47:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:36 smithi059 bash[29114]: cluster 2022-04-28T19:47:35.101489+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2022-04-28T19:47:36.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:36 smithi059 bash[23786]: cluster 2022-04-28T19:47:35.101489+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2022-04-28T19:47:37.125 INFO:tasks.rados.rados.0.smithi059.stdout:1034: finishing write tid 1 to smithi05955964-5 2022-04-28T19:47:37.231 INFO:tasks.rados.rados.0.smithi059.stdout:1034: finishing write tid 2 to smithi05955964-5 2022-04-28T19:47:37.231 INFO:tasks.rados.rados.0.smithi059.stdout:1034: finishing write tid 4 to smithi05955964-5 2022-04-28T19:47:37.231 INFO:tasks.rados.rados.0.smithi059.stdout:1034: finishing write tid 5 to smithi05955964-5 2022-04-28T19:47:37.231 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 463 (ObjNum 335 snap 101 seq_num 335) dirty exists 2022-04-28T19:47:37.231 INFO:tasks.rados.rados.0.smithi059.stdout:1034: left oid 5 (ObjNum 335 snap 101 seq_num 335) 2022-04-28T19:47:37.232 INFO:tasks.rados.rados.0.smithi059.stdout:1035: expect (ObjNum 281 snap 80 seq_num 281) 2022-04-28T19:47:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:37 smithi139 bash[23439]: cluster 2022-04-28T19:47:36.112200+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2022-04-28T19:47:37.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:37 smithi059 bash[29114]: cluster 2022-04-28T19:47:36.112200+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2022-04-28T19:47:37.558 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:37 smithi059 bash[23786]: cluster 2022-04-28T19:47:36.112200+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2022-04-28T19:47:37.568 INFO:tasks.rados.rados.0.smithi059.stdout:1034: done (2 left) 2022-04-28T19:47:37.568 INFO:tasks.rados.rados.0.smithi059.stdout:1035: done (1 left) 2022-04-28T19:47:37.569 INFO:tasks.rados.rados.0.smithi059.stdout:1037: done (0 left) 2022-04-28T19:47:37.569 INFO:tasks.rados.rados.0.smithi059.stdout:1038: snap_remove snap 99 2022-04-28T19:47:37.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:37 smithi139 bash[37584]: Warning: Permanently added 'smithi139,172.21.15.139' (ECDSA) to the list of known hosts. 2022-04-28T19:47:37.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:37 smithi139 bash[37584]: Warning: Permanently added '172.21.15.59' (ECDSA) to the list of known hosts. 2022-04-28T19:47:37.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:37 smithi139 bash[37584]: debug 2022-04-28T19:47:37.625+0000 7fdd0f812700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:47:37.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:37 smithi139 bash[37584]: debug 2022-04-28T19:47:37.625+0000 7fdd0f812700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:47:37.869 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:47:37 smithi139 bash[37584]: debug 2022-04-28T19:47:37.625+0000 7fdd0c80c700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:47:38.242 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:47:38.483 INFO:tasks.rados.rados.0.smithi059.stdout:1038: done (0 left) 2022-04-28T19:47:38.484 INFO:tasks.rados.rados.0.smithi059.stdout:1039: snap_create 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: cluster 2022-04-28T19:47:37.121638+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.258678+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.259105+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.259843+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.261493+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.262091+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:47:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.263910+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.264415+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.264819+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.265237+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.265635+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.265967+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.266359+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.266769+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.268823+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.269148+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.269628+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:47:38.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.271251+0000 mon.b (mon.2) 45 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:47:38.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: cluster 2022-04-28T19:47:37.121638+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2022-04-28T19:47:38.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.258678+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:47:38.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.259105+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:47:38.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.259843+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.261493+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.262091+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.263910+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.264415+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.264819+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.265237+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.265635+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:47:38.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.265967+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:47:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.266359+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:47:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.266769+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:47:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.268823+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:47:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.269148+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:47:38.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.269628+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:47:38.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.271251+0000 mon.b (mon.2) 45 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:47:38.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.272424+0000 mon.b (mon.2) 46 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:47:38.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.272929+0000 mon.b (mon.2) 47 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:47:38.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.273488+0000 mon.b (mon.2) 48 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:47:38.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.273993+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.274472+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.274934+0000 mon.b (mon.2) 51 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.275359+0000 mon.b (mon.2) 52 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.275821+0000 mon.b (mon.2) 53 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.276349+0000 mon.b (mon.2) 54 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.276862+0000 mon.b (mon.2) 55 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.277434+0000 mon.b (mon.2) 56 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.277977+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:47:38.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.278518+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.279041+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.279558+0000 mon.b (mon.2) 60 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.280082+0000 mon.b (mon.2) 61 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.280644+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.272424+0000 mon.b (mon.2) 46 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.272929+0000 mon.b (mon.2) 47 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:47:38.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.273488+0000 mon.b (mon.2) 48 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.273993+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.274472+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.274934+0000 mon.b (mon.2) 51 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.275359+0000 mon.b (mon.2) 52 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.275821+0000 mon.b (mon.2) 53 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.276349+0000 mon.b (mon.2) 54 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:47:38.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.276862+0000 mon.b (mon.2) 55 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.277434+0000 mon.b (mon.2) 56 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.277977+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.278518+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.279041+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.279558+0000 mon.b (mon.2) 60 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.280082+0000 mon.b (mon.2) 61 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:47:38.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.280644+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.281193+0000 mon.b (mon.2) 63 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.281725+0000 mon.b (mon.2) 64 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.282279+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.282790+0000 mon.b (mon.2) 66 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.283306+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.283828+0000 mon.b (mon.2) 68 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: cluster 2022-04-28T19:47:37.303142+0000 mon.a (mon.0) 890 : cluster [INF] Manager daemon x is now available 2022-04-28T19:47:38.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: cephadm 2022-04-28T19:47:37.322487+0000 mgr.x (mgr.34769) 1 : cephadm [INF] inventory: adjusted host smithi059 addr 'smithi059' -> '172.21.15.59' 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.330044+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.478063+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.480063+0000 mon.b (mon.2) 69 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.482196+0000 mon.b (mon.2) 70 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.557906+0000 mon.b (mon.2) 71 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.559035+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.571729+0000 mon.b (mon.2) 72 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.572954+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.818579+0000 mon.b (mon.2) 73 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:47:38.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[23786]: audit 2022-04-28T19:47:37.848870+0000 mon.b (mon.2) 74 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.281193+0000 mon.b (mon.2) 63 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.281725+0000 mon.b (mon.2) 64 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.282279+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.282790+0000 mon.b (mon.2) 66 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.283306+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.283828+0000 mon.b (mon.2) 68 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: cluster 2022-04-28T19:47:37.303142+0000 mon.a (mon.0) 890 : cluster [INF] Manager daemon x is now available 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: cephadm 2022-04-28T19:47:37.322487+0000 mgr.x (mgr.34769) 1 : cephadm [INF] inventory: adjusted host smithi059 addr 'smithi059' -> '172.21.15.59' 2022-04-28T19:47:38.604 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.330044+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.478063+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.480063+0000 mon.b (mon.2) 69 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.482196+0000 mon.b (mon.2) 70 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.557906+0000 mon.b (mon.2) 71 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.559035+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.571729+0000 mon.b (mon.2) 72 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.572954+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.818579+0000 mon.b (mon.2) 73 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:47:38.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:38 smithi059 bash[29114]: audit 2022-04-28T19:47:37.848870+0000 mon.b (mon.2) 74 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:47:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: cluster 2022-04-28T19:47:37.121638+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2022-04-28T19:47:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.258678+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.259105+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.259843+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.261493+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.262091+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.263910+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.264415+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.264819+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.265237+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.265635+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:47:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.265967+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.266359+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.266769+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.268823+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.269148+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.269628+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.271251+0000 mon.b (mon.2) 45 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.272424+0000 mon.b (mon.2) 46 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.272929+0000 mon.b (mon.2) 47 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.273488+0000 mon.b (mon.2) 48 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:47:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.273993+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.274472+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.274934+0000 mon.b (mon.2) 51 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.275359+0000 mon.b (mon.2) 52 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.275821+0000 mon.b (mon.2) 53 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.276349+0000 mon.b (mon.2) 54 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.276862+0000 mon.b (mon.2) 55 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.277434+0000 mon.b (mon.2) 56 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.277977+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.278518+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:47:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.279041+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.279558+0000 mon.b (mon.2) 60 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.280082+0000 mon.b (mon.2) 61 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.280644+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.281193+0000 mon.b (mon.2) 63 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.281725+0000 mon.b (mon.2) 64 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.282279+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:47:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.282790+0000 mon.b (mon.2) 66 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.283306+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.283828+0000 mon.b (mon.2) 68 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: cluster 2022-04-28T19:47:37.303142+0000 mon.a (mon.0) 890 : cluster [INF] Manager daemon x is now available 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: cephadm 2022-04-28T19:47:37.322487+0000 mgr.x (mgr.34769) 1 : cephadm [INF] inventory: adjusted host smithi059 addr 'smithi059' -> '172.21.15.59' 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.330044+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.478063+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.480063+0000 mon.b (mon.2) 69 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.482196+0000 mon.b (mon.2) 70 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.557906+0000 mon.b (mon.2) 71 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:38.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.559035+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:47:38.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.571729+0000 mon.b (mon.2) 72 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:38.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.572954+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2022-04-28T19:47:38.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.818579+0000 mon.b (mon.2) 73 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:47:38.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:38 smithi139 bash[23439]: audit 2022-04-28T19:47:37.848870+0000 mon.b (mon.2) 74 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:47:39.492 INFO:tasks.rados.rados.0.smithi059.stdout:1039: done (0 left) 2022-04-28T19:47:39.493 INFO:tasks.rados.rados.0.smithi059.stdout:1040: rollback oid 48 current snap is 102 2022-04-28T19:47:39.493 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 96 2022-04-28T19:47:39.493 INFO:tasks.rados.rados.0.smithi059.stdout:1041: copy_from oid 4 from oid 24 current snap is 102 2022-04-28T19:47:39.493 INFO:tasks.rados.rados.0.smithi059.stdout:1042: snap_create 2022-04-28T19:47:39.509 INFO:tasks.rados.rados.0.smithi059.stdout:1040: finishing rollback tid 0 to smithi05955964-48 2022-04-28T19:47:39.509 INFO:tasks.rados.rados.0.smithi059.stdout:1040: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:47:39.511 INFO:tasks.rados.rados.0.smithi059.stdout:1040: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:47:39.511 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 466 (ObjNum 88 snap 18 seq_num 88) dirty exists 2022-04-28T19:47:39.526 INFO:tasks.rados.rados.0.smithi059.stdout:1041: finishing copy_from to smithi05955964-4 2022-04-28T19:47:39.526 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 322 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:47:39.527 INFO:tasks.rados.rados.0.smithi059.stdout:1041: finishing copy_from racing read to smithi05955964-4 2022-04-28T19:47:39.527 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:47:39.528 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (6m) 5m ago 7m - - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:47:39.528 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (6m) 13s ago 6m - - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:47:39.528 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (5m) 5m ago 5m - - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:47:39.528 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (5m) 13s ago 5m - - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:47:39.529 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (16s) 13s ago 10m - - 16.2.7-969-g4fa079ba 863db77b5b5e 646fe4292ebe 2022-04-28T19:47:39.529 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (13m) 5m ago 13m - - 15.2.16-79-gb51d8665 6d41e73600e1 320b27bb479e 2022-04-28T19:47:39.529 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (13m) 5m ago 13m - 2048M 15.2.16-79-gb51d8665 6d41e73600e1 239497cf6769 2022-04-28T19:47:39.529 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (11m) 13s ago 11m - 2048M 15.2.16-79-gb51d8665 6d41e73600e1 0e12092e0bc3 2022-04-28T19:47:39.529 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (11m) 5m ago 11m - 2048M 15.2.16-79-gb51d8665 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:47:39.530 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (7m) 5m ago 7m - - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:47:39.530 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (7m) 13s ago 7m - - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:47:39.530 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (10m) 5m ago 10m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 547b5580e073 2022-04-28T19:47:39.530 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (10m) 5m ago 10m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:47:39.530 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (9m) 5m ago 9m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:47:39.531 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (9m) 5m ago 9m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:47:39.531 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (8m) 13s ago 8m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:47:39.531 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (8m) 13s ago 8m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:47:39.531 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (8m) 13s ago 8m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:47:39.531 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (7m) 13s ago 7m - 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:47:39.532 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (6m) 13s ago 7m - - 2.18.1 de242295e225 c763409947e4 2022-04-28T19:47:39.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: cluster 2022-04-28T19:47:38.221859+0000 mon.a (mon.0) 895 : cluster [DBG] mgrmap e18: x(active, since 7s), standbys: y 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: audit 2022-04-28T19:47:38.226502+0000 mgr.x (mgr.34769) 2 : audit [DBG] from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: cluster 2022-04-28T19:47:38.262801+0000 mgr.x (mgr.34769) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: cluster 2022-04-28T19:47:38.480100+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: audit 2022-04-28T19:47:38.483126+0000 mon.b (mon.2) 75 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: audit 2022-04-28T19:47:38.484079+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: audit 2022-04-28T19:47:38.484903+0000 mon.b (mon.2) 76 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:47:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[23786]: audit 2022-04-28T19:47:38.485732+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: cluster 2022-04-28T19:47:38.221859+0000 mon.a (mon.0) 895 : cluster [DBG] mgrmap e18: x(active, since 7s), standbys: y 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: audit 2022-04-28T19:47:38.226502+0000 mgr.x (mgr.34769) 2 : audit [DBG] from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: cluster 2022-04-28T19:47:38.262801+0000 mgr.x (mgr.34769) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: cluster 2022-04-28T19:47:38.480100+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: audit 2022-04-28T19:47:38.483126+0000 mon.b (mon.2) 75 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: audit 2022-04-28T19:47:38.484079+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: audit 2022-04-28T19:47:38.484903+0000 mon.b (mon.2) 76 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:47:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:39 smithi059 bash[29114]: audit 2022-04-28T19:47:38.485732+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:47:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: cluster 2022-04-28T19:47:38.221859+0000 mon.a (mon.0) 895 : cluster [DBG] mgrmap e18: x(active, since 7s), standbys: y 2022-04-28T19:47:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: audit 2022-04-28T19:47:38.226502+0000 mgr.x (mgr.34769) 2 : audit [DBG] from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:47:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: cluster 2022-04-28T19:47:38.262801+0000 mgr.x (mgr.34769) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: cluster 2022-04-28T19:47:38.480100+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2022-04-28T19:47:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: audit 2022-04-28T19:47:38.483126+0000 mon.b (mon.2) 75 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:47:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: audit 2022-04-28T19:47:38.484079+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:47:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: audit 2022-04-28T19:47:38.484903+0000 mon.b (mon.2) 76 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:47:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:39 smithi139 bash[23439]: audit 2022-04-28T19:47:38.485732+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:47:39.889 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:47:39.889 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:47:39.889 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 1, 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:47:39.890 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 14, 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:47:39.891 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:47:40.504 INFO:tasks.rados.rados.0.smithi059.stdout:1040: done (2 left) 2022-04-28T19:47:40.505 INFO:tasks.rados.rados.0.smithi059.stdout:1041: done (1 left) 2022-04-28T19:47:40.505 INFO:tasks.rados.rados.0.smithi059.stdout:1042: done (0 left) 2022-04-28T19:47:40.505 INFO:tasks.rados.rados.0.smithi059.stdout:1043: read oid 34 snap 97 2022-04-28T19:47:40.505 INFO:tasks.rados.rados.0.smithi059.stdout:1043: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:47:40.505 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 103 2022-04-28T19:47:40.505 INFO:tasks.rados.rados.0.smithi059.stdout:1044: seq_num 337 ranges {7585792=450560,8036352=401408} 2022-04-28T19:47:40.550 INFO:tasks.rados.rados.0.smithi059.stdout:1044: writing smithi05955964-45 from 7585792 to 8036352 tid 1 2022-04-28T19:47:40.553 INFO:tasks.rados.rados.0.smithi059.stdout:1044: writing smithi05955964-45 from 8036352 to 8437760 tid 2 2022-04-28T19:47:40.553 INFO:tasks.rados.rados.0.smithi059.stdout:1045: delete oid 48 current snap is 103 2022-04-28T19:47:40.554 INFO:tasks.rados.rados.0.smithi059.stdout:1043: expect (ObjNum 207 snap 59 seq_num 207) 2022-04-28T19:47:40.742 INFO:tasks.rados.rados.0.smithi059.stdout:1044: finishing write tid 1 to smithi05955964-45 2022-04-28T19:47:40.749 INFO:tasks.rados.rados.0.smithi059.stdout:1044: finishing write tid 2 to smithi05955964-45 2022-04-28T19:47:40.749 INFO:tasks.rados.rados.0.smithi059.stdout:1044: finishing write tid 4 to smithi05955964-45 2022-04-28T19:47:40.749 INFO:tasks.rados.rados.0.smithi059.stdout:1044: finishing write tid 5 to smithi05955964-45 2022-04-28T19:47:40.749 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 408 (ObjNum 337 snap 103 seq_num 337) dirty exists 2022-04-28T19:47:40.749 INFO:tasks.rados.rados.0.smithi059.stdout:1044: left oid 45 (ObjNum 337 snap 103 seq_num 337) 2022-04-28T19:47:40.749 INFO:tasks.rados.rados.0.smithi059.stdout:1043: done (2 left) 2022-04-28T19:47:40.750 INFO:tasks.rados.rados.0.smithi059.stdout:1044: done (1 left) 2022-04-28T19:47:40.750 INFO:tasks.rados.rados.0.smithi059.stdout:1045: done (0 left) 2022-04-28T19:47:40.750 INFO:tasks.rados.rados.0.smithi059.stdout:1046: setattr oid 29 current snap is 103 2022-04-28T19:47:40.750 INFO:tasks.rados.rados.0.smithi059.stdout:1047: rollback oid 3 current snap is 103 2022-04-28T19:47:40.751 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 95 2022-04-28T19:47:40.751 INFO:tasks.rados.rados.0.smithi059.stdout:1048: snap_create 2022-04-28T19:47:40.755 INFO:tasks.rados.rados.0.smithi059.stdout:1047: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:47:40.756 INFO:tasks.rados.rados.0.smithi059.stdout:1047: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:47:40.756 INFO:tasks.rados.rados.0.smithi059.stdout:1047: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:47:40.756 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 412 (ObjNum 317 snap 95 seq_num 317) dirty exists 2022-04-28T19:47:40.757 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 477 (ObjNum 305 snap 92 seq_num 305) dirty exists 2022-04-28T19:47:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:38.540891+0000 mgr.x (mgr.34769) 4 : audit [DBG] from='client.14955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:47:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: cluster 2022-04-28T19:47:39.269137+0000 mgr.x (mgr.34769) 5 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.490322+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: cluster 2022-04-28T19:47:39.490934+0000 mon.a (mon.0) 900 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.494601+0000 mon.b (mon.2) 77 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: cluster 2022-04-28T19:47:39.502031+0000 mon.a (mon.0) 901 : cluster [DBG] mgrmap e19: x(active, since 8s), standbys: y 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.505745+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.506415+0000 mon.b (mon.2) 79 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.507327+0000 mon.b (mon.2) 80 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.507910+0000 mon.b (mon.2) 81 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.508545+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.511701+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.512197+0000 mon.b (mon.2) 84 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.512683+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.513215+0000 mon.b (mon.2) 86 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.513717+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[23786]: audit 2022-04-28T19:47:39.887970+0000 mon.c (mon.1) 41 : audit [DBG] from='client.? 172.21.15.59:0/2395788882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:38.540891+0000 mgr.x (mgr.34769) 4 : audit [DBG] from='client.14955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: cluster 2022-04-28T19:47:39.269137+0000 mgr.x (mgr.34769) 5 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:40.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.490322+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: cluster 2022-04-28T19:47:39.490934+0000 mon.a (mon.0) 900 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.494601+0000 mon.b (mon.2) 77 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: cluster 2022-04-28T19:47:39.502031+0000 mon.a (mon.0) 901 : cluster [DBG] mgrmap e19: x(active, since 8s), standbys: y 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.505745+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.506415+0000 mon.b (mon.2) 79 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.507327+0000 mon.b (mon.2) 80 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.507910+0000 mon.b (mon.2) 81 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.508545+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.511701+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.512197+0000 mon.b (mon.2) 84 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.512683+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.513215+0000 mon.b (mon.2) 86 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.513717+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:40 smithi059 bash[29114]: audit 2022-04-28T19:47:39.887970+0000 mon.c (mon.1) 41 : audit [DBG] from='client.? 172.21.15.59:0/2395788882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:47:40.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:38.540891+0000 mgr.x (mgr.34769) 4 : audit [DBG] from='client.14955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:47:40.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: cluster 2022-04-28T19:47:39.269137+0000 mgr.x (mgr.34769) 5 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:40.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.490322+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: cluster 2022-04-28T19:47:39.490934+0000 mon.a (mon.0) 900 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.494601+0000 mon.b (mon.2) 77 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: cluster 2022-04-28T19:47:39.502031+0000 mon.a (mon.0) 901 : cluster [DBG] mgrmap e19: x(active, since 8s), standbys: y 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.505745+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.506415+0000 mon.b (mon.2) 79 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.507327+0000 mon.b (mon.2) 80 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.507910+0000 mon.b (mon.2) 81 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.508545+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.511701+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:47:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.512197+0000 mon.b (mon.2) 84 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.512683+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.513215+0000 mon.b (mon.2) 86 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.513717+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:40 smithi139 bash[23439]: audit 2022-04-28T19:47:39.887970+0000 mon.c (mon.1) 41 : audit [DBG] from='client.? 172.21.15.59:0/2395788882' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:47:41.511 INFO:tasks.rados.rados.0.smithi059.stdout:1046: done (2 left) 2022-04-28T19:47:41.511 INFO:tasks.rados.rados.0.smithi059.stdout:1047: done (1 left) 2022-04-28T19:47:41.511 INFO:tasks.rados.rados.0.smithi059.stdout:1048: done (0 left) 2022-04-28T19:47:41.512 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 104 2022-04-28T19:47:41.512 INFO:tasks.rados.rados.0.smithi059.stdout:1049: seq_num 339 ranges {0=483328,483328=401408} 2022-04-28T19:47:41.514 INFO:tasks.rados.rados.0.smithi059.stdout:1049: writing smithi05955964-36 from 0 to 483328 tid 1 2022-04-28T19:47:41.516 INFO:tasks.rados.rados.0.smithi059.stdout:1049: writing smithi05955964-36 from 483328 to 884736 tid 2 2022-04-28T19:47:41.516 INFO:tasks.rados.rados.0.smithi059.stdout:1050: read oid 24 snap -1 2022-04-28T19:47:41.516 INFO:tasks.rados.rados.0.smithi059.stdout:1050: expect (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:47:41.517 INFO:tasks.rados.rados.0.smithi059.stdout:1051: snap_remove snap 94 2022-04-28T19:47:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:41 smithi059 bash[29114]: cluster 2022-04-28T19:47:40.499108+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2022-04-28T19:47:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:41 smithi059 bash[23786]: cluster 2022-04-28T19:47:40.499108+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2022-04-28T19:47:41.867 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:41 smithi139 bash[23439]: cluster 2022-04-28T19:47:40.499108+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2022-04-28T19:47:42.219 INFO:tasks.rados.rados.0.smithi059.stdout:1051: done (2 left) 2022-04-28T19:47:42.219 INFO:tasks.rados.rados.0.smithi059.stdout:1052: read oid 22 snap -1 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:1052: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:1053: rollback oid 39 current snap is 104 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 39 to 96 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:1049: finishing write tid 1 to smithi05955964-36 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:1054: delete oid 17 current snap is 104 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:1049: finishing write tid 2 to smithi05955964-36 2022-04-28T19:47:42.220 INFO:tasks.rados.rados.0.smithi059.stdout:1050: expect (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:47:42.466 INFO:tasks.rados.rados.0.smithi059.stdout:1049: finishing write tid 4 to smithi05955964-36 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:1049: finishing write tid 5 to smithi05955964-36 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 403 (ObjNum 339 snap 104 seq_num 339) dirty exists 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:1049: left oid 36 (ObjNum 339 snap 104 seq_num 339) 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:1049: done (4 left) 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:1050: done (3 left) 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:1054: done (2 left) 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 104 2022-04-28T19:47:42.467 INFO:tasks.rados.rados.0.smithi059.stdout:1055: seq_num 341 ranges {212992=516096,729088=401408} 2022-04-28T19:47:42.478 INFO:tasks.rados.rados.0.smithi059.stdout:1055: writing smithi05955964-31 from 212992 to 729088 tid 1 2022-04-28T19:47:42.482 INFO:tasks.rados.rados.0.smithi059.stdout:1055: writing smithi05955964-31 from 729088 to 1130496 tid 2 2022-04-28T19:47:42.482 INFO:tasks.rados.rados.0.smithi059.stdout:1056: read oid 8 snap -1 2022-04-28T19:47:42.482 INFO:tasks.rados.rados.0.smithi059.stdout:1056: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:47:42.482 INFO:tasks.rados.rados.0.smithi059.stdout:1057: snap_remove snap 97 2022-04-28T19:47:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:42 smithi059 bash[29114]: cluster 2022-04-28T19:47:41.269736+0000 mgr.x (mgr.34769) 6 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:42 smithi059 bash[29114]: cluster 2022-04-28T19:47:41.509132+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2022-04-28T19:47:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:42 smithi059 bash[29114]: cluster 2022-04-28T19:47:42.217028+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2022-04-28T19:47:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:42 smithi059 bash[23786]: cluster 2022-04-28T19:47:41.269736+0000 mgr.x (mgr.34769) 6 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:42 smithi059 bash[23786]: cluster 2022-04-28T19:47:41.509132+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2022-04-28T19:47:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:42 smithi059 bash[23786]: cluster 2022-04-28T19:47:42.217028+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2022-04-28T19:47:42.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:42 smithi139 bash[23439]: cluster 2022-04-28T19:47:41.269736+0000 mgr.x (mgr.34769) 6 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 237 MiB data, 341 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:42.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:42 smithi139 bash[23439]: cluster 2022-04-28T19:47:41.509132+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2022-04-28T19:47:42.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:42 smithi139 bash[23439]: cluster 2022-04-28T19:47:42.217028+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2022-04-28T19:47:43.236 INFO:tasks.rados.rados.0.smithi059.stdout:1057: done (4 left) 2022-04-28T19:47:43.236 INFO:tasks.rados.rados.0.smithi059.stdout:1058: read oid 34 snap -1 2022-04-28T19:47:43.236 INFO:tasks.rados.rados.0.smithi059.stdout:1058: expect (ObjNum 328 snap 99 seq_num 328) 2022-04-28T19:47:43.237 INFO:tasks.rados.rados.0.smithi059.stdout:1059: rollback oid 13 current snap is 104 2022-04-28T19:47:43.237 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 13 to 73 2022-04-28T19:47:43.237 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 104 2022-04-28T19:47:43.237 INFO:tasks.rados.rados.0.smithi059.stdout:1060: seq_num 342 ranges {0=532480,532480=401408} 2022-04-28T19:47:43.244 INFO:tasks.rados.rados.0.smithi059.stdout:1060: writing smithi05955964-10 from 0 to 532480 tid 1 2022-04-28T19:47:43.244 INFO:tasks.rados.rados.0.smithi059.stdout:1060: writing smithi05955964-10 from 532480 to 933888 tid 2 2022-04-28T19:47:43.245 INFO:tasks.rados.rados.0.smithi059.stdout:1061: setattr oid 21 current snap is 104 2022-04-28T19:47:43.245 INFO:tasks.rados.rados.0.smithi059.stdout:1053: finishing rollback tid 1 to smithi05955964-39 2022-04-28T19:47:43.245 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 461 (ObjNum 1 snap 32762 seq_num 4161697872) dirty dne 2022-04-28T19:47:43.245 INFO:tasks.rados.rados.0.smithi059.stdout:1052: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:47:43.305 INFO:tasks.rados.rados.0.smithi059.stdout:1055: finishing write tid 1 to smithi05955964-31 2022-04-28T19:47:43.305 INFO:tasks.rados.rados.0.smithi059.stdout:1055: finishing write tid 2 to smithi05955964-31 2022-04-28T19:47:43.305 INFO:tasks.rados.rados.0.smithi059.stdout:1055: finishing write tid 4 to smithi05955964-31 2022-04-28T19:47:43.305 INFO:tasks.rados.rados.0.smithi059.stdout:1055: finishing write tid 5 to smithi05955964-31 2022-04-28T19:47:43.305 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 513 (ObjNum 341 snap 104 seq_num 341) dirty exists 2022-04-28T19:47:43.306 INFO:tasks.rados.rados.0.smithi059.stdout:1055: left oid 31 (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:47:43.306 INFO:tasks.rados.rados.0.smithi059.stdout:1056: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:47:43.569 INFO:tasks.rados.rados.0.smithi059.stdout:1060: finishing write tid 1 to smithi05955964-10 2022-04-28T19:47:43.569 INFO:tasks.rados.rados.0.smithi059.stdout:1060: finishing write tid 2 to smithi05955964-10 2022-04-28T19:47:43.569 INFO:tasks.rados.rados.0.smithi059.stdout:1060: finishing write tid 4 to smithi05955964-10 2022-04-28T19:47:43.569 INFO:tasks.rados.rados.0.smithi059.stdout:1060: finishing write tid 5 to smithi05955964-10 2022-04-28T19:47:43.569 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 457 (ObjNum 342 snap 104 seq_num 342) dirty exists 2022-04-28T19:47:43.570 INFO:tasks.rados.rados.0.smithi059.stdout:1060: left oid 10 (ObjNum 342 snap 104 seq_num 342) 2022-04-28T19:47:43.570 INFO:tasks.rados.rados.0.smithi059.stdout:1059: finishing rollback tid 0 to smithi05955964-13 2022-04-28T19:47:43.570 INFO:tasks.rados.rados.0.smithi059.stdout:1059: finishing rollback tid 1 to smithi05955964-13 2022-04-28T19:47:43.570 INFO:tasks.rados.rados.0.smithi059.stdout:1059: finishing rollback tid 2 to smithi05955964-13 2022-04-28T19:47:43.570 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 243 (ObjNum 161 snap 42 seq_num 161) dirty exists 2022-04-28T19:47:43.571 INFO:tasks.rados.rados.0.smithi059.stdout:1058: expect (ObjNum 328 snap 99 seq_num 328) 2022-04-28T19:47:43.835 INFO:tasks.rados.rados.0.smithi059.stdout:1052: done (7 left) 2022-04-28T19:47:43.835 INFO:tasks.rados.rados.0.smithi059.stdout:1053: done (6 left) 2022-04-28T19:47:43.835 INFO:tasks.rados.rados.0.smithi059.stdout:1055: done (5 left) 2022-04-28T19:47:43.836 INFO:tasks.rados.rados.0.smithi059.stdout:1056: done (4 left) 2022-04-28T19:47:43.836 INFO:tasks.rados.rados.0.smithi059.stdout:1058: done (3 left) 2022-04-28T19:47:43.836 INFO:tasks.rados.rados.0.smithi059.stdout:1059: done (2 left) 2022-04-28T19:47:43.836 INFO:tasks.rados.rados.0.smithi059.stdout:1060: done (1 left) 2022-04-28T19:47:43.836 INFO:tasks.rados.rados.0.smithi059.stdout:1062: setattr oid 9 current snap is 104 2022-04-28T19:47:43.838 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 365 (ObjNum 1767726955 snap 1668174681 seq_num 1701466958) dirty exists 2022-04-28T19:47:43.839 INFO:tasks.rados.rados.0.smithi059.stdout:1061: done (1 left) 2022-04-28T19:47:43.839 INFO:tasks.rados.rados.0.smithi059.stdout:1063: rmattr oid 4 current snap is 104 2022-04-28T19:47:43.839 INFO:tasks.rados.rados.0.smithi059.stdout:1063: done (1 left) 2022-04-28T19:47:43.839 INFO:tasks.rados.rados.0.smithi059.stdout:1064: read oid 25 snap -1 2022-04-28T19:47:43.839 INFO:tasks.rados.rados.0.smithi059.stdout:1064: expect deleted 2022-04-28T19:47:43.839 INFO:tasks.rados.rados.0.smithi059.stdout:1065: rmattr oid 36 current snap is 104 2022-04-28T19:47:43.840 INFO:tasks.rados.rados.0.smithi059.stdout:1065: done (2 left) 2022-04-28T19:47:43.840 INFO:tasks.rados.rados.0.smithi059.stdout:1066: setattr oid 4 current snap is 104 2022-04-28T19:47:43.840 INFO:tasks.rados.rados.0.smithi059.stdout:1064: done (2 left) 2022-04-28T19:47:43.840 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 104 2022-04-28T19:47:43.841 INFO:tasks.rados.rados.0.smithi059.stdout:1067: seq_num 343 ranges {2007040=548864,2555904=401408} 2022-04-28T19:47:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:43.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:47:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:43.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:47:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:43.520Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:47:43.857 INFO:tasks.rados.rados.0.smithi059.stdout:1067: writing smithi05955964-19 from 2007040 to 2555904 tid 1 2022-04-28T19:47:43.860 INFO:tasks.rados.rados.0.smithi059.stdout:1067: writing smithi05955964-19 from 2555904 to 2957312 tid 2 2022-04-28T19:47:43.860 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 104 2022-04-28T19:47:43.860 INFO:tasks.rados.rados.0.smithi059.stdout:1068: seq_num 344 ranges {8134656=565248,8699904=401408} 2022-04-28T19:47:43.909 INFO:tasks.rados.rados.0.smithi059.stdout:1068: writing smithi05955964-23 from 8134656 to 8699904 tid 1 2022-04-28T19:47:43.912 INFO:tasks.rados.rados.0.smithi059.stdout:1068: writing smithi05955964-23 from 8699904 to 9101312 tid 2 2022-04-28T19:47:43.912 INFO:tasks.rados.rados.0.smithi059.stdout:1069: setattr oid 25 current snap is 104 2022-04-28T19:47:43.912 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 297 (ObjNum 279 snap 80 seq_num 279) dirty exists 2022-04-28T19:47:43.912 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 408 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:47:43.912 INFO:tasks.rados.rados.0.smithi059.stdout:1067: finishing write tid 1 to smithi05955964-19 2022-04-28T19:47:43.913 INFO:tasks.rados.rados.0.smithi059.stdout:1067: finishing write tid 2 to smithi05955964-19 2022-04-28T19:47:43.913 INFO:tasks.rados.rados.0.smithi059.stdout:1067: finishing write tid 4 to smithi05955964-19 2022-04-28T19:47:43.913 INFO:tasks.rados.rados.0.smithi059.stdout:1067: finishing write tid 5 to smithi05955964-19 2022-04-28T19:47:43.913 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 484 (ObjNum 343 snap 104 seq_num 343) dirty exists 2022-04-28T19:47:43.913 INFO:tasks.rados.rados.0.smithi059.stdout:1067: left oid 19 (ObjNum 343 snap 104 seq_num 343) 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1062: done (4 left) 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1066: done (3 left) 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1067: done (2 left) 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1070: read oid 9 snap -1 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1070: expect (ObjNum 279 snap 80 seq_num 279) 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1071: read oid 30 snap 93 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1071: expect deleted 2022-04-28T19:47:43.914 INFO:tasks.rados.rados.0.smithi059.stdout:1072: setattr oid 3 current snap is 104 2022-04-28T19:47:43.915 INFO:tasks.rados.rados.0.smithi059.stdout:1073: read oid 47 snap -1 2022-04-28T19:47:43.915 INFO:tasks.rados.rados.0.smithi059.stdout:1073: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:47:43.915 INFO:tasks.rados.rados.0.smithi059.stdout:1074: copy_from oid 10 from oid 6 current snap is 104 2022-04-28T19:47:43.915 INFO:tasks.rados.rados.0.smithi059.stdout:1075: snap_remove snap 78 2022-04-28T19:47:44.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:42.855789+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.145037+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.146023+0000 mon.b (mon.2) 89 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.146811+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.147491+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.148258+0000 mon.b (mon.2) 92 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.149064+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.149873+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.150671+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.151511+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.152410+0000 mon.b (mon.2) 97 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.153578+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: cluster 2022-04-28T19:47:43.234884+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: cluster 2022-04-28T19:47:43.270348+0000 mgr.x (mgr.34769) 7 : cluster [DBG] pgmap v12: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.328353+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.610815+0000 mon.b (mon.2) 98 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.611861+0000 mon.b (mon.2) 99 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.612756+0000 mon.b (mon.2) 100 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.613504+0000 mon.b (mon.2) 101 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.614198+0000 mon.b (mon.2) 102 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.614953+0000 mon.b (mon.2) 103 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.615748+0000 mon.b (mon.2) 104 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.616509+0000 mon.b (mon.2) 105 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.617302+0000 mon.b (mon.2) 106 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.618179+0000 mon.b (mon.2) 107 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.619331+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.629949+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.638215+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.650329+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.660588+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:43 smithi139 bash[23439]: audit 2022-04-28T19:47:43.671785+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:42.855789+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.145037+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.146023+0000 mon.b (mon.2) 89 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.146811+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.147491+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.148258+0000 mon.b (mon.2) 92 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.149064+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.149873+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.150671+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.151511+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.152410+0000 mon.b (mon.2) 97 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.153578+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: cluster 2022-04-28T19:47:43.234884+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: cluster 2022-04-28T19:47:43.270348+0000 mgr.x (mgr.34769) 7 : cluster [DBG] pgmap v12: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.328353+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.610815+0000 mon.b (mon.2) 98 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.611861+0000 mon.b (mon.2) 99 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:42.855789+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.145037+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.146023+0000 mon.b (mon.2) 89 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.146811+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.147491+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.148258+0000 mon.b (mon.2) 92 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.149064+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.149873+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.150671+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.151511+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.152410+0000 mon.b (mon.2) 97 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.153578+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: cluster 2022-04-28T19:47:43.234884+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: cluster 2022-04-28T19:47:43.270348+0000 mgr.x (mgr.34769) 7 : cluster [DBG] pgmap v12: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.328353+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.610815+0000 mon.b (mon.2) 98 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.611861+0000 mon.b (mon.2) 99 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.612756+0000 mon.b (mon.2) 100 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.613504+0000 mon.b (mon.2) 101 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.614198+0000 mon.b (mon.2) 102 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.614953+0000 mon.b (mon.2) 103 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.615748+0000 mon.b (mon.2) 104 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.616509+0000 mon.b (mon.2) 105 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.617302+0000 mon.b (mon.2) 106 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.618179+0000 mon.b (mon.2) 107 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.619331+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.629949+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.638215+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.650329+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.660588+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[23786]: audit 2022-04-28T19:47:43.671785+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.612756+0000 mon.b (mon.2) 100 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.613504+0000 mon.b (mon.2) 101 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.614198+0000 mon.b (mon.2) 102 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.614953+0000 mon.b (mon.2) 103 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.615748+0000 mon.b (mon.2) 104 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.616509+0000 mon.b (mon.2) 105 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.617302+0000 mon.b (mon.2) 106 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.618179+0000 mon.b (mon.2) 107 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.619331+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.629949+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.638215+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.650329+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.660588+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:43 smithi059 bash[29114]: audit 2022-04-28T19:47:43.671785+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:44.867 INFO:tasks.rados.rados.0.smithi059.stdout:1075: done (7 left) 2022-04-28T19:47:44.867 INFO:tasks.rados.rados.0.smithi059.stdout:1076: copy_from oid 42 from oid 2 current snap is 104 2022-04-28T19:47:44.867 INFO:tasks.rados.rados.0.smithi059.stdout:1077: snap_remove snap 95 2022-04-28T19:47:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:45 smithi059 bash[29114]: cephadm 2022-04-28T19:47:43.686722+0000 mgr.x (mgr.34769) 8 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-04-28T19:47:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:45 smithi059 bash[29114]: cephadm 2022-04-28T19:47:43.703220+0000 mgr.x (mgr.34769) 9 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi139 2022-04-28T19:47:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:45 smithi059 bash[29114]: cluster 2022-04-28T19:47:44.864317+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2022-04-28T19:47:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:45 smithi059 bash[23786]: cephadm 2022-04-28T19:47:43.686722+0000 mgr.x (mgr.34769) 8 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-04-28T19:47:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:45 smithi059 bash[23786]: cephadm 2022-04-28T19:47:43.703220+0000 mgr.x (mgr.34769) 9 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi139 2022-04-28T19:47:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:45 smithi059 bash[23786]: cluster 2022-04-28T19:47:44.864317+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2022-04-28T19:47:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:45 smithi139 bash[23439]: cephadm 2022-04-28T19:47:43.686722+0000 mgr.x (mgr.34769) 8 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2022-04-28T19:47:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:45 smithi139 bash[23439]: cephadm 2022-04-28T19:47:43.703220+0000 mgr.x (mgr.34769) 9 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi139 2022-04-28T19:47:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:45 smithi139 bash[23439]: cluster 2022-04-28T19:47:44.864317+0000 mon.a (mon.0) 915 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:1077: done (8 left) 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:1078: snap_create 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:1068: finishing write tid 1 to smithi05955964-23 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 486 (ObjNum -133099664 snap 21893 seq_num 0) dirty exists 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 424 (ObjNum 317 snap 95 seq_num 317) dirty exists 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:1068: finishing write tid 2 to smithi05955964-23 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:1068: finishing write tid 4 to smithi05955964-23 2022-04-28T19:47:45.866 INFO:tasks.rados.rados.0.smithi059.stdout:1073: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:47:45.957 INFO:tasks.rados.rados.0.smithi059.stdout:1068: finishing write tid 5 to smithi05955964-23 2022-04-28T19:47:45.957 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 465 (ObjNum 344 snap 104 seq_num 344) dirty exists 2022-04-28T19:47:45.957 INFO:tasks.rados.rados.0.smithi059.stdout:1068: left oid 23 (ObjNum 344 snap 104 seq_num 344) 2022-04-28T19:47:45.957 INFO:tasks.rados.rados.0.smithi059.stdout:1070: expect (ObjNum 279 snap 80 seq_num 279) 2022-04-28T19:47:46.066 INFO:tasks.rados.rados.0.smithi059.stdout:1074: finishing copy_from to smithi05955964-10 2022-04-28T19:47:46.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:46.066 INFO:tasks.rados.rados.0.smithi059.stdout:1074: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:47:46.066 INFO:tasks.rados.rados.0.smithi059.stdout:1076: finishing copy_from to smithi05955964-42 2022-04-28T19:47:46.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 292 (ObjNum 293 snap 87 seq_num 293) dirty exists 2022-04-28T19:47:46.067 INFO:tasks.rados.rados.0.smithi059.stdout:1076: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:47:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:46 smithi059 bash[29114]: cluster 2022-04-28T19:47:45.271022+0000 mgr.x (mgr.34769) 10 : cluster [DBG] pgmap v14: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2022-04-28T19:47:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:46 smithi059 bash[29114]: cluster 2022-04-28T19:47:45.863967+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2022-04-28T19:47:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:46 smithi059 bash[23786]: cluster 2022-04-28T19:47:45.271022+0000 mgr.x (mgr.34769) 10 : cluster [DBG] pgmap v14: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2022-04-28T19:47:46.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:46 smithi059 bash[23786]: cluster 2022-04-28T19:47:45.863967+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2022-04-28T19:47:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:46 smithi139 bash[23439]: cluster 2022-04-28T19:47:45.271022+0000 mgr.x (mgr.34769) 10 : cluster [DBG] pgmap v14: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2022-04-28T19:47:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:46 smithi139 bash[23439]: cluster 2022-04-28T19:47:45.863967+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2022-04-28T19:47:46.872 INFO:tasks.rados.rados.0.smithi059.stdout:1068: done (8 left) 2022-04-28T19:47:46.872 INFO:tasks.rados.rados.0.smithi059.stdout:1069: done (7 left) 2022-04-28T19:47:46.872 INFO:tasks.rados.rados.0.smithi059.stdout:1070: done (6 left) 2022-04-28T19:47:46.872 INFO:tasks.rados.rados.0.smithi059.stdout:1071: done (5 left) 2022-04-28T19:47:46.873 INFO:tasks.rados.rados.0.smithi059.stdout:1072: done (4 left) 2022-04-28T19:47:46.873 INFO:tasks.rados.rados.0.smithi059.stdout:1073: done (3 left) 2022-04-28T19:47:46.873 INFO:tasks.rados.rados.0.smithi059.stdout:1074: done (2 left) 2022-04-28T19:47:46.873 INFO:tasks.rados.rados.0.smithi059.stdout:1076: done (1 left) 2022-04-28T19:47:46.873 INFO:tasks.rados.rados.0.smithi059.stdout:1078: done (0 left) 2022-04-28T19:47:46.873 INFO:tasks.rados.rados.0.smithi059.stdout:1079: snap_remove snap 88 2022-04-28T19:47:47.227 INFO:tasks.rados.rados.0.smithi059.stdout:1079: done (0 left) 2022-04-28T19:47:47.228 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 105 2022-04-28T19:47:47.228 INFO:tasks.rados.rados.0.smithi059.stdout:1080: seq_num 345 ranges {0=581632,581632=401408} 2022-04-28T19:47:47.231 INFO:tasks.rados.rados.0.smithi059.stdout:1080: writing smithi05955964-25 from 0 to 581632 tid 1 2022-04-28T19:47:47.234 INFO:tasks.rados.rados.0.smithi059.stdout:1080: writing smithi05955964-25 from 581632 to 983040 tid 2 2022-04-28T19:47:47.234 INFO:tasks.rados.rados.0.smithi059.stdout:1081: copy_from oid 1 from oid 10 current snap is 105 2022-04-28T19:47:47.234 INFO:tasks.rados.rados.0.smithi059.stdout:1082: read oid 46 snap -1 2022-04-28T19:47:47.234 INFO:tasks.rados.rados.0.smithi059.stdout:1082: expect deleted 2022-04-28T19:47:47.234 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 105 2022-04-28T19:47:47.235 INFO:tasks.rados.rados.0.smithi059.stdout:1083: seq_num 346 ranges {4972544=598016,5570560=401408} 2022-04-28T19:47:47.267 INFO:tasks.rados.rados.0.smithi059.stdout:1083: writing smithi05955964-24 from 4972544 to 5570560 tid 1 2022-04-28T19:47:47.270 INFO:tasks.rados.rados.0.smithi059.stdout:1083: writing smithi05955964-24 from 5570560 to 5971968 tid 2 2022-04-28T19:47:47.270 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 105 2022-04-28T19:47:47.271 INFO:tasks.rados.rados.0.smithi059.stdout:1084: seq_num 347 ranges {1572864=614400,2187264=401408} 2022-04-28T19:47:47.283 INFO:tasks.rados.rados.0.smithi059.stdout:1084: writing smithi05955964-5 from 1572864 to 2187264 tid 1 2022-04-28T19:47:47.286 INFO:tasks.rados.rados.0.smithi059.stdout:1084: writing smithi05955964-5 from 2187264 to 2588672 tid 2 2022-04-28T19:47:47.286 INFO:tasks.rados.rados.0.smithi059.stdout:1085: rollback oid 26 current snap is 105 2022-04-28T19:47:47.286 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 103 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1080: finishing write tid 1 to smithi05955964-25 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1080: finishing write tid 2 to smithi05955964-25 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1080: finishing write tid 4 to smithi05955964-25 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1080: finishing write tid 5 to smithi05955964-25 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 508 (ObjNum 345 snap 105 seq_num 345) dirty exists 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1080: left oid 25 (ObjNum 345 snap 105 seq_num 345) 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1081: finishing copy_from to smithi05955964-1 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1081: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:47:47.287 INFO:tasks.rados.rados.0.smithi059.stdout:1083: finishing write tid 1 to smithi05955964-24 2022-04-28T19:47:47.288 INFO:tasks.rados.rados.0.smithi059.stdout:1080: done (5 left) 2022-04-28T19:47:47.288 INFO:tasks.rados.rados.0.smithi059.stdout:1081: done (4 left) 2022-04-28T19:47:47.288 INFO:tasks.rados.rados.0.smithi059.stdout:1082: done (3 left) 2022-04-28T19:47:47.288 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 105 2022-04-28T19:47:47.288 INFO:tasks.rados.rados.0.smithi059.stdout:1086: seq_num 348 ranges {6037504=630784,6668288=401408} 2022-04-28T19:47:47.323 INFO:tasks.rados.rados.0.smithi059.stdout:1086: writing smithi05955964-12 from 6037504 to 6668288 tid 1 2022-04-28T19:47:47.326 INFO:tasks.rados.rados.0.smithi059.stdout:1086: writing smithi05955964-12 from 6668288 to 7069696 tid 2 2022-04-28T19:47:47.326 INFO:tasks.rados.rados.0.smithi059.stdout:1087: copy_from oid 9 from oid 3 current snap is 105 2022-04-28T19:47:47.326 INFO:tasks.rados.rados.0.smithi059.stdout:1083: finishing write tid 2 to smithi05955964-24 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1083: finishing write tid 4 to smithi05955964-24 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1088: copy_from oid 22 from oid 13 current snap is 105 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1083: finishing write tid 5 to smithi05955964-24 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 432 (ObjNum 346 snap 105 seq_num 346) dirty exists 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1083: left oid 24 (ObjNum 346 snap 105 seq_num 346) 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1084: finishing write tid 1 to smithi05955964-5 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1084: finishing write tid 2 to smithi05955964-5 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1084: finishing write tid 4 to smithi05955964-5 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1085: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1085: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:47:47.327 INFO:tasks.rados.rados.0.smithi059.stdout:1085: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 512 (ObjNum 332 snap 99 seq_num 332) dirty exists 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1084: finishing write tid 5 to smithi05955964-5 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 518 (ObjNum 347 snap 105 seq_num 347) dirty exists 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1084: left oid 5 (ObjNum 347 snap 105 seq_num 347) 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1083: done (5 left) 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1084: done (4 left) 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1085: done (3 left) 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1089: read oid 42 snap -1 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1089: expect (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:47:47.328 INFO:tasks.rados.rados.0.smithi059.stdout:1090: delete oid 46 current snap is 105 2022-04-28T19:47:47.329 INFO:tasks.rados.rados.0.smithi059.stdout:1090: done (4 left) 2022-04-28T19:47:47.329 INFO:tasks.rados.rados.0.smithi059.stdout:1091: snap_remove snap 93 2022-04-28T19:47:48.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:47 smithi139 bash[23439]: cluster 2022-04-28T19:47:46.869152+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2022-04-28T19:47:48.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:47 smithi139 bash[23439]: cluster 2022-04-28T19:47:47.223793+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2022-04-28T19:47:48.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:47 smithi139 bash[23439]: cluster 2022-04-28T19:47:47.271590+0000 mgr.x (mgr.34769) 11 : cluster [DBG] pgmap v18: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:48.247 INFO:tasks.rados.rados.0.smithi059.stdout:1091: done (4 left) 2022-04-28T19:47:48.247 INFO:tasks.rados.rados.0.smithi059.stdout:1092: rollback oid 33 current snap is 105 2022-04-28T19:47:48.247 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 73 2022-04-28T19:47:48.247 INFO:tasks.rados.rados.0.smithi059.stdout:1086: finishing write tid 1 to smithi05955964-12 2022-04-28T19:47:48.247 INFO:tasks.rados.rados.0.smithi059.stdout:1086: finishing write tid 2 to smithi05955964-12 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1086: finishing write tid 4 to smithi05955964-12 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1087: finishing copy_from to smithi05955964-9 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 424 (ObjNum 317 snap 95 seq_num 317) dirty exists 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1087: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1086: finishing write tid 5 to smithi05955964-12 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 529 (ObjNum 348 snap 105 seq_num 348) dirty exists 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1086: left oid 12 (ObjNum 348 snap 105 seq_num 348) 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1088: finishing copy_from to smithi05955964-22 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 243 (ObjNum 161 snap 42 seq_num 161) dirty exists 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1088: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:47:48.248 INFO:tasks.rados.rados.0.smithi059.stdout:1089: expect (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:47:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:47 smithi059 bash[23786]: cluster 2022-04-28T19:47:46.869152+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2022-04-28T19:47:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:47 smithi059 bash[23786]: cluster 2022-04-28T19:47:47.223793+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2022-04-28T19:47:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:47 smithi059 bash[23786]: cluster 2022-04-28T19:47:47.271590+0000 mgr.x (mgr.34769) 11 : cluster [DBG] pgmap v18: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:48.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:47 smithi059 bash[29114]: cluster 2022-04-28T19:47:46.869152+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2022-04-28T19:47:48.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:47 smithi059 bash[29114]: cluster 2022-04-28T19:47:47.223793+0000 mon.a (mon.0) 918 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2022-04-28T19:47:48.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:47 smithi059 bash[29114]: cluster 2022-04-28T19:47:47.271590+0000 mgr.x (mgr.34769) 11 : cluster [DBG] pgmap v18: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 245 MiB data, 352 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1092: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1092: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1086: done (4 left) 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1087: done (3 left) 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1088: done (2 left) 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1089: done (1 left) 2022-04-28T19:47:48.389 INFO:tasks.rados.rados.0.smithi059.stdout:1093: read oid 40 snap -1 2022-04-28T19:47:48.390 INFO:tasks.rados.rados.0.smithi059.stdout:1093: expect (ObjNum 319 snap 96 seq_num 319) 2022-04-28T19:47:48.390 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 105 2022-04-28T19:47:48.390 INFO:tasks.rados.rados.0.smithi059.stdout:1094: seq_num 350 ranges {2875392=671744,3547136=401408} 2022-04-28T19:47:48.416 INFO:tasks.rados.rados.0.smithi059.stdout:1094: writing smithi05955964-6 from 2875392 to 3547136 tid 1 2022-04-28T19:47:48.419 INFO:tasks.rados.rados.0.smithi059.stdout:1094: writing smithi05955964-6 from 3547136 to 3948544 tid 2 2022-04-28T19:47:48.419 INFO:tasks.rados.rados.0.smithi059.stdout:1095: read oid 48 snap -1 2022-04-28T19:47:48.419 INFO:tasks.rados.rados.0.smithi059.stdout:1095: expect deleted 2022-04-28T19:47:48.419 INFO:tasks.rados.rados.0.smithi059.stdout:1096: rollback oid 46 current snap is 105 2022-04-28T19:47:48.419 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 100 2022-04-28T19:47:48.419 INFO:tasks.rados.rados.0.smithi059.stdout:1097: delete oid 49 current snap is 105 2022-04-28T19:47:48.420 INFO:tasks.rados.rados.0.smithi059.stdout:1092: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:47:48.420 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 554 (ObjNum 217 snap 59 seq_num 217) dirty exists 2022-04-28T19:47:48.420 INFO:tasks.rados.rados.0.smithi059.stdout:1093: expect (ObjNum 319 snap 96 seq_num 319) 2022-04-28T19:47:48.513 INFO:tasks.rados.rados.0.smithi059.stdout:1096: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:47:48.513 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 360 (ObjNum -134743552 snap 21893 seq_num 4161438768) dirty dne 2022-04-28T19:47:48.513 INFO:tasks.rados.rados.0.smithi059.stdout:1094: finishing write tid 1 to smithi05955964-6 2022-04-28T19:47:48.513 INFO:tasks.rados.rados.0.smithi059.stdout:1094: finishing write tid 2 to smithi05955964-6 2022-04-28T19:47:48.513 INFO:tasks.rados.rados.0.smithi059.stdout:1094: finishing write tid 4 to smithi05955964-6 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1092: done (5 left) 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1093: done (4 left) 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1095: done (3 left) 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1096: done (2 left) 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1097: done (1 left) 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1098: rollback oid 33 current snap is 105 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 100 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1094: finishing write tid 5 to smithi05955964-6 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 446 (ObjNum 350 snap 105 seq_num 350) dirty exists 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1094: left oid 6 (ObjNum 350 snap 105 seq_num 350) 2022-04-28T19:47:48.514 INFO:tasks.rados.rados.0.smithi059.stdout:1094: done (1 left) 2022-04-28T19:47:48.515 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 105 2022-04-28T19:47:48.515 INFO:tasks.rados.rados.0.smithi059.stdout:1099: seq_num 352 ranges {0=704512,704512=401408} 2022-04-28T19:47:48.523 INFO:tasks.rados.rados.0.smithi059.stdout:1099: writing smithi05955964-15 from 0 to 704512 tid 1 2022-04-28T19:47:48.528 INFO:tasks.rados.rados.0.smithi059.stdout:1099: writing smithi05955964-15 from 704512 to 1105920 tid 2 2022-04-28T19:47:48.528 INFO:tasks.rados.rados.0.smithi059.stdout:1100: delete oid 18 current snap is 105 2022-04-28T19:47:48.528 INFO:tasks.rados.rados.0.smithi059.stdout:1098: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:47:48.528 INFO:tasks.rados.rados.0.smithi059.stdout:1098: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:47:48.529 INFO:tasks.rados.rados.0.smithi059.stdout:1098: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:47:48.529 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 559 (ObjNum 231 snap 60 seq_num 231) dirty exists 2022-04-28T19:47:48.529 INFO:tasks.rados.rados.0.smithi059.stdout:1099: finishing write tid 1 to smithi05955964-15 2022-04-28T19:47:48.532 INFO:tasks.rados.rados.0.smithi059.stdout:1099: finishing write tid 2 to smithi05955964-15 2022-04-28T19:47:48.532 INFO:tasks.rados.rados.0.smithi059.stdout:1099: finishing write tid 4 to smithi05955964-15 2022-04-28T19:47:48.534 INFO:tasks.rados.rados.0.smithi059.stdout:1098: done (2 left) 2022-04-28T19:47:48.534 INFO:tasks.rados.rados.0.smithi059.stdout:1100: done (1 left) 2022-04-28T19:47:48.534 INFO:tasks.rados.rados.0.smithi059.stdout:1101: rollback oid 47 current snap is 105 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 86 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:1102: read oid 29 snap -1 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:1102: expect (ObjNum 305 snap 92 seq_num 305) 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:1103: snap_create 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:1099: finishing write tid 5 to smithi05955964-15 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 538 (ObjNum 352 snap 105 seq_num 352) dirty exists 2022-04-28T19:47:48.535 INFO:tasks.rados.rados.0.smithi059.stdout:1099: left oid 15 (ObjNum 352 snap 105 seq_num 352) 2022-04-28T19:47:48.537 INFO:tasks.rados.rados.0.smithi059.stdout:1101: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:47:48.538 INFO:tasks.rados.rados.0.smithi059.stdout:1101: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:47:48.538 INFO:tasks.rados.rados.0.smithi059.stdout:1101: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:47:48.538 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 496 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:47:48.545 INFO:tasks.rados.rados.0.smithi059.stdout:1102: expect (ObjNum 305 snap 92 seq_num 305) 2022-04-28T19:47:49.257 INFO:tasks.rados.rados.0.smithi059.stdout:1099: done (3 left) 2022-04-28T19:47:49.257 INFO:tasks.rados.rados.0.smithi059.stdout:1101: done (2 left) 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1102: done (1 left) 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1103: done (0 left) 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1104: read oid 12 snap -1 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1104: expect (ObjNum 348 snap 105 seq_num 348) 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1105: rollback oid 47 current snap is 106 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 103 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1106: copy_from oid 22 from oid 35 current snap is 106 2022-04-28T19:47:49.258 INFO:tasks.rados.rados.0.smithi059.stdout:1107: delete oid 3 current snap is 106 2022-04-28T19:47:49.264 INFO:tasks.rados.rados.0.smithi059.stdout:1107: done (3 left) 2022-04-28T19:47:49.264 INFO:tasks.rados.rados.0.smithi059.stdout:1108: rmattr oid 24 current snap is 106 2022-04-28T19:47:49.264 INFO:tasks.rados.rados.0.smithi059.stdout:1108: done (3 left) 2022-04-28T19:47:49.264 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 106 2022-04-28T19:47:49.264 INFO:tasks.rados.rados.0.smithi059.stdout:1109: seq_num 355 ranges {2883584=753664,3637248=401408} 2022-04-28T19:47:49.285 INFO:tasks.rados.rados.0.smithi059.stdout:1109: writing smithi05955964-42 from 2883584 to 3637248 tid 1 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1109: writing smithi05955964-42 from 3637248 to 4038656 tid 2 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1110: snap_create 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1105: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1105: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1105: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 500 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1106: finishing copy_from to smithi05955964-22 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 147 (ObjNum 265 snap 77 seq_num 265) dirty exists 2022-04-28T19:47:49.288 INFO:tasks.rados.rados.0.smithi059.stdout:1106: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:47:49.295 INFO:tasks.rados.rados.0.smithi059.stdout:1104: expect (ObjNum 348 snap 105 seq_num 348) 2022-04-28T19:47:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:49 smithi059 bash[29114]: cluster 2022-04-28T19:47:48.240866+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2022-04-28T19:47:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:49 smithi059 bash[23786]: cluster 2022-04-28T19:47:48.240866+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2022-04-28T19:47:49.686 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:49 smithi139 bash[23439]: cluster 2022-04-28T19:47:48.240866+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2022-04-28T19:47:49.754 INFO:tasks.rados.rados.0.smithi059.stdout:1109: finishing write tid 1 to smithi05955964-42 2022-04-28T19:47:49.754 INFO:tasks.rados.rados.0.smithi059.stdout:1109: finishing write tid 2 to smithi05955964-42 2022-04-28T19:47:49.755 INFO:tasks.rados.rados.0.smithi059.stdout:1109: finishing write tid 4 to smithi05955964-42 2022-04-28T19:47:49.755 INFO:tasks.rados.rados.0.smithi059.stdout:1109: finishing write tid 5 to smithi05955964-42 2022-04-28T19:47:49.755 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 470 (ObjNum 355 snap 106 seq_num 355) dirty exists 2022-04-28T19:47:49.755 INFO:tasks.rados.rados.0.smithi059.stdout:1109: left oid 42 (ObjNum 355 snap 106 seq_num 355) 2022-04-28T19:47:50.268 INFO:tasks.rados.rados.0.smithi059.stdout:1104: done (4 left) 2022-04-28T19:47:50.268 INFO:tasks.rados.rados.0.smithi059.stdout:1105: done (3 left) 2022-04-28T19:47:50.268 INFO:tasks.rados.rados.0.smithi059.stdout:1106: done (2 left) 2022-04-28T19:47:50.268 INFO:tasks.rados.rados.0.smithi059.stdout:1109: done (1 left) 2022-04-28T19:47:50.268 INFO:tasks.rados.rados.0.smithi059.stdout:1110: done (0 left) 2022-04-28T19:47:50.268 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 107 2022-04-28T19:47:50.269 INFO:tasks.rados.rados.0.smithi059.stdout:1111: seq_num 356 ranges {0=770048,770048=401408} 2022-04-28T19:47:50.274 INFO:tasks.rados.rados.0.smithi059.stdout:1111: writing smithi05955964-39 from 0 to 770048 tid 1 2022-04-28T19:47:50.277 INFO:tasks.rados.rados.0.smithi059.stdout:1111: writing smithi05955964-39 from 770048 to 1171456 tid 2 2022-04-28T19:47:50.277 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 107 2022-04-28T19:47:50.277 INFO:tasks.rados.rados.0.smithi059.stdout:1112: seq_num 357 ranges {2899968=786432,3686400=401408} 2022-04-28T19:47:50.298 INFO:tasks.rados.rados.0.smithi059.stdout:1112: writing smithi05955964-43 from 2899968 to 3686400 tid 1 2022-04-28T19:47:50.300 INFO:tasks.rados.rados.0.smithi059.stdout:1112: writing smithi05955964-43 from 3686400 to 4087808 tid 2 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1113: read oid 2 snap -1 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1113: expect (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1114: setattr oid 34 current snap is 107 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1111: finishing write tid 1 to smithi05955964-39 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1111: finishing write tid 2 to smithi05955964-39 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1111: finishing write tid 4 to smithi05955964-39 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1111: finishing write tid 5 to smithi05955964-39 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 562 (ObjNum 356 snap 107 seq_num 356) dirty exists 2022-04-28T19:47:50.301 INFO:tasks.rados.rados.0.smithi059.stdout:1111: left oid 39 (ObjNum 356 snap 107 seq_num 356) 2022-04-28T19:47:50.302 INFO:tasks.rados.rados.0.smithi059.stdout:1111: done (3 left) 2022-04-28T19:47:50.302 INFO:tasks.rados.rados.0.smithi059.stdout:1115: read oid 32 snap -1 2022-04-28T19:47:50.303 INFO:tasks.rados.rados.0.smithi059.stdout:1115: expect deleted 2022-04-28T19:47:50.303 INFO:tasks.rados.rados.0.smithi059.stdout:1116: copy_from oid 35 from oid 9 current snap is 107 2022-04-28T19:47:50.303 INFO:tasks.rados.rados.0.smithi059.stdout:1117: setattr oid 50 current snap is 107 2022-04-28T19:47:50.303 INFO:tasks.rados.rados.0.smithi059.stdout:1118: delete oid 23 current snap is 107 2022-04-28T19:47:50.308 INFO:tasks.rados.rados.0.smithi059.stdout:1112: finishing write tid 1 to smithi05955964-43 2022-04-28T19:47:50.308 INFO:tasks.rados.rados.0.smithi059.stdout:1112: finishing write tid 2 to smithi05955964-43 2022-04-28T19:47:50.309 INFO:tasks.rados.rados.0.smithi059.stdout:1112: finishing write tid 4 to smithi05955964-43 2022-04-28T19:47:50.309 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 472 (ObjNum 328 snap 99 seq_num 328) dirty exists 2022-04-28T19:47:50.309 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 568 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:47:50.310 INFO:tasks.rados.rados.0.smithi059.stdout:1112: finishing write tid 5 to smithi05955964-43 2022-04-28T19:47:50.310 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 566 (ObjNum 357 snap 107 seq_num 357) dirty exists 2022-04-28T19:47:50.310 INFO:tasks.rados.rados.0.smithi059.stdout:1112: left oid 43 (ObjNum 357 snap 107 seq_num 357) 2022-04-28T19:47:50.311 INFO:tasks.rados.rados.0.smithi059.stdout:1112: done (6 left) 2022-04-28T19:47:50.311 INFO:tasks.rados.rados.0.smithi059.stdout:1114: done (5 left) 2022-04-28T19:47:50.311 INFO:tasks.rados.rados.0.smithi059.stdout:1115: done (4 left) 2022-04-28T19:47:50.311 INFO:tasks.rados.rados.0.smithi059.stdout:1117: done (3 left) 2022-04-28T19:47:50.312 INFO:tasks.rados.rados.0.smithi059.stdout:1118: done (2 left) 2022-04-28T19:47:50.312 INFO:tasks.rados.rados.0.smithi059.stdout:1119: setattr oid 20 current snap is 107 2022-04-28T19:47:50.313 INFO:tasks.rados.rados.0.smithi059.stdout:1120: snap_create 2022-04-28T19:47:50.313 INFO:tasks.rados.rados.0.smithi059.stdout:1113: expect (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:47:50.449 INFO:tasks.rados.rados.0.smithi059.stdout:1116: finishing copy_from to smithi05955964-35 2022-04-28T19:47:50.450 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 424 (ObjNum 317 snap 95 seq_num 317) dirty exists 2022-04-28T19:47:50.450 INFO:tasks.rados.rados.0.smithi059.stdout:1116: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:47:50.450 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 450 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:47:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:50 smithi059 bash[23786]: cluster 2022-04-28T19:47:49.251001+0000 mon.a (mon.0) 920 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2022-04-28T19:47:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:50 smithi059 bash[23786]: cluster 2022-04-28T19:47:49.272060+0000 mgr.x (mgr.34769) 12 : cluster [DBG] pgmap v21: 89 pgs: 1 active+clean+snaptrim_wait, 83 active+clean, 5 active+clean+snaptrim; 247 MiB data, 360 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 4.4 MiB/s wr, 45 op/s 2022-04-28T19:47:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:50 smithi059 bash[29114]: cluster 2022-04-28T19:47:49.251001+0000 mon.a (mon.0) 920 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2022-04-28T19:47:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:50 smithi059 bash[29114]: cluster 2022-04-28T19:47:49.272060+0000 mgr.x (mgr.34769) 12 : cluster [DBG] pgmap v21: 89 pgs: 1 active+clean+snaptrim_wait, 83 active+clean, 5 active+clean+snaptrim; 247 MiB data, 360 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 4.4 MiB/s wr, 45 op/s 2022-04-28T19:47:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:50 smithi139 bash[23439]: cluster 2022-04-28T19:47:49.251001+0000 mon.a (mon.0) 920 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2022-04-28T19:47:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:50 smithi139 bash[23439]: cluster 2022-04-28T19:47:49.272060+0000 mgr.x (mgr.34769) 12 : cluster [DBG] pgmap v21: 89 pgs: 1 active+clean+snaptrim_wait, 83 active+clean, 5 active+clean+snaptrim; 247 MiB data, 360 MiB used, 707 GiB / 715 GiB avail; 12 MiB/s rd, 4.4 MiB/s wr, 45 op/s 2022-04-28T19:47:51.269 INFO:tasks.rados.rados.0.smithi059.stdout:1113: done (3 left) 2022-04-28T19:47:51.269 INFO:tasks.rados.rados.0.smithi059.stdout:1116: done (2 left) 2022-04-28T19:47:51.269 INFO:tasks.rados.rados.0.smithi059.stdout:1119: done (1 left) 2022-04-28T19:47:51.269 INFO:tasks.rados.rados.0.smithi059.stdout:1120: done (0 left) 2022-04-28T19:47:51.269 INFO:tasks.rados.rados.0.smithi059.stdout:1121: read oid 42 snap -1 2022-04-28T19:47:51.270 INFO:tasks.rados.rados.0.smithi059.stdout:1121: expect (ObjNum 355 snap 106 seq_num 355) 2022-04-28T19:47:51.270 INFO:tasks.rados.rados.0.smithi059.stdout:1122: snap_remove snap 86 2022-04-28T19:47:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:51 smithi059 bash[23786]: cluster 2022-04-28T19:47:50.265312+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2022-04-28T19:47:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:51 smithi059 bash[29114]: cluster 2022-04-28T19:47:50.265312+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2022-04-28T19:47:51.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:51 smithi139 bash[23439]: cluster 2022-04-28T19:47:50.265312+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2022-04-28T19:47:52.225 INFO:tasks.rados.rados.0.smithi059.stdout:1122: done (1 left) 2022-04-28T19:47:52.225 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 108 2022-04-28T19:47:52.225 INFO:tasks.rados.rados.0.smithi059.stdout:1123: seq_num 359 ranges {4677632=417792,5095424=417792,5513216=385024} 2022-04-28T19:47:52.255 INFO:tasks.rados.rados.0.smithi059.stdout:1123: writing smithi05955964-37 from 4677632 to 5095424 tid 1 2022-04-28T19:47:52.258 INFO:tasks.rados.rados.0.smithi059.stdout:1123: writing smithi05955964-37 from 5095424 to 5513216 tid 2 2022-04-28T19:47:52.261 INFO:tasks.rados.rados.0.smithi059.stdout:1123: writing smithi05955964-37 from 5513216 to 5898240 tid 3 2022-04-28T19:47:52.261 INFO:tasks.rados.rados.0.smithi059.stdout:1124: read oid 4 snap -1 2022-04-28T19:47:52.261 INFO:tasks.rados.rados.0.smithi059.stdout:1124: expect (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:47:52.261 INFO:tasks.rados.rados.0.smithi059.stdout:1125: read oid 49 snap 98 2022-04-28T19:47:52.262 INFO:tasks.rados.rados.0.smithi059.stdout:1125: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:47:52.262 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 108 2022-04-28T19:47:52.262 INFO:tasks.rados.rados.0.smithi059.stdout:1126: seq_num 360 ranges {0=434176,434176=704512,1138688=98304} 2022-04-28T19:47:52.264 INFO:tasks.rados.rados.0.smithi059.stdout:1126: writing smithi05955964-16 from 0 to 434176 tid 1 2022-04-28T19:47:52.269 INFO:tasks.rados.rados.0.smithi059.stdout:1126: writing smithi05955964-16 from 434176 to 1138688 tid 2 2022-04-28T19:47:52.270 INFO:tasks.rados.rados.0.smithi059.stdout:1126: writing smithi05955964-16 from 1138688 to 1236992 tid 3 2022-04-28T19:47:52.270 INFO:tasks.rados.rados.0.smithi059.stdout:1127: delete oid 28 current snap is 108 2022-04-28T19:47:52.270 INFO:tasks.rados.rados.0.smithi059.stdout:1121: expect (ObjNum 355 snap 106 seq_num 355) 2022-04-28T19:47:52.480 INFO:tasks.rados.rados.0.smithi059.stdout:1123: finishing write tid 1 to smithi05955964-37 2022-04-28T19:47:52.481 INFO:tasks.rados.rados.0.smithi059.stdout:1123: finishing write tid 2 to smithi05955964-37 2022-04-28T19:47:52.481 INFO:tasks.rados.rados.0.smithi059.stdout:1121: done (5 left) 2022-04-28T19:47:52.481 INFO:tasks.rados.rados.0.smithi059.stdout:1127: done (4 left) 2022-04-28T19:47:52.481 INFO:tasks.rados.rados.0.smithi059.stdout:1128: read oid 48 snap -1 2022-04-28T19:47:52.481 INFO:tasks.rados.rados.0.smithi059.stdout:1128: expect deleted 2022-04-28T19:47:52.481 INFO:tasks.rados.rados.0.smithi059.stdout:1129: snap_remove snap 103 2022-04-28T19:47:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:52 smithi059 bash[23786]: cluster 2022-04-28T19:47:51.266792+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2022-04-28T19:47:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:52 smithi059 bash[23786]: cluster 2022-04-28T19:47:51.272902+0000 mgr.x (mgr.34769) 13 : cluster [DBG] pgmap v24: 89 pgs: 1 active+clean+snaptrim_wait, 83 active+clean, 5 active+clean+snaptrim; 247 MiB data, 360 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 38 op/s 2022-04-28T19:47:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:52 smithi059 bash[23786]: cluster 2022-04-28T19:47:52.222504+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2022-04-28T19:47:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:52 smithi059 bash[29114]: cluster 2022-04-28T19:47:51.266792+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2022-04-28T19:47:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:52 smithi059 bash[29114]: cluster 2022-04-28T19:47:51.272902+0000 mgr.x (mgr.34769) 13 : cluster [DBG] pgmap v24: 89 pgs: 1 active+clean+snaptrim_wait, 83 active+clean, 5 active+clean+snaptrim; 247 MiB data, 360 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 38 op/s 2022-04-28T19:47:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:52 smithi059 bash[29114]: cluster 2022-04-28T19:47:52.222504+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2022-04-28T19:47:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:52 smithi139 bash[23439]: cluster 2022-04-28T19:47:51.266792+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2022-04-28T19:47:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:52 smithi139 bash[23439]: cluster 2022-04-28T19:47:51.272902+0000 mgr.x (mgr.34769) 13 : cluster [DBG] pgmap v24: 89 pgs: 1 active+clean+snaptrim_wait, 83 active+clean, 5 active+clean+snaptrim; 247 MiB data, 360 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 38 op/s 2022-04-28T19:47:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:52 smithi139 bash[23439]: cluster 2022-04-28T19:47:52.222504+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2022-04-28T19:47:53.295 INFO:tasks.rados.rados.0.smithi059.stdout:1129: done (5 left) 2022-04-28T19:47:53.295 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 108 2022-04-28T19:47:53.295 INFO:tasks.rados.rados.0.smithi059.stdout:1130: seq_num 362 ranges {4751360=466944,5218304=466944,5685248=335872} 2022-04-28T19:47:53.325 INFO:tasks.rados.rados.0.smithi059.stdout:1130: writing smithi05955964-34 from 4751360 to 5218304 tid 1 2022-04-28T19:47:53.328 INFO:tasks.rados.rados.0.smithi059.stdout:1130: writing smithi05955964-34 from 5218304 to 5685248 tid 2 2022-04-28T19:47:53.330 INFO:tasks.rados.rados.0.smithi059.stdout:1130: writing smithi05955964-34 from 5685248 to 6021120 tid 3 2022-04-28T19:47:53.331 INFO:tasks.rados.rados.0.smithi059.stdout:1131: read oid 27 snap -1 2022-04-28T19:47:53.331 INFO:tasks.rados.rados.0.smithi059.stdout:1131: expect (ObjNum 302 snap 92 seq_num 302) 2022-04-28T19:47:53.331 INFO:tasks.rados.rados.0.smithi059.stdout:1132: delete oid 25 current snap is 108 2022-04-28T19:47:53.331 INFO:tasks.rados.rados.0.smithi059.stdout:1125: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:47:53.367 INFO:tasks.rados.rados.0.smithi059.stdout:1123: finishing write tid 3 to smithi05955964-37 2022-04-28T19:47:53.367 INFO:tasks.rados.rados.0.smithi059.stdout:1123: finishing write tid 5 to smithi05955964-37 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1123: finishing write tid 6 to smithi05955964-37 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 487 (ObjNum 359 snap 108 seq_num 359) dirty exists 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1123: left oid 37 (ObjNum 359 snap 108 seq_num 359) 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1126: finishing write tid 1 to smithi05955964-16 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1126: finishing write tid 2 to smithi05955964-16 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1126: finishing write tid 3 to smithi05955964-16 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1126: finishing write tid 5 to smithi05955964-16 2022-04-28T19:47:53.368 INFO:tasks.rados.rados.0.smithi059.stdout:1124: expect (ObjNum 231 snap 60 seq_num 231) 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1126: finishing write tid 6 to smithi05955964-16 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 376 (ObjNum 360 snap 108 seq_num 360) dirty exists 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1126: left oid 16 (ObjNum 360 snap 108 seq_num 360) 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1130: finishing write tid 1 to smithi05955964-34 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1130: finishing write tid 2 to smithi05955964-34 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1130: finishing write tid 3 to smithi05955964-34 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1130: finishing write tid 5 to smithi05955964-34 2022-04-28T19:47:53.580 INFO:tasks.rados.rados.0.smithi059.stdout:1123: done (7 left) 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1124: done (6 left) 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1125: done (5 left) 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1126: done (4 left) 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1128: done (3 left) 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1132: done (2 left) 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1133: snap_create 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:1130: finishing write tid 6 to smithi05955964-34 2022-04-28T19:47:53.581 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 500 (ObjNum 362 snap 108 seq_num 362) dirty exists 2022-04-28T19:47:53.582 INFO:tasks.rados.rados.0.smithi059.stdout:1130: left oid 34 (ObjNum 362 snap 108 seq_num 362) 2022-04-28T19:47:53.582 INFO:tasks.rados.rados.0.smithi059.stdout:1131: expect (ObjNum 302 snap 92 seq_num 302) 2022-04-28T19:47:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:53.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:47:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:53.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:47:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:47:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:47:53.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:47:54.308 INFO:tasks.rados.rados.0.smithi059.stdout:1130: done (2 left) 2022-04-28T19:47:54.309 INFO:tasks.rados.rados.0.smithi059.stdout:1131: done (1 left) 2022-04-28T19:47:54.309 INFO:tasks.rados.rados.0.smithi059.stdout:1133: done (0 left) 2022-04-28T19:47:54.309 INFO:tasks.rados.rados.0.smithi059.stdout:1134: snap_remove snap 105 2022-04-28T19:47:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:54 smithi059 bash[23786]: cluster 2022-04-28T19:47:53.274188+0000 mgr.x (mgr.34769) 14 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2022-04-28T19:47:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:54 smithi059 bash[23786]: cluster 2022-04-28T19:47:53.293745+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2022-04-28T19:47:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:54 smithi059 bash[29114]: cluster 2022-04-28T19:47:53.274188+0000 mgr.x (mgr.34769) 14 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2022-04-28T19:47:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:54 smithi059 bash[29114]: cluster 2022-04-28T19:47:53.293745+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2022-04-28T19:47:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:54 smithi139 bash[23439]: cluster 2022-04-28T19:47:53.274188+0000 mgr.x (mgr.34769) 14 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2022-04-28T19:47:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:54 smithi139 bash[23439]: cluster 2022-04-28T19:47:53.293745+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2022-04-28T19:47:55.309 INFO:tasks.rados.rados.0.smithi059.stdout:1134: done (0 left) 2022-04-28T19:47:55.309 INFO:tasks.rados.rados.0.smithi059.stdout:1135: copy_from oid 33 from oid 1 current snap is 109 2022-04-28T19:47:55.309 INFO:tasks.rados.rados.0.smithi059.stdout:1136: read oid 46 snap -1 2022-04-28T19:47:55.309 INFO:tasks.rados.rados.0.smithi059.stdout:1136: expect deleted 2022-04-28T19:47:55.310 INFO:tasks.rados.rados.0.smithi059.stdout:1137: snap_remove snap 108 2022-04-28T19:47:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:55 smithi059 bash[23786]: cluster 2022-04-28T19:47:54.303751+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2022-04-28T19:47:55.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:55 smithi059 bash[29114]: cluster 2022-04-28T19:47:54.303751+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2022-04-28T19:47:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[23439]: cluster 2022-04-28T19:47:54.303751+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2022-04-28T19:47:55.618 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 systemd[1]: Stopping Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:47:56.118 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=warn ts=2022-04-28T19:47:55.621Z caller=main.go:524 msg="Received SIGTERM, exiting gracefully..." 2022-04-28T19:47:56.118 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.621Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2022-04-28T19:47:56.118 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.621Z caller=main.go:561 msg="Stopping notify discovery manager..." 2022-04-28T19:47:56.118 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=main.go:583 msg="Stopping scrape manager..." 2022-04-28T19:47:56.118 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=main.go:543 msg="Scrape discovery manager stopped" 2022-04-28T19:47:56.118 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2022-04-28T19:47:56.119 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=main.go:577 msg="Scrape manager stopped" 2022-04-28T19:47:56.119 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2022-04-28T19:47:56.119 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=main.go:557 msg="Notify discovery manager stopped" 2022-04-28T19:47:56.119 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2022-04-28T19:47:56.119 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.622Z caller=main.go:749 msg="Notifier manager stopped" 2022-04-28T19:47:56.119 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:55 smithi139 bash[35873]: level=info ts=2022-04-28T19:47:55.623Z caller=main.go:761 msg="See you next time!" 2022-04-28T19:47:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:1137: done (2 left) 2022-04-28T19:47:56.320 INFO:tasks.rados.rados.0.smithi059.stdout:1138: snap_remove snap 73 2022-04-28T19:47:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:56 smithi059 bash[29114]: cluster 2022-04-28T19:47:55.274637+0000 mgr.x (mgr.34769) 15 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2022-04-28T19:47:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:56 smithi059 bash[29114]: cluster 2022-04-28T19:47:55.305524+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2022-04-28T19:47:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:56 smithi059 bash[23786]: cluster 2022-04-28T19:47:55.274637+0000 mgr.x (mgr.34769) 15 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2022-04-28T19:47:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:56 smithi059 bash[23786]: cluster 2022-04-28T19:47:55.305524+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2022-04-28T19:47:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:56 smithi139 bash[23439]: cluster 2022-04-28T19:47:55.274637+0000 mgr.x (mgr.34769) 15 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail; 18 MiB/s rd, 2.8 MiB/s wr, 33 op/s 2022-04-28T19:47:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:56 smithi139 bash[23439]: cluster 2022-04-28T19:47:55.305524+0000 mon.a (mon.0) 926 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2022-04-28T19:47:56.618 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:56 smithi139 docker[39365]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-prometheus.a 2022-04-28T19:47:56.619 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:56 smithi139 systemd[1]: Stopped Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:47:56.619 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:56 smithi139 systemd[1]: Started Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:47:57.228 INFO:tasks.rados.rados.0.smithi059.stdout:1138: done (2 left) 2022-04-28T19:47:57.228 INFO:tasks.rados.rados.0.smithi059.stdout:1139: snap_create 2022-04-28T19:47:57.228 INFO:tasks.rados.rados.0.smithi059.stdout:1135: finishing copy_from to smithi05955964-33 2022-04-28T19:47:57.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 71 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:47:57.228 INFO:tasks.rados.rados.0.smithi059.stdout:1135: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:47:57.535 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: cluster 2022-04-28T19:47:56.317346+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2022-04-28T19:47:57.535 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: audit 2022-04-28T19:47:56.325885+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:57.535 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: audit 2022-04-28T19:47:56.330228+0000 mon.b (mon.2) 108 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:47:57.535 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: audit 2022-04-28T19:47:56.330990+0000 mgr.x (mgr.34769) 16 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:47:57.535 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: audit 2022-04-28T19:47:56.333528+0000 mon.b (mon.2) 109 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:57.536 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: audit 2022-04-28T19:47:56.341890+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:57.536 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: cephadm 2022-04-28T19:47:56.343317+0000 mgr.x (mgr.34769) 17 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-04-28T19:47:57.536 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[23439]: cluster 2022-04-28T19:47:57.224424+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2022-04-28T19:47:57.536 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.480Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2022-04-28T19:47:57.536 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.480Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2022-04-28T19:47:57.536 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.480Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2022-04-28T19:47:57.536 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.480Z caller=main.go:339 host_details="(Linux 4.15.0-176-generic #185-Ubuntu SMP Tue Mar 29 17:40:04 UTC 2022 x86_64 smithi139 (none))" 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.480Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.480Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.482Z caller=main.go:678 msg="Starting TSDB ..." 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.482Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.490Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.491Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.505Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.505Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.505Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=14.431865ms 2022-04-28T19:47:57.537 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.508Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2022-04-28T19:47:57.538 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.508Z caller=main.go:695 msg="TSDB started" 2022-04-28T19:47:57.538 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.509Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:47:57.538 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.533Z caller=main.go:827 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: cluster 2022-04-28T19:47:56.317346+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: audit 2022-04-28T19:47:56.325885+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: audit 2022-04-28T19:47:56.330228+0000 mon.b (mon.2) 108 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: audit 2022-04-28T19:47:56.330990+0000 mgr.x (mgr.34769) 16 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: audit 2022-04-28T19:47:56.333528+0000 mon.b (mon.2) 109 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: audit 2022-04-28T19:47:56.341890+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: cephadm 2022-04-28T19:47:56.343317+0000 mgr.x (mgr.34769) 17 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-04-28T19:47:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[29114]: cluster 2022-04-28T19:47:57.224424+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: cluster 2022-04-28T19:47:56.317346+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: audit 2022-04-28T19:47:56.325885+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: audit 2022-04-28T19:47:56.330228+0000 mon.b (mon.2) 108 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: audit 2022-04-28T19:47:56.330990+0000 mgr.x (mgr.34769) 16 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: audit 2022-04-28T19:47:56.333528+0000 mon.b (mon.2) 109 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: audit 2022-04-28T19:47:56.341890+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: cephadm 2022-04-28T19:47:56.343317+0000 mgr.x (mgr.34769) 17 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-04-28T19:47:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:57 smithi059 bash[23786]: cluster 2022-04-28T19:47:57.224424+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2022-04-28T19:47:57.868 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:47:57 smithi139 bash[39417]: level=info ts=2022-04-28T19:47:57.533Z caller=main.go:646 msg="Server is ready to receive web requests." 2022-04-28T19:47:58.230 INFO:tasks.rados.rados.0.smithi059.stdout:1135: done (2 left) 2022-04-28T19:47:58.230 INFO:tasks.rados.rados.0.smithi059.stdout:1136: done (1 left) 2022-04-28T19:47:58.230 INFO:tasks.rados.rados.0.smithi059.stdout:1139: done (0 left) 2022-04-28T19:47:58.230 INFO:tasks.rados.rados.0.smithi059.stdout:1140: rmattr oid 26 current snap is 110 2022-04-28T19:47:58.230 INFO:tasks.rados.rados.0.smithi059.stdout:1140: done (0 left) 2022-04-28T19:47:58.230 INFO:tasks.rados.rados.0.smithi059.stdout:1141: rmattr oid 47 current snap is 110 2022-04-28T19:47:58.231 INFO:tasks.rados.rados.0.smithi059.stdout:1142: rmattr oid 11 current snap is 110 2022-04-28T19:47:58.231 INFO:tasks.rados.rados.0.smithi059.stdout:1143: snap_remove snap 96 2022-04-28T19:47:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:58 smithi059 bash[23786]: cluster 2022-04-28T19:47:57.275118+0000 mgr.x (mgr.34769) 18 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:58 smithi059 bash[23786]: cluster 2022-04-28T19:47:58.227344+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2022-04-28T19:47:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:58 smithi059 bash[29114]: cluster 2022-04-28T19:47:57.275118+0000 mgr.x (mgr.34769) 18 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:58 smithi059 bash[29114]: cluster 2022-04-28T19:47:58.227344+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2022-04-28T19:47:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:58 smithi139 bash[23439]: cluster 2022-04-28T19:47:57.275118+0000 mgr.x (mgr.34769) 18 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 259 MiB data, 374 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:47:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:58 smithi139 bash[23439]: cluster 2022-04-28T19:47:58.227344+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2022-04-28T19:47:59.233 INFO:tasks.rados.rados.0.smithi059.stdout:1143: done (2 left) 2022-04-28T19:47:59.233 INFO:tasks.rados.rados.0.smithi059.stdout:1144: delete oid 10 current snap is 110 2022-04-28T19:47:59.233 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 462 (ObjNum 325 snap 98 seq_num 325) dirty exists 2022-04-28T19:47:59.233 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 530 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:47:59.248 INFO:tasks.rados.rados.0.smithi059.stdout:1141: done (2 left) 2022-04-28T19:47:59.248 INFO:tasks.rados.rados.0.smithi059.stdout:1142: done (1 left) 2022-04-28T19:47:59.248 INFO:tasks.rados.rados.0.smithi059.stdout:1144: done (0 left) 2022-04-28T19:47:59.248 INFO:tasks.rados.rados.0.smithi059.stdout:1145: rollback oid 3 current snap is 110 2022-04-28T19:47:59.248 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 106 2022-04-28T19:47:59.248 INFO:tasks.rados.rados.0.smithi059.stdout:1146: setattr oid 37 current snap is 110 2022-04-28T19:47:59.249 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 110 2022-04-28T19:47:59.249 INFO:tasks.rados.rados.0.smithi059.stdout:1147: seq_num 365 ranges {0=516096,516096=573440,1089536=229376} 2022-04-28T19:47:59.252 INFO:tasks.rados.rados.0.smithi059.stdout:1147: writing smithi05955964-18 from 0 to 516096 tid 1 2022-04-28T19:47:59.256 INFO:tasks.rados.rados.0.smithi059.stdout:1147: writing smithi05955964-18 from 516096 to 1089536 tid 2 2022-04-28T19:47:59.258 INFO:tasks.rados.rados.0.smithi059.stdout:1147: writing smithi05955964-18 from 1089536 to 1318912 tid 3 2022-04-28T19:47:59.258 INFO:tasks.rados.rados.0.smithi059.stdout:1148: snap_create 2022-04-28T19:47:59.258 INFO:tasks.rados.rados.0.smithi059.stdout:1145: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:47:59.258 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 448 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty dne 2022-04-28T19:47:59.259 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 524 (ObjNum 359 snap 108 seq_num 359) dirty exists 2022-04-28T19:47:59.263 INFO:tasks.rados.rados.0.smithi059.stdout:1147: finishing write tid 1 to smithi05955964-18 2022-04-28T19:47:59.263 INFO:tasks.rados.rados.0.smithi059.stdout:1147: finishing write tid 2 to smithi05955964-18 2022-04-28T19:47:59.264 INFO:tasks.rados.rados.0.smithi059.stdout:1147: finishing write tid 3 to smithi05955964-18 2022-04-28T19:47:59.264 INFO:tasks.rados.rados.0.smithi059.stdout:1147: finishing write tid 5 to smithi05955964-18 2022-04-28T19:47:59.265 INFO:tasks.rados.rados.0.smithi059.stdout:1147: finishing write tid 6 to smithi05955964-18 2022-04-28T19:47:59.265 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 528 (ObjNum 365 snap 110 seq_num 365) dirty exists 2022-04-28T19:47:59.265 INFO:tasks.rados.rados.0.smithi059.stdout:1147: left oid 18 (ObjNum 365 snap 110 seq_num 365) 2022-04-28T19:48:00.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: cephadm 2022-04-28T19:47:58.800076+0000 mgr.x (mgr.34769) 19 : cephadm [INF] Upgrade: Updating mgr.y 2022-04-28T19:48:00.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: audit 2022-04-28T19:47:58.810175+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:00.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: audit 2022-04-28T19:47:58.811713+0000 mon.b (mon.2) 110 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:00.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: audit 2022-04-28T19:47:58.812948+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:00.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: audit 2022-04-28T19:47:58.814571+0000 mon.b (mon.2) 111 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:48:00.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: audit 2022-04-28T19:47:58.816311+0000 mon.b (mon.2) 112 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:00.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: cephadm 2022-04-28T19:47:58.817726+0000 mgr.x (mgr.34769) 20 : cephadm [INF] Deploying daemon mgr.y on smithi059 2022-04-28T19:48:00.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: audit 2022-04-28T19:47:58.818708+0000 mon.b (mon.2) 113 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mgr.y","key": "container_image"}]: dispatch 2022-04-28T19:48:00.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: cluster 2022-04-28T19:47:59.230461+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2022-04-28T19:48:00.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[23786]: cluster 2022-04-28T19:47:59.275530+0000 mgr.x (mgr.34769) 21 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 259 MiB data, 375 MiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 1019 KiB/s wr, 3 op/s 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: cephadm 2022-04-28T19:47:58.800076+0000 mgr.x (mgr.34769) 19 : cephadm [INF] Upgrade: Updating mgr.y 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: audit 2022-04-28T19:47:58.810175+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: audit 2022-04-28T19:47:58.811713+0000 mon.b (mon.2) 110 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: audit 2022-04-28T19:47:58.812948+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: audit 2022-04-28T19:47:58.814571+0000 mon.b (mon.2) 111 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: audit 2022-04-28T19:47:58.816311+0000 mon.b (mon.2) 112 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: cephadm 2022-04-28T19:47:58.817726+0000 mgr.x (mgr.34769) 20 : cephadm [INF] Deploying daemon mgr.y on smithi059 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: audit 2022-04-28T19:47:58.818708+0000 mon.b (mon.2) 113 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mgr.y","key": "container_image"}]: dispatch 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: cluster 2022-04-28T19:47:59.230461+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2022-04-28T19:48:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:47:59 smithi059 bash[29114]: cluster 2022-04-28T19:47:59.275530+0000 mgr.x (mgr.34769) 21 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 259 MiB data, 375 MiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 1019 KiB/s wr, 3 op/s 2022-04-28T19:48:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: cephadm 2022-04-28T19:47:58.800076+0000 mgr.x (mgr.34769) 19 : cephadm [INF] Upgrade: Updating mgr.y 2022-04-28T19:48:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: audit 2022-04-28T19:47:58.810175+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: audit 2022-04-28T19:47:58.811713+0000 mon.b (mon.2) 110 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: audit 2022-04-28T19:47:58.812948+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: audit 2022-04-28T19:47:58.814571+0000 mon.b (mon.2) 111 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:48:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: audit 2022-04-28T19:47:58.816311+0000 mon.b (mon.2) 112 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: cephadm 2022-04-28T19:47:58.817726+0000 mgr.x (mgr.34769) 20 : cephadm [INF] Deploying daemon mgr.y on smithi059 2022-04-28T19:48:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: audit 2022-04-28T19:47:58.818708+0000 mon.b (mon.2) 113 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mgr.y","key": "container_image"}]: dispatch 2022-04-28T19:48:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: cluster 2022-04-28T19:47:59.230461+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2022-04-28T19:48:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:47:59 smithi139 bash[23439]: cluster 2022-04-28T19:47:59.275530+0000 mgr.x (mgr.34769) 21 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 259 MiB data, 375 MiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 1019 KiB/s wr, 3 op/s 2022-04-28T19:48:00.241 INFO:tasks.rados.rados.0.smithi059.stdout:1145: done (3 left) 2022-04-28T19:48:00.242 INFO:tasks.rados.rados.0.smithi059.stdout:1146: done (2 left) 2022-04-28T19:48:00.243 INFO:tasks.rados.rados.0.smithi059.stdout:1147: done (1 left) 2022-04-28T19:48:00.243 INFO:tasks.rados.rados.0.smithi059.stdout:1148: done (0 left) 2022-04-28T19:48:00.243 INFO:tasks.rados.rados.0.smithi059.stdout:1149: read oid 7 snap 104 2022-04-28T19:48:00.243 INFO:tasks.rados.rados.0.smithi059.stdout:1149: expect deleted 2022-04-28T19:48:00.243 INFO:tasks.rados.rados.0.smithi059.stdout:1150: rmattr oid 2 current snap is 111 2022-04-28T19:48:00.244 INFO:tasks.rados.rados.0.smithi059.stdout:1151: copy_from oid 46 from oid 47 current snap is 111 2022-04-28T19:48:00.244 INFO:tasks.rados.rados.0.smithi059.stdout:1152: copy_from oid 38 from oid 20 current snap is 111 2022-04-28T19:48:00.244 INFO:tasks.rados.rados.0.smithi059.stdout:1153: copy_from oid 44 from oid 39 current snap is 111 2022-04-28T19:48:00.244 INFO:tasks.rados.rados.0.smithi059.stdout:1154: delete oid 25 current snap is 111 2022-04-28T19:48:00.245 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 293 (ObjNum 293 snap 87 seq_num 293) dirty exists 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:1149: done (5 left) 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:1150: done (4 left) 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:1154: done (3 left) 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:1155: rollback oid 40 current snap is 111 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 104 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 111 2022-04-28T19:48:00.248 INFO:tasks.rados.rados.0.smithi059.stdout:1156: seq_num 367 ranges {0=548864,548864=737280,1286144=65536} 2022-04-28T19:48:00.252 INFO:tasks.rados.rados.0.smithi059.stdout:1156: writing smithi05955964-49 from 0 to 548864 tid 1 2022-04-28T19:48:00.257 INFO:tasks.rados.rados.0.smithi059.stdout:1156: writing smithi05955964-49 from 548864 to 1286144 tid 2 2022-04-28T19:48:00.257 INFO:tasks.rados.rados.0.smithi059.stdout:1156: writing smithi05955964-49 from 1286144 to 1351680 tid 3 2022-04-28T19:48:00.257 INFO:tasks.rados.rados.0.smithi059.stdout:1157: snap_remove snap 106 2022-04-28T19:48:01.246 INFO:tasks.rados.rados.0.smithi059.stdout:1157: done (5 left) 2022-04-28T19:48:01.246 INFO:tasks.rados.rados.0.smithi059.stdout:1158: snap_remove snap 107 2022-04-28T19:48:01.252 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:01 smithi059 systemd[1]: Stopping Ceph mgr.y for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:48:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:01 smithi059 bash[23786]: cluster 2022-04-28T19:48:00.238993+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2022-04-28T19:48:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:01 smithi059 bash[29114]: cluster 2022-04-28T19:48:00.238993+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2022-04-28T19:48:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:01 smithi139 bash[23439]: cluster 2022-04-28T19:48:00.238993+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2022-04-28T19:48:02.231 INFO:tasks.rados.rados.0.smithi059.stdout:1158: done (5 left) 2022-04-28T19:48:02.232 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 111 2022-04-28T19:48:02.232 INFO:tasks.rados.rados.0.smithi059.stdout:1159: seq_num 368 ranges {0=573440,573440=622592,1196032=180224} 2022-04-28T19:48:02.236 INFO:tasks.rados.rados.0.smithi059.stdout:1159: writing smithi05955964-17 from 0 to 573440 tid 1 2022-04-28T19:48:02.241 INFO:tasks.rados.rados.0.smithi059.stdout:1159: writing smithi05955964-17 from 573440 to 1196032 tid 2 2022-04-28T19:48:02.243 INFO:tasks.rados.rados.0.smithi059.stdout:1159: writing smithi05955964-17 from 1196032 to 1376256 tid 3 2022-04-28T19:48:02.243 INFO:tasks.rados.rados.0.smithi059.stdout:1160: read oid 11 snap -1 2022-04-28T19:48:02.243 INFO:tasks.rados.rados.0.smithi059.stdout:1160: expect (ObjNum 325 snap 98 seq_num 325) 2022-04-28T19:48:02.243 INFO:tasks.rados.rados.0.smithi059.stdout:1161: rollback oid 33 current snap is 111 2022-04-28T19:48:02.243 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 71 2022-04-28T19:48:02.243 INFO:tasks.rados.rados.0.smithi059.stdout:1152: finishing copy_from to smithi05955964-38 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 450 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1152: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1155: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1155: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1155: finishing rollback tid 2 to smithi05955964-40 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 450 (ObjNum 319 snap 96 seq_num 319) dirty exists 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1153: finishing copy_from to smithi05955964-44 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 562 (ObjNum 356 snap 107 seq_num 356) dirty exists 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1153: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1156: finishing write tid 1 to smithi05955964-49 2022-04-28T19:48:02.244 INFO:tasks.rados.rados.0.smithi059.stdout:1156: finishing write tid 2 to smithi05955964-49 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1156: finishing write tid 3 to smithi05955964-49 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1151: finishing copy_from to smithi05955964-46 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 530 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1156: finishing write tid 5 to smithi05955964-49 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1151: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1156: finishing write tid 6 to smithi05955964-49 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 454 (ObjNum 367 snap 111 seq_num 367) dirty exists 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1156: left oid 49 (ObjNum 367 snap 111 seq_num 367) 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1151: done (7 left) 2022-04-28T19:48:02.245 INFO:tasks.rados.rados.0.smithi059.stdout:1152: done (6 left) 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:1153: done (5 left) 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:1155: done (4 left) 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:1156: done (3 left) 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:1162: rollback oid 26 current snap is 111 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 71 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:1163: rollback oid 23 current snap is 111 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 98 2022-04-28T19:48:02.246 INFO:tasks.rados.rados.0.smithi059.stdout:1164: snap_create 2022-04-28T19:48:02.247 INFO:tasks.rados.rados.0.smithi059.stdout:1159: finishing write tid 1 to smithi05955964-17 2022-04-28T19:48:02.248 INFO:tasks.rados.rados.0.smithi059.stdout:1161: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:1162: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:1159: finishing write tid 2 to smithi05955964-17 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:1161: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:1159: finishing write tid 3 to smithi05955964-17 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:1159: finishing write tid 5 to smithi05955964-17 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:1161: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:48:02.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 614 (ObjNum 217 snap 59 seq_num 217) dirty exists 2022-04-28T19:48:02.250 INFO:tasks.rados.rados.0.smithi059.stdout:1162: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:48:02.250 INFO:tasks.rados.rados.0.smithi059.stdout:1162: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:48:02.250 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 562 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:48:02.250 INFO:tasks.rados.rados.0.smithi059.stdout:1159: finishing write tid 6 to smithi05955964-17 2022-04-28T19:48:02.251 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 279 (ObjNum 368 snap 111 seq_num 368) dirty exists 2022-04-28T19:48:02.251 INFO:tasks.rados.rados.0.smithi059.stdout:1159: left oid 17 (ObjNum 368 snap 111 seq_num 368) 2022-04-28T19:48:02.253 INFO:tasks.rados.rados.0.smithi059.stdout:1163: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:48:02.254 INFO:tasks.rados.rados.0.smithi059.stdout:1163: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:48:02.254 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 546 (ObjNum 281 snap 80 seq_num 281) dirty exists 2022-04-28T19:48:02.271 INFO:tasks.rados.rados.0.smithi059.stdout:1160: expect (ObjNum 325 snap 98 seq_num 325) 2022-04-28T19:48:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:02 smithi059 bash[29114]: cluster 2022-04-28T19:48:01.242044+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2022-04-28T19:48:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:02 smithi059 bash[29114]: cluster 2022-04-28T19:48:01.275988+0000 mgr.x (mgr.34769) 22 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 259 MiB data, 375 MiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 1012 KiB/s wr, 3 op/s 2022-04-28T19:48:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:02 smithi059 bash[29114]: cluster 2022-04-28T19:48:02.229846+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2022-04-28T19:48:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:02 smithi059 bash[23786]: cluster 2022-04-28T19:48:01.242044+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2022-04-28T19:48:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:02 smithi059 bash[23786]: cluster 2022-04-28T19:48:01.275988+0000 mgr.x (mgr.34769) 22 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 259 MiB data, 375 MiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 1012 KiB/s wr, 3 op/s 2022-04-28T19:48:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:02 smithi059 bash[23786]: cluster 2022-04-28T19:48:02.229846+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2022-04-28T19:48:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:02 smithi139 bash[23439]: cluster 2022-04-28T19:48:01.242044+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2022-04-28T19:48:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:02 smithi139 bash[23439]: cluster 2022-04-28T19:48:01.275988+0000 mgr.x (mgr.34769) 22 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 259 MiB data, 375 MiB used, 707 GiB / 715 GiB avail; 2.3 MiB/s rd, 1012 KiB/s wr, 3 op/s 2022-04-28T19:48:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:02 smithi139 bash[23439]: cluster 2022-04-28T19:48:02.229846+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1159: done (5 left) 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1160: done (4 left) 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1161: done (3 left) 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1162: done (2 left) 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1163: done (1 left) 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1164: done (0 left) 2022-04-28T19:48:03.247 INFO:tasks.rados.rados.0.smithi059.stdout:1165: snap_create 2022-04-28T19:48:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:03.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:03.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:48:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:03.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:48:04.249 INFO:tasks.rados.rados.0.smithi059.stdout:1165: done (0 left) 2022-04-28T19:48:04.250 INFO:tasks.rados.rados.0.smithi059.stdout:1166: delete oid 39 current snap is 113 2022-04-28T19:48:04.260 INFO:tasks.rados.rados.0.smithi059.stdout:1166: done (0 left) 2022-04-28T19:48:04.260 INFO:tasks.rados.rados.0.smithi059.stdout:1167: rollback oid 33 current snap is 113 2022-04-28T19:48:04.260 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 110 2022-04-28T19:48:04.260 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 113 2022-04-28T19:48:04.261 INFO:tasks.rados.rados.0.smithi059.stdout:1168: seq_num 370 ranges {7069696=606208,7675904=794624,8470528=8192} 2022-04-28T19:48:04.302 INFO:tasks.rados.rados.0.smithi059.stdout:1168: writing smithi05955964-12 from 7069696 to 7675904 tid 1 2022-04-28T19:48:04.307 INFO:tasks.rados.rados.0.smithi059.stdout:1168: writing smithi05955964-12 from 7675904 to 8470528 tid 2 2022-04-28T19:48:04.307 INFO:tasks.rados.rados.0.smithi059.stdout:1168: writing smithi05955964-12 from 8470528 to 8478720 tid 3 2022-04-28T19:48:04.308 INFO:tasks.rados.rados.0.smithi059.stdout:1169: rollback oid 42 current snap is 113 2022-04-28T19:48:04.308 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 83 2022-04-28T19:48:04.308 INFO:tasks.rados.rados.0.smithi059.stdout:1167: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:48:04.308 INFO:tasks.rados.rados.0.smithi059.stdout:1170: snap_remove snap 104 2022-04-28T19:48:04.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:04 smithi059 bash[29114]: cluster 2022-04-28T19:48:03.244218+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2022-04-28T19:48:04.677 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:04 smithi059 bash[29114]: cluster 2022-04-28T19:48:03.276638+0000 mgr.x (mgr.34769) 23 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2022-04-28T19:48:04.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:04 smithi059 bash[23786]: cluster 2022-04-28T19:48:03.244218+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2022-04-28T19:48:04.678 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:04 smithi059 bash[23786]: cluster 2022-04-28T19:48:03.276638+0000 mgr.x (mgr.34769) 23 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2022-04-28T19:48:04.678 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:04 smithi139 bash[23439]: cluster 2022-04-28T19:48:03.244218+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2022-04-28T19:48:04.679 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:04 smithi139 bash[23439]: cluster 2022-04-28T19:48:03.276638+0000 mgr.x (mgr.34769) 23 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2022-04-28T19:48:05.270 INFO:tasks.rados.rados.0.smithi059.stdout:1170: done (3 left) 2022-04-28T19:48:05.270 INFO:tasks.rados.rados.0.smithi059.stdout:1171: read oid 49 snap -1 2022-04-28T19:48:05.270 INFO:tasks.rados.rados.0.smithi059.stdout:1171: expect (ObjNum 367 snap 111 seq_num 367) 2022-04-28T19:48:05.270 INFO:tasks.rados.rados.0.smithi059.stdout:1172: setattr oid 1 current snap is 113 2022-04-28T19:48:05.270 INFO:tasks.rados.rados.0.smithi059.stdout:1167: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:48:05.270 INFO:tasks.rados.rados.0.smithi059.stdout:1167: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 620 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1168: finishing write tid 1 to smithi05955964-12 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1168: finishing write tid 2 to smithi05955964-12 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1168: finishing write tid 3 to smithi05955964-12 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1168: finishing write tid 5 to smithi05955964-12 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1168: finishing write tid 6 to smithi05955964-12 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 563 (ObjNum 370 snap 113 seq_num 370) dirty exists 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1168: left oid 12 (ObjNum 370 snap 113 seq_num 370) 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1169: finishing rollback tid 0 to smithi05955964-42 2022-04-28T19:48:05.271 INFO:tasks.rados.rados.0.smithi059.stdout:1169: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:48:05.272 INFO:tasks.rados.rados.0.smithi059.stdout:1169: finishing rollback tid 2 to smithi05955964-42 2022-04-28T19:48:05.272 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 540 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:48:05.272 INFO:tasks.rados.rados.0.smithi059.stdout:1167: done (4 left) 2022-04-28T19:48:05.272 INFO:tasks.rados.rados.0.smithi059.stdout:1168: done (3 left) 2022-04-28T19:48:05.272 INFO:tasks.rados.rados.0.smithi059.stdout:1169: done (2 left) 2022-04-28T19:48:05.272 INFO:tasks.rados.rados.0.smithi059.stdout:1173: delete oid 3 current snap is 113 2022-04-28T19:48:05.277 INFO:tasks.rados.rados.0.smithi059.stdout:1173: done (2 left) 2022-04-28T19:48:05.278 INFO:tasks.rados.rados.0.smithi059.stdout:1174: snap_create 2022-04-28T19:48:05.283 INFO:tasks.rados.rados.0.smithi059.stdout:1171: expect (ObjNum 367 snap 111 seq_num 367) 2022-04-28T19:48:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 460 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:48:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:05 smithi059 bash[29114]: cluster 2022-04-28T19:48:04.245897+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2022-04-28T19:48:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:05 smithi059 bash[23786]: cluster 2022-04-28T19:48:04.245897+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2022-04-28T19:48:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:05 smithi139 bash[23439]: cluster 2022-04-28T19:48:04.245897+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:1171: done (2 left) 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:1172: done (1 left) 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:1174: done (0 left) 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:1175: rollback oid 41 current snap is 114 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 101 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 114 2022-04-28T19:48:06.275 INFO:tasks.rados.rados.0.smithi059.stdout:1176: seq_num 372 ranges {4972544=638976,5611520=565248,6176768=237568} 2022-04-28T19:48:06.307 INFO:tasks.rados.rados.0.smithi059.stdout:1176: writing smithi05955964-4 from 4972544 to 5611520 tid 1 2022-04-28T19:48:06.310 INFO:tasks.rados.rados.0.smithi059.stdout:1176: writing smithi05955964-4 from 5611520 to 6176768 tid 2 2022-04-28T19:48:06.312 INFO:tasks.rados.rados.0.smithi059.stdout:1176: writing smithi05955964-4 from 6176768 to 6414336 tid 3 2022-04-28T19:48:06.312 INFO:tasks.rados.rados.0.smithi059.stdout:1177: delete oid 19 current snap is 114 2022-04-28T19:48:06.312 INFO:tasks.rados.rados.0.smithi059.stdout:1175: finishing rollback tid 0 to smithi05955964-41 2022-04-28T19:48:06.313 INFO:tasks.rados.rados.0.smithi059.stdout:1175: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:48:06.313 INFO:tasks.rados.rados.0.smithi059.stdout:1175: finishing rollback tid 2 to smithi05955964-41 2022-04-28T19:48:06.313 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 252 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:48:06.314 INFO:tasks.rados.rados.0.smithi059.stdout:1176: finishing write tid 1 to smithi05955964-4 2022-04-28T19:48:06.316 INFO:tasks.rados.rados.0.smithi059.stdout:1176: finishing write tid 2 to smithi05955964-4 2022-04-28T19:48:06.317 INFO:tasks.rados.rados.0.smithi059.stdout:1175: done (2 left) 2022-04-28T19:48:06.317 INFO:tasks.rados.rados.0.smithi059.stdout:1177: done (1 left) 2022-04-28T19:48:06.317 INFO:tasks.rados.rados.0.smithi059.stdout:1178: rollback oid 6 current snap is 114 2022-04-28T19:48:06.317 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 101 2022-04-28T19:48:06.317 INFO:tasks.rados.rados.0.smithi059.stdout:1179: snap_remove snap 113 2022-04-28T19:48:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:06 smithi059 bash[29114]: cluster 2022-04-28T19:48:05.266995+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2022-04-28T19:48:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:06 smithi059 bash[29114]: cluster 2022-04-28T19:48:05.277220+0000 mgr.x (mgr.34769) 24 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2022-04-28T19:48:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:06 smithi059 bash[23786]: cluster 2022-04-28T19:48:05.266995+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2022-04-28T19:48:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:06 smithi059 bash[23786]: cluster 2022-04-28T19:48:05.277220+0000 mgr.x (mgr.34769) 24 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2022-04-28T19:48:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:06 smithi139 bash[23439]: cluster 2022-04-28T19:48:05.266995+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2022-04-28T19:48:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:06 smithi139 bash[23439]: cluster 2022-04-28T19:48:05.277220+0000 mgr.x (mgr.34769) 24 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.2 MiB/s wr, 15 op/s 2022-04-28T19:48:07.236 INFO:tasks.rados.rados.0.smithi059.stdout:1179: done (2 left) 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1180: read oid 7 snap -1 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1180: expect deleted 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1181: delete oid 35 current snap is 114 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1176: finishing write tid 3 to smithi05955964-4 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1176: finishing write tid 5 to smithi05955964-4 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1176: finishing write tid 6 to smithi05955964-4 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 491 (ObjNum 372 snap 114 seq_num 372) dirty exists 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1176: left oid 4 (ObjNum 372 snap 114 seq_num 372) 2022-04-28T19:48:07.237 INFO:tasks.rados.rados.0.smithi059.stdout:1178: finishing rollback tid 0 to smithi05955964-6 2022-04-28T19:48:07.238 INFO:tasks.rados.rados.0.smithi059.stdout:1178: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:48:07.238 INFO:tasks.rados.rados.0.smithi059.stdout:1178: finishing rollback tid 2 to smithi05955964-6 2022-04-28T19:48:07.238 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 494 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:48:07.241 INFO:tasks.rados.rados.0.smithi059.stdout:1176: done (3 left) 2022-04-28T19:48:07.242 INFO:tasks.rados.rados.0.smithi059.stdout:1178: done (2 left) 2022-04-28T19:48:07.242 INFO:tasks.rados.rados.0.smithi059.stdout:1181: done (1 left) 2022-04-28T19:48:07.242 INFO:tasks.rados.rados.0.smithi059.stdout:1182: rollback oid 16 current snap is 114 2022-04-28T19:48:07.242 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 110 2022-04-28T19:48:07.242 INFO:tasks.rados.rados.0.smithi059.stdout:1183: rmattr oid 43 current snap is 114 2022-04-28T19:48:07.242 INFO:tasks.rados.rados.0.smithi059.stdout:1184: delete oid 18 current snap is 114 2022-04-28T19:48:07.248 INFO:tasks.rados.rados.0.smithi059.stdout:1182: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:48:07.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 628 (ObjNum 357 snap 107 seq_num 357) dirty exists 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:1182: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:1182: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 380 (ObjNum 360 snap 108 seq_num 360) dirty exists 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:1180: done (3 left) 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:1182: done (2 left) 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:1183: done (1 left) 2022-04-28T19:48:07.251 INFO:tasks.rados.rados.0.smithi059.stdout:1184: done (0 left) 2022-04-28T19:48:07.252 INFO:tasks.rados.rados.0.smithi059.stdout:1185: read oid 26 snap -1 2022-04-28T19:48:07.252 INFO:tasks.rados.rados.0.smithi059.stdout:1185: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:48:07.252 INFO:tasks.rados.rados.0.smithi059.stdout:1186: snap_create 2022-04-28T19:48:07.259 INFO:tasks.rados.rados.0.smithi059.stdout:1185: expect (ObjNum 132 snap 31 seq_num 132) 2022-04-28T19:48:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:07 smithi059 bash[29114]: cluster 2022-04-28T19:48:06.272625+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2022-04-28T19:48:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:07 smithi059 bash[29114]: cluster 2022-04-28T19:48:07.231905+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2022-04-28T19:48:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:07 smithi059 bash[23786]: cluster 2022-04-28T19:48:06.272625+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2022-04-28T19:48:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:07 smithi059 bash[23786]: cluster 2022-04-28T19:48:07.231905+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2022-04-28T19:48:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:07 smithi139 bash[23439]: cluster 2022-04-28T19:48:06.272625+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2022-04-28T19:48:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:07 smithi139 bash[23439]: cluster 2022-04-28T19:48:07.231905+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2022-04-28T19:48:08.248 INFO:tasks.rados.rados.0.smithi059.stdout:1185: done (1 left) 2022-04-28T19:48:08.248 INFO:tasks.rados.rados.0.smithi059.stdout:1186: done (0 left) 2022-04-28T19:48:08.248 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 115 2022-04-28T19:48:08.249 INFO:tasks.rados.rados.0.smithi059.stdout:1187: seq_num 376 ranges {2547712=704512,3252224=548864,3801088=253952} 2022-04-28T19:48:08.266 INFO:tasks.rados.rados.0.smithi059.stdout:1187: writing smithi05955964-40 from 2547712 to 3252224 tid 1 2022-04-28T19:48:08.270 INFO:tasks.rados.rados.0.smithi059.stdout:1187: writing smithi05955964-40 from 3252224 to 3801088 tid 2 2022-04-28T19:48:08.271 INFO:tasks.rados.rados.0.smithi059.stdout:1187: writing smithi05955964-40 from 3801088 to 4055040 tid 3 2022-04-28T19:48:08.271 INFO:tasks.rados.rados.0.smithi059.stdout:1188: delete oid 2 current snap is 115 2022-04-28T19:48:08.276 INFO:tasks.rados.rados.0.smithi059.stdout:1188: done (1 left) 2022-04-28T19:48:08.277 INFO:tasks.rados.rados.0.smithi059.stdout:1189: snap_remove snap 100 2022-04-28T19:48:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:08 smithi059 bash[23786]: cluster 2022-04-28T19:48:07.277686+0000 mgr.x (mgr.34769) 25 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:08.688 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:08 smithi059 bash[23786]: cluster 2022-04-28T19:48:08.245273+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2022-04-28T19:48:08.689 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:08 smithi059 bash[29114]: cluster 2022-04-28T19:48:07.277686+0000 mgr.x (mgr.34769) 25 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:08.690 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:08 smithi059 bash[29114]: cluster 2022-04-28T19:48:08.245273+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2022-04-28T19:48:08.690 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:08 smithi139 bash[23439]: cluster 2022-04-28T19:48:07.277686+0000 mgr.x (mgr.34769) 25 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 273 MiB data, 387 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:08.690 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:08 smithi139 bash[23439]: cluster 2022-04-28T19:48:08.245273+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2022-04-28T19:48:09.290 INFO:tasks.rados.rados.0.smithi059.stdout:1189: done (1 left) 2022-04-28T19:48:09.290 INFO:tasks.rados.rados.0.smithi059.stdout:1190: copy_from oid 43 from oid 39 current snap is 115 2022-04-28T19:48:09.290 INFO:tasks.rados.rados.0.smithi059.stdout:1187: finishing write tid 1 to smithi05955964-40 2022-04-28T19:48:09.290 INFO:tasks.rados.rados.0.smithi059.stdout:1187: finishing write tid 2 to smithi05955964-40 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1187: finishing write tid 3 to smithi05955964-40 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1187: finishing write tid 5 to smithi05955964-40 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1187: finishing write tid 6 to smithi05955964-40 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 469 (ObjNum 376 snap 115 seq_num 376) dirty exists 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1187: left oid 40 (ObjNum 376 snap 115 seq_num 376) 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1187: done (1 left) 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 115 2022-04-28T19:48:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1191: seq_num 378 ranges {1351680=737280,2088960=720896,2809856=81920} 2022-04-28T19:48:09.303 INFO:tasks.rados.rados.0.smithi059.stdout:1191: writing smithi05955964-49 from 1351680 to 2088960 tid 1 2022-04-28T19:48:09.308 INFO:tasks.rados.rados.0.smithi059.stdout:1191: writing smithi05955964-49 from 2088960 to 2809856 tid 2 2022-04-28T19:48:09.309 INFO:tasks.rados.rados.0.smithi059.stdout:1191: writing smithi05955964-49 from 2809856 to 2891776 tid 3 2022-04-28T19:48:09.309 INFO:tasks.rados.rados.0.smithi059.stdout:1192: delete oid 26 current snap is 115 2022-04-28T19:48:09.309 INFO:tasks.rados.rados.0.smithi059.stdout:1190: finishing copy_from to smithi05955964-43 2022-04-28T19:48:09.309 INFO:tasks.rados.rados.0.smithi059.stdout:1190: got expected ENOENT (src dne) 2022-04-28T19:48:09.312 INFO:tasks.rados.rados.0.smithi059.stdout:1190: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:48:09.315 INFO:tasks.rados.rados.0.smithi059.stdout:1190: done (2 left) 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:1192: done (1 left) 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:1193: read oid 7 snap 111 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:1193: expect deleted 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:1194: rollback oid 18 current snap is 115 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 112 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 115 2022-04-28T19:48:09.316 INFO:tasks.rados.rados.0.smithi059.stdout:1195: seq_num 380 ranges {4055040=770048,4825088=483328,5308416=319488} 2022-04-28T19:48:09.343 INFO:tasks.rados.rados.0.smithi059.stdout:1195: writing smithi05955964-40 from 4055040 to 4825088 tid 1 2022-04-28T19:48:09.346 INFO:tasks.rados.rados.0.smithi059.stdout:1195: writing smithi05955964-40 from 4825088 to 5308416 tid 2 2022-04-28T19:48:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1195: writing smithi05955964-40 from 5308416 to 5627904 tid 3 2022-04-28T19:48:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1196: read oid 31 snap -1 2022-04-28T19:48:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1196: expect (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:48:09.349 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 115 2022-04-28T19:48:09.349 INFO:tasks.rados.rados.0.smithi059.stdout:1197: seq_num 381 ranges {0=786432,786432=417792,1204224=385024} 2022-04-28T19:48:09.353 INFO:tasks.rados.rados.0.smithi059.stdout:1197: writing smithi05955964-32 from 0 to 786432 tid 1 2022-04-28T19:48:09.355 INFO:tasks.rados.rados.0.smithi059.stdout:1197: writing smithi05955964-32 from 786432 to 1204224 tid 2 2022-04-28T19:48:09.358 INFO:tasks.rados.rados.0.smithi059.stdout:1197: writing smithi05955964-32 from 1204224 to 1589248 tid 3 2022-04-28T19:48:09.358 INFO:tasks.rados.rados.0.smithi059.stdout:1198: read oid 24 snap -1 2022-04-28T19:48:09.358 INFO:tasks.rados.rados.0.smithi059.stdout:1198: expect (ObjNum 346 snap 105 seq_num 346) 2022-04-28T19:48:09.358 INFO:tasks.rados.rados.0.smithi059.stdout:1199: snap_create 2022-04-28T19:48:09.358 INFO:tasks.rados.rados.0.smithi059.stdout:1194: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1194: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 560 (ObjNum 365 snap 110 seq_num 365) dirty exists 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1191: finishing write tid 1 to smithi05955964-49 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1191: finishing write tid 2 to smithi05955964-49 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1191: finishing write tid 3 to smithi05955964-49 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1191: finishing write tid 5 to smithi05955964-49 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1191: finishing write tid 6 to smithi05955964-49 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 474 (ObjNum 378 snap 115 seq_num 378) dirty exists 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1191: left oid 49 (ObjNum 378 snap 115 seq_num 378) 2022-04-28T19:48:09.359 INFO:tasks.rados.rados.0.smithi059.stdout:1195: finishing write tid 1 to smithi05955964-40 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:1195: finishing write tid 2 to smithi05955964-40 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:1195: finishing write tid 3 to smithi05955964-40 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:1195: finishing write tid 5 to smithi05955964-40 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:1195: finishing write tid 6 to smithi05955964-40 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 482 (ObjNum 380 snap 115 seq_num 380) dirty exists 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:1195: left oid 40 (ObjNum 380 snap 115 seq_num 380) 2022-04-28T19:48:09.360 INFO:tasks.rados.rados.0.smithi059.stdout:1196: expect (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1197: finishing write tid 1 to smithi05955964-32 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1197: finishing write tid 2 to smithi05955964-32 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1197: finishing write tid 3 to smithi05955964-32 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1197: finishing write tid 5 to smithi05955964-32 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1197: finishing write tid 6 to smithi05955964-32 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 638 (ObjNum 381 snap 115 seq_num 381) dirty exists 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1197: left oid 32 (ObjNum 381 snap 115 seq_num 381) 2022-04-28T19:48:09.391 INFO:tasks.rados.rados.0.smithi059.stdout:1198: expect (ObjNum 346 snap 105 seq_num 346) 2022-04-28T19:48:10.221 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:48:10.341 INFO:tasks.rados.rados.0.smithi059.stdout:1191: done (7 left) 2022-04-28T19:48:10.341 INFO:tasks.rados.rados.0.smithi059.stdout:1193: done (6 left) 2022-04-28T19:48:10.341 INFO:tasks.rados.rados.0.smithi059.stdout:1194: done (5 left) 2022-04-28T19:48:10.341 INFO:tasks.rados.rados.0.smithi059.stdout:1195: done (4 left) 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1196: done (3 left) 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1197: done (2 left) 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1198: done (1 left) 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1199: done (0 left) 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1200: read oid 18 snap -1 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1200: expect (ObjNum 365 snap 110 seq_num 365) 2022-04-28T19:48:10.342 INFO:tasks.rados.rados.0.smithi059.stdout:1201: setattr oid 1 current snap is 116 2022-04-28T19:48:10.344 INFO:tasks.rados.rados.0.smithi059.stdout:1202: snap_create 2022-04-28T19:48:10.349 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 484 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:48:10.357 INFO:tasks.rados.rados.0.smithi059.stdout:1200: expect (ObjNum 365 snap 110 seq_num 365) 2022-04-28T19:48:10.551 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (7m) 27s ago 7m 13.7M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (7m) 27s ago 7m 25.4M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (5m) 27s ago 5m 13.4M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (5m) 27s ago 5m 15.8M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (47s) 27s ago 11m 403M - 16.2.7-969-g4fa079ba 863db77b5b5e 646fe4292ebe 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 running (13m) 27s ago 13m 299M - 15.2.16-79-gb51d8665 6d41e73600e1 320b27bb479e 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (13m) 27s ago 13m 57.6M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 239497cf6769 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (11m) 27s ago 11m 51.8M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 0e12092e0bc3 2022-04-28T19:48:10.552 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (11m) 27s ago 11m 49.4M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (7m) 27s ago 7m 10.5M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (7m) 27s ago 7m 10.3M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (10m) 27s ago 11m 141M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 547b5580e073 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (10m) 27s ago 10m 123M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (10m) 27s ago 10m 156M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (9m) 27s ago 9m 128M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (9m) 27s ago 9m 139M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (9m) 27s ago 9m 161M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (8m) 27s ago 8m 111M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:48:10.553 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (8m) 27s ago 8m 107M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:48:10.554 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 starting - - - - 2022-04-28T19:48:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:10 smithi059 bash[29114]: cluster 2022-04-28T19:48:09.278689+0000 mgr.x (mgr.34769) 26 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 278 MiB data, 390 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 693 KiB/s wr, 13 op/s 2022-04-28T19:48:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:10 smithi059 bash[29114]: cluster 2022-04-28T19:48:09.287457+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2022-04-28T19:48:10.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:10 smithi059 bash[23786]: cluster 2022-04-28T19:48:09.278689+0000 mgr.x (mgr.34769) 26 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 278 MiB data, 390 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 693 KiB/s wr, 13 op/s 2022-04-28T19:48:10.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:10 smithi059 bash[23786]: cluster 2022-04-28T19:48:09.287457+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2022-04-28T19:48:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:10 smithi139 bash[23439]: cluster 2022-04-28T19:48:09.278689+0000 mgr.x (mgr.34769) 26 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 278 MiB data, 390 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 693 KiB/s wr, 13 op/s 2022-04-28T19:48:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:10 smithi139 bash[23439]: cluster 2022-04-28T19:48:09.287457+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2022-04-28T19:48:10.916 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:48:10.916 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:48:10.916 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:48:10.916 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 1, 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:48:10.917 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:10.918 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:48:10.918 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 14, 2022-04-28T19:48:10.918 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:48:10.918 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:48:10.918 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:48:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:1200: done (2 left) 2022-04-28T19:48:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:1201: done (1 left) 2022-04-28T19:48:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:1202: done (0 left) 2022-04-28T19:48:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:1203: setattr oid 40 current snap is 117 2022-04-28T19:48:11.352 INFO:tasks.rados.rados.0.smithi059.stdout:1204: read oid 8 snap -1 2022-04-28T19:48:11.352 INFO:tasks.rados.rados.0.smithi059.stdout:1204: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:48:11.352 INFO:tasks.rados.rados.0.smithi059.stdout:1205: snap_remove snap 102 2022-04-28T19:48:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.206523+0000 mgr.x (mgr.34769) 27 : audit [DBG] from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: cluster 2022-04-28T19:48:10.338083+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2022-04-28T19:48:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.518660+0000 mgr.x (mgr.34769) 28 : audit [DBG] from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.524508+0000 mon.b (mon.2) 114 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.525582+0000 mon.b (mon.2) 115 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.526528+0000 mon.b (mon.2) 116 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.527525+0000 mon.b (mon.2) 117 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.528490+0000 mon.b (mon.2) 118 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.529518+0000 mon.b (mon.2) 119 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.533084+0000 mon.b (mon.2) 120 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.533876+0000 mon.b (mon.2) 121 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.534476+0000 mon.b (mon.2) 122 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.535087+0000 mon.b (mon.2) 123 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.535670+0000 mon.b (mon.2) 124 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[23786]: audit 2022-04-28T19:48:10.914842+0000 mon.c (mon.1) 42 : audit [DBG] from='client.? 172.21.15.59:0/2015805010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.206523+0000 mgr.x (mgr.34769) 27 : audit [DBG] from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: cluster 2022-04-28T19:48:10.338083+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.518660+0000 mgr.x (mgr.34769) 28 : audit [DBG] from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.524508+0000 mon.b (mon.2) 114 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.525582+0000 mon.b (mon.2) 115 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.526528+0000 mon.b (mon.2) 116 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.527525+0000 mon.b (mon.2) 117 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.528490+0000 mon.b (mon.2) 118 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.529518+0000 mon.b (mon.2) 119 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.533084+0000 mon.b (mon.2) 120 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.533876+0000 mon.b (mon.2) 121 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.534476+0000 mon.b (mon.2) 122 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.535087+0000 mon.b (mon.2) 123 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.535670+0000 mon.b (mon.2) 124 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:11 smithi059 bash[29114]: audit 2022-04-28T19:48:10.914842+0000 mon.c (mon.1) 42 : audit [DBG] from='client.? 172.21.15.59:0/2015805010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.206523+0000 mgr.x (mgr.34769) 27 : audit [DBG] from='client.14967 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: cluster 2022-04-28T19:48:10.338083+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2022-04-28T19:48:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.518660+0000 mgr.x (mgr.34769) 28 : audit [DBG] from='client.24889 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.524508+0000 mon.b (mon.2) 114 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.525582+0000 mon.b (mon.2) 115 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.526528+0000 mon.b (mon.2) 116 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.527525+0000 mon.b (mon.2) 117 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.528490+0000 mon.b (mon.2) 118 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.529518+0000 mon.b (mon.2) 119 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.533084+0000 mon.b (mon.2) 120 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.533876+0000 mon.b (mon.2) 121 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.534476+0000 mon.b (mon.2) 122 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.535087+0000 mon.b (mon.2) 123 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.535670+0000 mon.b (mon.2) 124 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:11 smithi139 bash[23439]: audit 2022-04-28T19:48:10.914842+0000 mon.c (mon.1) 42 : audit [DBG] from='client.? 172.21.15.59:0/2015805010' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:12.235 INFO:tasks.rados.rados.0.smithi059.stdout:1205: done (2 left) 2022-04-28T19:48:12.235 INFO:tasks.rados.rados.0.smithi059.stdout:1206: read oid 6 snap -1 2022-04-28T19:48:12.235 INFO:tasks.rados.rados.0.smithi059.stdout:1206: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:48:12.235 INFO:tasks.rados.rados.0.smithi059.stdout:1207: snap_remove snap 83 2022-04-28T19:48:12.466 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[29114]: cluster 2022-04-28T19:48:11.279280+0000 mgr.x (mgr.34769) 29 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 278 MiB data, 390 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 687 KiB/s wr, 13 op/s 2022-04-28T19:48:12.466 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[29114]: cluster 2022-04-28T19:48:11.347416+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2022-04-28T19:48:12.466 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[29114]: cluster 2022-04-28T19:48:12.230514+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[57577]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr.y 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[57577]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr-y 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y.service: Main process exited, code=exited, status=137/n/a 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[57577]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr.y 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.y.service: Failed with result 'exit-code'. 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 systemd[1]: Stopped Ceph mgr.y for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:48:12.467 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[23786]: cluster 2022-04-28T19:48:11.279280+0000 mgr.x (mgr.34769) 29 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 278 MiB data, 390 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 687 KiB/s wr, 13 op/s 2022-04-28T19:48:12.468 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[23786]: cluster 2022-04-28T19:48:11.347416+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2022-04-28T19:48:12.468 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:12 smithi059 bash[23786]: cluster 2022-04-28T19:48:12.230514+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2022-04-28T19:48:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:12 smithi139 bash[23439]: cluster 2022-04-28T19:48:11.279280+0000 mgr.x (mgr.34769) 29 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 278 MiB data, 390 MiB used, 707 GiB / 715 GiB avail; 2.5 MiB/s rd, 687 KiB/s wr, 13 op/s 2022-04-28T19:48:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:12 smithi139 bash[23439]: cluster 2022-04-28T19:48:11.347416+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2022-04-28T19:48:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:12 smithi139 bash[23439]: cluster 2022-04-28T19:48:12.230514+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2022-04-28T19:48:13.247 INFO:tasks.rados.rados.0.smithi059.stdout:1207: done (3 left) 2022-04-28T19:48:13.263 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 117 2022-04-28T19:48:13.263 INFO:tasks.rados.rados.0.smithi059.stdout:1208: seq_num 382 ranges {0=802816,802816=704512,1507328=98304} 2022-04-28T19:48:13.264 INFO:tasks.rados.rados.0.smithi059.stdout:1208: writing smithi05955964-19 from 0 to 802816 tid 1 2022-04-28T19:48:13.264 INFO:tasks.rados.rados.0.smithi059.stdout:1208: writing smithi05955964-19 from 802816 to 1507328 tid 2 2022-04-28T19:48:13.264 INFO:tasks.rados.rados.0.smithi059.stdout:1208: writing smithi05955964-19 from 1507328 to 1605632 tid 3 2022-04-28T19:48:13.267 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 486 (ObjNum 380 snap 115 seq_num 380) dirty exists 2022-04-28T19:48:13.267 INFO:tasks.rados.rados.0.smithi059.stdout:1204: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:48:13.533 INFO:tasks.rados.rados.0.smithi059.stdout:1206: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:48:13.613 INFO:tasks.rados.rados.0.smithi059.stdout:1208: finishing write tid 1 to smithi05955964-19 2022-04-28T19:48:13.613 INFO:tasks.rados.rados.0.smithi059.stdout:1208: finishing write tid 2 to smithi05955964-19 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1208: finishing write tid 3 to smithi05955964-19 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1208: finishing write tid 5 to smithi05955964-19 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1208: finishing write tid 6 to smithi05955964-19 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 596 (ObjNum 382 snap 117 seq_num 382) dirty exists 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1208: left oid 19 (ObjNum 382 snap 117 seq_num 382) 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1203: done (3 left) 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1204: done (2 left) 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1206: done (1 left) 2022-04-28T19:48:13.614 INFO:tasks.rados.rados.0.smithi059.stdout:1208: done (0 left) 2022-04-28T19:48:13.615 INFO:tasks.rados.rados.0.smithi059.stdout:1209: setattr oid 17 current snap is 117 2022-04-28T19:48:13.615 INFO:tasks.rados.rados.0.smithi059.stdout:1210: rollback oid 9 current snap is 117 2022-04-28T19:48:13.616 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 116 2022-04-28T19:48:13.616 INFO:tasks.rados.rados.0.smithi059.stdout:1211: snap_remove snap 109 2022-04-28T19:48:13.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:12 smithi059 systemd[1]: Started Ceph mgr.y for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:48:13.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[57771]: debug 2022-04-28T19:48:13.714+0000 7fd31ac81700 1 -- 172.21.15.59:0/746114313 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a85d74a4e0 con 0x55a85d83f000 2022-04-28T19:48:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: audit 2022-04-28T19:48:12.510117+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: audit 2022-04-28T19:48:12.511788+0000 mon.b (mon.2) 125 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: audit 2022-04-28T19:48:12.513860+0000 mon.b (mon.2) 126 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: audit 2022-04-28T19:48:12.515311+0000 mon.b (mon.2) 127 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: cephadm 2022-04-28T19:48:12.516604+0000 mgr.x (mgr.34769) 30 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: audit 2022-04-28T19:48:12.518100+0000 mon.b (mon.2) 128 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: audit 2022-04-28T19:48:12.518708+0000 mon.b (mon.2) 129 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[29114]: cluster 2022-04-28T19:48:13.243898+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2022-04-28T19:48:13.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:13.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:48:13.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:13.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:13.845 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:13.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:48:13.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: audit 2022-04-28T19:48:12.510117+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:13.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: audit 2022-04-28T19:48:12.511788+0000 mon.b (mon.2) 125 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:13.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: audit 2022-04-28T19:48:12.513860+0000 mon.b (mon.2) 126 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:13.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: audit 2022-04-28T19:48:12.515311+0000 mon.b (mon.2) 127 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:13.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: cephadm 2022-04-28T19:48:12.516604+0000 mgr.x (mgr.34769) 30 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:13.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: audit 2022-04-28T19:48:12.518100+0000 mon.b (mon.2) 128 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:13.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: audit 2022-04-28T19:48:12.518708+0000 mon.b (mon.2) 129 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:13.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[23786]: cluster 2022-04-28T19:48:13.243898+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2022-04-28T19:48:13.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: audit 2022-04-28T19:48:12.510117+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:13.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: audit 2022-04-28T19:48:12.511788+0000 mon.b (mon.2) 125 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: audit 2022-04-28T19:48:12.513860+0000 mon.b (mon.2) 126 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: audit 2022-04-28T19:48:12.515311+0000 mon.b (mon.2) 127 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: cephadm 2022-04-28T19:48:12.516604+0000 mgr.x (mgr.34769) 30 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: audit 2022-04-28T19:48:12.518100+0000 mon.b (mon.2) 128 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: audit 2022-04-28T19:48:12.518708+0000 mon.b (mon.2) 129 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:13 smithi139 bash[23439]: cluster 2022-04-28T19:48:13.243898+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2022-04-28T19:48:14.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[57771]: debug 2022-04-28T19:48:13.874+0000 7fd32b03b680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-04-28T19:48:14.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:13 smithi059 bash[57771]: debug 2022-04-28T19:48:13.974+0000 7fd32b03b680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-04-28T19:48:14.530 INFO:tasks.rados.rados.0.smithi059.stdout:1211: done (2 left) 2022-04-28T19:48:14.530 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 117 2022-04-28T19:48:14.530 INFO:tasks.rados.rados.0.smithi059.stdout:1212: seq_num 383 ranges {2105344=417792,2523136=589824,3112960=417792,3530752=196608} 2022-04-28T19:48:14.548 INFO:tasks.rados.rados.0.smithi059.stdout:1212: writing smithi05955964-29 from 2105344 to 2523136 tid 1 2022-04-28T19:48:14.556 INFO:tasks.rados.rados.0.smithi059.stdout:1212: writing smithi05955964-29 from 2523136 to 3112960 tid 2 2022-04-28T19:48:14.562 INFO:tasks.rados.rados.0.smithi059.stdout:1212: writing smithi05955964-29 from 3112960 to 3530752 tid 3 2022-04-28T19:48:14.566 INFO:tasks.rados.rados.0.smithi059.stdout:1212: writing smithi05955964-29 from 3530752 to 3727360 tid 4 2022-04-28T19:48:14.566 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 117 2022-04-28T19:48:14.566 INFO:tasks.rados.rados.0.smithi059.stdout:1213: seq_num 384 ranges {6414336=434176,6848512=475136,7323648=630784,7954432=98304} 2022-04-28T19:48:14.604 INFO:tasks.rados.rados.0.smithi059.stdout:1213: writing smithi05955964-4 from 6414336 to 6848512 tid 1 2022-04-28T19:48:14.606 INFO:tasks.rados.rados.0.smithi059.stdout:1213: writing smithi05955964-4 from 6848512 to 7323648 tid 2 2022-04-28T19:48:14.610 INFO:tasks.rados.rados.0.smithi059.stdout:1213: writing smithi05955964-4 from 7323648 to 7954432 tid 3 2022-04-28T19:48:14.613 INFO:tasks.rados.rados.0.smithi059.stdout:1213: writing smithi05955964-4 from 7954432 to 8052736 tid 4 2022-04-28T19:48:14.613 INFO:tasks.rados.rados.0.smithi059.stdout:1214: read oid 39 snap -1 2022-04-28T19:48:14.613 INFO:tasks.rados.rados.0.smithi059.stdout:1214: expect deleted 2022-04-28T19:48:14.613 INFO:tasks.rados.rados.0.smithi059.stdout:1210: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:48:14.613 INFO:tasks.rados.rados.0.smithi059.stdout:1210: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:48:14.613 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 293 (ObjNum 368 snap 111 seq_num 368) dirty exists 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1210: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 565 (ObjNum 317 snap 95 seq_num 317) dirty exists 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1212: finishing write tid 1 to smithi05955964-29 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1212: finishing write tid 2 to smithi05955964-29 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1212: finishing write tid 3 to smithi05955964-29 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1212: finishing write tid 4 to smithi05955964-29 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1212: finishing write tid 6 to smithi05955964-29 2022-04-28T19:48:14.614 INFO:tasks.rados.rados.0.smithi059.stdout:1212: finishing write tid 7 to smithi05955964-29 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 588 (ObjNum 383 snap 117 seq_num 383) dirty exists 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1212: left oid 29 (ObjNum 383 snap 117 seq_num 383) 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1213: finishing write tid 1 to smithi05955964-4 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1213: finishing write tid 2 to smithi05955964-4 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1209: done (4 left) 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1210: done (3 left) 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1212: done (2 left) 2022-04-28T19:48:14.615 INFO:tasks.rados.rados.0.smithi059.stdout:1215: copy_from oid 11 from oid 16 current snap is 117 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1216: read oid 44 snap -1 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1216: expect (ObjNum 356 snap 107 seq_num 356) 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1217: rollback oid 47 current snap is 117 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 110 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1218: rmattr oid 20 current snap is 117 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1219: read oid 10 snap -1 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1219: expect deleted 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1220: rmattr oid 33 current snap is 117 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1220: done (7 left) 2022-04-28T19:48:14.616 INFO:tasks.rados.rados.0.smithi059.stdout:1221: rmattr oid 40 current snap is 117 2022-04-28T19:48:14.617 INFO:tasks.rados.rados.0.smithi059.stdout:1222: copy_from oid 14 from oid 15 current snap is 117 2022-04-28T19:48:14.617 INFO:tasks.rados.rados.0.smithi059.stdout:1223: snap_create 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:1213: finishing write tid 3 to smithi05955964-4 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:1213: finishing write tid 4 to smithi05955964-4 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:1217: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:1213: finishing write tid 6 to smithi05955964-4 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:1217: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 522 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:48:14.620 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 501 (ObjNum 380 snap 115 seq_num 380) dirty exists 2022-04-28T19:48:14.621 INFO:tasks.rados.rados.0.smithi059.stdout:1213: finishing write tid 7 to smithi05955964-4 2022-04-28T19:48:14.621 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 529 (ObjNum 384 snap 117 seq_num 384) dirty exists 2022-04-28T19:48:14.621 INFO:tasks.rados.rados.0.smithi059.stdout:1213: left oid 4 (ObjNum 384 snap 117 seq_num 384) 2022-04-28T19:48:14.621 INFO:tasks.rados.rados.0.smithi059.stdout:1217: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:48:14.621 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 584 (ObjNum 257 snap 74 seq_num 257) dirty exists 2022-04-28T19:48:14.623 INFO:tasks.rados.rados.0.smithi059.stdout:1216: expect (ObjNum 356 snap 107 seq_num 356) 2022-04-28T19:48:14.652 INFO:tasks.rados.rados.0.smithi059.stdout:1222: finishing copy_from to smithi05955964-14 2022-04-28T19:48:14.652 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 538 (ObjNum 352 snap 105 seq_num 352) dirty exists 2022-04-28T19:48:14.652 INFO:tasks.rados.rados.0.smithi059.stdout:1222: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:48:14.652 INFO:tasks.rados.rados.0.smithi059.stdout:1215: finishing copy_from to smithi05955964-11 2022-04-28T19:48:14.652 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 380 (ObjNum 360 snap 108 seq_num 360) dirty exists 2022-04-28T19:48:14.652 INFO:tasks.rados.rados.0.smithi059.stdout:1215: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:48:14.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:14 smithi059 bash[23786]: cluster 2022-04-28T19:48:13.280013+0000 mgr.x (mgr.34769) 31 : cluster [DBG] pgmap v57: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2022-04-28T19:48:14.842 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:14 smithi059 bash[57771]: debug 2022-04-28T19:48:14.426+0000 7fd32b03b680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-04-28T19:48:14.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:14 smithi059 bash[29114]: cluster 2022-04-28T19:48:13.280013+0000 mgr.x (mgr.34769) 31 : cluster [DBG] pgmap v57: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2022-04-28T19:48:14.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:14 smithi139 bash[23439]: cluster 2022-04-28T19:48:13.280013+0000 mgr.x (mgr.34769) 31 : cluster [DBG] pgmap v57: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2022-04-28T19:48:15.532 INFO:tasks.rados.rados.0.smithi059.stdout:1213: done (9 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1214: done (8 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1215: done (7 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1216: done (6 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1217: done (5 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1218: done (4 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1219: done (3 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1221: done (2 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1222: done (1 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:1223: done (0 left) 2022-04-28T19:48:15.533 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 118 2022-04-28T19:48:15.534 INFO:tasks.rados.rados.0.smithi059.stdout:1224: seq_num 385 ranges {0=450560,450560=761856,1212416=442368} 2022-04-28T19:48:15.534 INFO:tasks.rados.rados.0.smithi059.stdout:1224: writing smithi05955964-28 from 0 to 450560 tid 1 2022-04-28T19:48:15.539 INFO:tasks.rados.rados.0.smithi059.stdout:1224: writing smithi05955964-28 from 450560 to 1212416 tid 2 2022-04-28T19:48:15.542 INFO:tasks.rados.rados.0.smithi059.stdout:1224: writing smithi05955964-28 from 1212416 to 1654784 tid 3 2022-04-28T19:48:15.543 INFO:tasks.rados.rados.0.smithi059.stdout:1225: rmattr oid 39 current snap is 118 2022-04-28T19:48:15.543 INFO:tasks.rados.rados.0.smithi059.stdout:1225: done (1 left) 2022-04-28T19:48:15.543 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 118 2022-04-28T19:48:15.543 INFO:tasks.rados.rados.0.smithi059.stdout:1226: seq_num 386 ranges {6021120=466944,6488064=638976,7127040=565248} 2022-04-28T19:48:15.580 INFO:tasks.rados.rados.0.smithi059.stdout:1226: writing smithi05955964-34 from 6021120 to 6488064 tid 1 2022-04-28T19:48:15.583 INFO:tasks.rados.rados.0.smithi059.stdout:1226: writing smithi05955964-34 from 6488064 to 7127040 tid 2 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1226: writing smithi05955964-34 from 7127040 to 7692288 tid 3 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1227: copy_from oid 3 from oid 13 current snap is 118 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1224: finishing write tid 1 to smithi05955964-28 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1224: finishing write tid 2 to smithi05955964-28 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1224: finishing write tid 3 to smithi05955964-28 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1224: finishing write tid 5 to smithi05955964-28 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:1224: finishing write tid 6 to smithi05955964-28 2022-04-28T19:48:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 384 (ObjNum 385 snap 118 seq_num 385) dirty exists 2022-04-28T19:48:15.589 INFO:tasks.rados.rados.0.smithi059.stdout:1224: left oid 28 (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:48:15.589 INFO:tasks.rados.rados.0.smithi059.stdout:1224: done (2 left) 2022-04-28T19:48:15.589 INFO:tasks.rados.rados.0.smithi059.stdout:1228: copy_from oid 5 from oid 45 current snap is 118 2022-04-28T19:48:15.589 INFO:tasks.rados.rados.0.smithi059.stdout:1229: snap_create 2022-04-28T19:48:15.589 INFO:tasks.rados.rados.0.smithi059.stdout:1226: finishing write tid 1 to smithi05955964-34 2022-04-28T19:48:15.594 INFO:tasks.rados.rados.0.smithi059.stdout:1226: finishing write tid 2 to smithi05955964-34 2022-04-28T19:48:15.596 INFO:tasks.rados.rados.0.smithi059.stdout:1226: finishing write tid 3 to smithi05955964-34 2022-04-28T19:48:15.596 INFO:tasks.rados.rados.0.smithi059.stdout:1226: finishing write tid 5 to smithi05955964-34 2022-04-28T19:48:15.599 INFO:tasks.rados.rados.0.smithi059.stdout:1226: finishing write tid 6 to smithi05955964-34 2022-04-28T19:48:15.599 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 587 (ObjNum 386 snap 118 seq_num 386) dirty exists 2022-04-28T19:48:15.599 INFO:tasks.rados.rados.0.smithi059.stdout:1226: left oid 34 (ObjNum 386 snap 118 seq_num 386) 2022-04-28T19:48:15.603 INFO:tasks.rados.rados.0.smithi059.stdout:1227: finishing copy_from to smithi05955964-3 2022-04-28T19:48:15.603 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 243 (ObjNum 161 snap 42 seq_num 161) dirty exists 2022-04-28T19:48:15.603 INFO:tasks.rados.rados.0.smithi059.stdout:1227: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:48:15.639 INFO:tasks.rados.rados.0.smithi059.stdout:1228: finishing copy_from to smithi05955964-5 2022-04-28T19:48:15.639 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 408 (ObjNum 337 snap 103 seq_num 337) dirty exists 2022-04-28T19:48:15.640 INFO:tasks.rados.rados.0.smithi059.stdout:1228: finishing copy_from racing read to smithi05955964-5 2022-04-28T19:48:15.787 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:15 smithi059 bash[57771]: debug 2022-04-28T19:48:15.402+0000 7fd32b03b680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-04-28T19:48:15.787 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:15 smithi059 bash[57771]: debug 2022-04-28T19:48:15.506+0000 7fd32b03b680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-04-28T19:48:15.787 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:15 smithi059 bash[29114]: cluster 2022-04-28T19:48:14.524294+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2022-04-28T19:48:15.788 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:15 smithi059 bash[23786]: cluster 2022-04-28T19:48:14.524294+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2022-04-28T19:48:15.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:15 smithi139 bash[23439]: cluster 2022-04-28T19:48:14.524294+0000 mon.a (mon.0) 950 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2022-04-28T19:48:16.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:15 smithi059 bash[57771]: debug 2022-04-28T19:48:15.782+0000 7fd32b03b680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-04-28T19:48:16.542 INFO:tasks.rados.rados.0.smithi059.stdout:1226: done (3 left) 2022-04-28T19:48:16.542 INFO:tasks.rados.rados.0.smithi059.stdout:1227: done (2 left) 2022-04-28T19:48:16.542 INFO:tasks.rados.rados.0.smithi059.stdout:1228: done (1 left) 2022-04-28T19:48:16.542 INFO:tasks.rados.rados.0.smithi059.stdout:1229: done (0 left) 2022-04-28T19:48:16.542 INFO:tasks.rados.rados.0.smithi059.stdout:1230: snap_create 2022-04-28T19:48:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:16 smithi059 bash[29114]: cluster 2022-04-28T19:48:15.280764+0000 mgr.x (mgr.34769) 32 : cluster [DBG] pgmap v59: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2022-04-28T19:48:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:16 smithi059 bash[29114]: cluster 2022-04-28T19:48:15.527971+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2022-04-28T19:48:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:16 smithi059 bash[23786]: cluster 2022-04-28T19:48:15.280764+0000 mgr.x (mgr.34769) 32 : cluster [DBG] pgmap v59: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2022-04-28T19:48:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:16 smithi059 bash[23786]: cluster 2022-04-28T19:48:15.527971+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2022-04-28T19:48:16.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:16 smithi139 bash[23439]: cluster 2022-04-28T19:48:15.280764+0000 mgr.x (mgr.34769) 32 : cluster [DBG] pgmap v59: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2022-04-28T19:48:16.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:16 smithi139 bash[23439]: cluster 2022-04-28T19:48:15.527971+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2022-04-28T19:48:17.237 INFO:tasks.rados.rados.0.smithi059.stdout:1230: done (0 left) 2022-04-28T19:48:17.237 INFO:tasks.rados.rados.0.smithi059.stdout:1231: snap_remove snap 118 2022-04-28T19:48:17.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[57771]: debug 2022-04-28T19:48:17.014+0000 7fd32b03b680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-04-28T19:48:17.301 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[57771]: debug 2022-04-28T19:48:17.194+0000 7fd32b03b680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:48:17.587 INFO:tasks.rados.rados.0.smithi059.stdout:1231: done (0 left) 2022-04-28T19:48:17.711 INFO:tasks.rados.rados.0.smithi059.stdout:1232: rollback oid 5 current snap is 120 2022-04-28T19:48:17.711 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 111 2022-04-28T19:48:17.711 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 120 2022-04-28T19:48:17.712 INFO:tasks.rados.rados.0.smithi059.stdout:1233: seq_num 387 ranges {3727360=483328,4210688=524288,4734976=679936} 2022-04-28T19:48:17.712 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[57771]: debug 2022-04-28T19:48:17.298+0000 7fd32b03b680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-04-28T19:48:17.712 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[57771]: debug 2022-04-28T19:48:17.414+0000 7fd32b03b680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-04-28T19:48:17.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[23786]: cluster 2022-04-28T19:48:16.536753+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2022-04-28T19:48:17.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[23786]: cluster 2022-04-28T19:48:17.233493+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2022-04-28T19:48:17.713 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[23786]: audit 2022-04-28T19:48:17.247507+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:17.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[29114]: cluster 2022-04-28T19:48:16.536753+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2022-04-28T19:48:17.713 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[29114]: cluster 2022-04-28T19:48:17.233493+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2022-04-28T19:48:17.714 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[29114]: audit 2022-04-28T19:48:17.247507+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:17.715 INFO:tasks.rados.rados.0.smithi059.stdout:1233: writing smithi05955964-29 from 3727360 to 4210688 tid 1 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1233: writing smithi05955964-29 from 4210688 to 4734976 tid 2 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1233: writing smithi05955964-29 from 4734976 to 5414912 tid 3 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 120 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1234: seq_num 388 ranges {0=507904,507904=458752,966656=606208,1572864=139264} 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1234: writing smithi05955964-10 from 0 to 507904 tid 1 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1234: writing smithi05955964-10 from 507904 to 966656 tid 2 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1234: writing smithi05955964-10 from 966656 to 1572864 tid 3 2022-04-28T19:48:17.716 INFO:tasks.rados.rados.0.smithi059.stdout:1234: writing smithi05955964-10 from 1572864 to 1712128 tid 4 2022-04-28T19:48:17.717 INFO:tasks.rados.rados.0.smithi059.stdout:1235: delete oid 26 current snap is 120 2022-04-28T19:48:17.717 INFO:tasks.rados.rados.0.smithi059.stdout:1232: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:48:17.717 INFO:tasks.rados.rados.0.smithi059.stdout:1232: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:48:17.717 INFO:tasks.rados.rados.0.smithi059.stdout:1232: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:48:17.717 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 594 (ObjNum 347 snap 105 seq_num 347) dirty exists 2022-04-28T19:48:17.717 INFO:tasks.rados.rados.0.smithi059.stdout:1233: finishing write tid 1 to smithi05955964-29 2022-04-28T19:48:17.718 INFO:tasks.rados.rados.0.smithi059.stdout:1233: finishing write tid 2 to smithi05955964-29 2022-04-28T19:48:17.718 INFO:tasks.rados.rados.0.smithi059.stdout:1233: finishing write tid 3 to smithi05955964-29 2022-04-28T19:48:17.718 INFO:tasks.rados.rados.0.smithi059.stdout:1233: finishing write tid 5 to smithi05955964-29 2022-04-28T19:48:17.718 INFO:tasks.rados.rados.0.smithi059.stdout:1233: finishing write tid 6 to smithi05955964-29 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 599 (ObjNum 387 snap 120 seq_num 387) dirty exists 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1233: left oid 29 (ObjNum 387 snap 120 seq_num 387) 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1234: finishing write tid 1 to smithi05955964-10 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1234: finishing write tid 2 to smithi05955964-10 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1232: done (3 left) 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1233: done (2 left) 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1235: done (1 left) 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1236: read oid 39 snap -1 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1236: expect deleted 2022-04-28T19:48:17.719 INFO:tasks.rados.rados.0.smithi059.stdout:1237: snap_remove snap 71 2022-04-28T19:48:17.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:17 smithi139 bash[23439]: cluster 2022-04-28T19:48:16.536753+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2022-04-28T19:48:17.955 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:17 smithi139 bash[23439]: cluster 2022-04-28T19:48:17.233493+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2022-04-28T19:48:17.955 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:17 smithi139 bash[23439]: audit 2022-04-28T19:48:17.247507+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.028 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:17 smithi059 bash[57771]: debug 2022-04-28T19:48:17.646+0000 7fd32b03b680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-04-28T19:48:18.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[57771]: debug 2022-04-28T19:48:18.021+0000 7fd32b03b680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-04-28T19:48:18.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[57771]: debug 2022-04-28T19:48:18.145+0000 7fd32b03b680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-04-28T19:48:18.588 INFO:tasks.rados.rados.0.smithi059.stdout:1234: finishing write tid 3 to smithi05955964-10 2022-04-28T19:48:18.627 INFO:tasks.rados.rados.0.smithi059.stdout:1234: finishing write tid 4 to smithi05955964-10 2022-04-28T19:48:18.627 INFO:tasks.rados.rados.0.smithi059.stdout:1234: finishing write tid 6 to smithi05955964-10 2022-04-28T19:48:18.628 INFO:tasks.rados.rados.0.smithi059.stdout:1234: finishing write tid 7 to smithi05955964-10 2022-04-28T19:48:18.628 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 589 (ObjNum 388 snap 120 seq_num 388) dirty exists 2022-04-28T19:48:18.628 INFO:tasks.rados.rados.0.smithi059.stdout:1234: left oid 10 (ObjNum 388 snap 120 seq_num 388) 2022-04-28T19:48:18.628 INFO:tasks.rados.rados.0.smithi059.stdout:1234: done (2 left) 2022-04-28T19:48:18.628 INFO:tasks.rados.rados.0.smithi059.stdout:1236: done (1 left) 2022-04-28T19:48:18.629 INFO:tasks.rados.rados.0.smithi059.stdout:1237: done (0 left) 2022-04-28T19:48:18.629 INFO:tasks.rados.rados.0.smithi059.stdout:1238: snap_remove snap 117 2022-04-28T19:48:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: cluster 2022-04-28T19:48:17.281431+0000 mgr.x (mgr.34769) 33 : cluster [DBG] pgmap v63: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: audit 2022-04-28T19:48:17.551881+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: audit 2022-04-28T19:48:17.561203+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: audit 2022-04-28T19:48:17.567627+0000 mon.b (mon.2) 130 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: audit 2022-04-28T19:48:17.573642+0000 mon.b (mon.2) 131 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-04-28T19:48:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: audit 2022-04-28T19:48:17.574653+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[29114]: cluster 2022-04-28T19:48:17.583921+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: cluster 2022-04-28T19:48:17.281431+0000 mgr.x (mgr.34769) 33 : cluster [DBG] pgmap v63: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: audit 2022-04-28T19:48:17.551881+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: audit 2022-04-28T19:48:17.561203+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: audit 2022-04-28T19:48:17.567627+0000 mon.b (mon.2) 130 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: audit 2022-04-28T19:48:17.573642+0000 mon.b (mon.2) 131 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: audit 2022-04-28T19:48:17.574653+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-04-28T19:48:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:18 smithi059 bash[23786]: cluster 2022-04-28T19:48:17.583921+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: cluster 2022-04-28T19:48:17.281431+0000 mgr.x (mgr.34769) 33 : cluster [DBG] pgmap v63: 89 pgs: 2 active+clean+snaptrim_wait, 6 active+clean+snaptrim, 81 active+clean; 280 MiB data, 399 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: audit 2022-04-28T19:48:17.551881+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: audit 2022-04-28T19:48:17.561203+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.34769 ' entity='mgr.x' 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: audit 2022-04-28T19:48:17.567627+0000 mon.b (mon.2) 130 : audit [DBG] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: audit 2022-04-28T19:48:17.573642+0000 mon.b (mon.2) 131 : audit [INF] from='mgr.34769 172.21.15.139:0/3169458192' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: audit 2022-04-28T19:48:17.574653+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2022-04-28T19:48:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[23439]: cluster 2022-04-28T19:48:17.583921+0000 mon.a (mon.0) 958 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2022-04-28T19:48:18.869 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[37584]: debug ignoring --setuser ceph since I am not root 2022-04-28T19:48:18.869 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[37584]: ignoring --setgroup ceph since I am not root 2022-04-28T19:48:18.869 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[37584]: debug 2022-04-28T19:48:18.616+0000 7f40585dd700 1 -- 172.21.15.139:0/2059934225 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5589024744e0 con 0x558902568800 2022-04-28T19:48:18.869 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[37584]: debug 2022-04-28T19:48:18.772+0000 7f4068784680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-04-28T19:48:19.313 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:18 smithi139 bash[37584]: debug 2022-04-28T19:48:18.880+0000 7f4068784680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-04-28T19:48:19.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[57771]: debug 2022-04-28T19:48:19.037+0000 7fd32b03b680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-04-28T19:48:19.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[57771]: debug 2022-04-28T19:48:19.141+0000 7fd32b03b680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-04-28T19:48:19.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[57771]: debug 2022-04-28T19:48:19.261+0000 7fd32b03b680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-04-28T19:48:19.596 INFO:tasks.rados.rados.0.smithi059.stdout:1238: done (0 left) 2022-04-28T19:48:19.597 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 120 2022-04-28T19:48:19.597 INFO:tasks.rados.rados.0.smithi059.stdout:1239: seq_num 390 ranges {8134656=540672,8675328=630784,9306112=573440} 2022-04-28T19:48:19.617 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:19 smithi139 bash[23439]: cephadm 2022-04-28T19:48:17.570998+0000 mgr.x (mgr.34769) 34 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-04-28T19:48:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:19 smithi139 bash[23439]: cephadm 2022-04-28T19:48:17.573403+0000 mgr.x (mgr.34769) 35 : cephadm [INF] Failing over to other MGR 2022-04-28T19:48:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:19 smithi139 bash[23439]: audit 2022-04-28T19:48:18.566865+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2022-04-28T19:48:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:19 smithi139 bash[23439]: cluster 2022-04-28T19:48:18.567053+0000 mon.a (mon.0) 960 : cluster [DBG] mgrmap e20: y(active, starting, since 0.991129s) 2022-04-28T19:48:19.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:19 smithi139 bash[23439]: cluster 2022-04-28T19:48:18.585097+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2022-04-28T19:48:19.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:19 smithi139 bash[37584]: debug 2022-04-28T19:48:19.308+0000 7f4068784680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-04-28T19:48:19.645 INFO:tasks.rados.rados.0.smithi059.stdout:1239: writing smithi05955964-23 from 8134656 to 8675328 tid 1 2022-04-28T19:48:19.649 INFO:tasks.rados.rados.0.smithi059.stdout:1239: writing smithi05955964-23 from 8675328 to 9306112 tid 2 2022-04-28T19:48:19.653 INFO:tasks.rados.rados.0.smithi059.stdout:1239: writing smithi05955964-23 from 9306112 to 9879552 tid 3 2022-04-28T19:48:19.653 INFO:tasks.rados.rados.0.smithi059.stdout:1240: copy_from oid 40 from oid 35 current snap is 120 2022-04-28T19:48:19.653 INFO:tasks.rados.rados.0.smithi059.stdout:1239: finishing write tid 1 to smithi05955964-23 2022-04-28T19:48:19.653 INFO:tasks.rados.rados.0.smithi059.stdout:1241: snap_create 2022-04-28T19:48:19.654 INFO:tasks.rados.rados.0.smithi059.stdout:1240: finishing copy_from to smithi05955964-40 2022-04-28T19:48:19.654 INFO:tasks.rados.rados.0.smithi059.stdout:1240: got expected ENOENT (src dne) 2022-04-28T19:48:19.654 INFO:tasks.rados.rados.0.smithi059.stdout:1240: finishing copy_from racing read to smithi05955964-40 2022-04-28T19:48:19.656 INFO:tasks.rados.rados.0.smithi059.stdout:1239: finishing write tid 2 to smithi05955964-23 2022-04-28T19:48:19.656 INFO:tasks.rados.rados.0.smithi059.stdout:1239: finishing write tid 3 to smithi05955964-23 2022-04-28T19:48:19.657 INFO:tasks.rados.rados.0.smithi059.stdout:1239: finishing write tid 5 to smithi05955964-23 2022-04-28T19:48:19.658 INFO:tasks.rados.rados.0.smithi059.stdout:1239: finishing write tid 6 to smithi05955964-23 2022-04-28T19:48:19.658 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 602 (ObjNum 390 snap 120 seq_num 390) dirty exists 2022-04-28T19:48:19.658 INFO:tasks.rados.rados.0.smithi059.stdout:1239: left oid 23 (ObjNum 390 snap 120 seq_num 390) 2022-04-28T19:48:19.792 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[57771]: debug 2022-04-28T19:48:19.489+0000 7fd32b03b680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[29114]: cephadm 2022-04-28T19:48:17.570998+0000 mgr.x (mgr.34769) 34 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[29114]: cephadm 2022-04-28T19:48:17.573403+0000 mgr.x (mgr.34769) 35 : cephadm [INF] Failing over to other MGR 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[29114]: audit 2022-04-28T19:48:18.566865+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[29114]: cluster 2022-04-28T19:48:18.567053+0000 mon.a (mon.0) 960 : cluster [DBG] mgrmap e20: y(active, starting, since 0.991129s) 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[29114]: cluster 2022-04-28T19:48:18.585097+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[23786]: cephadm 2022-04-28T19:48:17.570998+0000 mgr.x (mgr.34769) 34 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2022-04-28T19:48:19.793 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[23786]: cephadm 2022-04-28T19:48:17.573403+0000 mgr.x (mgr.34769) 35 : cephadm [INF] Failing over to other MGR 2022-04-28T19:48:19.794 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[23786]: audit 2022-04-28T19:48:18.566865+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.34769 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2022-04-28T19:48:19.794 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[23786]: cluster 2022-04-28T19:48:18.567053+0000 mon.a (mon.0) 960 : cluster [DBG] mgrmap e20: y(active, starting, since 0.991129s) 2022-04-28T19:48:19.794 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[23786]: cluster 2022-04-28T19:48:18.585097+0000 mon.a (mon.0) 961 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2022-04-28T19:48:20.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[57771]: debug 2022-04-28T19:48:19.789+0000 7fd32b03b680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-04-28T19:48:20.092 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:19 smithi059 bash[57771]: debug 2022-04-28T19:48:19.957+0000 7fd32b03b680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-04-28T19:48:20.458 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[57771]: debug 2022-04-28T19:48:20.133+0000 7fd32b03b680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:48:20.459 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[57771]: debug 2022-04-28T19:48:20.345+0000 7fd32b03b680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-04-28T19:48:20.487 INFO:tasks.rados.rados.0.smithi059.stdout:1239: done (2 left) 2022-04-28T19:48:20.488 INFO:tasks.rados.rados.0.smithi059.stdout:1240: done (1 left) 2022-04-28T19:48:20.488 INFO:tasks.rados.rados.0.smithi059.stdout:1241: done (0 left) 2022-04-28T19:48:20.488 INFO:tasks.rados.rados.0.smithi059.stdout:1242: delete oid 3 current snap is 121 2022-04-28T19:48:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:1242: done (0 left) 2022-04-28T19:48:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:1243: read oid 6 snap -1 2022-04-28T19:48:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:1243: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:48:20.491 INFO:tasks.rados.rados.0.smithi059.stdout:1244: delete oid 30 current snap is 121 2022-04-28T19:48:20.493 INFO:tasks.rados.rados.0.smithi059.stdout:1244: done (1 left) 2022-04-28T19:48:20.493 INFO:tasks.rados.rados.0.smithi059.stdout:1245: snap_remove snap 119 2022-04-28T19:48:20.547 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[37584]: debug 2022-04-28T19:48:20.204+0000 7f4068784680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-04-28T19:48:20.547 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[37584]: debug 2022-04-28T19:48:20.296+0000 7f4068784680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-04-28T19:48:20.547 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[37584]: debug 2022-04-28T19:48:20.544+0000 7f4068784680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-04-28T19:48:20.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[57771]: debug 2022-04-28T19:48:20.453+0000 7fd32b03b680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-04-28T19:48:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: cluster 2022-04-28T19:48:19.591635+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2022-04-28T19:48:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: cluster 2022-04-28T19:48:20.477531+0000 mon.a (mon.0) 963 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:48:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: cluster 2022-04-28T19:48:20.478999+0000 mon.a (mon.0) 964 : cluster [INF] Activating manager daemon y 2022-04-28T19:48:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: cluster 2022-04-28T19:48:20.485633+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2022-04-28T19:48:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: audit 2022-04-28T19:48:20.486433+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: audit 2022-04-28T19:48:20.487257+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: audit 2022-04-28T19:48:20.488269+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[23786]: audit 2022-04-28T19:48:20.488634+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: cluster 2022-04-28T19:48:19.591635+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: cluster 2022-04-28T19:48:20.477531+0000 mon.a (mon.0) 963 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: cluster 2022-04-28T19:48:20.478999+0000 mon.a (mon.0) 964 : cluster [INF] Activating manager daemon y 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: cluster 2022-04-28T19:48:20.485633+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2022-04-28T19:48:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: audit 2022-04-28T19:48:20.486433+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-04-28T19:48:20.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: audit 2022-04-28T19:48:20.487257+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:20.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: audit 2022-04-28T19:48:20.488269+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-04-28T19:48:20.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:20 smithi059 bash[29114]: audit 2022-04-28T19:48:20.488634+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: cluster 2022-04-28T19:48:19.591635+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: cluster 2022-04-28T19:48:20.477531+0000 mon.a (mon.0) 963 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: cluster 2022-04-28T19:48:20.478999+0000 mon.a (mon.0) 964 : cluster [INF] Activating manager daemon y 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: cluster 2022-04-28T19:48:20.485633+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: audit 2022-04-28T19:48:20.486433+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: audit 2022-04-28T19:48:20.487257+0000 mon.a (mon.0) 967 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:20.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: audit 2022-04-28T19:48:20.488269+0000 mon.a (mon.0) 968 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2022-04-28T19:48:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:20 smithi139 bash[23439]: audit 2022-04-28T19:48:20.488634+0000 mon.a (mon.0) 969 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:21.491 INFO:tasks.rados.rados.0.smithi059.stdout:1243: expect (ObjNum 134 snap 32 seq_num 134) 2022-04-28T19:48:21.570 INFO:tasks.rados.rados.0.smithi059.stdout:1243: done (1 left) 2022-04-28T19:48:21.571 INFO:tasks.rados.rados.0.smithi059.stdout:1245: done (0 left) 2022-04-28T19:48:21.571 INFO:tasks.rados.rados.0.smithi059.stdout:1246: read oid 40 snap -1 2022-04-28T19:48:21.571 INFO:tasks.rados.rados.0.smithi059.stdout:1246: expect (ObjNum 380 snap 115 seq_num 380) 2022-04-28T19:48:21.571 INFO:tasks.rados.rados.0.smithi059.stdout:1247: delete oid 2 current snap is 121 2022-04-28T19:48:21.573 INFO:tasks.rados.rados.0.smithi059.stdout:1247: done (1 left) 2022-04-28T19:48:21.573 INFO:tasks.rados.rados.0.smithi059.stdout:1248: delete oid 29 current snap is 121 2022-04-28T19:48:21.580 INFO:tasks.rados.rados.0.smithi059.stdout:1248: done (1 left) 2022-04-28T19:48:21.580 INFO:tasks.rados.rados.0.smithi059.stdout:1249: read oid 18 snap -1 2022-04-28T19:48:21.581 INFO:tasks.rados.rados.0.smithi059.stdout:1249: expect (ObjNum 365 snap 110 seq_num 365) 2022-04-28T19:48:21.581 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 121 2022-04-28T19:48:21.581 INFO:tasks.rados.rados.0.smithi059.stdout:1250: seq_num 395 ranges {1105920=622592,1728512=450560,2179072=753664} 2022-04-28T19:48:21.592 INFO:tasks.rados.rados.0.smithi059.stdout:1250: writing smithi05955964-15 from 1105920 to 1728512 tid 1 2022-04-28T19:48:21.595 INFO:tasks.rados.rados.0.smithi059.stdout:1250: writing smithi05955964-15 from 1728512 to 2179072 tid 2 2022-04-28T19:48:21.600 INFO:tasks.rados.rados.0.smithi059.stdout:1250: writing smithi05955964-15 from 2179072 to 2932736 tid 3 2022-04-28T19:48:21.601 INFO:tasks.rados.rados.0.smithi059.stdout:1251: read oid 49 snap -1 2022-04-28T19:48:21.601 INFO:tasks.rados.rados.0.smithi059.stdout:1251: expect (ObjNum 378 snap 115 seq_num 378) 2022-04-28T19:48:21.601 INFO:tasks.rados.rados.0.smithi059.stdout:1252: read oid 31 snap -1 2022-04-28T19:48:21.601 INFO:tasks.rados.rados.0.smithi059.stdout:1252: expect (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:48:21.601 INFO:tasks.rados.rados.0.smithi059.stdout:1253: snap_remove snap 120 2022-04-28T19:48:21.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:21 smithi139 bash[23439]: cluster 2022-04-28T19:48:20.598209+0000 mon.a (mon.0) 970 : cluster [DBG] mgrmap e21: y(active, starting, since 0.11942s) 2022-04-28T19:48:21.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:21 smithi139 bash[23439]: cluster 2022-04-28T19:48:21.489027+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2022-04-28T19:48:21.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:21 smithi139 bash[37584]: debug 2022-04-28T19:48:21.660+0000 7f4068784680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-04-28T19:48:21.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:21 smithi139 bash[37584]: debug 2022-04-28T19:48:21.820+0000 7f4068784680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:48:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:21 smithi059 bash[29114]: cluster 2022-04-28T19:48:20.598209+0000 mon.a (mon.0) 970 : cluster [DBG] mgrmap e21: y(active, starting, since 0.11942s) 2022-04-28T19:48:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:21 smithi059 bash[29114]: cluster 2022-04-28T19:48:21.489027+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2022-04-28T19:48:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:21 smithi059 bash[23786]: cluster 2022-04-28T19:48:20.598209+0000 mon.a (mon.0) 970 : cluster [DBG] mgrmap e21: y(active, starting, since 0.11942s) 2022-04-28T19:48:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:21 smithi059 bash[23786]: cluster 2022-04-28T19:48:21.489027+0000 mon.a (mon.0) 971 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2022-04-28T19:48:22.247 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:21 smithi139 bash[37584]: debug 2022-04-28T19:48:21.912+0000 7f4068784680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-04-28T19:48:22.274 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:22 smithi139 bash[37584]: debug 2022-04-28T19:48:22.024+0000 7f4068784680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-04-28T19:48:22.608 INFO:tasks.rados.rados.0.smithi059.stdout:1253: done (5 left) 2022-04-28T19:48:22.608 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 121 2022-04-28T19:48:22.608 INFO:tasks.rados.rados.0.smithi059.stdout:1254: seq_num 396 ranges {1171456=638976,1810432=786432,2596864=417792} 2022-04-28T19:48:22.613 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:22 smithi139 bash[37584]: debug 2022-04-28T19:48:22.244+0000 7f4068784680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-04-28T19:48:22.613 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:22 smithi139 bash[37584]: debug 2022-04-28T19:48:22.608+0000 7f4068784680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-04-28T19:48:22.618 INFO:tasks.rados.rados.0.smithi059.stdout:1254: writing smithi05955964-44 from 1171456 to 1810432 tid 1 2022-04-28T19:48:22.623 INFO:tasks.rados.rados.0.smithi059.stdout:1254: writing smithi05955964-44 from 1810432 to 2596864 tid 2 2022-04-28T19:48:22.626 INFO:tasks.rados.rados.0.smithi059.stdout:1254: writing smithi05955964-44 from 2596864 to 3014656 tid 3 2022-04-28T19:48:22.627 INFO:tasks.rados.rados.0.smithi059.stdout:1255: snap_create 2022-04-28T19:48:22.627 INFO:tasks.rados.rados.0.smithi059.stdout:1246: expect (ObjNum 380 snap 115 seq_num 380) 2022-04-28T19:48:22.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:22 smithi139 bash[37584]: debug 2022-04-28T19:48:22.728+0000 7f4068784680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-04-28T19:48:22.902 INFO:tasks.rados.rados.0.smithi059.stdout:1249: expect (ObjNum 365 snap 110 seq_num 365) 2022-04-28T19:48:22.931 INFO:tasks.rados.rados.0.smithi059.stdout:1250: finishing write tid 1 to smithi05955964-15 2022-04-28T19:48:22.931 INFO:tasks.rados.rados.0.smithi059.stdout:1250: finishing write tid 2 to smithi05955964-15 2022-04-28T19:48:22.932 INFO:tasks.rados.rados.0.smithi059.stdout:1250: finishing write tid 3 to smithi05955964-15 2022-04-28T19:48:22.932 INFO:tasks.rados.rados.0.smithi059.stdout:1250: finishing write tid 5 to smithi05955964-15 2022-04-28T19:48:22.932 INFO:tasks.rados.rados.0.smithi059.stdout:1250: finishing write tid 6 to smithi05955964-15 2022-04-28T19:48:22.932 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 590 (ObjNum 395 snap 121 seq_num 395) dirty exists 2022-04-28T19:48:22.933 INFO:tasks.rados.rados.0.smithi059.stdout:1250: left oid 15 (ObjNum 395 snap 121 seq_num 395) 2022-04-28T19:48:22.933 INFO:tasks.rados.rados.0.smithi059.stdout:1251: expect (ObjNum 378 snap 115 seq_num 378) 2022-04-28T19:48:23.017 INFO:tasks.rados.rados.0.smithi059.stdout:1252: expect (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:48:23.046 INFO:tasks.rados.rados.0.smithi059.stdout:1254: finishing write tid 1 to smithi05955964-44 2022-04-28T19:48:23.046 INFO:tasks.rados.rados.0.smithi059.stdout:1254: finishing write tid 2 to smithi05955964-44 2022-04-28T19:48:23.046 INFO:tasks.rados.rados.0.smithi059.stdout:1254: finishing write tid 3 to smithi05955964-44 2022-04-28T19:48:23.046 INFO:tasks.rados.rados.0.smithi059.stdout:1254: finishing write tid 5 to smithi05955964-44 2022-04-28T19:48:23.046 INFO:tasks.rados.rados.0.smithi059.stdout:1254: finishing write tid 6 to smithi05955964-44 2022-04-28T19:48:23.047 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 569 (ObjNum 396 snap 121 seq_num 396) dirty exists 2022-04-28T19:48:23.047 INFO:tasks.rados.rados.0.smithi059.stdout:1254: left oid 44 (ObjNum 396 snap 121 seq_num 396) 2022-04-28T19:48:23.611 INFO:tasks.rados.rados.0.smithi059.stdout:1246: done (6 left) 2022-04-28T19:48:23.611 INFO:tasks.rados.rados.0.smithi059.stdout:1249: done (5 left) 2022-04-28T19:48:23.611 INFO:tasks.rados.rados.0.smithi059.stdout:1250: done (4 left) 2022-04-28T19:48:23.612 INFO:tasks.rados.rados.0.smithi059.stdout:1251: done (3 left) 2022-04-28T19:48:23.612 INFO:tasks.rados.rados.0.smithi059.stdout:1252: done (2 left) 2022-04-28T19:48:23.612 INFO:tasks.rados.rados.0.smithi059.stdout:1254: done (1 left) 2022-04-28T19:48:23.612 INFO:tasks.rados.rados.0.smithi059.stdout:1255: done (0 left) 2022-04-28T19:48:23.612 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 122 2022-04-28T19:48:23.612 INFO:tasks.rados.rados.0.smithi059.stdout:1256: seq_num 397 ranges {7692288=655360,8347648=671744,9019392=532480} 2022-04-28T19:48:23.658 INFO:tasks.rados.rados.0.smithi059.stdout:1256: writing smithi05955964-34 from 7692288 to 8347648 tid 1 2022-04-28T19:48:23.661 INFO:tasks.rados.rados.0.smithi059.stdout:1256: writing smithi05955964-34 from 8347648 to 9019392 tid 2 2022-04-28T19:48:23.665 INFO:tasks.rados.rados.0.smithi059.stdout:1256: writing smithi05955964-34 from 9019392 to 9551872 tid 3 2022-04-28T19:48:23.665 INFO:tasks.rados.rados.0.smithi059.stdout:1257: read oid 37 snap -1 2022-04-28T19:48:23.665 INFO:tasks.rados.rados.0.smithi059.stdout:1257: expect (ObjNum 359 snap 108 seq_num 359) 2022-04-28T19:48:23.665 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 122 2022-04-28T19:48:23.666 INFO:tasks.rados.rados.0.smithi059.stdout:1258: seq_num 398 ranges {524288=671744,1196032=557056,1753088=532480,2285568=114688} 2022-04-28T19:48:23.672 INFO:tasks.rados.rados.0.smithi059.stdout:1258: writing smithi05955964-13 from 524288 to 1196032 tid 1 2022-04-28T19:48:23.676 INFO:tasks.rados.rados.0.smithi059.stdout:1258: writing smithi05955964-13 from 1196032 to 1753088 tid 2 2022-04-28T19:48:23.679 INFO:tasks.rados.rados.0.smithi059.stdout:1258: writing smithi05955964-13 from 1753088 to 2285568 tid 3 2022-04-28T19:48:23.681 INFO:tasks.rados.rados.0.smithi059.stdout:1258: writing smithi05955964-13 from 2285568 to 2400256 tid 4 2022-04-28T19:48:23.681 INFO:tasks.rados.rados.0.smithi059.stdout:1259: copy_from oid 44 from oid 28 current snap is 122 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: finishing write tid 1 to smithi05955964-34 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: finishing write tid 2 to smithi05955964-34 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: finishing write tid 3 to smithi05955964-34 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: finishing write tid 5 to smithi05955964-34 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: finishing write tid 6 to smithi05955964-34 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 603 (ObjNum 397 snap 122 seq_num 397) dirty exists 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: left oid 34 (ObjNum 397 snap 122 seq_num 397) 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1258: finishing write tid 1 to smithi05955964-13 2022-04-28T19:48:23.682 INFO:tasks.rados.rados.0.smithi059.stdout:1256: done (3 left) 2022-04-28T19:48:23.683 INFO:tasks.rados.rados.0.smithi059.stdout:1260: snap_create 2022-04-28T19:48:23.683 INFO:tasks.rados.rados.0.smithi059.stdout:1258: finishing write tid 2 to smithi05955964-13 2022-04-28T19:48:23.685 INFO:tasks.rados.rados.0.smithi059.stdout:1258: finishing write tid 3 to smithi05955964-13 2022-04-28T19:48:23.685 INFO:tasks.rados.rados.0.smithi059.stdout:1258: finishing write tid 4 to smithi05955964-13 2022-04-28T19:48:23.685 INFO:tasks.rados.rados.0.smithi059.stdout:1258: finishing write tid 6 to smithi05955964-13 2022-04-28T19:48:23.687 INFO:tasks.rados.rados.0.smithi059.stdout:1258: finishing write tid 7 to smithi05955964-13 2022-04-28T19:48:23.687 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 303 (ObjNum 398 snap 122 seq_num 398) dirty exists 2022-04-28T19:48:23.688 INFO:tasks.rados.rados.0.smithi059.stdout:1258: left oid 13 (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:48:23.704 INFO:tasks.rados.rados.0.smithi059.stdout:1257: expect (ObjNum 359 snap 108 seq_num 359) 2022-04-28T19:48:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[29114]: cluster 2022-04-28T19:48:22.606493+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2022-04-28T19:48:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:23.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:23.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:48:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:23.516Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" 2022-04-28T19:48:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:23.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:48:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:23.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:23.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:48:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:23 smithi059 bash[23786]: cluster 2022-04-28T19:48:22.606493+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2022-04-28T19:48:23.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:23 smithi139 bash[23439]: cluster 2022-04-28T19:48:22.606493+0000 mon.a (mon.0) 972 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2022-04-28T19:48:23.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:23 smithi139 bash[37584]: debug 2022-04-28T19:48:23.580+0000 7f4068784680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-04-28T19:48:23.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:23 smithi139 bash[37584]: debug 2022-04-28T19:48:23.696+0000 7f4068784680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-04-28T19:48:23.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:23 smithi139 bash[37584]: debug 2022-04-28T19:48:23.812+0000 7f4068784680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-04-28T19:48:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:1259: finishing copy_from to smithi05955964-44 2022-04-28T19:48:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 384 (ObjNum 385 snap 118 seq_num 385) dirty exists 2022-04-28T19:48:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:1259: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:48:24.356 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:24 smithi139 bash[37584]: debug 2022-04-28T19:48:24.044+0000 7f4068784680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-04-28T19:48:24.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:24 smithi139 bash[37584]: debug 2022-04-28T19:48:24.352+0000 7f4068784680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-04-28T19:48:24.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:24 smithi139 bash[37584]: debug 2022-04-28T19:48:24.524+0000 7f4068784680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-04-28T19:48:24.621 INFO:tasks.rados.rados.0.smithi059.stdout:1257: done (3 left) 2022-04-28T19:48:24.621 INFO:tasks.rados.rados.0.smithi059.stdout:1258: done (2 left) 2022-04-28T19:48:24.621 INFO:tasks.rados.rados.0.smithi059.stdout:1259: done (1 left) 2022-04-28T19:48:24.621 INFO:tasks.rados.rados.0.smithi059.stdout:1260: done (0 left) 2022-04-28T19:48:24.621 INFO:tasks.rados.rados.0.smithi059.stdout:1261: delete oid 25 current snap is 123 2022-04-28T19:48:24.626 INFO:tasks.rados.rados.0.smithi059.stdout:1261: done (0 left) 2022-04-28T19:48:24.626 INFO:tasks.rados.rados.0.smithi059.stdout:1262: delete oid 12 current snap is 123 2022-04-28T19:48:24.630 INFO:tasks.rados.rados.0.smithi059.stdout:1262: done (0 left) 2022-04-28T19:48:24.630 INFO:tasks.rados.rados.0.smithi059.stdout:1263: rollback oid 34 current snap is 123 2022-04-28T19:48:24.630 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 34 to 112 2022-04-28T19:48:24.630 INFO:tasks.rados.rados.0.smithi059.stdout:1264: snap_remove snap 115 2022-04-28T19:48:24.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:24 smithi139 bash[23439]: cluster 2022-04-28T19:48:23.608563+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2022-04-28T19:48:24.919 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:24 smithi139 bash[37584]: debug 2022-04-28T19:48:24.708+0000 7f4068784680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:48:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:24 smithi059 bash[29114]: cluster 2022-04-28T19:48:23.608563+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2022-04-28T19:48:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:24 smithi059 bash[23786]: cluster 2022-04-28T19:48:23.608563+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2022-04-28T19:48:25.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:24 smithi139 bash[37584]: debug 2022-04-28T19:48:24.916+0000 7f4068784680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-04-28T19:48:25.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[37584]: debug 2022-04-28T19:48:25.020+0000 7f4068784680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-04-28T19:48:25.650 INFO:tasks.rados.rados.0.smithi059.stdout:1264: done (1 left) 2022-04-28T19:48:25.650 INFO:tasks.rados.rados.0.smithi059.stdout:1265: copy_from oid 15 from oid 41 current snap is 123 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:1263: finishing rollback tid 0 to smithi05955964-34 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:1263: finishing rollback tid 1 to smithi05955964-34 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:1263: finishing rollback tid 2 to smithi05955964-34 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 607 (ObjNum 362 snap 108 seq_num 362) dirty exists 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:1263: done (1 left) 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:1266: rollback oid 24 current snap is 123 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 122 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:1267: rollback oid 49 current snap is 123 2022-04-28T19:48:25.651 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 110 2022-04-28T19:48:25.652 INFO:tasks.rados.rados.0.smithi059.stdout:1268: snap_remove snap 98 2022-04-28T19:48:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[29114]: cluster 2022-04-28T19:48:24.619723+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2022-04-28T19:48:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[29114]: audit 2022-04-28T19:48:25.044852+0000 mon.b (mon.2) 132 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:48:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[29114]: cluster 2022-04-28T19:48:25.045113+0000 mon.a (mon.0) 975 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:48:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[29114]: audit 2022-04-28T19:48:25.045620+0000 mon.b (mon.2) 133 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[29114]: audit 2022-04-28T19:48:25.047464+0000 mon.b (mon.2) 134 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:48:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[29114]: audit 2022-04-28T19:48:25.048121+0000 mon.b (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[23786]: cluster 2022-04-28T19:48:24.619723+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2022-04-28T19:48:26.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[23786]: audit 2022-04-28T19:48:25.044852+0000 mon.b (mon.2) 132 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:48:26.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[23786]: cluster 2022-04-28T19:48:25.045113+0000 mon.a (mon.0) 975 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:48:26.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[23786]: audit 2022-04-28T19:48:25.045620+0000 mon.b (mon.2) 133 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:26.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[23786]: audit 2022-04-28T19:48:25.047464+0000 mon.b (mon.2) 134 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:48:26.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:25 smithi059 bash[23786]: audit 2022-04-28T19:48:25.048121+0000 mon.b (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[23439]: cluster 2022-04-28T19:48:24.619723+0000 mon.a (mon.0) 974 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2022-04-28T19:48:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[23439]: audit 2022-04-28T19:48:25.044852+0000 mon.b (mon.2) 132 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:48:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[23439]: cluster 2022-04-28T19:48:25.045113+0000 mon.a (mon.0) 975 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:48:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[23439]: audit 2022-04-28T19:48:25.045620+0000 mon.b (mon.2) 133 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[23439]: audit 2022-04-28T19:48:25.047464+0000 mon.b (mon.2) 134 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:48:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:25 smithi139 bash[23439]: audit 2022-04-28T19:48:25.048121+0000 mon.b (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.139:0/1815238910' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:26.647 INFO:tasks.rados.rados.0.smithi059.stdout:1268: done (3 left) 2022-04-28T19:48:26.648 INFO:tasks.rados.rados.0.smithi059.stdout:1269: copy_from oid 2 from oid 11 current snap is 123 2022-04-28T19:48:26.648 INFO:tasks.rados.rados.0.smithi059.stdout:1270: snap_create 2022-04-28T19:48:26.648 INFO:tasks.rados.rados.0.smithi059.stdout:1266: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:48:26.648 INFO:tasks.rados.rados.0.smithi059.stdout:1266: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:48:26.649 INFO:tasks.rados.rados.0.smithi059.stdout:1266: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:48:26.649 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 547 (ObjNum 346 snap 105 seq_num 346) dirty exists 2022-04-28T19:48:26.649 INFO:tasks.rados.rados.0.smithi059.stdout:1267: finishing rollback tid 0 to smithi05955964-49 2022-04-28T19:48:26.649 INFO:tasks.rados.rados.0.smithi059.stdout:1267: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:48:26.649 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 506 (ObjNum -133126496 snap 21893 seq_num 4161617664) dirty dne 2022-04-28T19:48:26.650 INFO:tasks.rados.rados.0.smithi059.stdout:1265: finishing copy_from to smithi05955964-15 2022-04-28T19:48:26.650 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 252 (ObjNum 187 snap 53 seq_num 187) dirty exists 2022-04-28T19:48:26.650 INFO:tasks.rados.rados.0.smithi059.stdout:1265: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:48:26.688 INFO:tasks.rados.rados.0.smithi059.stdout:1269: finishing copy_from to smithi05955964-2 2022-04-28T19:48:26.688 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 380 (ObjNum 360 snap 108 seq_num 360) dirty exists 2022-04-28T19:48:26.689 INFO:tasks.rados.rados.0.smithi059.stdout:1269: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:48:26.907 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: cluster 2022-04-28T19:48:25.644693+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2022-04-28T19:48:26.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: cluster 2022-04-28T19:48:25.646886+0000 mon.a (mon.0) 977 : cluster [DBG] mgrmap e22: y(active, starting, since 5s), standbys: x 2022-04-28T19:48:26.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:25.650728+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:48:26.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:25.651084+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:48:26.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.602978+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.603510+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.604224+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.605841+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.606826+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.608449+0000 mon.c (mon.1) 48 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.608942+0000 mon.c (mon.1) 49 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.609369+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.609910+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:48:26.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.610379+0000 mon.c (mon.1) 52 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.610874+0000 mon.c (mon.1) 53 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.611352+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.611947+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.612510+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.612985+0000 mon.c (mon.1) 57 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.613748+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.615599+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.617045+0000 mon.c (mon.1) 60 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/client_keyrings"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.617534+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/config_checks"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.618000+0000 mon.c (mon.1) 62 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:48:26.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.618599+0000 mon.c (mon.1) 63 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.619186+0000 mon.c (mon.1) 64 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.619777+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.620340+0000 mon.c (mon.1) 66 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.620795+0000 mon.c (mon.1) 67 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.621250+0000 mon.c (mon.1) 68 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.621759+0000 mon.c (mon.1) 69 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.622265+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.622740+0000 mon.c (mon.1) 71 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.623235+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:48:26.918 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.623724+0000 mon.c (mon.1) 73 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.624226+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.624767+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.625295+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.625811+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.626294+0000 mon.c (mon.1) 78 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.626790+0000 mon.c (mon.1) 79 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.627415+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.628030+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:48:26.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.628747+0000 mon.c (mon.1) 82 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:48:26.920 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.629295+0000 mon.c (mon.1) 83 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:48:26.920 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:26 smithi139 bash[23439]: audit 2022-04-28T19:48:26.629812+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:48:26.920 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: cluster 2022-04-28T19:48:25.644693+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: cluster 2022-04-28T19:48:25.646886+0000 mon.a (mon.0) 977 : cluster [DBG] mgrmap e22: y(active, starting, since 5s), standbys: x 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:25.650728+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:25.651084+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.602978+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.603510+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.604224+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.605841+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.606826+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.608449+0000 mon.c (mon.1) 48 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:48:26.921 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.608942+0000 mon.c (mon.1) 49 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.609369+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.609910+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.610379+0000 mon.c (mon.1) 52 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.610874+0000 mon.c (mon.1) 53 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.611352+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.611947+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.612510+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.612985+0000 mon.c (mon.1) 57 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:48:26.922 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.613748+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.615599+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.617045+0000 mon.c (mon.1) 60 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/client_keyrings"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.617534+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/config_checks"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.618000+0000 mon.c (mon.1) 62 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.618599+0000 mon.c (mon.1) 63 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.619186+0000 mon.c (mon.1) 64 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.619777+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.620340+0000 mon.c (mon.1) 66 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:48:26.923 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.620795+0000 mon.c (mon.1) 67 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:48:26.924 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.621250+0000 mon.c (mon.1) 68 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:48:26.924 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.621759+0000 mon.c (mon.1) 69 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:48:26.924 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.622265+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:48:26.924 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.622740+0000 mon.c (mon.1) 71 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:48:26.924 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.623235+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:48:26.924 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.623724+0000 mon.c (mon.1) 73 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: cluster 2022-04-28T19:48:25.644693+0000 mon.a (mon.0) 976 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: cluster 2022-04-28T19:48:25.646886+0000 mon.a (mon.0) 977 : cluster [DBG] mgrmap e22: y(active, starting, since 5s), standbys: x 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:25.650728+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:25.651084+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.? 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.602978+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.603510+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.604224+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:48:26.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.605841+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.606826+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.608449+0000 mon.c (mon.1) 48 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.608942+0000 mon.c (mon.1) 49 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.609369+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.609910+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.610379+0000 mon.c (mon.1) 52 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.610874+0000 mon.c (mon.1) 53 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.611352+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.611947+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:48:26.926 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.612510+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.612985+0000 mon.c (mon.1) 57 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.624226+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.624767+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.625295+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.625811+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.626294+0000 mon.c (mon.1) 78 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:48:26.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.626790+0000 mon.c (mon.1) 79 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:26.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.627415+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:26.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.628030+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:48:26.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.628747+0000 mon.c (mon.1) 82 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:48:26.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.629295+0000 mon.c (mon.1) 83 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:48:26.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[29114]: audit 2022-04-28T19:48:26.629812+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:48:26.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.613748+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.615599+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.617045+0000 mon.c (mon.1) 60 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/client_keyrings"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.617534+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/config_checks"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.618000+0000 mon.c (mon.1) 62 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.618599+0000 mon.c (mon.1) 63 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.619186+0000 mon.c (mon.1) 64 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.619777+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.620340+0000 mon.c (mon.1) 66 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:48:26.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.620795+0000 mon.c (mon.1) 67 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.621250+0000 mon.c (mon.1) 68 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.621759+0000 mon.c (mon.1) 69 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.622265+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.622740+0000 mon.c (mon.1) 71 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.623235+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.623724+0000 mon.c (mon.1) 73 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.624226+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.624767+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.625295+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:48:26.930 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.625811+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.626294+0000 mon.c (mon.1) 78 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.626790+0000 mon.c (mon.1) 79 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.627415+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.628030+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.628747+0000 mon.c (mon.1) 82 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.629295+0000 mon.c (mon.1) 83 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:48:26.931 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[23786]: audit 2022-04-28T19:48:26.629812+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:48:27.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: Warning: Permanently added '172.21.15.59' (ECDSA) to the list of known hosts. 2022-04-28T19:48:27.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: Warning: Permanently added '172.21.15.139' (ECDSA) to the list of known hosts. 2022-04-28T19:48:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: debug 2022-04-28T19:48:26.961+0000 7fd298315700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:48:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: debug 2022-04-28T19:48:26.961+0000 7fd298315700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:48:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: debug 2022-04-28T19:48:26.961+0000 7fd29b31b700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:48:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: debug 2022-04-28T19:48:26.961+0000 7fd29b31b700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:48:27.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:48:26 smithi059 bash[57771]: debug 2022-04-28T19:48:26.961+0000 7fd29b31b700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1265: done (4 left) 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1266: done (3 left) 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1267: done (2 left) 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1269: done (1 left) 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1270: done (0 left) 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1271: copy_from oid 35 from oid 22 current snap is 124 2022-04-28T19:48:27.675 INFO:tasks.rados.rados.0.smithi059.stdout:1272: rollback oid 20 current snap is 124 2022-04-28T19:48:27.676 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 121 2022-04-28T19:48:27.676 INFO:tasks.rados.rados.0.smithi059.stdout:1273: delete oid 27 current snap is 124 2022-04-28T19:48:27.684 INFO:tasks.rados.rados.0.smithi059.stdout:1272: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:48:27.685 INFO:tasks.rados.rados.0.smithi059.stdout:1273: done (2 left) 2022-04-28T19:48:27.685 INFO:tasks.rados.rados.0.smithi059.stdout:1274: rollback oid 36 current snap is 124 2022-04-28T19:48:27.685 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 121 2022-04-28T19:48:27.685 INFO:tasks.rados.rados.0.smithi059.stdout:1275: delete oid 7 current snap is 124 2022-04-28T19:48:27.686 INFO:tasks.rados.rados.0.smithi059.stdout:1272: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:48:27.686 INFO:tasks.rados.rados.0.smithi059.stdout:1272: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:48:27.686 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 549 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:48:27.689 INFO:tasks.rados.rados.0.smithi059.stdout:1274: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:48:27.690 INFO:tasks.rados.rados.0.smithi059.stdout:1274: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:48:27.691 INFO:tasks.rados.rados.0.smithi059.stdout:1272: done (3 left) 2022-04-28T19:48:27.691 INFO:tasks.rados.rados.0.smithi059.stdout:1275: done (2 left) 2022-04-28T19:48:27.691 INFO:tasks.rados.rados.0.smithi059.stdout:1276: copy_from oid 20 from oid 34 current snap is 124 2022-04-28T19:48:27.691 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 124 2022-04-28T19:48:27.692 INFO:tasks.rados.rados.0.smithi059.stdout:1277: seq_num 403 ranges {0=753664,753664=425984,1179648=720896,1900544=57344} 2022-04-28T19:48:27.694 INFO:tasks.rados.rados.0.smithi059.stdout:1277: writing smithi05955964-7 from 0 to 753664 tid 1 2022-04-28T19:48:27.697 INFO:tasks.rados.rados.0.smithi059.stdout:1277: writing smithi05955964-7 from 753664 to 1179648 tid 2 2022-04-28T19:48:27.702 INFO:tasks.rados.rados.0.smithi059.stdout:1277: writing smithi05955964-7 from 1179648 to 1900544 tid 3 2022-04-28T19:48:27.703 INFO:tasks.rados.rados.0.smithi059.stdout:1277: writing smithi05955964-7 from 1900544 to 1957888 tid 4 2022-04-28T19:48:27.703 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 124 2022-04-28T19:48:27.703 INFO:tasks.rados.rados.0.smithi059.stdout:1278: seq_num 404 ranges {0=770048,770048=712704,1482752=491520} 2022-04-28T19:48:27.707 INFO:tasks.rados.rados.0.smithi059.stdout:1278: writing smithi05955964-3 from 0 to 770048 tid 1 2022-04-28T19:48:27.712 INFO:tasks.rados.rados.0.smithi059.stdout:1278: writing smithi05955964-3 from 770048 to 1482752 tid 2 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:1278: writing smithi05955964-3 from 1482752 to 1974272 tid 3 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:1279: rollback oid 6 current snap is 124 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 114 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:1274: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 520 (ObjNum 339 snap 104 seq_num 339) dirty exists 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:1274: done (5 left) 2022-04-28T19:48:27.717 INFO:tasks.rados.rados.0.smithi059.stdout:1280: read oid 40 snap -1 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1280: expect (ObjNum 380 snap 115 seq_num 380) 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1281: snap_create 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1271: finishing copy_from to smithi05955964-35 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 147 (ObjNum 265 snap 77 seq_num 265) dirty exists 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1271: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1277: finishing write tid 1 to smithi05955964-7 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1277: finishing write tid 2 to smithi05955964-7 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1277: finishing write tid 3 to smithi05955964-7 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1277: finishing write tid 4 to smithi05955964-7 2022-04-28T19:48:27.718 INFO:tasks.rados.rados.0.smithi059.stdout:1277: finishing write tid 6 to smithi05955964-7 2022-04-28T19:48:27.719 INFO:tasks.rados.rados.0.smithi059.stdout:1277: finishing write tid 7 to smithi05955964-7 2022-04-28T19:48:27.719 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 612 (ObjNum 403 snap 124 seq_num 403) dirty exists 2022-04-28T19:48:27.719 INFO:tasks.rados.rados.0.smithi059.stdout:1277: left oid 7 (ObjNum 403 snap 124 seq_num 403) 2022-04-28T19:48:27.731 INFO:tasks.rados.rados.0.smithi059.stdout:1278: finishing write tid 1 to smithi05955964-3 2022-04-28T19:48:27.737 INFO:tasks.rados.rados.0.smithi059.stdout:1280: expect (ObjNum 380 snap 115 seq_num 380) 2022-04-28T19:48:28.014 INFO:tasks.rados.rados.0.smithi059.stdout:1276: finishing copy_from to smithi05955964-20 2022-04-28T19:48:28.014 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 607 (ObjNum 362 snap 108 seq_num 362) dirty exists 2022-04-28T19:48:28.014 INFO:tasks.rados.rados.0.smithi059.stdout:1278: finishing write tid 2 to smithi05955964-3 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1278: finishing write tid 3 to smithi05955964-3 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1276: finishing copy_from racing read to smithi05955964-20 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1278: finishing write tid 5 to smithi05955964-3 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1279: finishing rollback tid 0 to smithi05955964-6 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1279: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1279: finishing rollback tid 2 to smithi05955964-6 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 613 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1278: finishing write tid 6 to smithi05955964-3 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 610 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:48:28.015 INFO:tasks.rados.rados.0.smithi059.stdout:1278: left oid 3 (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:48:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: cluster 2022-04-28T19:48:26.644685+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2022-04-28T19:48:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: cluster 2022-04-28T19:48:26.667249+0000 mon.a (mon.0) 981 : cluster [INF] Manager daemon y is now available 2022-04-28T19:48:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: cephadm 2022-04-28T19:48:26.681655+0000 mgr.y (mgr.14985) 1 : cephadm [INF] inventory: adjusted host smithi139 addr 'smithi139' -> '172.21.15.139' 2022-04-28T19:48:28.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.687316+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:28.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.821457+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:28.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.824739+0000 mon.c (mon.1) 86 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:28.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.829979+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:28.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.836412+0000 mon.c (mon.1) 88 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:28.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.876896+0000 mon.c (mon.1) 89 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:48:28.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.877439+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:48:28.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.910741+0000 mon.c (mon.1) 90 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:48:28.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:26.911459+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:48:28.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:27.162999+0000 mon.c (mon.1) 91 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:28.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[23786]: audit 2022-04-28T19:48:27.167417+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:28.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: cluster 2022-04-28T19:48:26.644685+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2022-04-28T19:48:28.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: cluster 2022-04-28T19:48:26.667249+0000 mon.a (mon.0) 981 : cluster [INF] Manager daemon y is now available 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: cephadm 2022-04-28T19:48:26.681655+0000 mgr.y (mgr.14985) 1 : cephadm [INF] inventory: adjusted host smithi139 addr 'smithi139' -> '172.21.15.139' 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.687316+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.821457+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.824739+0000 mon.c (mon.1) 86 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.829979+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.836412+0000 mon.c (mon.1) 88 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.876896+0000 mon.c (mon.1) 89 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.877439+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:48:28.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.910741+0000 mon.c (mon.1) 90 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:48:28.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:26.911459+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:48:28.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:27.162999+0000 mon.c (mon.1) 91 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:28.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:27 smithi059 bash[29114]: audit 2022-04-28T19:48:27.167417+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: cluster 2022-04-28T19:48:26.644685+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2022-04-28T19:48:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: cluster 2022-04-28T19:48:26.667249+0000 mon.a (mon.0) 981 : cluster [INF] Manager daemon y is now available 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: cephadm 2022-04-28T19:48:26.681655+0000 mgr.y (mgr.14985) 1 : cephadm [INF] inventory: adjusted host smithi139 addr 'smithi139' -> '172.21.15.139' 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.687316+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.821457+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.824739+0000 mon.c (mon.1) 86 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.829979+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.836412+0000 mon.c (mon.1) 88 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.876896+0000 mon.c (mon.1) 89 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:48:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.877439+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:48:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.910741+0000 mon.c (mon.1) 90 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:48:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:26.911459+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:48:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:27.162999+0000 mon.c (mon.1) 91 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:27 smithi139 bash[23439]: audit 2022-04-28T19:48:27.167417+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:28.690 INFO:tasks.rados.rados.0.smithi059.stdout:1271: done (6 left) 2022-04-28T19:48:28.691 INFO:tasks.rados.rados.0.smithi059.stdout:1276: done (5 left) 2022-04-28T19:48:28.691 INFO:tasks.rados.rados.0.smithi059.stdout:1277: done (4 left) 2022-04-28T19:48:28.691 INFO:tasks.rados.rados.0.smithi059.stdout:1278: done (3 left) 2022-04-28T19:48:28.691 INFO:tasks.rados.rados.0.smithi059.stdout:1279: done (2 left) 2022-04-28T19:48:28.692 INFO:tasks.rados.rados.0.smithi059.stdout:1280: done (1 left) 2022-04-28T19:48:28.692 INFO:tasks.rados.rados.0.smithi059.stdout:1281: done (0 left) 2022-04-28T19:48:28.692 INFO:tasks.rados.rados.0.smithi059.stdout:1282: read oid 49 snap -1 2022-04-28T19:48:28.693 INFO:tasks.rados.rados.0.smithi059.stdout:1282: expect deleted 2022-04-28T19:48:28.693 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 125 2022-04-28T19:48:28.693 INFO:tasks.rados.rados.0.smithi059.stdout:1283: seq_num 405 ranges {8437760=786432,9224192=589824,9814016=614400} 2022-04-28T19:48:28.742 INFO:tasks.rados.rados.0.smithi059.stdout:1283: writing smithi05955964-45 from 8437760 to 9224192 tid 1 2022-04-28T19:48:28.746 INFO:tasks.rados.rados.0.smithi059.stdout:1283: writing smithi05955964-45 from 9224192 to 9814016 tid 2 2022-04-28T19:48:28.750 INFO:tasks.rados.rados.0.smithi059.stdout:1283: writing smithi05955964-45 from 9814016 to 10428416 tid 3 2022-04-28T19:48:28.750 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 125 2022-04-28T19:48:28.750 INFO:tasks.rados.rados.0.smithi059.stdout:1284: seq_num 406 ranges {1318912=802816,2121728=475136,2596864=499712,3096576=229376} 2022-04-28T19:48:28.762 INFO:tasks.rados.rados.0.smithi059.stdout:1284: writing smithi05955964-18 from 1318912 to 2121728 tid 1 2022-04-28T19:48:28.765 INFO:tasks.rados.rados.0.smithi059.stdout:1284: writing smithi05955964-18 from 2121728 to 2596864 tid 2 2022-04-28T19:48:28.768 INFO:tasks.rados.rados.0.smithi059.stdout:1284: writing smithi05955964-18 from 2596864 to 3096576 tid 3 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:1284: writing smithi05955964-18 from 3096576 to 3325952 tid 4 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:1285: rollback oid 27 current snap is 125 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 122 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:1283: finishing write tid 1 to smithi05955964-45 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:1283: finishing write tid 2 to smithi05955964-45 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:1283: finishing write tid 3 to smithi05955964-45 2022-04-28T19:48:28.770 INFO:tasks.rados.rados.0.smithi059.stdout:1282: done (3 left) 2022-04-28T19:48:28.771 INFO:tasks.rados.rados.0.smithi059.stdout:1286: read oid 34 snap -1 2022-04-28T19:48:28.771 INFO:tasks.rados.rados.0.smithi059.stdout:1286: expect (ObjNum 362 snap 108 seq_num 362) 2022-04-28T19:48:28.771 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 125 2022-04-28T19:48:28.771 INFO:tasks.rados.rados.0.smithi059.stdout:1287: seq_num 407 ranges {1236992=425984,1662976=761856,2424832=704512,3129344=139264} 2022-04-28T19:48:28.779 INFO:tasks.rados.rados.0.smithi059.stdout:1287: writing smithi05955964-11 from 1236992 to 1662976 tid 1 2022-04-28T19:48:28.784 INFO:tasks.rados.rados.0.smithi059.stdout:1287: writing smithi05955964-11 from 1662976 to 2424832 tid 2 2022-04-28T19:48:28.788 INFO:tasks.rados.rados.0.smithi059.stdout:1287: writing smithi05955964-11 from 2424832 to 3129344 tid 3 2022-04-28T19:48:28.791 INFO:tasks.rados.rados.0.smithi059.stdout:1287: writing smithi05955964-11 from 3129344 to 3268608 tid 4 2022-04-28T19:48:28.791 INFO:tasks.rados.rados.0.smithi059.stdout:1288: rmattr oid 1 current snap is 125 2022-04-28T19:48:28.791 INFO:tasks.rados.rados.0.smithi059.stdout:1283: finishing write tid 5 to smithi05955964-45 2022-04-28T19:48:28.791 INFO:tasks.rados.rados.0.smithi059.stdout:1283: finishing write tid 6 to smithi05955964-45 2022-04-28T19:48:28.792 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 617 (ObjNum 405 snap 125 seq_num 405) dirty exists 2022-04-28T19:48:28.792 INFO:tasks.rados.rados.0.smithi059.stdout:1283: left oid 45 (ObjNum 405 snap 125 seq_num 405) 2022-04-28T19:48:28.792 INFO:tasks.rados.rados.0.smithi059.stdout:1285: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:48:28.792 INFO:tasks.rados.rados.0.smithi059.stdout:1285: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:48:28.792 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 618 (ObjNum 302 snap 92 seq_num 302) dirty exists 2022-04-28T19:48:28.792 INFO:tasks.rados.rados.0.smithi059.stdout:1284: finishing write tid 1 to smithi05955964-18 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:1284: finishing write tid 2 to smithi05955964-18 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:1284: finishing write tid 3 to smithi05955964-18 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:1284: finishing write tid 4 to smithi05955964-18 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:1284: finishing write tid 6 to smithi05955964-18 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:1287: finishing write tid 1 to smithi05955964-11 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:1284: finishing write tid 7 to smithi05955964-18 2022-04-28T19:48:28.793 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 623 (ObjNum 406 snap 125 seq_num 406) dirty exists 2022-04-28T19:48:28.794 INFO:tasks.rados.rados.0.smithi059.stdout:1284: left oid 18 (ObjNum 406 snap 125 seq_num 406) 2022-04-28T19:48:28.794 INFO:tasks.rados.rados.0.smithi059.stdout:1287: finishing write tid 2 to smithi05955964-11 2022-04-28T19:48:28.794 INFO:tasks.rados.rados.0.smithi059.stdout:1283: done (5 left) 2022-04-28T19:48:28.794 INFO:tasks.rados.rados.0.smithi059.stdout:1284: done (4 left) 2022-04-28T19:48:28.794 INFO:tasks.rados.rados.0.smithi059.stdout:1285: done (3 left) 2022-04-28T19:48:28.794 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 125 2022-04-28T19:48:28.795 INFO:tasks.rados.rados.0.smithi059.stdout:1289: seq_num 408 ranges {5971968=442368,6414336=647168,7061504=507904,7569408=450560} 2022-04-28T19:48:28.826 INFO:tasks.rados.rados.0.smithi059.stdout:1289: writing smithi05955964-24 from 5971968 to 6414336 tid 1 2022-04-28T19:48:28.830 INFO:tasks.rados.rados.0.smithi059.stdout:1289: writing smithi05955964-24 from 6414336 to 7061504 tid 2 2022-04-28T19:48:28.833 INFO:tasks.rados.rados.0.smithi059.stdout:1289: writing smithi05955964-24 from 7061504 to 7569408 tid 3 2022-04-28T19:48:28.836 INFO:tasks.rados.rados.0.smithi059.stdout:1289: writing smithi05955964-24 from 7569408 to 8019968 tid 4 2022-04-28T19:48:28.836 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 125 2022-04-28T19:48:28.836 INFO:tasks.rados.rados.0.smithi059.stdout:1290: seq_num 409 ranges {2875392=458752,3334144=532480,3866624=720896,4587520=352256} 2022-04-28T19:48:28.855 INFO:tasks.rados.rados.0.smithi059.stdout:1290: writing smithi05955964-6 from 2875392 to 3334144 tid 1 2022-04-28T19:48:28.858 INFO:tasks.rados.rados.0.smithi059.stdout:1290: writing smithi05955964-6 from 3334144 to 3866624 tid 2 2022-04-28T19:48:28.863 INFO:tasks.rados.rados.0.smithi059.stdout:1290: writing smithi05955964-6 from 3866624 to 4587520 tid 3 2022-04-28T19:48:28.865 INFO:tasks.rados.rados.0.smithi059.stdout:1290: writing smithi05955964-6 from 4587520 to 4939776 tid 4 2022-04-28T19:48:28.865 INFO:tasks.rados.rados.0.smithi059.stdout:1291: snap_remove snap 112 2022-04-28T19:48:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: cephadm 2022-04-28T19:48:26.838571+0000 mgr.y (mgr.14985) 2 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: cluster 2022-04-28T19:48:27.671603+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2022-04-28T19:48:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: cluster 2022-04-28T19:48:27.672373+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e23: y(active, since 7s), standbys: x 2022-04-28T19:48:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: audit 2022-04-28T19:48:27.673357+0000 mon.c (mon.1) 93 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:48:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: audit 2022-04-28T19:48:27.674082+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:48:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: audit 2022-04-28T19:48:27.680301+0000 mon.c (mon.1) 94 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:48:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: audit 2022-04-28T19:48:27.681017+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[29114]: cluster 2022-04-28T19:48:27.704690+0000 mgr.y (mgr.14985) 3 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: cephadm 2022-04-28T19:48:26.838571+0000 mgr.y (mgr.14985) 2 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: cluster 2022-04-28T19:48:27.671603+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: cluster 2022-04-28T19:48:27.672373+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e23: y(active, since 7s), standbys: x 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: audit 2022-04-28T19:48:27.673357+0000 mon.c (mon.1) 93 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: audit 2022-04-28T19:48:27.674082+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:48:29.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: audit 2022-04-28T19:48:27.680301+0000 mon.c (mon.1) 94 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:48:29.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: audit 2022-04-28T19:48:27.681017+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:48:29.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:28 smithi059 bash[23786]: cluster 2022-04-28T19:48:27.704690+0000 mgr.y (mgr.14985) 3 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:29.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: cephadm 2022-04-28T19:48:26.838571+0000 mgr.y (mgr.14985) 2 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:29.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: cluster 2022-04-28T19:48:27.671603+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2022-04-28T19:48:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: cluster 2022-04-28T19:48:27.672373+0000 mon.a (mon.0) 986 : cluster [DBG] mgrmap e23: y(active, since 7s), standbys: x 2022-04-28T19:48:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: audit 2022-04-28T19:48:27.673357+0000 mon.c (mon.1) 93 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:48:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: audit 2022-04-28T19:48:27.674082+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:48:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: audit 2022-04-28T19:48:27.680301+0000 mon.c (mon.1) 94 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:48:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: audit 2022-04-28T19:48:27.681017+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:48:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:28 smithi139 bash[23439]: cluster 2022-04-28T19:48:27.704690+0000 mgr.y (mgr.14985) 3 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:29.690 INFO:tasks.rados.rados.0.smithi059.stdout:1291: done (5 left) 2022-04-28T19:48:29.691 INFO:tasks.rados.rados.0.smithi059.stdout:1292: snap_remove snap 114 2022-04-28T19:48:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[23786]: cluster 2022-04-28T19:48:28.610824+0000 mgr.y (mgr.14985) 4 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[23786]: audit 2022-04-28T19:48:28.682433+0000 mon.a (mon.0) 989 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:48:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[23786]: cluster 2022-04-28T19:48:28.682622+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2022-04-28T19:48:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[23786]: cluster 2022-04-28T19:48:28.683759+0000 mon.a (mon.0) 991 : cluster [DBG] mgrmap e24: y(active, since 8s), standbys: x 2022-04-28T19:48:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[29114]: cluster 2022-04-28T19:48:28.610824+0000 mgr.y (mgr.14985) 4 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[29114]: audit 2022-04-28T19:48:28.682433+0000 mon.a (mon.0) 989 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:48:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[29114]: cluster 2022-04-28T19:48:28.682622+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2022-04-28T19:48:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:29 smithi059 bash[29114]: cluster 2022-04-28T19:48:28.683759+0000 mon.a (mon.0) 991 : cluster [DBG] mgrmap e24: y(active, since 8s), standbys: x 2022-04-28T19:48:30.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:29 smithi139 bash[23439]: cluster 2022-04-28T19:48:28.610824+0000 mgr.y (mgr.14985) 4 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:30.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:29 smithi139 bash[23439]: audit 2022-04-28T19:48:28.682433+0000 mon.a (mon.0) 989 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:48:30.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:29 smithi139 bash[23439]: cluster 2022-04-28T19:48:28.682622+0000 mon.a (mon.0) 990 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2022-04-28T19:48:30.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:29 smithi139 bash[23439]: cluster 2022-04-28T19:48:28.683759+0000 mon.a (mon.0) 991 : cluster [DBG] mgrmap e24: y(active, since 8s), standbys: x 2022-04-28T19:48:30.708 INFO:tasks.rados.rados.0.smithi059.stdout:1292: done (5 left) 2022-04-28T19:48:30.708 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 125 2022-04-28T19:48:30.708 INFO:tasks.rados.rados.0.smithi059.stdout:1293: seq_num 410 ranges {1605632=475136,2080768=417792,2498560=475136,2973696=425984,3399680=286720} 2022-04-28T19:48:30.721 INFO:tasks.rados.rados.0.smithi059.stdout:1293: writing smithi05955964-19 from 1605632 to 2080768 tid 1 2022-04-28T19:48:30.723 INFO:tasks.rados.rados.0.smithi059.stdout:1293: writing smithi05955964-19 from 2080768 to 2498560 tid 2 2022-04-28T19:48:30.727 INFO:tasks.rados.rados.0.smithi059.stdout:1293: writing smithi05955964-19 from 2498560 to 2973696 tid 3 2022-04-28T19:48:30.729 INFO:tasks.rados.rados.0.smithi059.stdout:1293: writing smithi05955964-19 from 2973696 to 3399680 tid 4 2022-04-28T19:48:30.732 INFO:tasks.rados.rados.0.smithi059.stdout:1293: writing smithi05955964-19 from 3399680 to 3686400 tid 5 2022-04-28T19:48:30.732 INFO:tasks.rados.rados.0.smithi059.stdout:1294: read oid 47 snap 101 2022-04-28T19:48:30.732 INFO:tasks.rados.rados.0.smithi059.stdout:1294: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:48:30.732 INFO:tasks.rados.rados.0.smithi059.stdout:1287: finishing write tid 3 to smithi05955964-11 2022-04-28T19:48:30.732 INFO:tasks.rados.rados.0.smithi059.stdout:1287: finishing write tid 4 to smithi05955964-11 2022-04-28T19:48:30.732 INFO:tasks.rados.rados.0.smithi059.stdout:1287: finishing write tid 6 to smithi05955964-11 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:1295: read oid 37 snap 110 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:1295: expect (ObjNum 359 snap 108 seq_num 359) 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:1296: setattr oid 38 current snap is 125 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:1287: finishing write tid 7 to smithi05955964-11 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 561 (ObjNum 407 snap 125 seq_num 407) dirty exists 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:1287: left oid 11 (ObjNum 407 snap 125 seq_num 407) 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 522 (ObjNum 134 snap 32 seq_num 134) dirty exists 2022-04-28T19:48:30.733 INFO:tasks.rados.rados.0.smithi059.stdout:1286: expect (ObjNum 362 snap 108 seq_num 362) 2022-04-28T19:48:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:30 smithi059 bash[23786]: cluster 2022-04-28T19:48:29.686670+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2022-04-28T19:48:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:30 smithi059 bash[29114]: cluster 2022-04-28T19:48:29.686670+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2022-04-28T19:48:31.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:30 smithi139 bash[23439]: cluster 2022-04-28T19:48:29.686670+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2022-04-28T19:48:31.133 INFO:tasks.rados.rados.0.smithi059.stdout:1289: finishing write tid 1 to smithi05955964-24 2022-04-28T19:48:31.133 INFO:tasks.rados.rados.0.smithi059.stdout:1289: finishing write tid 2 to smithi05955964-24 2022-04-28T19:48:31.133 INFO:tasks.rados.rados.0.smithi059.stdout:1289: finishing write tid 3 to smithi05955964-24 2022-04-28T19:48:31.133 INFO:tasks.rados.rados.0.smithi059.stdout:1289: finishing write tid 4 to smithi05955964-24 2022-04-28T19:48:31.133 INFO:tasks.rados.rados.0.smithi059.stdout:1289: finishing write tid 6 to smithi05955964-24 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1289: finishing write tid 7 to smithi05955964-24 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 567 (ObjNum 408 snap 125 seq_num 408) dirty exists 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1289: left oid 24 (ObjNum 408 snap 125 seq_num 408) 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1290: finishing write tid 1 to smithi05955964-6 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1290: finishing write tid 2 to smithi05955964-6 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1290: finishing write tid 3 to smithi05955964-6 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1290: finishing write tid 4 to smithi05955964-6 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1290: finishing write tid 6 to smithi05955964-6 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:1290: finishing write tid 7 to smithi05955964-6 2022-04-28T19:48:31.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 622 (ObjNum 409 snap 125 seq_num 409) dirty exists 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1290: left oid 6 (ObjNum 409 snap 125 seq_num 409) 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 1 to smithi05955964-19 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 2 to smithi05955964-19 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 3 to smithi05955964-19 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 4 to smithi05955964-19 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 5 to smithi05955964-19 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 7 to smithi05955964-19 2022-04-28T19:48:31.135 INFO:tasks.rados.rados.0.smithi059.stdout:1294: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:48:31.221 INFO:tasks.rados.rados.0.smithi059.stdout:1293: finishing write tid 8 to smithi05955964-19 2022-04-28T19:48:31.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 637 (ObjNum 410 snap 125 seq_num 410) dirty exists 2022-04-28T19:48:31.221 INFO:tasks.rados.rados.0.smithi059.stdout:1293: left oid 19 (ObjNum 410 snap 125 seq_num 410) 2022-04-28T19:48:31.221 INFO:tasks.rados.rados.0.smithi059.stdout:1286: done (8 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1287: done (7 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1288: done (6 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1289: done (5 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1290: done (4 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1293: done (3 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1294: done (2 left) 2022-04-28T19:48:31.222 INFO:tasks.rados.rados.0.smithi059.stdout:1297: snap_remove snap 116 2022-04-28T19:48:31.719 INFO:tasks.rados.rados.0.smithi059.stdout:1297: done (2 left) 2022-04-28T19:48:31.720 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 125 2022-04-28T19:48:31.720 INFO:tasks.rados.rados.0.smithi059.stdout:1298: seq_num 411 ranges {0=491520,491520=753664,1245184=688128,1933312=163840} 2022-04-28T19:48:31.723 INFO:tasks.rados.rados.0.smithi059.stdout:1298: writing smithi05955964-26 from 0 to 491520 tid 1 2022-04-28T19:48:31.730 INFO:tasks.rados.rados.0.smithi059.stdout:1298: writing smithi05955964-26 from 491520 to 1245184 tid 2 2022-04-28T19:48:31.734 INFO:tasks.rados.rados.0.smithi059.stdout:1298: writing smithi05955964-26 from 1245184 to 1933312 tid 3 2022-04-28T19:48:31.736 INFO:tasks.rados.rados.0.smithi059.stdout:1298: writing smithi05955964-26 from 1933312 to 2097152 tid 4 2022-04-28T19:48:31.736 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 125 2022-04-28T19:48:31.736 INFO:tasks.rados.rados.0.smithi059.stdout:1299: seq_num 412 ranges {1376256=507904,1884160=630784,2514944=491520,3006464=483328} 2022-04-28T19:48:31.747 INFO:tasks.rados.rados.0.smithi059.stdout:1299: writing smithi05955964-17 from 1376256 to 1884160 tid 1 2022-04-28T19:48:31.750 INFO:tasks.rados.rados.0.smithi059.stdout:1299: writing smithi05955964-17 from 1884160 to 2514944 tid 2 2022-04-28T19:48:31.753 INFO:tasks.rados.rados.0.smithi059.stdout:1299: writing smithi05955964-17 from 2514944 to 3006464 tid 3 2022-04-28T19:48:31.756 INFO:tasks.rados.rados.0.smithi059.stdout:1299: writing smithi05955964-17 from 3006464 to 3489792 tid 4 2022-04-28T19:48:31.757 INFO:tasks.rados.rados.0.smithi059.stdout:1300: rollback oid 16 current snap is 125 2022-04-28T19:48:31.757 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 123 2022-04-28T19:48:31.757 INFO:tasks.rados.rados.0.smithi059.stdout:1295: expect (ObjNum 359 snap 108 seq_num 359) 2022-04-28T19:48:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:31 smithi059 bash[29114]: cluster 2022-04-28T19:48:30.611607+0000 mgr.y (mgr.14985) 5 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:31 smithi059 bash[29114]: cluster 2022-04-28T19:48:30.707235+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2022-04-28T19:48:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:31 smithi059 bash[23786]: cluster 2022-04-28T19:48:30.611607+0000 mgr.y (mgr.14985) 5 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:31 smithi059 bash[23786]: cluster 2022-04-28T19:48:30.707235+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2022-04-28T19:48:32.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:31 smithi139 bash[23439]: cluster 2022-04-28T19:48:30.611607+0000 mgr.y (mgr.14985) 5 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:32.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:31 smithi139 bash[23439]: cluster 2022-04-28T19:48:30.707235+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2022-04-28T19:48:32.120 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 581 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:48:32.120 INFO:tasks.rados.rados.0.smithi059.stdout:1298: finishing write tid 1 to smithi05955964-26 2022-04-28T19:48:32.120 INFO:tasks.rados.rados.0.smithi059.stdout:1298: finishing write tid 2 to smithi05955964-26 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1298: finishing write tid 3 to smithi05955964-26 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1298: finishing write tid 4 to smithi05955964-26 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1298: finishing write tid 6 to smithi05955964-26 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1298: finishing write tid 7 to smithi05955964-26 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 642 (ObjNum 411 snap 125 seq_num 411) dirty exists 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1298: left oid 26 (ObjNum 411 snap 125 seq_num 411) 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1299: finishing write tid 1 to smithi05955964-17 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1299: finishing write tid 2 to smithi05955964-17 2022-04-28T19:48:32.121 INFO:tasks.rados.rados.0.smithi059.stdout:1295: done (4 left) 2022-04-28T19:48:32.122 INFO:tasks.rados.rados.0.smithi059.stdout:1296: done (3 left) 2022-04-28T19:48:32.122 INFO:tasks.rados.rados.0.smithi059.stdout:1298: done (2 left) 2022-04-28T19:48:32.122 INFO:tasks.rados.rados.0.smithi059.stdout:1301: rollback oid 26 current snap is 125 2022-04-28T19:48:32.122 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 111 2022-04-28T19:48:32.122 INFO:tasks.rados.rados.0.smithi059.stdout:1299: finishing write tid 3 to smithi05955964-17 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1299: finishing write tid 4 to smithi05955964-17 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1300: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1299: finishing write tid 6 to smithi05955964-17 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1300: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1300: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 387 (ObjNum 360 snap 108 seq_num 360) dirty exists 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1299: finishing write tid 7 to smithi05955964-17 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 329 (ObjNum 412 snap 125 seq_num 412) dirty exists 2022-04-28T19:48:32.123 INFO:tasks.rados.rados.0.smithi059.stdout:1299: left oid 17 (ObjNum 412 snap 125 seq_num 412) 2022-04-28T19:48:32.124 INFO:tasks.rados.rados.0.smithi059.stdout:1299: done (2 left) 2022-04-28T19:48:32.124 INFO:tasks.rados.rados.0.smithi059.stdout:1300: done (1 left) 2022-04-28T19:48:32.124 INFO:tasks.rados.rados.0.smithi059.stdout:1302: rmattr oid 32 current snap is 125 2022-04-28T19:48:32.124 INFO:tasks.rados.rados.0.smithi059.stdout:1303: rmattr oid 27 current snap is 125 2022-04-28T19:48:32.124 INFO:tasks.rados.rados.0.smithi059.stdout:1304: delete oid 42 current snap is 125 2022-04-28T19:48:32.124 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 0 (ObjNum 381 snap 115 seq_num 381) dirty exists 2022-04-28T19:48:32.125 INFO:tasks.rados.rados.0.smithi059.stdout:1302: done (3 left) 2022-04-28T19:48:32.125 INFO:tasks.rados.rados.0.smithi059.stdout:1304: done (2 left) 2022-04-28T19:48:32.125 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 125 2022-04-28T19:48:32.125 INFO:tasks.rados.rados.0.smithi059.stdout:1305: seq_num 414 ranges {2768896=540672,3309568=802816,4112384=458752,4571136=344064} 2022-04-28T19:48:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:1305: writing smithi05955964-47 from 2768896 to 3309568 tid 1 2022-04-28T19:48:32.153 INFO:tasks.rados.rados.0.smithi059.stdout:1305: writing smithi05955964-47 from 3309568 to 4112384 tid 2 2022-04-28T19:48:32.156 INFO:tasks.rados.rados.0.smithi059.stdout:1305: writing smithi05955964-47 from 4112384 to 4571136 tid 3 2022-04-28T19:48:32.158 INFO:tasks.rados.rados.0.smithi059.stdout:1305: writing smithi05955964-47 from 4571136 to 4915200 tid 4 2022-04-28T19:48:32.158 INFO:tasks.rados.rados.0.smithi059.stdout:1306: snap_remove snap 122 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1306: done (3 left) 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1307: read oid 30 snap -1 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1307: expect deleted 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1308: copy_from oid 24 from oid 43 current snap is 125 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1309: delete oid 36 current snap is 125 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1301: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1301: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:48:32.248 INFO:tasks.rados.rados.0.smithi059.stdout:1301: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 645 (ObjNum 132 snap 31 seq_num 132) dirty exists 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 637 (ObjNum 302 snap 92 seq_num 302) dirty exists 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: finishing write tid 1 to smithi05955964-47 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: finishing write tid 2 to smithi05955964-47 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: finishing write tid 3 to smithi05955964-47 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: finishing write tid 4 to smithi05955964-47 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: finishing write tid 6 to smithi05955964-47 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: finishing write tid 7 to smithi05955964-47 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 643 (ObjNum 414 snap 125 seq_num 414) dirty exists 2022-04-28T19:48:32.249 INFO:tasks.rados.rados.0.smithi059.stdout:1305: left oid 47 (ObjNum 414 snap 125 seq_num 414) 2022-04-28T19:48:32.257 INFO:tasks.rados.rados.0.smithi059.stdout:1301: done (5 left) 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:1303: done (4 left) 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:1305: done (3 left) 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:1307: done (2 left) 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:1309: done (1 left) 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:1310: copy_from oid 25 from oid 3 current snap is 125 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 125 2022-04-28T19:48:32.258 INFO:tasks.rados.rados.0.smithi059.stdout:1311: seq_num 416 ranges {2875392=573440,3448832=573440,4022272=475136,4497408=557056} 2022-04-28T19:48:32.278 INFO:tasks.rados.rados.0.smithi059.stdout:1311: writing smithi05955964-1 from 2875392 to 3448832 tid 1 2022-04-28T19:48:32.282 INFO:tasks.rados.rados.0.smithi059.stdout:1311: writing smithi05955964-1 from 3448832 to 4022272 tid 2 2022-04-28T19:48:32.285 INFO:tasks.rados.rados.0.smithi059.stdout:1311: writing smithi05955964-1 from 4022272 to 4497408 tid 3 2022-04-28T19:48:32.289 INFO:tasks.rados.rados.0.smithi059.stdout:1311: writing smithi05955964-1 from 4497408 to 5054464 tid 4 2022-04-28T19:48:32.289 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 125 2022-04-28T19:48:32.289 INFO:tasks.rados.rados.0.smithi059.stdout:1312: seq_num 417 ranges {3072000=589824,3661824=458752,4120576=679936,4800512=466944} 2022-04-28T19:48:32.310 INFO:tasks.rados.rados.0.smithi059.stdout:1312: writing smithi05955964-27 from 3072000 to 3661824 tid 1 2022-04-28T19:48:32.312 INFO:tasks.rados.rados.0.smithi059.stdout:1312: writing smithi05955964-27 from 3661824 to 4120576 tid 2 2022-04-28T19:48:32.316 INFO:tasks.rados.rados.0.smithi059.stdout:1312: writing smithi05955964-27 from 4120576 to 4800512 tid 3 2022-04-28T19:48:32.320 INFO:tasks.rados.rados.0.smithi059.stdout:1312: writing smithi05955964-27 from 4800512 to 5267456 tid 4 2022-04-28T19:48:32.320 INFO:tasks.rados.rados.0.smithi059.stdout:1310: finishing copy_from to smithi05955964-25 2022-04-28T19:48:32.320 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 610 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:48:32.320 INFO:tasks.rados.rados.0.smithi059.stdout:1313: read oid 17 snap -1 2022-04-28T19:48:32.320 INFO:tasks.rados.rados.0.smithi059.stdout:1313: expect (ObjNum 412 snap 125 seq_num 412) 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1314: setattr oid 47 current snap is 125 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1310: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1308: finishing copy_from to smithi05955964-24 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 628 (ObjNum 357 snap 107 seq_num 357) dirty exists 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1308: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1311: finishing write tid 1 to smithi05955964-1 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1311: finishing write tid 2 to smithi05955964-1 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1311: finishing write tid 3 to smithi05955964-1 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1311: finishing write tid 4 to smithi05955964-1 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1311: finishing write tid 6 to smithi05955964-1 2022-04-28T19:48:32.321 INFO:tasks.rados.rados.0.smithi059.stdout:1311: finishing write tid 7 to smithi05955964-1 2022-04-28T19:48:32.322 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 559 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:48:32.322 INFO:tasks.rados.rados.0.smithi059.stdout:1311: left oid 1 (ObjNum 416 snap 125 seq_num 416) 2022-04-28T19:48:32.322 INFO:tasks.rados.rados.0.smithi059.stdout:1312: finishing write tid 1 to smithi05955964-27 2022-04-28T19:48:32.322 INFO:tasks.rados.rados.0.smithi059.stdout:1312: finishing write tid 2 to smithi05955964-27 2022-04-28T19:48:32.322 INFO:tasks.rados.rados.0.smithi059.stdout:1312: finishing write tid 3 to smithi05955964-27 2022-04-28T19:48:32.322 INFO:tasks.rados.rados.0.smithi059.stdout:1308: done (5 left) 2022-04-28T19:48:32.323 INFO:tasks.rados.rados.0.smithi059.stdout:1310: done (4 left) 2022-04-28T19:48:32.323 INFO:tasks.rados.rados.0.smithi059.stdout:1311: done (3 left) 2022-04-28T19:48:32.323 INFO:tasks.rados.rados.0.smithi059.stdout:1315: delete oid 19 current snap is 125 2022-04-28T19:48:32.323 INFO:tasks.rados.rados.0.smithi059.stdout:1315: done (3 left) 2022-04-28T19:48:32.323 INFO:tasks.rados.rados.0.smithi059.stdout:1316: delete oid 35 current snap is 125 2022-04-28T19:48:32.324 INFO:tasks.rados.rados.0.smithi059.stdout:1312: finishing write tid 4 to smithi05955964-27 2022-04-28T19:48:32.324 INFO:tasks.rados.rados.0.smithi059.stdout:1312: finishing write tid 6 to smithi05955964-27 2022-04-28T19:48:32.325 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 653 (ObjNum 414 snap 125 seq_num 414) dirty exists 2022-04-28T19:48:32.326 INFO:tasks.rados.rados.0.smithi059.stdout:1312: finishing write tid 7 to smithi05955964-27 2022-04-28T19:48:32.326 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 652 (ObjNum 417 snap 125 seq_num 417) dirty exists 2022-04-28T19:48:32.326 INFO:tasks.rados.rados.0.smithi059.stdout:1312: left oid 27 (ObjNum 417 snap 125 seq_num 417) 2022-04-28T19:48:32.331 INFO:tasks.rados.rados.0.smithi059.stdout:1312: done (3 left) 2022-04-28T19:48:32.332 INFO:tasks.rados.rados.0.smithi059.stdout:1314: done (2 left) 2022-04-28T19:48:32.332 INFO:tasks.rados.rados.0.smithi059.stdout:1316: done (1 left) 2022-04-28T19:48:32.332 INFO:tasks.rados.rados.0.smithi059.stdout:1317: copy_from oid 31 from oid 38 current snap is 125 2022-04-28T19:48:32.332 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 125 2022-04-28T19:48:32.332 INFO:tasks.rados.rados.0.smithi059.stdout:1318: seq_num 420 ranges {0=638976,638976=557056,1196032=450560,1646592=598016} 2022-04-28T19:48:32.335 INFO:tasks.rados.rados.0.smithi059.stdout:1318: writing smithi05955964-35 from 0 to 638976 tid 1 2022-04-28T19:48:32.339 INFO:tasks.rados.rados.0.smithi059.stdout:1318: writing smithi05955964-35 from 638976 to 1196032 tid 2 2022-04-28T19:48:32.343 INFO:tasks.rados.rados.0.smithi059.stdout:1318: writing smithi05955964-35 from 1196032 to 1646592 tid 3 2022-04-28T19:48:32.347 INFO:tasks.rados.rados.0.smithi059.stdout:1318: writing smithi05955964-35 from 1646592 to 2244608 tid 4 2022-04-28T19:48:32.347 INFO:tasks.rados.rados.0.smithi059.stdout:1319: copy_from oid 27 from oid 28 current snap is 125 2022-04-28T19:48:32.347 INFO:tasks.rados.rados.0.smithi059.stdout:1313: expect (ObjNum 412 snap 125 seq_num 412) 2022-04-28T19:48:32.452 INFO:tasks.rados.rados.0.smithi059.stdout:1318: finishing write tid 1 to smithi05955964-35 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1318: finishing write tid 2 to smithi05955964-35 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1317: finishing copy_from to smithi05955964-31 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 581 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1317: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1318: finishing write tid 3 to smithi05955964-35 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1318: finishing write tid 4 to smithi05955964-35 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1318: finishing write tid 6 to smithi05955964-35 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:1318: finishing write tid 7 to smithi05955964-35 2022-04-28T19:48:32.499 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 431 (ObjNum 420 snap 125 seq_num 420) dirty exists 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1318: left oid 35 (ObjNum 420 snap 125 seq_num 420) 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1319: finishing copy_from to smithi05955964-27 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 384 (ObjNum 385 snap 118 seq_num 385) dirty exists 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1313: done (3 left) 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1317: done (2 left) 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1318: done (1 left) 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1320: read oid 13 snap -1 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1320: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1321: rollback oid 49 current snap is 125 2022-04-28T19:48:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 123 2022-04-28T19:48:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1319: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:48:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1319: done (2 left) 2022-04-28T19:48:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1322: delete oid 37 current snap is 125 2022-04-28T19:48:32.502 INFO:tasks.rados.rados.0.smithi059.stdout:1321: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:48:32.624 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 506 (ObjNum 288 snap 0 seq_num 97) dirty dne 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1321: done (2 left) 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1322: done (1 left) 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1323: read oid 10 snap 124 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1323: expect (ObjNum 388 snap 120 seq_num 388) 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1324: read oid 18 snap -1 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1324: expect (ObjNum 406 snap 125 seq_num 406) 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1325: rollback oid 48 current snap is 125 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 124 2022-04-28T19:48:32.625 INFO:tasks.rados.rados.0.smithi059.stdout:1326: rollback oid 26 current snap is 125 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 124 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:1327: copy_from oid 14 from oid 32 current snap is 125 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:1328: snap_create 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:1325: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 468 (ObjNum -132984304 snap 21893 seq_num 4162281536) dirty dne 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:1326: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:1326: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:48:32.626 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 651 (ObjNum 1 snap 21893 seq_num 4161612640) dirty dne 2022-04-28T19:48:32.628 INFO:tasks.rados.rados.0.smithi059.stdout:1323: expect (ObjNum 388 snap 120 seq_num 388) 2022-04-28T19:48:32.630 INFO:tasks.rados.rados.0.smithi059.stdout:1320: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:48:32.630 INFO:tasks.rados.rados.0.smithi059.stdout:1327: finishing copy_from to smithi05955964-14 2022-04-28T19:48:32.630 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 638 (ObjNum 381 snap 115 seq_num 381) dirty exists 2022-04-28T19:48:32.630 INFO:tasks.rados.rados.0.smithi059.stdout:1327: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:48:32.630 INFO:tasks.rados.rados.0.smithi059.stdout:1324: expect (ObjNum 406 snap 125 seq_num 406) 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: cluster 2022-04-28T19:48:31.717840+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:31.996510+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.214753+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: cluster 2022-04-28T19:48:32.243666+0000 mon.a (mon.0) 997 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.295775+0000 mon.c (mon.1) 95 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.296241+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.296661+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.297045+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.297439+0000 mon.c (mon.1) 99 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.297804+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.298206+0000 mon.c (mon.1) 101 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.298574+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.298949+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.299353+0000 mon.c (mon.1) 104 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.299708+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.522896+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.523620+0000 mon.c (mon.1) 106 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: cluster 2022-04-28T19:48:31.717840+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:31.996510+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.214753+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: cluster 2022-04-28T19:48:32.243666+0000 mon.a (mon.0) 997 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.295775+0000 mon.c (mon.1) 95 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.296241+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.296661+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.297045+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.297439+0000 mon.c (mon.1) 99 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.297804+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.298206+0000 mon.c (mon.1) 101 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.298574+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.298949+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:48:33.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.299353+0000 mon.c (mon.1) 104 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.299708+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.522896+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.523620+0000 mon.c (mon.1) 106 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.524430+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.525236+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.525996+0000 mon.c (mon.1) 109 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.526737+0000 mon.c (mon.1) 110 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.527503+0000 mon.c (mon.1) 111 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.528128+0000 mon.c (mon.1) 112 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.528726+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:48:33.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.529350+0000 mon.c (mon.1) 114 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.098 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.529832+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.098 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.536732+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[23786]: audit 2022-04-28T19:48:32.549959+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.524430+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.525236+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.525996+0000 mon.c (mon.1) 109 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.526737+0000 mon.c (mon.1) 110 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.527503+0000 mon.c (mon.1) 111 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.528128+0000 mon.c (mon.1) 112 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.528726+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:48:33.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.529350+0000 mon.c (mon.1) 114 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.529832+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.536732+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:32 smithi059 bash[29114]: audit 2022-04-28T19:48:32.549959+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:33.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: cluster 2022-04-28T19:48:31.717840+0000 mon.a (mon.0) 994 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2022-04-28T19:48:33.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:31.996510+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.214753+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: cluster 2022-04-28T19:48:32.243666+0000 mon.a (mon.0) 997 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.295775+0000 mon.c (mon.1) 95 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.296241+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.296661+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.297045+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.297439+0000 mon.c (mon.1) 99 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.297804+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.298206+0000 mon.c (mon.1) 101 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.298574+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.298949+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.299353+0000 mon.c (mon.1) 104 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.299708+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.522896+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.523620+0000 mon.c (mon.1) 106 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.524430+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.525236+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.525996+0000 mon.c (mon.1) 109 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.526737+0000 mon.c (mon.1) 110 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.527503+0000 mon.c (mon.1) 111 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.528128+0000 mon.c (mon.1) 112 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.528726+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.529350+0000 mon.c (mon.1) 114 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.529832+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.536732+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:32 smithi139 bash[23439]: audit 2022-04-28T19:48:32.549959+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:33.265 INFO:tasks.rados.rados.0.smithi059.stdout:1320: done (6 left) 2022-04-28T19:48:33.265 INFO:tasks.rados.rados.0.smithi059.stdout:1323: done (5 left) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1324: done (4 left) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1325: done (3 left) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1326: done (2 left) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1327: done (1 left) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1328: done (0 left) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1329: read oid 31 snap 111 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1329: expect (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:48:33.266 INFO:tasks.rados.rados.0.smithi059.stdout:1330: delete oid 4 current snap is 126 2022-04-28T19:48:33.269 INFO:tasks.rados.rados.0.smithi059.stdout:1330: done (1 left) 2022-04-28T19:48:33.269 INFO:tasks.rados.rados.0.smithi059.stdout:1331: snap_remove snap 121 2022-04-28T19:48:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[29114]: cluster 2022-04-28T19:48:32.612300+0000 mgr.y (mgr.14985) 6 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[29114]: cluster 2022-04-28T19:48:33.262657+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2022-04-28T19:48:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[23786]: cluster 2022-04-28T19:48:32.612300+0000 mgr.y (mgr.14985) 6 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[23786]: cluster 2022-04-28T19:48:33.262657+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2022-04-28T19:48:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:33.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:33.516Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:48:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:33.516Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:48:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:33.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:48:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:33.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:33.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:48:34.047 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:33 smithi139 bash[23439]: cluster 2022-04-28T19:48:32.612300+0000 mgr.y (mgr.14985) 6 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 203 MiB data, 309 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:34.047 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:33 smithi139 bash[23439]: cluster 2022-04-28T19:48:33.262657+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2022-04-28T19:48:34.267 INFO:tasks.rados.rados.0.smithi059.stdout:1331: done (1 left) 2022-04-28T19:48:34.267 INFO:tasks.rados.rados.0.smithi059.stdout:1332: read oid 9 snap -1 2022-04-28T19:48:34.267 INFO:tasks.rados.rados.0.smithi059.stdout:1332: expect (ObjNum 317 snap 95 seq_num 317) 2022-04-28T19:48:34.267 INFO:tasks.rados.rados.0.smithi059.stdout:1333: delete oid 2 current snap is 126 2022-04-28T19:48:34.267 INFO:tasks.rados.rados.0.smithi059.stdout:1329: expect (ObjNum 341 snap 104 seq_num 341) 2022-04-28T19:48:34.303 INFO:tasks.rados.rados.0.smithi059.stdout:1332: expect (ObjNum 317 snap 95 seq_num 317) 2022-04-28T19:48:34.316 INFO:tasks.rados.rados.0.smithi059.stdout:1329: done (2 left) 2022-04-28T19:48:34.316 INFO:tasks.rados.rados.0.smithi059.stdout:1332: done (1 left) 2022-04-28T19:48:34.317 INFO:tasks.rados.rados.0.smithi059.stdout:1333: done (0 left) 2022-04-28T19:48:34.317 INFO:tasks.rados.rados.0.smithi059.stdout:1334: snap_create 2022-04-28T19:48:35.273 INFO:tasks.rados.rados.0.smithi059.stdout:1334: done (0 left) 2022-04-28T19:48:35.274 INFO:tasks.rados.rados.0.smithi059.stdout:1335: snap_create 2022-04-28T19:48:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: cluster 2022-04-28T19:48:34.260681+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2022-04-28T19:48:35.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: cluster 2022-04-28T19:48:34.613019+0000 mgr.y (mgr.14985) 7 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 240 MiB data, 353 MiB used, 707 GiB / 715 GiB avail; 23 MiB/s rd, 12 MiB/s wr, 59 op/s 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: audit 2022-04-28T19:48:34.991082+0000 mon.c (mon.1) 116 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: audit 2022-04-28T19:48:34.991864+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: audit 2022-04-28T19:48:34.994322+0000 mon.c (mon.1) 117 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: audit 2022-04-28T19:48:34.996222+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[29114]: audit 2022-04-28T19:48:34.998703+0000 mon.c (mon.1) 119 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.x","key": "container_image"}]: dispatch 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: cluster 2022-04-28T19:48:34.260681+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2022-04-28T19:48:35.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: cluster 2022-04-28T19:48:34.613019+0000 mgr.y (mgr.14985) 7 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 240 MiB data, 353 MiB used, 707 GiB / 715 GiB avail; 23 MiB/s rd, 12 MiB/s wr, 59 op/s 2022-04-28T19:48:35.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: audit 2022-04-28T19:48:34.991082+0000 mon.c (mon.1) 116 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:35.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: audit 2022-04-28T19:48:34.991864+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:35.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: audit 2022-04-28T19:48:34.994322+0000 mon.c (mon.1) 117 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:48:35.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: audit 2022-04-28T19:48:34.996222+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:35.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:35 smithi059 bash[23786]: audit 2022-04-28T19:48:34.998703+0000 mon.c (mon.1) 119 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.x","key": "container_image"}]: dispatch 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: cluster 2022-04-28T19:48:34.260681+0000 mon.a (mon.0) 1002 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: cluster 2022-04-28T19:48:34.613019+0000 mgr.y (mgr.14985) 7 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 240 MiB data, 353 MiB used, 707 GiB / 715 GiB avail; 23 MiB/s rd, 12 MiB/s wr, 59 op/s 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: audit 2022-04-28T19:48:34.991082+0000 mon.c (mon.1) 116 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: audit 2022-04-28T19:48:34.991864+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: audit 2022-04-28T19:48:34.994322+0000 mon.c (mon.1) 117 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: audit 2022-04-28T19:48:34.996222+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:35 smithi139 bash[23439]: audit 2022-04-28T19:48:34.998703+0000 mon.c (mon.1) 119 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.x","key": "container_image"}]: dispatch 2022-04-28T19:48:36.279 INFO:tasks.rados.rados.0.smithi059.stdout:1335: done (0 left) 2022-04-28T19:48:36.279 INFO:tasks.rados.rados.0.smithi059.stdout:1336: rollback oid 2 current snap is 128 2022-04-28T19:48:36.279 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 2 to 111 2022-04-28T19:48:36.279 INFO:tasks.rados.rados.0.smithi059.stdout:1337: read oid 21 snap -1 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:1337: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:1338: rollback oid 16 current snap is 128 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 101 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:1339: read oid 19 snap -1 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:1339: expect deleted 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:1340: delete oid 38 current snap is 128 2022-04-28T19:48:36.280 INFO:tasks.rados.rados.0.smithi059.stdout:1337: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:36.287 INFO:tasks.rados.rados.0.smithi059.stdout:1336: finishing rollback tid 1 to smithi05955964-2 2022-04-28T19:48:36.289 INFO:tasks.rados.rados.0.smithi059.stdout:1338: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:48:36.290 INFO:tasks.rados.rados.0.smithi059.stdout:1336: finishing rollback tid 2 to smithi05955964-2 2022-04-28T19:48:36.290 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 383 (ObjNum 293 snap 87 seq_num 293) dirty exists 2022-04-28T19:48:36.291 INFO:tasks.rados.rados.0.smithi059.stdout:1338: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:48:36.291 INFO:tasks.rados.rados.0.smithi059.stdout:1338: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:48:36.291 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 390 (ObjNum 1 snap 32762 seq_num 4162246576) dirty exists 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:1336: done (4 left) 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:1337: done (3 left) 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:1338: done (2 left) 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:1339: done (1 left) 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:1340: done (0 left) 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:1341: rollback oid 26 current snap is 128 2022-04-28T19:48:36.293 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 101 2022-04-28T19:48:36.294 INFO:tasks.rados.rados.0.smithi059.stdout:1342: read oid 44 snap 124 2022-04-28T19:48:36.294 INFO:tasks.rados.rados.0.smithi059.stdout:1342: expect (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:48:36.294 INFO:tasks.rados.rados.0.smithi059.stdout:1343: read oid 16 snap -1 2022-04-28T19:48:36.294 INFO:tasks.rados.rados.0.smithi059.stdout:1343: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:36.294 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 128 2022-04-28T19:48:36.294 INFO:tasks.rados.rados.0.smithi059.stdout:1344: seq_num 425 ranges {6021120=720896,6742016=778240,7520256=638976,8159232=188416} 2022-04-28T19:48:36.331 INFO:tasks.rados.rados.0.smithi059.stdout:1344: writing smithi05955964-34 from 6021120 to 6742016 tid 1 2022-04-28T19:48:36.336 INFO:tasks.rados.rados.0.smithi059.stdout:1344: writing smithi05955964-34 from 6742016 to 7520256 tid 2 2022-04-28T19:48:36.340 INFO:tasks.rados.rados.0.smithi059.stdout:1344: writing smithi05955964-34 from 7520256 to 8159232 tid 3 2022-04-28T19:48:36.341 INFO:tasks.rados.rados.0.smithi059.stdout:1344: writing smithi05955964-34 from 8159232 to 8347648 tid 4 2022-04-28T19:48:36.341 INFO:tasks.rados.rados.0.smithi059.stdout:1345: copy_from oid 22 from oid 4 current snap is 128 2022-04-28T19:48:36.341 INFO:tasks.rados.rados.0.smithi059.stdout:1343: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:36.347 INFO:tasks.rados.rados.0.smithi059.stdout:1341: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:48:36.347 INFO:tasks.rados.rados.0.smithi059.stdout:1341: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:48:36.347 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 659 (ObjNum 332 snap 99 seq_num 332) dirty exists 2022-04-28T19:48:36.347 INFO:tasks.rados.rados.0.smithi059.stdout:1342: expect (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:48:36.387 INFO:tasks.rados.rados.0.smithi059.stdout:1344: finishing write tid 1 to smithi05955964-34 2022-04-28T19:48:36.387 INFO:tasks.rados.rados.0.smithi059.stdout:1344: finishing write tid 2 to smithi05955964-34 2022-04-28T19:48:36.387 INFO:tasks.rados.rados.0.smithi059.stdout:1341: done (4 left) 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1342: done (3 left) 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1343: done (2 left) 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1345: finishing copy_from to smithi05955964-22 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1345: got expected ENOENT (src dne) 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1346: read oid 43 snap -1 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1346: expect (ObjNum 357 snap 107 seq_num 357) 2022-04-28T19:48:36.388 INFO:tasks.rados.rados.0.smithi059.stdout:1345: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:48:36.389 INFO:tasks.rados.rados.0.smithi059.stdout:1344: finishing write tid 3 to smithi05955964-34 2022-04-28T19:48:36.389 INFO:tasks.rados.rados.0.smithi059.stdout:1344: finishing write tid 4 to smithi05955964-34 2022-04-28T19:48:36.389 INFO:tasks.rados.rados.0.smithi059.stdout:1345: done (2 left) 2022-04-28T19:48:36.389 INFO:tasks.rados.rados.0.smithi059.stdout:1347: snap_remove snap 110 2022-04-28T19:48:36.528 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:36 smithi139 bash[23439]: cephadm 2022-04-28T19:48:34.990300+0000 mgr.y (mgr.14985) 8 : cephadm [INF] Upgrade: Updating mgr.x 2022-04-28T19:48:36.528 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:36 smithi139 bash[23439]: cephadm 2022-04-28T19:48:34.997675+0000 mgr.y (mgr.14985) 9 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:48:36.528 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:36 smithi139 bash[23439]: cluster 2022-04-28T19:48:35.271378+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2022-04-28T19:48:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:36 smithi059 bash[29114]: cephadm 2022-04-28T19:48:34.990300+0000 mgr.y (mgr.14985) 8 : cephadm [INF] Upgrade: Updating mgr.x 2022-04-28T19:48:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:36 smithi059 bash[29114]: cephadm 2022-04-28T19:48:34.997675+0000 mgr.y (mgr.14985) 9 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:48:36.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:36 smithi059 bash[29114]: cluster 2022-04-28T19:48:35.271378+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2022-04-28T19:48:36.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:36 smithi059 bash[23786]: cephadm 2022-04-28T19:48:34.990300+0000 mgr.y (mgr.14985) 8 : cephadm [INF] Upgrade: Updating mgr.x 2022-04-28T19:48:36.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:36 smithi059 bash[23786]: cephadm 2022-04-28T19:48:34.997675+0000 mgr.y (mgr.14985) 9 : cephadm [INF] Deploying daemon mgr.x on smithi139 2022-04-28T19:48:36.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:36 smithi059 bash[23786]: cluster 2022-04-28T19:48:35.271378+0000 mon.a (mon.0) 1004 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2022-04-28T19:48:37.246 INFO:tasks.rados.rados.0.smithi059.stdout:1347: done (2 left) 2022-04-28T19:48:37.246 INFO:tasks.rados.rados.0.smithi059.stdout:1348: snap_remove snap 125 2022-04-28T19:48:37.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:37 smithi139 systemd[1]: Stopping Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:48:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:37 smithi139 bash[23439]: cluster 2022-04-28T19:48:36.276117+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2022-04-28T19:48:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:37 smithi139 bash[23439]: cluster 2022-04-28T19:48:36.613801+0000 mgr.y (mgr.14985) 10 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 240 MiB data, 353 MiB used, 707 GiB / 715 GiB avail; 22 MiB/s rd, 12 MiB/s wr, 58 op/s 2022-04-28T19:48:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:37 smithi139 bash[23439]: cluster 2022-04-28T19:48:37.242382+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2022-04-28T19:48:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:37 smithi059 bash[29114]: cluster 2022-04-28T19:48:36.276117+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2022-04-28T19:48:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:37 smithi059 bash[29114]: cluster 2022-04-28T19:48:36.613801+0000 mgr.y (mgr.14985) 10 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 240 MiB data, 353 MiB used, 707 GiB / 715 GiB avail; 22 MiB/s rd, 12 MiB/s wr, 58 op/s 2022-04-28T19:48:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:37 smithi059 bash[29114]: cluster 2022-04-28T19:48:37.242382+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2022-04-28T19:48:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:37 smithi059 bash[23786]: cluster 2022-04-28T19:48:36.276117+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2022-04-28T19:48:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:37 smithi059 bash[23786]: cluster 2022-04-28T19:48:36.613801+0000 mgr.y (mgr.14985) 10 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 240 MiB data, 353 MiB used, 707 GiB / 715 GiB avail; 22 MiB/s rd, 12 MiB/s wr, 58 op/s 2022-04-28T19:48:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:37 smithi059 bash[23786]: cluster 2022-04-28T19:48:37.242382+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2022-04-28T19:48:38.257 INFO:tasks.rados.rados.0.smithi059.stdout:1348: done (2 left) 2022-04-28T19:48:38.258 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 128 2022-04-28T19:48:38.258 INFO:tasks.rados.rados.0.smithi059.stdout:1349: seq_num 426 ranges {1654784=737280,2392064=712704,3104768=802816,3907584=90112} 2022-04-28T19:48:38.271 INFO:tasks.rados.rados.0.smithi059.stdout:1349: writing smithi05955964-28 from 1654784 to 2392064 tid 1 2022-04-28T19:48:38.275 INFO:tasks.rados.rados.0.smithi059.stdout:1349: writing smithi05955964-28 from 2392064 to 3104768 tid 2 2022-04-28T19:48:38.280 INFO:tasks.rados.rados.0.smithi059.stdout:1349: writing smithi05955964-28 from 3104768 to 3907584 tid 3 2022-04-28T19:48:38.281 INFO:tasks.rados.rados.0.smithi059.stdout:1349: writing smithi05955964-28 from 3907584 to 3997696 tid 4 2022-04-28T19:48:38.281 INFO:tasks.rados.rados.0.smithi059.stdout:1350: snap_remove snap 101 2022-04-28T19:48:39.266 INFO:tasks.rados.rados.0.smithi059.stdout:1350: done (3 left) 2022-04-28T19:48:39.267 INFO:tasks.rados.rados.0.smithi059.stdout:1351: delete oid 12 current snap is 128 2022-04-28T19:48:39.267 INFO:tasks.rados.rados.0.smithi059.stdout:1344: finishing write tid 6 to smithi05955964-34 2022-04-28T19:48:39.267 INFO:tasks.rados.rados.0.smithi059.stdout:1344: finishing write tid 7 to smithi05955964-34 2022-04-28T19:48:39.267 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 659 (ObjNum 425 snap 128 seq_num 425) dirty exists 2022-04-28T19:48:39.267 INFO:tasks.rados.rados.0.smithi059.stdout:1344: left oid 34 (ObjNum 425 snap 128 seq_num 425) 2022-04-28T19:48:39.267 INFO:tasks.rados.rados.0.smithi059.stdout:1346: expect (ObjNum 357 snap 107 seq_num 357) 2022-04-28T19:48:39.467 INFO:tasks.rados.rados.0.smithi059.stdout:1349: finishing write tid 1 to smithi05955964-28 2022-04-28T19:48:39.467 INFO:tasks.rados.rados.0.smithi059.stdout:1349: finishing write tid 2 to smithi05955964-28 2022-04-28T19:48:39.467 INFO:tasks.rados.rados.0.smithi059.stdout:1349: finishing write tid 3 to smithi05955964-28 2022-04-28T19:48:39.467 INFO:tasks.rados.rados.0.smithi059.stdout:1349: finishing write tid 4 to smithi05955964-28 2022-04-28T19:48:39.467 INFO:tasks.rados.rados.0.smithi059.stdout:1349: finishing write tid 6 to smithi05955964-28 2022-04-28T19:48:39.467 INFO:tasks.rados.rados.0.smithi059.stdout:1349: finishing write tid 7 to smithi05955964-28 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 395 (ObjNum 426 snap 128 seq_num 426) dirty exists 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:1349: left oid 28 (ObjNum 426 snap 128 seq_num 426) 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:1344: done (3 left) 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:1346: done (2 left) 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:1349: done (1 left) 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:1351: done (0 left) 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 128 2022-04-28T19:48:39.468 INFO:tasks.rados.rados.0.smithi059.stdout:1352: seq_num 428 ranges {1130496=778240,1908736=483328,2392064=409600,2801664=417792,3219456=294912} 2022-04-28T19:48:39.486 INFO:tasks.rados.rados.0.smithi059.stdout:1352: writing smithi05955964-15 from 1130496 to 1908736 tid 1 2022-04-28T19:48:39.488 INFO:tasks.rados.rados.0.smithi059.stdout:1352: writing smithi05955964-15 from 1908736 to 2392064 tid 2 2022-04-28T19:48:39.491 INFO:tasks.rados.rados.0.smithi059.stdout:1352: writing smithi05955964-15 from 2392064 to 2801664 tid 3 2022-04-28T19:48:39.493 INFO:tasks.rados.rados.0.smithi059.stdout:1352: writing smithi05955964-15 from 2801664 to 3219456 tid 4 2022-04-28T19:48:39.495 INFO:tasks.rados.rados.0.smithi059.stdout:1352: writing smithi05955964-15 from 3219456 to 3514368 tid 5 2022-04-28T19:48:39.495 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 128 2022-04-28T19:48:39.495 INFO:tasks.rados.rados.0.smithi059.stdout:1353: seq_num 429 ranges {3203072=794624,3997696=770048,4767744=622592,5390336=212992} 2022-04-28T19:48:39.517 INFO:tasks.rados.rados.0.smithi059.stdout:1353: writing smithi05955964-22 from 3203072 to 3997696 tid 1 2022-04-28T19:48:39.522 INFO:tasks.rados.rados.0.smithi059.stdout:1353: writing smithi05955964-22 from 3997696 to 4767744 tid 2 2022-04-28T19:48:39.526 INFO:tasks.rados.rados.0.smithi059.stdout:1353: writing smithi05955964-22 from 4767744 to 5390336 tid 3 2022-04-28T19:48:39.527 INFO:tasks.rados.rados.0.smithi059.stdout:1353: writing smithi05955964-22 from 5390336 to 5603328 tid 4 2022-04-28T19:48:39.527 INFO:tasks.rados.rados.0.smithi059.stdout:1354: read oid 30 snap -1 2022-04-28T19:48:39.527 INFO:tasks.rados.rados.0.smithi059.stdout:1354: expect deleted 2022-04-28T19:48:39.528 INFO:tasks.rados.rados.0.smithi059.stdout:1355: read oid 50 snap -1 2022-04-28T19:48:39.528 INFO:tasks.rados.rados.0.smithi059.stdout:1355: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:48:39.528 INFO:tasks.rados.rados.0.smithi059.stdout:1356: snap_remove snap 127 2022-04-28T19:48:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:39 smithi059 bash[29114]: cluster 2022-04-28T19:48:38.254888+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2022-04-28T19:48:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:39 smithi059 bash[29114]: cluster 2022-04-28T19:48:38.614536+0000 mgr.y (mgr.14985) 11 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail; 5.4 MiB/s rd, 568 KiB/s wr, 9 op/s 2022-04-28T19:48:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:39 smithi059 bash[23786]: cluster 2022-04-28T19:48:38.254888+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2022-04-28T19:48:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:39 smithi059 bash[23786]: cluster 2022-04-28T19:48:38.614536+0000 mgr.y (mgr.14985) 11 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail; 5.4 MiB/s rd, 568 KiB/s wr, 9 op/s 2022-04-28T19:48:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:39 smithi139 bash[23439]: cluster 2022-04-28T19:48:38.254888+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2022-04-28T19:48:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:39 smithi139 bash[23439]: cluster 2022-04-28T19:48:38.614536+0000 mgr.y (mgr.14985) 11 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail; 5.4 MiB/s rd, 568 KiB/s wr, 9 op/s 2022-04-28T19:48:40.282 INFO:tasks.rados.rados.0.smithi059.stdout:1356: done (4 left) 2022-04-28T19:48:40.282 INFO:tasks.rados.rados.0.smithi059.stdout:1357: snap_create 2022-04-28T19:48:40.282 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 1 to smithi05955964-15 2022-04-28T19:48:40.282 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 2 to smithi05955964-15 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 3 to smithi05955964-15 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 4 to smithi05955964-15 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 5 to smithi05955964-15 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 7 to smithi05955964-15 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1352: finishing write tid 8 to smithi05955964-15 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 654 (ObjNum 428 snap 128 seq_num 428) dirty exists 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1352: left oid 15 (ObjNum 428 snap 128 seq_num 428) 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1353: finishing write tid 1 to smithi05955964-22 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1353: finishing write tid 2 to smithi05955964-22 2022-04-28T19:48:40.283 INFO:tasks.rados.rados.0.smithi059.stdout:1353: finishing write tid 3 to smithi05955964-22 2022-04-28T19:48:40.284 INFO:tasks.rados.rados.0.smithi059.stdout:1353: finishing write tid 4 to smithi05955964-22 2022-04-28T19:48:40.284 INFO:tasks.rados.rados.0.smithi059.stdout:1353: finishing write tid 6 to smithi05955964-22 2022-04-28T19:48:40.284 INFO:tasks.rados.rados.0.smithi059.stdout:1353: finishing write tid 7 to smithi05955964-22 2022-04-28T19:48:40.284 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 677 (ObjNum 429 snap 128 seq_num 429) dirty exists 2022-04-28T19:48:40.284 INFO:tasks.rados.rados.0.smithi059.stdout:1353: left oid 22 (ObjNum 429 snap 128 seq_num 429) 2022-04-28T19:48:40.284 INFO:tasks.rados.rados.0.smithi059.stdout:1355: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:48:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:40 smithi059 bash[29114]: cluster 2022-04-28T19:48:39.264449+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2022-04-28T19:48:40.710 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:40 smithi059 bash[23786]: cluster 2022-04-28T19:48:39.264449+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2022-04-28T19:48:40.711 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:40 smithi139 bash[23439]: cluster 2022-04-28T19:48:39.264449+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2022-04-28T19:48:41.263 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:48:41.282 INFO:tasks.rados.rados.0.smithi059.stdout:1352: done (4 left) 2022-04-28T19:48:41.282 INFO:tasks.rados.rados.0.smithi059.stdout:1353: done (3 left) 2022-04-28T19:48:41.282 INFO:tasks.rados.rados.0.smithi059.stdout:1354: done (2 left) 2022-04-28T19:48:41.283 INFO:tasks.rados.rados.0.smithi059.stdout:1355: done (1 left) 2022-04-28T19:48:41.283 INFO:tasks.rados.rados.0.smithi059.stdout:1357: done (0 left) 2022-04-28T19:48:41.283 INFO:tasks.rados.rados.0.smithi059.stdout:1358: copy_from oid 48 from oid 8 current snap is 129 2022-04-28T19:48:41.283 INFO:tasks.rados.rados.0.smithi059.stdout:1359: read oid 41 snap -1 2022-04-28T19:48:41.283 INFO:tasks.rados.rados.0.smithi059.stdout:1359: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:48:41.283 INFO:tasks.rados.rados.0.smithi059.stdout:1360: snap_remove snap 111 2022-04-28T19:48:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:41 smithi059 bash[29114]: cluster 2022-04-28T19:48:40.280118+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2022-04-28T19:48:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:41 smithi059 bash[29114]: cluster 2022-04-28T19:48:40.615225+0000 mgr.y (mgr.14985) 12 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail; 5.4 MiB/s rd, 568 KiB/s wr, 9 op/s 2022-04-28T19:48:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:41 smithi059 bash[23786]: cluster 2022-04-28T19:48:40.280118+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2022-04-28T19:48:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:41 smithi059 bash[23786]: cluster 2022-04-28T19:48:40.615225+0000 mgr.y (mgr.14985) 12 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail; 5.4 MiB/s rd, 568 KiB/s wr, 9 op/s 2022-04-28T19:48:41.594 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:48:41.594 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (7m) 9s ago 8m 14.0M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (7m) 9s ago 7m 25.6M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (6m) 9s ago 6m 13.7M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (6m) 9s ago 6m 16.6M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 running (78s) 9s ago 11m 374M - 16.2.7-969-g4fa079ba 863db77b5b5e 646fe4292ebe 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (28s) 9s ago 14m 402M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (14m) 9s ago 14m 63.3M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 239497cf6769 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (12m) 9s ago 12m 60.0M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 0e12092e0bc3 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (12m) 9s ago 12m 57.8M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:48:41.595 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (8m) 9s ago 8m 10.7M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (8m) 9s ago 8m 10.5M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (11m) 9s ago 11m 185M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 547b5580e073 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (11m) 9s ago 11m 151M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (10m) 9s ago 10m 195M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (10m) 9s ago 10m 160M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (9m) 9s ago 9m 165M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (9m) 9s ago 9m 178M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (9m) 9s ago 9m 132M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (8m) 9s ago 8m 118M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:48:41.596 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (45s) 9s ago 8m 30.1M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:48:41.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:41 smithi139 bash[23439]: cluster 2022-04-28T19:48:40.280118+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2022-04-28T19:48:41.721 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:41 smithi139 bash[23439]: cluster 2022-04-28T19:48:40.615225+0000 mgr.y (mgr.14985) 12 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail; 5.4 MiB/s rd, 568 KiB/s wr, 9 op/s 2022-04-28T19:48:41.960 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:48:41.960 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:48:41.961 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:48:41.962 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:48:41.962 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:48:41.962 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 13, 2022-04-28T19:48:41.962 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:48:41.962 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:48:41.962 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:48:42.248 INFO:tasks.rados.rados.0.smithi059.stdout:1360: done (2 left) 2022-04-28T19:48:42.248 INFO:tasks.rados.rados.0.smithi059.stdout:1361: snap_remove snap 123 2022-04-28T19:48:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.247229+0000 mgr.y (mgr.14985) 13 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: cluster 2022-04-28T19:48:41.276794+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2022-04-28T19:48:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.566488+0000 mgr.y (mgr.14985) 14 : audit [DBG] from='client.24940 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.572207+0000 mon.c (mon.1) 120 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.572995+0000 mon.c (mon.1) 121 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.573867+0000 mon.c (mon.1) 122 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.574569+0000 mon.c (mon.1) 123 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.575204+0000 mon.c (mon.1) 124 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.575878+0000 mon.c (mon.1) 125 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.578849+0000 mon.c (mon.1) 126 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.579613+0000 mon.c (mon.1) 127 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.580387+0000 mon.c (mon.1) 128 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.581120+0000 mon.c (mon.1) 129 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.581824+0000 mon.c (mon.1) 130 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: audit 2022-04-28T19:48:41.959127+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.59:0/477761365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[23786]: cluster 2022-04-28T19:48:42.245631+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.247229+0000 mgr.y (mgr.14985) 13 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: cluster 2022-04-28T19:48:41.276794+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.566488+0000 mgr.y (mgr.14985) 14 : audit [DBG] from='client.24940 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.572207+0000 mon.c (mon.1) 120 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.572995+0000 mon.c (mon.1) 121 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.573867+0000 mon.c (mon.1) 122 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.574569+0000 mon.c (mon.1) 123 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.575204+0000 mon.c (mon.1) 124 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.575878+0000 mon.c (mon.1) 125 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.578849+0000 mon.c (mon.1) 126 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.579613+0000 mon.c (mon.1) 127 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.580387+0000 mon.c (mon.1) 128 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.581120+0000 mon.c (mon.1) 129 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.581824+0000 mon.c (mon.1) 130 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: audit 2022-04-28T19:48:41.959127+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.59:0/477761365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:42.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:42 smithi059 bash[29114]: cluster 2022-04-28T19:48:42.245631+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2022-04-28T19:48:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.247229+0000 mgr.y (mgr.14985) 13 : audit [DBG] from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: cluster 2022-04-28T19:48:41.276794+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.566488+0000 mgr.y (mgr.14985) 14 : audit [DBG] from='client.24940 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.572207+0000 mon.c (mon.1) 120 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.572995+0000 mon.c (mon.1) 121 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.573867+0000 mon.c (mon.1) 122 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.574569+0000 mon.c (mon.1) 123 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.575204+0000 mon.c (mon.1) 124 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.575878+0000 mon.c (mon.1) 125 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.578849+0000 mon.c (mon.1) 126 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:48:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.579613+0000 mon.c (mon.1) 127 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.580387+0000 mon.c (mon.1) 128 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.581120+0000 mon.c (mon.1) 129 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.581824+0000 mon.c (mon.1) 130 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:48:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: audit 2022-04-28T19:48:41.959127+0000 mon.a (mon.0) 1011 : audit [DBG] from='client.? 172.21.15.59:0/477761365' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:42 smithi139 bash[23439]: cluster 2022-04-28T19:48:42.245631+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2022-04-28T19:48:43.265 INFO:tasks.rados.rados.0.smithi059.stdout:1361: done (2 left) 2022-04-28T19:48:43.265 INFO:tasks.rados.rados.0.smithi059.stdout:1362: copy_from oid 5 from oid 45 current snap is 129 2022-04-28T19:48:43.265 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 129 2022-04-28T19:48:43.265 INFO:tasks.rados.rados.0.smithi059.stdout:1363: seq_num 430 ranges {2875392=8192} 2022-04-28T19:48:43.282 INFO:tasks.rados.rados.0.smithi059.stdout:1363: writing smithi05955964-33 from 2875392 to 2883584 tid 1 2022-04-28T19:48:43.282 INFO:tasks.rados.rados.0.smithi059.stdout:1359: expect (ObjNum 187 snap 53 seq_num 187) 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:1358: finishing copy_from to smithi05955964-48 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 372 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:1358: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:1363: finishing write tid 1 to smithi05955964-33 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:1363: finishing write tid 3 to smithi05955964-33 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:1363: finishing write tid 4 to smithi05955964-33 2022-04-28T19:48:43.323 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 697 (ObjNum 430 snap 129 seq_num 430) dirty exists 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1363: left oid 33 (ObjNum 430 snap 129 seq_num 430) 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1362: finishing copy_from to smithi05955964-5 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 617 (ObjNum 405 snap 125 seq_num 405) dirty exists 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1362: finishing copy_from racing read to smithi05955964-5 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1358: done (3 left) 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1359: done (2 left) 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1362: done (1 left) 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1363: done (0 left) 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 129 2022-04-28T19:48:43.324 INFO:tasks.rados.rados.0.smithi059.stdout:1364: seq_num 431 ranges {4538368=24576} 2022-04-28T19:48:43.350 INFO:tasks.rados.rados.0.smithi059.stdout:1364: writing smithi05955964-8 from 4538368 to 4562944 tid 1 2022-04-28T19:48:43.351 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 129 2022-04-28T19:48:43.351 INFO:tasks.rados.rados.0.smithi059.stdout:1365: seq_num 432 ranges {3325952=40960} 2022-04-28T19:48:43.370 INFO:tasks.rados.rados.0.smithi059.stdout:1365: writing smithi05955964-18 from 3325952 to 3366912 tid 1 2022-04-28T19:48:43.370 INFO:tasks.rados.rados.0.smithi059.stdout:1366: rmattr oid 15 current snap is 129 2022-04-28T19:48:43.370 INFO:tasks.rados.rados.0.smithi059.stdout:1364: finishing write tid 1 to smithi05955964-8 2022-04-28T19:48:43.370 INFO:tasks.rados.rados.0.smithi059.stdout:1364: finishing write tid 3 to smithi05955964-8 2022-04-28T19:48:43.370 INFO:tasks.rados.rados.0.smithi059.stdout:1364: finishing write tid 4 to smithi05955964-8 2022-04-28T19:48:43.371 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 658 (ObjNum 431 snap 129 seq_num 431) dirty exists 2022-04-28T19:48:43.371 INFO:tasks.rados.rados.0.smithi059.stdout:1364: left oid 8 (ObjNum 431 snap 129 seq_num 431) 2022-04-28T19:48:43.371 INFO:tasks.rados.rados.0.smithi059.stdout:1364: done (2 left) 2022-04-28T19:48:43.371 INFO:tasks.rados.rados.0.smithi059.stdout:1367: delete oid 22 current snap is 129 2022-04-28T19:48:43.373 INFO:tasks.rados.rados.0.smithi059.stdout:1367: done (2 left) 2022-04-28T19:48:43.373 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 129 2022-04-28T19:48:43.373 INFO:tasks.rados.rados.0.smithi059.stdout:1368: seq_num 434 ranges {1974272=73728} 2022-04-28T19:48:43.385 INFO:tasks.rados.rados.0.smithi059.stdout:1368: writing smithi05955964-3 from 1974272 to 2048000 tid 1 2022-04-28T19:48:43.385 INFO:tasks.rados.rados.0.smithi059.stdout:1369: delete oid 35 current snap is 129 2022-04-28T19:48:43.385 INFO:tasks.rados.rados.0.smithi059.stdout:1365: finishing write tid 1 to smithi05955964-18 2022-04-28T19:48:43.386 INFO:tasks.rados.rados.0.smithi059.stdout:1365: finishing write tid 3 to smithi05955964-18 2022-04-28T19:48:43.386 INFO:tasks.rados.rados.0.smithi059.stdout:1365: finishing write tid 4 to smithi05955964-18 2022-04-28T19:48:43.386 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 698 (ObjNum 432 snap 129 seq_num 432) dirty exists 2022-04-28T19:48:43.386 INFO:tasks.rados.rados.0.smithi059.stdout:1365: left oid 18 (ObjNum 432 snap 129 seq_num 432) 2022-04-28T19:48:43.386 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 668 (ObjNum 428 snap 128 seq_num 428) dirty exists 2022-04-28T19:48:43.387 INFO:tasks.rados.rados.0.smithi059.stdout:1365: done (3 left) 2022-04-28T19:48:43.387 INFO:tasks.rados.rados.0.smithi059.stdout:1366: done (2 left) 2022-04-28T19:48:43.387 INFO:tasks.rados.rados.0.smithi059.stdout:1369: done (1 left) 2022-04-28T19:48:43.387 INFO:tasks.rados.rados.0.smithi059.stdout:1370: setattr oid 17 current snap is 129 2022-04-28T19:48:43.389 INFO:tasks.rados.rados.0.smithi059.stdout:1368: finishing write tid 1 to smithi05955964-3 2022-04-28T19:48:43.390 INFO:tasks.rados.rados.0.smithi059.stdout:1371: read oid 21 snap -1 2022-04-28T19:48:43.390 INFO:tasks.rados.rados.0.smithi059.stdout:1371: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:43.391 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 129 2022-04-28T19:48:43.391 INFO:tasks.rados.rados.0.smithi059.stdout:1372: seq_num 436 ranges {3645440=106496} 2022-04-28T19:48:43.411 INFO:tasks.rados.rados.0.smithi059.stdout:1372: writing smithi05955964-26 from 3645440 to 3751936 tid 1 2022-04-28T19:48:43.411 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 129 2022-04-28T19:48:43.411 INFO:tasks.rados.rados.0.smithi059.stdout:1373: seq_num 437 ranges {1957888=122880} 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:1373: writing smithi05955964-7 from 1957888 to 2080768 tid 1 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:1374: rollback oid 35 current snap is 129 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 35 to 124 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:1368: finishing write tid 3 to smithi05955964-3 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:1375: rollback oid 4 current snap is 129 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 124 2022-04-28T19:48:43.423 INFO:tasks.rados.rados.0.smithi059.stdout:1371: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:43.427 INFO:tasks.rados.rados.0.smithi059.stdout:1368: finishing write tid 4 to smithi05955964-3 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 628 (ObjNum 434 snap 129 seq_num 434) dirty exists 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1368: left oid 3 (ObjNum 434 snap 129 seq_num 434) 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 349 (ObjNum 412 snap 125 seq_num 412) dirty exists 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1372: finishing write tid 1 to smithi05955964-26 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1372: finishing write tid 3 to smithi05955964-26 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1372: finishing write tid 4 to smithi05955964-26 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 697 (ObjNum 436 snap 129 seq_num 436) dirty exists 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1372: left oid 26 (ObjNum 436 snap 129 seq_num 436) 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1375: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:48:43.428 INFO:tasks.rados.rados.0.smithi059.stdout:1375: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 648 (ObjNum 384 snap 117 seq_num 384) dirty exists 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1368: done (6 left) 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1370: done (5 left) 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1371: done (4 left) 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1372: done (3 left) 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1375: done (2 left) 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1376: snap_create 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1373: finishing write tid 1 to smithi05955964-7 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1374: finishing rollback tid 1 to smithi05955964-35 2022-04-28T19:48:43.429 INFO:tasks.rados.rados.0.smithi059.stdout:1374: finishing rollback tid 2 to smithi05955964-35 2022-04-28T19:48:43.430 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 434 (ObjNum 265 snap 77 seq_num 265) dirty exists 2022-04-28T19:48:43.430 INFO:tasks.rados.rados.0.smithi059.stdout:1373: finishing write tid 3 to smithi05955964-7 2022-04-28T19:48:43.430 INFO:tasks.rados.rados.0.smithi059.stdout:1373: finishing write tid 4 to smithi05955964-7 2022-04-28T19:48:43.430 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 671 (ObjNum 437 snap 129 seq_num 437) dirty exists 2022-04-28T19:48:43.430 INFO:tasks.rados.rados.0.smithi059.stdout:1373: left oid 7 (ObjNum 437 snap 129 seq_num 437) 2022-04-28T19:48:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[29114]: cluster 2022-04-28T19:48:42.616114+0000 mgr.y (mgr.14985) 15 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[29114]: cluster 2022-04-28T19:48:43.260230+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2022-04-28T19:48:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[23786]: cluster 2022-04-28T19:48:42.616114+0000 mgr.y (mgr.14985) 15 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[23786]: cluster 2022-04-28T19:48:43.260230+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2022-04-28T19:48:43.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:43.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:43.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:43.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:48:43.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:43.517Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:48:43.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:43.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:48:43.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:43.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:43.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:43.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:48:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:43 smithi139 bash[23439]: cluster 2022-04-28T19:48:42.616114+0000 mgr.y (mgr.14985) 15 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 238 MiB data, 357 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:43 smithi139 bash[23439]: cluster 2022-04-28T19:48:43.260230+0000 mon.a (mon.0) 1013 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2022-04-28T19:48:44.303 INFO:tasks.rados.rados.0.smithi059.stdout:1373: done (2 left) 2022-04-28T19:48:44.303 INFO:tasks.rados.rados.0.smithi059.stdout:1374: done (1 left) 2022-04-28T19:48:44.303 INFO:tasks.rados.rados.0.smithi059.stdout:1376: done (0 left) 2022-04-28T19:48:44.304 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 130 2022-04-28T19:48:44.304 INFO:tasks.rados.rados.0.smithi059.stdout:1377: seq_num 438 ranges {10428416=139264} 2022-04-28T19:48:44.368 INFO:tasks.rados.rados.0.smithi059.stdout:1377: writing smithi05955964-5 from 10428416 to 10567680 tid 1 2022-04-28T19:48:44.369 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 130 2022-04-28T19:48:44.369 INFO:tasks.rados.rados.0.smithi059.stdout:1378: seq_num 439 ranges {1712128=155648} 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1378: writing smithi05955964-10 from 1712128 to 1867776 tid 1 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1379: rmattr oid 30 current snap is 130 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1379: done (2 left) 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1380: copy_from oid 33 from oid 50 current snap is 130 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1377: finishing write tid 1 to smithi05955964-5 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1381: read oid 28 snap -1 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:1381: expect (ObjNum 426 snap 128 seq_num 426) 2022-04-28T19:48:44.380 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 130 2022-04-28T19:48:44.381 INFO:tasks.rados.rados.0.smithi059.stdout:1382: seq_num 440 ranges {0=172032} 2022-04-28T19:48:44.381 INFO:tasks.rados.rados.0.smithi059.stdout:1382: writing smithi05955964-22 from 0 to 172032 tid 1 2022-04-28T19:48:44.381 INFO:tasks.rados.rados.0.smithi059.stdout:1383: copy_from oid 41 from oid 25 current snap is 130 2022-04-28T19:48:44.382 INFO:tasks.rados.rados.0.smithi059.stdout:1377: finishing write tid 3 to smithi05955964-5 2022-04-28T19:48:44.382 INFO:tasks.rados.rados.0.smithi059.stdout:1384: snap_remove snap 126 2022-04-28T19:48:45.308 INFO:tasks.rados.rados.0.smithi059.stdout:1384: done (6 left) 2022-04-28T19:48:45.309 INFO:tasks.rados.rados.0.smithi059.stdout:1385: snap_remove snap 124 2022-04-28T19:48:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:45 smithi059 bash[29114]: cluster 2022-04-28T19:48:44.301923+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2022-04-28T19:48:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:45 smithi059 bash[29114]: cluster 2022-04-28T19:48:44.616689+0000 mgr.y (mgr.14985) 16 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 166 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2022-04-28T19:48:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:45 smithi059 bash[23786]: cluster 2022-04-28T19:48:44.301923+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2022-04-28T19:48:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:45 smithi059 bash[23786]: cluster 2022-04-28T19:48:44.616689+0000 mgr.y (mgr.14985) 16 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 166 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2022-04-28T19:48:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:45 smithi139 bash[23439]: cluster 2022-04-28T19:48:44.301923+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2022-04-28T19:48:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:45 smithi139 bash[23439]: cluster 2022-04-28T19:48:44.616689+0000 mgr.y (mgr.14985) 16 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 166 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2022-04-28T19:48:46.315 INFO:tasks.rados.rados.0.smithi059.stdout:1385: done (6 left) 2022-04-28T19:48:46.315 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 130 2022-04-28T19:48:46.315 INFO:tasks.rados.rados.0.smithi059.stdout:1386: seq_num 441 ranges {4087808=188416} 2022-04-28T19:48:46.341 INFO:tasks.rados.rados.0.smithi059.stdout:1386: writing smithi05955964-43 from 4087808 to 4276224 tid 1 2022-04-28T19:48:46.341 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 130 2022-04-28T19:48:46.342 INFO:tasks.rados.rados.0.smithi059.stdout:1387: seq_num 442 ranges {3751936=204800} 2022-04-28T19:48:46.364 INFO:tasks.rados.rados.0.smithi059.stdout:1387: writing smithi05955964-26 from 3751936 to 3956736 tid 1 2022-04-28T19:48:46.364 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 130 2022-04-28T19:48:46.364 INFO:tasks.rados.rados.0.smithi059.stdout:1388: seq_num 443 ranges {0=221184} 2022-04-28T19:48:46.366 INFO:tasks.rados.rados.0.smithi059.stdout:1388: writing smithi05955964-37 from 0 to 221184 tid 1 2022-04-28T19:48:46.366 INFO:tasks.rados.rados.0.smithi059.stdout:1389: copy_from oid 44 from oid 12 current snap is 130 2022-04-28T19:48:46.366 INFO:tasks.rados.rados.0.smithi059.stdout:1390: copy_from oid 9 from oid 24 current snap is 130 2022-04-28T19:48:46.366 INFO:tasks.rados.rados.0.smithi059.stdout:1391: rollback oid 13 current snap is 130 2022-04-28T19:48:46.367 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 13 to 128 2022-04-28T19:48:46.367 INFO:tasks.rados.rados.0.smithi059.stdout:1392: read oid 46 snap -1 2022-04-28T19:48:46.367 INFO:tasks.rados.rados.0.smithi059.stdout:1392: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:48:46.367 INFO:tasks.rados.rados.0.smithi059.stdout:1393: snap_create 2022-04-28T19:48:46.367 INFO:tasks.rados.rados.0.smithi059.stdout:1377: finishing write tid 4 to smithi05955964-5 2022-04-28T19:48:46.368 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 661 (ObjNum 438 snap 130 seq_num 438) dirty exists 2022-04-28T19:48:46.368 INFO:tasks.rados.rados.0.smithi059.stdout:1377: left oid 5 (ObjNum 438 snap 130 seq_num 438) 2022-04-28T19:48:46.368 INFO:tasks.rados.rados.0.smithi059.stdout:1378: finishing write tid 1 to smithi05955964-10 2022-04-28T19:48:46.368 INFO:tasks.rados.rados.0.smithi059.stdout:1378: finishing write tid 3 to smithi05955964-10 2022-04-28T19:48:46.368 INFO:tasks.rados.rados.0.smithi059.stdout:1382: finishing write tid 1 to smithi05955964-22 2022-04-28T19:48:46.368 INFO:tasks.rados.rados.0.smithi059.stdout:1382: finishing write tid 3 to smithi05955964-22 2022-04-28T19:48:46.369 INFO:tasks.rados.rados.0.smithi059.stdout:1378: finishing write tid 4 to smithi05955964-10 2022-04-28T19:48:46.369 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 683 (ObjNum 439 snap 130 seq_num 439) dirty exists 2022-04-28T19:48:46.369 INFO:tasks.rados.rados.0.smithi059.stdout:1378: left oid 10 (ObjNum 439 snap 130 seq_num 439) 2022-04-28T19:48:46.369 INFO:tasks.rados.rados.0.smithi059.stdout:1382: finishing write tid 4 to smithi05955964-22 2022-04-28T19:48:46.369 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 699 (ObjNum 440 snap 130 seq_num 440) dirty exists 2022-04-28T19:48:46.369 INFO:tasks.rados.rados.0.smithi059.stdout:1382: left oid 22 (ObjNum 440 snap 130 seq_num 440) 2022-04-28T19:48:46.370 INFO:tasks.rados.rados.0.smithi059.stdout:1383: finishing copy_from to smithi05955964-41 2022-04-28T19:48:46.370 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 610 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:48:46.370 INFO:tasks.rados.rados.0.smithi059.stdout:1380: finishing copy_from to smithi05955964-33 2022-04-28T19:48:46.370 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 568 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:48:46.370 INFO:tasks.rados.rados.0.smithi059.stdout:1383: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:48:46.370 INFO:tasks.rados.rados.0.smithi059.stdout:1380: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:48:46.371 INFO:tasks.rados.rados.0.smithi059.stdout:1381: expect (ObjNum 426 snap 128 seq_num 426) 2022-04-28T19:48:46.490 INFO:tasks.rados.rados.0.smithi059.stdout:1386: finishing write tid 1 to smithi05955964-43 2022-04-28T19:48:46.490 INFO:tasks.rados.rados.0.smithi059.stdout:1386: finishing write tid 3 to smithi05955964-43 2022-04-28T19:48:46.490 INFO:tasks.rados.rados.0.smithi059.stdout:1386: finishing write tid 4 to smithi05955964-43 2022-04-28T19:48:46.490 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 706 (ObjNum 441 snap 130 seq_num 441) dirty exists 2022-04-28T19:48:46.490 INFO:tasks.rados.rados.0.smithi059.stdout:1386: left oid 43 (ObjNum 441 snap 130 seq_num 441) 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1389: finishing copy_from to smithi05955964-44 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1389: got expected ENOENT (src dne) 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1389: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1387: finishing write tid 1 to smithi05955964-26 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1387: finishing write tid 3 to smithi05955964-26 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1388: finishing write tid 1 to smithi05955964-37 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1388: finishing write tid 3 to smithi05955964-37 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1387: finishing write tid 4 to smithi05955964-26 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 710 (ObjNum 442 snap 130 seq_num 442) dirty exists 2022-04-28T19:48:46.491 INFO:tasks.rados.rados.0.smithi059.stdout:1387: left oid 26 (ObjNum 442 snap 130 seq_num 442) 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:1391: finishing rollback tid 0 to smithi05955964-13 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:1391: finishing rollback tid 1 to smithi05955964-13 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:1391: finishing rollback tid 2 to smithi05955964-13 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 357 (ObjNum 398 snap 122 seq_num 398) dirty exists 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:1388: finishing write tid 4 to smithi05955964-37 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 712 (ObjNum 443 snap 130 seq_num 443) dirty exists 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:1388: left oid 37 (ObjNum 443 snap 130 seq_num 443) 2022-04-28T19:48:46.492 INFO:tasks.rados.rados.0.smithi059.stdout:1392: expect (ObjNum 257 snap 74 seq_num 257) 2022-04-28T19:48:46.575 INFO:tasks.rados.rados.0.smithi059.stdout:1390: finishing copy_from to smithi05955964-9 2022-04-28T19:48:46.575 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 628 (ObjNum 357 snap 107 seq_num 357) dirty exists 2022-04-28T19:48:46.575 INFO:tasks.rados.rados.0.smithi059.stdout:1390: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:48:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:46 smithi059 bash[29114]: cluster 2022-04-28T19:48:45.307007+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2022-04-28T19:48:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:46 smithi059 bash[23786]: cluster 2022-04-28T19:48:45.307007+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2022-04-28T19:48:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:46 smithi139 bash[23439]: cluster 2022-04-28T19:48:45.307007+0000 mon.a (mon.0) 1015 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2022-04-28T19:48:47.250 INFO:tasks.rados.rados.0.smithi059.stdout:1377: done (13 left) 2022-04-28T19:48:47.250 INFO:tasks.rados.rados.0.smithi059.stdout:1378: done (12 left) 2022-04-28T19:48:47.250 INFO:tasks.rados.rados.0.smithi059.stdout:1380: done (11 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1381: done (10 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1382: done (9 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1383: done (8 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1386: done (7 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1387: done (6 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1388: done (5 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1389: done (4 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1390: done (3 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1391: done (2 left) 2022-04-28T19:48:47.251 INFO:tasks.rados.rados.0.smithi059.stdout:1392: done (1 left) 2022-04-28T19:48:47.252 INFO:tasks.rados.rados.0.smithi059.stdout:1393: done (0 left) 2022-04-28T19:48:47.252 INFO:tasks.rados.rados.0.smithi059.stdout:1394: snap_create 2022-04-28T19:48:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:47 smithi059 bash[23786]: cluster 2022-04-28T19:48:46.310106+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2022-04-28T19:48:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:47 smithi059 bash[23786]: cluster 2022-04-28T19:48:46.617337+0000 mgr.y (mgr.14985) 17 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 166 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2022-04-28T19:48:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:47 smithi059 bash[23786]: cluster 2022-04-28T19:48:47.247738+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2022-04-28T19:48:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:47 smithi059 bash[29114]: cluster 2022-04-28T19:48:46.310106+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2022-04-28T19:48:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:47 smithi059 bash[29114]: cluster 2022-04-28T19:48:46.617337+0000 mgr.y (mgr.14985) 17 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 166 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2022-04-28T19:48:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:47 smithi059 bash[29114]: cluster 2022-04-28T19:48:47.247738+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2022-04-28T19:48:47.639 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:47 smithi139 bash[23439]: cluster 2022-04-28T19:48:46.310106+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2022-04-28T19:48:47.639 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:47 smithi139 bash[23439]: cluster 2022-04-28T19:48:46.617337+0000 mgr.y (mgr.14985) 17 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 166 MiB data, 309 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 2.8 MiB/s wr, 7 op/s 2022-04-28T19:48:47.639 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:47 smithi139 bash[23439]: cluster 2022-04-28T19:48:47.247738+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2022-04-28T19:48:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:1394: done (0 left) 2022-04-28T19:48:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:1395: setattr oid 12 current snap is 132 2022-04-28T19:48:48.262 INFO:tasks.rados.rados.0.smithi059.stdout:1396: delete oid 11 current snap is 132 2022-04-28T19:48:48.267 INFO:tasks.rados.rados.0.smithi059.stdout:1396: done (1 left) 2022-04-28T19:48:48.267 INFO:tasks.rados.rados.0.smithi059.stdout:1397: snap_remove snap 129 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 bash[41228]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr.x 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=137/n/a 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 bash[41228]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr-x 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 bash[41228]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mgr.x 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 systemd[1]: Stopped Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:48:48.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:48 smithi139 systemd[1]: Started Ceph mgr.x for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:48:49.271 INFO:tasks.rados.rados.0.smithi059.stdout:1397: done (1 left) 2022-04-28T19:48:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:1398: rmattr oid 47 current snap is 132 2022-04-28T19:48:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 682 (ObjNum -133450976 snap 21893 seq_num 2280311808) dirty exists 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1395: done (1 left) 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1399: read oid 14 snap 128 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1399: expect (ObjNum 381 snap 115 seq_num 381) 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1400: read oid 22 snap -1 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1400: expect (ObjNum 440 snap 130 seq_num 440) 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1401: read oid 18 snap -1 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1401: expect (ObjNum 432 snap 129 seq_num 432) 2022-04-28T19:48:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:1402: rmattr oid 29 current snap is 132 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:1402: done (4 left) 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:1403: copy_from oid 24 from oid 20 current snap is 132 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 132 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:1404: seq_num 445 ranges {0=253952} 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:1404: writing smithi05955964-12 from 0 to 253952 tid 1 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 132 2022-04-28T19:48:49.359 INFO:tasks.rados.rados.0.smithi059.stdout:1405: seq_num 446 ranges {3489792=278528} 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:1405: writing smithi05955964-17 from 3489792 to 3768320 tid 1 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:1406: setattr oid 1 current snap is 132 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:1399: expect (ObjNum 381 snap 115 seq_num 381) 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:1400: expect (ObjNum 440 snap 130 seq_num 440) 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 691 (ObjNum 414 snap 125 seq_num 414) dirty exists 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:1404: finishing write tid 1 to smithi05955964-12 2022-04-28T19:48:49.361 INFO:tasks.rados.rados.0.smithi059.stdout:1404: finishing write tid 3 to smithi05955964-12 2022-04-28T19:48:49.362 INFO:tasks.rados.rados.0.smithi059.stdout:1401: expect (ObjNum 432 snap 129 seq_num 432) 2022-04-28T19:48:49.493 INFO:tasks.rados.rados.0.smithi059.stdout:1404: finishing write tid 4 to smithi05955964-12 2022-04-28T19:48:49.493 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 684 (ObjNum 445 snap 132 seq_num 445) dirty exists 2022-04-28T19:48:49.493 INFO:tasks.rados.rados.0.smithi059.stdout:1404: left oid 12 (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1405: finishing write tid 1 to smithi05955964-17 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1405: finishing write tid 3 to smithi05955964-17 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1405: finishing write tid 4 to smithi05955964-17 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 362 (ObjNum 446 snap 132 seq_num 446) dirty exists 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1405: left oid 17 (ObjNum 446 snap 132 seq_num 446) 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1403: finishing copy_from to smithi05955964-24 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 607 (ObjNum 362 snap 108 seq_num 362) dirty exists 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1403: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1398: done (7 left) 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1399: done (6 left) 2022-04-28T19:48:49.494 INFO:tasks.rados.rados.0.smithi059.stdout:1400: done (5 left) 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:1401: done (4 left) 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:1403: done (3 left) 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:1404: done (2 left) 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:1405: done (1 left) 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:1407: rollback oid 20 current snap is 132 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 128 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 132 2022-04-28T19:48:49.495 INFO:tasks.rados.rados.0.smithi059.stdout:1408: seq_num 447 ranges {4538368=294912} 2022-04-28T19:48:49.516 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: cluster 2022-04-28T19:48:48.258067+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2022-04-28T19:48:49.516 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: audit 2022-04-28T19:48:48.483321+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:49.516 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: audit 2022-04-28T19:48:48.486052+0000 mon.c (mon.1) 131 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:49.517 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: audit 2022-04-28T19:48:48.488251+0000 mon.c (mon.1) 132 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:49.517 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: audit 2022-04-28T19:48:48.489715+0000 mon.c (mon.1) 133 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:49.517 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: cephadm 2022-04-28T19:48:48.491007+0000 mgr.y (mgr.14985) 18 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:49.517 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: audit 2022-04-28T19:48:48.492584+0000 mon.c (mon.1) 134 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:49.517 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[23439]: cluster 2022-04-28T19:48:48.617855+0000 mgr.y (mgr.14985) 19 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.4 MiB/s wr, 23 op/s 2022-04-28T19:48:49.524 INFO:tasks.rados.rados.0.smithi059.stdout:1408: writing smithi05955964-48 from 4538368 to 4833280 tid 1 2022-04-28T19:48:49.524 INFO:tasks.rados.rados.0.smithi059.stdout:1409: rmattr oid 17 current snap is 132 2022-04-28T19:48:49.524 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 594 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:48:49.524 INFO:tasks.rados.rados.0.smithi059.stdout:1406: done (3 left) 2022-04-28T19:48:49.524 INFO:tasks.rados.rados.0.smithi059.stdout:1410: read oid 9 snap -1 2022-04-28T19:48:49.525 INFO:tasks.rados.rados.0.smithi059.stdout:1410: expect (ObjNum 357 snap 107 seq_num 357) 2022-04-28T19:48:49.525 INFO:tasks.rados.rados.0.smithi059.stdout:1411: snap_remove snap 131 2022-04-28T19:48:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: cluster 2022-04-28T19:48:48.258067+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2022-04-28T19:48:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: audit 2022-04-28T19:48:48.483321+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: audit 2022-04-28T19:48:48.486052+0000 mon.c (mon.1) 131 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: audit 2022-04-28T19:48:48.488251+0000 mon.c (mon.1) 132 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: audit 2022-04-28T19:48:48.489715+0000 mon.c (mon.1) 133 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: cephadm 2022-04-28T19:48:48.491007+0000 mgr.y (mgr.14985) 18 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: audit 2022-04-28T19:48:48.492584+0000 mon.c (mon.1) 134 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[23786]: cluster 2022-04-28T19:48:48.617855+0000 mgr.y (mgr.14985) 19 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.4 MiB/s wr, 23 op/s 2022-04-28T19:48:49.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: cluster 2022-04-28T19:48:48.258067+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2022-04-28T19:48:49.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: audit 2022-04-28T19:48:48.483321+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:49.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: audit 2022-04-28T19:48:48.486052+0000 mon.c (mon.1) 131 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:49.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: audit 2022-04-28T19:48:48.488251+0000 mon.c (mon.1) 132 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:49.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: audit 2022-04-28T19:48:48.489715+0000 mon.c (mon.1) 133 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:48:49.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: cephadm 2022-04-28T19:48:48.491007+0000 mgr.y (mgr.14985) 18 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:48:49.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: audit 2022-04-28T19:48:48.492584+0000 mon.c (mon.1) 134 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:48:49.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:49 smithi059 bash[29114]: cluster 2022-04-28T19:48:48.617855+0000 mgr.y (mgr.14985) 19 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.4 MiB/s wr, 23 op/s 2022-04-28T19:48:49.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[41342]: debug 2022-04-28T19:48:49.711+0000 7effcb25a700 1 -- 172.21.15.139:0/2135339791 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56172fb104e0 con 0x56172fc05000 2022-04-28T19:48:50.268 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[41342]: debug 2022-04-28T19:48:49.875+0000 7effdb614680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-04-28T19:48:50.269 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:49 smithi139 bash[41342]: debug 2022-04-28T19:48:49.975+0000 7effdb614680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-04-28T19:48:50.493 INFO:tasks.rados.rados.0.smithi059.stdout:1411: done (4 left) 2022-04-28T19:48:50.494 INFO:tasks.rados.rados.0.smithi059.stdout:1412: delete oid 28 current snap is 132 2022-04-28T19:48:50.494 INFO:tasks.rados.rados.0.smithi059.stdout:1407: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:48:50.494 INFO:tasks.rados.rados.0.smithi059.stdout:1407: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:48:50.494 INFO:tasks.rados.rados.0.smithi059.stdout:1407: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:48:50.494 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 640 (ObjNum 362 snap 108 seq_num 362) dirty exists 2022-04-28T19:48:50.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 363 (ObjNum 446 snap 132 seq_num 446) dirty exists 2022-04-28T19:48:50.495 INFO:tasks.rados.rados.0.smithi059.stdout:1408: finishing write tid 1 to smithi05955964-48 2022-04-28T19:48:50.495 INFO:tasks.rados.rados.0.smithi059.stdout:1408: finishing write tid 3 to smithi05955964-48 2022-04-28T19:48:50.496 INFO:tasks.rados.rados.0.smithi059.stdout:1408: finishing write tid 4 to smithi05955964-48 2022-04-28T19:48:50.496 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 715 (ObjNum 447 snap 132 seq_num 447) dirty exists 2022-04-28T19:48:50.496 INFO:tasks.rados.rados.0.smithi059.stdout:1408: left oid 48 (ObjNum 447 snap 132 seq_num 447) 2022-04-28T19:48:50.496 INFO:tasks.rados.rados.0.smithi059.stdout:1410: expect (ObjNum 357 snap 107 seq_num 357) 2022-04-28T19:48:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:50 smithi059 bash[23786]: cluster 2022-04-28T19:48:49.268172+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2022-04-28T19:48:50.712 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:50 smithi059 bash[29114]: cluster 2022-04-28T19:48:49.268172+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2022-04-28T19:48:50.712 INFO:tasks.rados.rados.0.smithi059.stdout:1407: done (4 left) 2022-04-28T19:48:50.713 INFO:tasks.rados.rados.0.smithi059.stdout:1408: done (3 left) 2022-04-28T19:48:50.713 INFO:tasks.rados.rados.0.smithi059.stdout:1409: done (2 left) 2022-04-28T19:48:50.713 INFO:tasks.rados.rados.0.smithi059.stdout:1410: done (1 left) 2022-04-28T19:48:50.713 INFO:tasks.rados.rados.0.smithi059.stdout:1412: done (0 left) 2022-04-28T19:48:50.713 INFO:tasks.rados.rados.0.smithi059.stdout:1413: copy_from oid 23 from oid 1 current snap is 132 2022-04-28T19:48:50.713 INFO:tasks.rados.rados.0.smithi059.stdout:1414: snap_remove snap 130 2022-04-28T19:48:50.714 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:50 smithi139 bash[23439]: cluster 2022-04-28T19:48:49.268172+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2022-04-28T19:48:50.714 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:50 smithi139 bash[41342]: debug 2022-04-28T19:48:50.411+0000 7effdb614680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-04-28T19:48:51.506 INFO:tasks.rados.rados.0.smithi059.stdout:1414: done (1 left) 2022-04-28T19:48:51.507 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 132 2022-04-28T19:48:51.507 INFO:tasks.rados.rados.0.smithi059.stdout:1415: seq_num 449 ranges {0=327680} 2022-04-28T19:48:51.509 INFO:tasks.rados.rados.0.smithi059.stdout:1415: writing smithi05955964-39 from 0 to 327680 tid 1 2022-04-28T19:48:51.509 INFO:tasks.rados.rados.0.smithi059.stdout:1416: copy_from oid 34 from oid 31 current snap is 132 2022-04-28T19:48:51.509 INFO:tasks.rados.rados.0.smithi059.stdout:1413: finishing copy_from to smithi05955964-23 2022-04-28T19:48:51.509 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 594 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:48:51.509 INFO:tasks.rados.rados.0.smithi059.stdout:1413: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:48:51.509 INFO:tasks.rados.rados.0.smithi059.stdout:1413: done (2 left) 2022-04-28T19:48:51.510 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 132 2022-04-28T19:48:51.510 INFO:tasks.rados.rados.0.smithi059.stdout:1417: seq_num 450 ranges {0=344064} 2022-04-28T19:48:51.511 INFO:tasks.rados.rados.0.smithi059.stdout:1417: writing smithi05955964-38 from 0 to 344064 tid 1 2022-04-28T19:48:51.511 INFO:tasks.rados.rados.0.smithi059.stdout:1418: read oid 17 snap -1 2022-04-28T19:48:51.511 INFO:tasks.rados.rados.0.smithi059.stdout:1418: expect (ObjNum 446 snap 132 seq_num 446) 2022-04-28T19:48:51.511 INFO:tasks.rados.rados.0.smithi059.stdout:1419: delete oid 43 current snap is 132 2022-04-28T19:48:51.515 INFO:tasks.rados.rados.0.smithi059.stdout:1417: finishing write tid 1 to smithi05955964-38 2022-04-28T19:48:51.515 INFO:tasks.rados.rados.0.smithi059.stdout:1417: finishing write tid 3 to smithi05955964-38 2022-04-28T19:48:51.518 INFO:tasks.rados.rados.0.smithi059.stdout:1415: finishing write tid 1 to smithi05955964-39 2022-04-28T19:48:51.518 INFO:tasks.rados.rados.0.smithi059.stdout:1417: finishing write tid 4 to smithi05955964-38 2022-04-28T19:48:51.518 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 674 (ObjNum 450 snap 132 seq_num 450) dirty exists 2022-04-28T19:48:51.519 INFO:tasks.rados.rados.0.smithi059.stdout:1417: left oid 38 (ObjNum 450 snap 132 seq_num 450) 2022-04-28T19:48:51.521 INFO:tasks.rados.rados.0.smithi059.stdout:1415: finishing write tid 3 to smithi05955964-39 2022-04-28T19:48:51.521 INFO:tasks.rados.rados.0.smithi059.stdout:1417: done (4 left) 2022-04-28T19:48:51.521 INFO:tasks.rados.rados.0.smithi059.stdout:1419: done (3 left) 2022-04-28T19:48:51.522 INFO:tasks.rados.rados.0.smithi059.stdout:1420: copy_from oid 40 from oid 26 current snap is 132 2022-04-28T19:48:51.522 INFO:tasks.rados.rados.0.smithi059.stdout:1415: finishing write tid 4 to smithi05955964-39 2022-04-28T19:48:51.522 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 712 (ObjNum 449 snap 132 seq_num 449) dirty exists 2022-04-28T19:48:51.522 INFO:tasks.rados.rados.0.smithi059.stdout:1415: left oid 39 (ObjNum 449 snap 132 seq_num 449) 2022-04-28T19:48:51.522 INFO:tasks.rados.rados.0.smithi059.stdout:1415: done (3 left) 2022-04-28T19:48:51.523 INFO:tasks.rados.rados.0.smithi059.stdout:1421: snap_remove snap 128 2022-04-28T19:48:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:51 smithi139 bash[23439]: cluster 2022-04-28T19:48:50.488703+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2022-04-28T19:48:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:51 smithi139 bash[23439]: cluster 2022-04-28T19:48:50.618528+0000 mgr.y (mgr.14985) 20 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.4 MiB/s wr, 23 op/s 2022-04-28T19:48:51.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:51 smithi139 bash[41342]: debug 2022-04-28T19:48:51.315+0000 7effdb614680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-04-28T19:48:51.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:51 smithi139 bash[41342]: debug 2022-04-28T19:48:51.411+0000 7effdb614680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-04-28T19:48:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:51 smithi059 bash[23786]: cluster 2022-04-28T19:48:50.488703+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2022-04-28T19:48:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:51 smithi059 bash[23786]: cluster 2022-04-28T19:48:50.618528+0000 mgr.y (mgr.14985) 20 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.4 MiB/s wr, 23 op/s 2022-04-28T19:48:51.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:51 smithi059 bash[29114]: cluster 2022-04-28T19:48:50.488703+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2022-04-28T19:48:51.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:51 smithi059 bash[29114]: cluster 2022-04-28T19:48:50.618528+0000 mgr.y (mgr.14985) 20 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.4 MiB/s wr, 23 op/s 2022-04-28T19:48:52.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:51 smithi139 bash[41342]: debug 2022-04-28T19:48:51.663+0000 7effdb614680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-04-28T19:48:52.252 INFO:tasks.rados.rados.0.smithi059.stdout:1421: done (3 left) 2022-04-28T19:48:52.252 INFO:tasks.rados.rados.0.smithi059.stdout:1422: snap_create 2022-04-28T19:48:52.252 INFO:tasks.rados.rados.0.smithi059.stdout:1416: finishing copy_from to smithi05955964-34 2022-04-28T19:48:52.252 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 581 (ObjNum 226 snap 60 seq_num 226) dirty exists 2022-04-28T19:48:52.252 INFO:tasks.rados.rados.0.smithi059.stdout:1416: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:48:52.253 INFO:tasks.rados.rados.0.smithi059.stdout:1418: expect (ObjNum 446 snap 132 seq_num 446) 2022-04-28T19:48:52.429 INFO:tasks.rados.rados.0.smithi059.stdout:1420: finishing copy_from to smithi05955964-40 2022-04-28T19:48:52.429 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 710 (ObjNum 442 snap 130 seq_num 442) dirty exists 2022-04-28T19:48:52.429 INFO:tasks.rados.rados.0.smithi059.stdout:1420: finishing copy_from racing read to smithi05955964-40 2022-04-28T19:48:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:52 smithi059 bash[29114]: cluster 2022-04-28T19:48:51.499196+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2022-04-28T19:48:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:52 smithi059 bash[29114]: cluster 2022-04-28T19:48:52.249621+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2022-04-28T19:48:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:52 smithi059 bash[23786]: cluster 2022-04-28T19:48:51.499196+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2022-04-28T19:48:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:52 smithi059 bash[23786]: cluster 2022-04-28T19:48:52.249621+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2022-04-28T19:48:52.853 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:52 smithi139 bash[23439]: cluster 2022-04-28T19:48:51.499196+0000 mon.a (mon.0) 1022 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2022-04-28T19:48:52.853 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:52 smithi139 bash[23439]: cluster 2022-04-28T19:48:52.249621+0000 mon.a (mon.0) 1023 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2022-04-28T19:48:53.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:52 smithi139 bash[41342]: debug 2022-04-28T19:48:52.847+0000 7effdb614680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-04-28T19:48:53.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[41342]: debug 2022-04-28T19:48:53.043+0000 7effdb614680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:48:53.268 INFO:tasks.rados.rados.0.smithi059.stdout:1416: done (3 left) 2022-04-28T19:48:53.268 INFO:tasks.rados.rados.0.smithi059.stdout:1418: done (2 left) 2022-04-28T19:48:53.269 INFO:tasks.rados.rados.0.smithi059.stdout:1420: done (1 left) 2022-04-28T19:48:53.269 INFO:tasks.rados.rados.0.smithi059.stdout:1422: done (0 left) 2022-04-28T19:48:53.269 INFO:tasks.rados.rados.0.smithi059.stdout:1423: snap_create 2022-04-28T19:48:53.506 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[23439]: cluster 2022-04-28T19:48:52.619229+0000 mgr.y (mgr.14985) 21 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:53.507 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[23439]: cluster 2022-04-28T19:48:53.265853+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2022-04-28T19:48:53.507 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[41342]: debug 2022-04-28T19:48:53.147+0000 7effdb614680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-04-28T19:48:53.507 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[41342]: debug 2022-04-28T19:48:53.279+0000 7effdb614680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-04-28T19:48:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[29114]: cluster 2022-04-28T19:48:52.619229+0000 mgr.y (mgr.14985) 21 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[29114]: cluster 2022-04-28T19:48:53.265853+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2022-04-28T19:48:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:53.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:53.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:53.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:48:53.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:53.517Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" 2022-04-28T19:48:53.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:53.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:48:53.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:53.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:48:53.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:48:53.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:48:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[23786]: cluster 2022-04-28T19:48:52.619229+0000 mgr.y (mgr.14985) 21 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 163 MiB data, 273 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:48:53.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:53 smithi059 bash[23786]: cluster 2022-04-28T19:48:53.265853+0000 mon.a (mon.0) 1024 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2022-04-28T19:48:53.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[41342]: debug 2022-04-28T19:48:53.543+0000 7effdb614680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-04-28T19:48:54.265 INFO:tasks.rados.rados.0.smithi059.stdout:1423: done (0 left) 2022-04-28T19:48:54.265 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 134 2022-04-28T19:48:54.265 INFO:tasks.rados.rados.0.smithi059.stdout:1424: seq_num 452 ranges {2080768=376832} 2022-04-28T19:48:54.280 INFO:tasks.rados.rados.0.smithi059.stdout:1424: writing smithi05955964-7 from 2080768 to 2457600 tid 1 2022-04-28T19:48:54.280 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 134 2022-04-28T19:48:54.280 INFO:tasks.rados.rados.0.smithi059.stdout:1425: seq_num 453 ranges {1589248=393216} 2022-04-28T19:48:54.291 INFO:tasks.rados.rados.0.smithi059.stdout:1425: writing smithi05955964-32 from 1589248 to 1982464 tid 1 2022-04-28T19:48:54.291 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 134 2022-04-28T19:48:54.291 INFO:tasks.rados.rados.0.smithi059.stdout:1426: seq_num 454 ranges {1392640=409600} 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1426: writing smithi05955964-34 from 1392640 to 1802240 tid 1 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1427: snap_create 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1424: finishing write tid 1 to smithi05955964-7 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1424: finishing write tid 3 to smithi05955964-7 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1424: finishing write tid 4 to smithi05955964-7 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 691 (ObjNum 452 snap 134 seq_num 452) dirty exists 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1424: left oid 7 (ObjNum 452 snap 134 seq_num 452) 2022-04-28T19:48:54.302 INFO:tasks.rados.rados.0.smithi059.stdout:1425: finishing write tid 1 to smithi05955964-32 2022-04-28T19:48:54.303 INFO:tasks.rados.rados.0.smithi059.stdout:1425: finishing write tid 3 to smithi05955964-32 2022-04-28T19:48:54.303 INFO:tasks.rados.rados.0.smithi059.stdout:1425: finishing write tid 4 to smithi05955964-32 2022-04-28T19:48:54.303 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 720 (ObjNum 453 snap 134 seq_num 453) dirty exists 2022-04-28T19:48:54.304 INFO:tasks.rados.rados.0.smithi059.stdout:1425: left oid 32 (ObjNum 453 snap 134 seq_num 453) 2022-04-28T19:48:54.307 INFO:tasks.rados.rados.0.smithi059.stdout:1426: finishing write tid 1 to smithi05955964-34 2022-04-28T19:48:54.307 INFO:tasks.rados.rados.0.smithi059.stdout:1426: finishing write tid 3 to smithi05955964-34 2022-04-28T19:48:54.309 INFO:tasks.rados.rados.0.smithi059.stdout:1426: finishing write tid 4 to smithi05955964-34 2022-04-28T19:48:54.309 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 721 (ObjNum 454 snap 134 seq_num 454) dirty exists 2022-04-28T19:48:54.309 INFO:tasks.rados.rados.0.smithi059.stdout:1426: left oid 34 (ObjNum 454 snap 134 seq_num 454) 2022-04-28T19:48:54.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:53 smithi139 bash[41342]: debug 2022-04-28T19:48:53.979+0000 7effdb614680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-04-28T19:48:54.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:54 smithi139 bash[41342]: debug 2022-04-28T19:48:54.131+0000 7effdb614680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-04-28T19:48:55.270 INFO:tasks.rados.rados.0.smithi059.stdout:1424: done (3 left) 2022-04-28T19:48:55.270 INFO:tasks.rados.rados.0.smithi059.stdout:1425: done (2 left) 2022-04-28T19:48:55.270 INFO:tasks.rados.rados.0.smithi059.stdout:1426: done (1 left) 2022-04-28T19:48:55.271 INFO:tasks.rados.rados.0.smithi059.stdout:1427: done (0 left) 2022-04-28T19:48:55.271 INFO:tasks.rados.rados.0.smithi059.stdout:1428: delete oid 40 current snap is 135 2022-04-28T19:48:55.276 INFO:tasks.rados.rados.0.smithi059.stdout:1428: done (0 left) 2022-04-28T19:48:55.276 INFO:tasks.rados.rados.0.smithi059.stdout:1429: snap_remove snap 132 2022-04-28T19:48:55.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[41342]: debug 2022-04-28T19:48:55.035+0000 7effdb614680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-04-28T19:48:55.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[41342]: debug 2022-04-28T19:48:55.135+0000 7effdb614680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-04-28T19:48:55.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[41342]: debug 2022-04-28T19:48:55.247+0000 7effdb614680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: cluster 2022-04-28T19:48:54.263800+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.425784+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.435148+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.445162+0000 mon.c (mon.1) 135 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.448897+0000 mon.c (mon.1) 136 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: cephadm 2022-04-28T19:48:54.450008+0000 mgr.y (mgr.14985) 22 : cephadm [INF] Upgrade: Setting container_image for all mgr 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.457376+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.459117+0000 mon.c (mon.1) 137 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.459591+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.467128+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.468832+0000 mon.c (mon.1) 138 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.469284+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.476601+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.478627+0000 mon.c (mon.1) 139 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: audit 2022-04-28T19:48:54.480248+0000 mon.c (mon.1) 140 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: cephadm 2022-04-28T19:48:54.481380+0000 mgr.y (mgr.14985) 23 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2022-04-28T19:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[23439]: cluster 2022-04-28T19:48:54.619953+0000 mgr.y (mgr.14985) 24 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 128 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 29 op/s 2022-04-28T19:48:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: cluster 2022-04-28T19:48:54.263800+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.425784+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.435148+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.445162+0000 mon.c (mon.1) 135 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.448897+0000 mon.c (mon.1) 136 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: cephadm 2022-04-28T19:48:54.450008+0000 mgr.y (mgr.14985) 22 : cephadm [INF] Upgrade: Setting container_image for all mgr 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.457376+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.459117+0000 mon.c (mon.1) 137 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.459591+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.467128+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2022-04-28T19:48:55.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.468832+0000 mon.c (mon.1) 138 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.469284+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.476601+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.478627+0000 mon.c (mon.1) 139 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: audit 2022-04-28T19:48:54.480248+0000 mon.c (mon.1) 140 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: cephadm 2022-04-28T19:48:54.481380+0000 mgr.y (mgr.14985) 23 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[29114]: cluster 2022-04-28T19:48:54.619953+0000 mgr.y (mgr.14985) 24 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 128 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 29 op/s 2022-04-28T19:48:55.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: cluster 2022-04-28T19:48:54.263800+0000 mon.a (mon.0) 1025 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.425784+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.435148+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.445162+0000 mon.c (mon.1) 135 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.448897+0000 mon.c (mon.1) 136 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: cephadm 2022-04-28T19:48:54.450008+0000 mgr.y (mgr.14985) 22 : cephadm [INF] Upgrade: Setting container_image for all mgr 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.457376+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.459117+0000 mon.c (mon.1) 137 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.459591+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2022-04-28T19:48:55.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.467128+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.468832+0000 mon.c (mon.1) 138 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.469284+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.476601+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.14985 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.478627+0000 mon.c (mon.1) 139 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: audit 2022-04-28T19:48:54.480248+0000 mon.c (mon.1) 140 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: cephadm 2022-04-28T19:48:54.481380+0000 mgr.y (mgr.14985) 23 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2022-04-28T19:48:55.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:55 smithi059 bash[23786]: cluster 2022-04-28T19:48:54.619953+0000 mgr.y (mgr.14985) 24 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 128 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 29 op/s 2022-04-28T19:48:55.764 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[41342]: debug 2022-04-28T19:48:55.467+0000 7effdb614680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-04-28T19:48:56.097 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[41342]: debug 2022-04-28T19:48:55.759+0000 7effdb614680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-04-28T19:48:56.098 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:55 smithi139 bash[41342]: debug 2022-04-28T19:48:55.923+0000 7effdb614680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-04-28T19:48:56.288 INFO:tasks.rados.rados.0.smithi059.stdout:1429: done (0 left) 2022-04-28T19:48:56.288 INFO:tasks.rados.rados.0.smithi059.stdout:1430: snap_remove snap 134 2022-04-28T19:48:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:56 smithi139 bash[23439]: cluster 2022-04-28T19:48:55.268152+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2022-04-28T19:48:56.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:56 smithi139 bash[41342]: debug 2022-04-28T19:48:56.091+0000 7effdb614680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:48:56.368 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:56 smithi139 bash[41342]: debug 2022-04-28T19:48:56.303+0000 7effdb614680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-04-28T19:48:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:56 smithi059 bash[29114]: cluster 2022-04-28T19:48:55.268152+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2022-04-28T19:48:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:56 smithi059 bash[23786]: cluster 2022-04-28T19:48:55.268152+0000 mon.a (mon.0) 1033 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2022-04-28T19:48:56.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:48:56 smithi139 bash[41342]: debug 2022-04-28T19:48:56.407+0000 7effdb614680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-04-28T19:48:57.255 INFO:tasks.rados.rados.0.smithi059.stdout:1430: done (0 left) 2022-04-28T19:48:57.255 INFO:tasks.rados.rados.0.smithi059.stdout:1431: snap_remove snap 133 2022-04-28T19:48:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: cluster 2022-04-28T19:48:56.285110+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2022-04-28T19:48:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: cluster 2022-04-28T19:48:56.433036+0000 mon.a (mon.0) 1035 : cluster [DBG] Standby manager daemon x restarted 2022-04-28T19:48:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: cluster 2022-04-28T19:48:56.433228+0000 mon.a (mon.0) 1036 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:48:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:56.435839+0000 mon.b (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:48:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:56.436839+0000 mon.b (mon.2) 137 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:56.437825+0000 mon.b (mon.2) 138 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:48:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:56.438202+0000 mon.b (mon.2) 139 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: cluster 2022-04-28T19:48:56.620581+0000 mgr.y (mgr.14985) 25 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 128 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 29 op/s 2022-04-28T19:48:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:57.233912+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:57.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:57.237856+0000 mon.c (mon.1) 141 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:48:57.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:57.239156+0000 mon.c (mon.1) 142 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:48:57.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:57.240634+0000 mon.c (mon.1) 143 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:57.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: audit 2022-04-28T19:48:57.242629+0000 mon.c (mon.1) 144 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "container_image"}]: dispatch 2022-04-28T19:48:57.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[23786]: cluster 2022-04-28T19:48:57.251326+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: cluster 2022-04-28T19:48:56.285110+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: cluster 2022-04-28T19:48:56.433036+0000 mon.a (mon.0) 1035 : cluster [DBG] Standby manager daemon x restarted 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: cluster 2022-04-28T19:48:56.433228+0000 mon.a (mon.0) 1036 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:56.435839+0000 mon.b (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:56.436839+0000 mon.b (mon.2) 137 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:56.437825+0000 mon.b (mon.2) 138 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:56.438202+0000 mon.b (mon.2) 139 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:57.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: cluster 2022-04-28T19:48:56.620581+0000 mgr.y (mgr.14985) 25 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 128 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 29 op/s 2022-04-28T19:48:57.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:57.233912+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:57.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:57.237856+0000 mon.c (mon.1) 141 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:48:57.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:57.239156+0000 mon.c (mon.1) 142 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:48:57.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:57.240634+0000 mon.c (mon.1) 143 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:57.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: audit 2022-04-28T19:48:57.242629+0000 mon.c (mon.1) 144 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "container_image"}]: dispatch 2022-04-28T19:48:57.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:57 smithi059 bash[29114]: cluster 2022-04-28T19:48:57.251326+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2022-04-28T19:48:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: cluster 2022-04-28T19:48:56.285110+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2022-04-28T19:48:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: cluster 2022-04-28T19:48:56.433036+0000 mon.a (mon.0) 1035 : cluster [DBG] Standby manager daemon x restarted 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: cluster 2022-04-28T19:48:56.433228+0000 mon.a (mon.0) 1036 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:56.435839+0000 mon.b (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:56.436839+0000 mon.b (mon.2) 137 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:56.437825+0000 mon.b (mon.2) 138 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:56.438202+0000 mon.b (mon.2) 139 : audit [DBG] from='mgr.? 172.21.15.139:0/420811030' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: cluster 2022-04-28T19:48:56.620581+0000 mgr.y (mgr.14985) 25 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 128 MiB data, 273 MiB used, 707 GiB / 715 GiB avail; 9.3 MiB/s rd, 4.3 MiB/s wr, 29 op/s 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:57.233912+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.14985 ' entity='mgr.y' 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:57.237856+0000 mon.c (mon.1) 141 : audit [INF] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:57.239156+0000 mon.c (mon.1) 142 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:48:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:57.240634+0000 mon.c (mon.1) 143 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:48:57.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: audit 2022-04-28T19:48:57.242629+0000 mon.c (mon.1) 144 : audit [DBG] from='mgr.14985 172.21.15.59:0/2770718532' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "container_image"}]: dispatch 2022-04-28T19:48:57.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:57 smithi139 bash[23439]: cluster 2022-04-28T19:48:57.251326+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2022-04-28T19:48:58.268 INFO:tasks.rados.rados.0.smithi059.stdout:1431: done (0 left) 2022-04-28T19:48:58.269 INFO:tasks.rados.rados.0.smithi059.stdout:1432: delete oid 34 current snap is 135 2022-04-28T19:48:58.275 INFO:tasks.rados.rados.0.smithi059.stdout:1432: done (0 left) 2022-04-28T19:48:58.275 INFO:tasks.rados.rados.0.smithi059.stdout:1433: read oid 33 snap -1 2022-04-28T19:48:58.275 INFO:tasks.rados.rados.0.smithi059.stdout:1433: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:48:58.275 INFO:tasks.rados.rados.0.smithi059.stdout:1434: setattr oid 2 current snap is 135 2022-04-28T19:48:58.276 INFO:tasks.rados.rados.0.smithi059.stdout:1435: copy_from oid 28 from oid 27 current snap is 135 2022-04-28T19:48:58.276 INFO:tasks.rados.rados.0.smithi059.stdout:1436: read oid 31 snap -1 2022-04-28T19:48:58.276 INFO:tasks.rados.rados.0.smithi059.stdout:1436: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:48:58.280 INFO:tasks.rados.rados.0.smithi059.stdout:1437: read oid 30 snap -1 2022-04-28T19:48:58.280 INFO:tasks.rados.rados.0.smithi059.stdout:1437: expect deleted 2022-04-28T19:48:58.280 INFO:tasks.rados.rados.0.smithi059.stdout:1438: read oid 35 snap -1 2022-04-28T19:48:58.280 INFO:tasks.rados.rados.0.smithi059.stdout:1438: expect (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:48:58.280 INFO:tasks.rados.rados.0.smithi059.stdout:1439: read oid 16 snap -1 2022-04-28T19:48:58.281 INFO:tasks.rados.rados.0.smithi059.stdout:1439: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:58.281 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 135 2022-04-28T19:48:58.281 INFO:tasks.rados.rados.0.smithi059.stdout:1440: seq_num 457 ranges {1867776=458752} 2022-04-28T19:48:58.292 INFO:tasks.rados.rados.0.smithi059.stdout:1440: writing smithi05955964-10 from 1867776 to 2326528 tid 1 2022-04-28T19:48:58.292 INFO:tasks.rados.rados.0.smithi059.stdout:1441: copy_from oid 21 from oid 7 current snap is 135 2022-04-28T19:48:58.292 INFO:tasks.rados.rados.0.smithi059.stdout:1439: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:48:58.299 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 384 (ObjNum 293 snap 87 seq_num 293) dirty exists 2022-04-28T19:48:58.299 INFO:tasks.rados.rados.0.smithi059.stdout:1436: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:48:58.345 INFO:tasks.rados.rados.0.smithi059.stdout:1435: finishing copy_from to smithi05955964-28 2022-04-28T19:48:58.345 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 384 (ObjNum 385 snap 118 seq_num 385) dirty exists 2022-04-28T19:48:58.345 INFO:tasks.rados.rados.0.smithi059.stdout:1435: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:48:58.345 INFO:tasks.rados.rados.0.smithi059.stdout:1440: finishing write tid 1 to smithi05955964-10 2022-04-28T19:48:58.345 INFO:tasks.rados.rados.0.smithi059.stdout:1440: finishing write tid 3 to smithi05955964-10 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1440: finishing write tid 4 to smithi05955964-10 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 709 (ObjNum 457 snap 135 seq_num 457) dirty exists 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1440: left oid 10 (ObjNum 457 snap 135 seq_num 457) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1434: done (8 left) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1435: done (7 left) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1436: done (6 left) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1437: done (5 left) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1439: done (4 left) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1440: done (3 left) 2022-04-28T19:48:58.346 INFO:tasks.rados.rados.0.smithi059.stdout:1442: setattr oid 2 current snap is 135 2022-04-28T19:48:58.347 INFO:tasks.rados.rados.0.smithi059.stdout:1438: expect (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:48:58.439 INFO:tasks.rados.rados.0.smithi059.stdout:1433: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:48:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[29114]: cephadm 2022-04-28T19:48:57.224684+0000 mgr.y (mgr.14985) 26 : cephadm [INF] Upgrade: Updating mon.a 2022-04-28T19:48:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[29114]: cephadm 2022-04-28T19:48:57.241957+0000 mgr.y (mgr.14985) 27 : cephadm [INF] Deploying daemon mon.a on smithi059 2022-04-28T19:48:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[29114]: cluster 2022-04-28T19:48:57.299677+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e25: y(active, since 36s), standbys: x 2022-04-28T19:48:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[29114]: cluster 2022-04-28T19:48:58.264982+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2022-04-28T19:48:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[23786]: cephadm 2022-04-28T19:48:57.224684+0000 mgr.y (mgr.14985) 26 : cephadm [INF] Upgrade: Updating mon.a 2022-04-28T19:48:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[23786]: cephadm 2022-04-28T19:48:57.241957+0000 mgr.y (mgr.14985) 27 : cephadm [INF] Deploying daemon mon.a on smithi059 2022-04-28T19:48:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[23786]: cluster 2022-04-28T19:48:57.299677+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e25: y(active, since 36s), standbys: x 2022-04-28T19:48:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:58 smithi059 bash[23786]: cluster 2022-04-28T19:48:58.264982+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2022-04-28T19:48:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:58 smithi139 bash[23439]: cephadm 2022-04-28T19:48:57.224684+0000 mgr.y (mgr.14985) 26 : cephadm [INF] Upgrade: Updating mon.a 2022-04-28T19:48:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:58 smithi139 bash[23439]: cephadm 2022-04-28T19:48:57.241957+0000 mgr.y (mgr.14985) 27 : cephadm [INF] Deploying daemon mon.a on smithi059 2022-04-28T19:48:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:58 smithi139 bash[23439]: cluster 2022-04-28T19:48:57.299677+0000 mon.a (mon.0) 1039 : cluster [DBG] mgrmap e25: y(active, since 36s), standbys: x 2022-04-28T19:48:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:58 smithi139 bash[23439]: cluster 2022-04-28T19:48:58.264982+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2022-04-28T19:48:58.701 INFO:tasks.rados.rados.0.smithi059.stdout:1441: finishing copy_from to smithi05955964-21 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 691 (ObjNum 452 snap 134 seq_num 452) dirty exists 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1441: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1433: done (3 left) 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1438: done (2 left) 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1441: done (1 left) 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1443: rollback oid 41 current snap is 135 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1443: done (1 left) 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1444: rollback oid 46 current snap is 135 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1444: done (1 left) 2022-04-28T19:48:58.775 INFO:tasks.rados.rados.0.smithi059.stdout:1445: snap_create 2022-04-28T19:48:58.776 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 385 (ObjNum 293 snap 87 seq_num 293) dirty exists 2022-04-28T19:48:59.307 INFO:tasks.rados.rados.0.smithi059.stdout:1442: done (1 left) 2022-04-28T19:48:59.307 INFO:tasks.rados.rados.0.smithi059.stdout:1445: done (0 left) 2022-04-28T19:48:59.307 INFO:tasks.rados.rados.0.smithi059.stdout:1446: delete oid 14 current snap is 136 2022-04-28T19:48:59.311 INFO:tasks.rados.rados.0.smithi059.stdout:1446: done (0 left) 2022-04-28T19:48:59.312 INFO:tasks.rados.rados.0.smithi059.stdout:1447: read oid 24 snap -1 2022-04-28T19:48:59.312 INFO:tasks.rados.rados.0.smithi059.stdout:1447: expect (ObjNum 362 snap 108 seq_num 362) 2022-04-28T19:48:59.312 INFO:tasks.rados.rados.0.smithi059.stdout:1448: delete oid 15 current snap is 136 2022-04-28T19:48:59.316 INFO:tasks.rados.rados.0.smithi059.stdout:1448: done (1 left) 2022-04-28T19:48:59.316 INFO:tasks.rados.rados.0.smithi059.stdout:1449: snap_remove snap 135 2022-04-28T19:48:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:48:59 smithi059 bash[29114]: cluster 2022-04-28T19:48:58.621257+0000 mgr.y (mgr.14985) 28 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 288 KiB/s wr, 2 op/s 2022-04-28T19:48:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:59 smithi059 bash[23786]: cluster 2022-04-28T19:48:58.621257+0000 mgr.y (mgr.14985) 28 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 288 KiB/s wr, 2 op/s 2022-04-28T19:48:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:59 smithi059 systemd[1]: Stopping Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:48:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:59 smithi059 bash[23786]: debug 2022-04-28T19:48:59.545+0000 7f64270ca700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:48:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:48:59 smithi059 bash[23786]: debug 2022-04-28T19:48:59.545+0000 7f64270ca700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2022-04-28T19:48:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:48:59 smithi139 bash[23439]: cluster 2022-04-28T19:48:58.621257+0000 mgr.y (mgr.14985) 28 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 288 KiB/s wr, 2 op/s 2022-04-28T19:49:00.490 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:00 smithi059 bash[59629]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.a 2022-04-28T19:49:00.490 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:00 smithi059 bash[59629]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon-a 2022-04-28T19:49:00.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:00 smithi059 bash[59629]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.a 2022-04-28T19:49:00.491 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:00 smithi059 systemd[1]: Stopped Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:49:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:00 smithi059 systemd[1]: Started Ceph mon.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:49:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.705+0000 7efc0f39c880 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:49:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.705+0000 7efc0f39c880 0 ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable), process ceph-mon, pid 1 2022-04-28T19:49:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.705+0000 7efc0f39c880 0 pidfile_write: ignore empty --pid-file 2022-04-28T19:49:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 0 load: jerasure load: lrc load: isa 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: RocksDB version: 6.8.1 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Compile date Apr 26 2022 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: DB SUMMARY 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: CURRENT file: CURRENT 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: IDENTITY file: IDENTITY 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 712 Bytes 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 14616650 ; 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.create_if_missing: 0 2022-04-28T19:49:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.env: 0x55f2aacb7000 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.fs: Posix File System 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.info_log: 0x55f2ac7fa440 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:49:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:49:02.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.db_log_dir: 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.709+0000 7efc0f39c880 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:49:02.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.write_buffer_manager: 0x55f2ad4fa090 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:49:02.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:49:02.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:49:02.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.unordered_write: 0 2022-04-28T19:49:02.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:49:02.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:49:02.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.row_cache: None 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.wal_filter: None 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:49:02.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.persist_stats_to_disk: 0 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.write_dbid_to_manifest: 0 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.log_readahead_size: 0 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:49:02.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.strict_bytes_per_sync: 0 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Compression algorithms supported: 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:49:02.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kZSTD supported: 0 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2022-04-28T19:49:02.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.merge_operator: 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f2ac79dd10) 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: cache_index_and_filter_blocks: 1 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: cache_index_and_filter_blocks_with_high_priority: 0 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:49:02.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: pin_top_level_index_and_filter: 1 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: index_type: 0 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: data_block_index_type: 0 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: index_shortening: 1 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: hash_index_allow_collision: 1 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: checksum: 1 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: no_block_cache: 0 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_cache: 0x55f2ac7d4f10 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_cache_name: BinnedLRUCache 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_cache_options: 2022-04-28T19:49:02.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: capacity : 536870912 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: num_shard_bits : 4 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: strict_capacity_limit : 0 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: high_pri_pool_ratio: 0.000 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_cache_compressed: (nil) 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: persistent_cache: (nil) 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_size: 4096 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_size_deviation: 10 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_restart_interval: 16 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: index_block_restart_interval: 1 2022-04-28T19:49:02.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: metadata_block_size: 4096 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: partition_filters: 0 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: use_delta_encoding: 1 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: whole_key_filtering: 1 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: verify_compression: 0 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: read_amp_bytes_per_bit: 0 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: format_version: 2 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: enable_index_compression: 1 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: block_align: 0 2022-04-28T19:49:02.107 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.num_levels: 7 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:49:02.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:49:02.109 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:49:02.110 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:49:02.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:49:02.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:49:02.113 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.ttl: 2592000 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: Options.periodic_compaction_seconds: 0 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11364, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 21 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175341718508, "job": 1, "event": "recovery_started", "log_files": [21]} 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.713+0000 7efc0f39c880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #21 mode 2 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.733+0000 7efc0f39c880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-04-28T19:49:02.114 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.757+0000 7efc0f39c880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175341760790, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 25, "file_size": 14361996, "table_properties": {"data_size": 14330681, "index_size": 26456, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 4037, "raw_key_size": 34451, "raw_average_key_size": 21, "raw_value_size": 14289748, "raw_average_value_size": 9113, "num_data_blocks": 808, "num_entries": 1568, "num_deletions": 364, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1651175341, "oldest_key_time": 3, "file_creation_time": 0}} 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.757+0000 7efc0f39c880 4 rocksdb: [version_set.cc:3826] Creating manifest 26 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.757+0000 7efc0f39c880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.757+0000 7efc0f39c880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175341763062, "job": 1, "event": "recovery_finished"} 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.765+0000 7efc0f39c880 4 rocksdb: DB pointer 0x55f2ac89d800 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.765+0000 7efbf63ce700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.765+0000 7efbf63ce700 4 rocksdb: [db_impl/db_impl.cc:851] 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** DB Stats ** 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Uptime(secs): 0.1 total, 0.1 interval 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:49:02.115 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** Compaction Stats [default] ** 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: L0 1/0 13.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: L6 1/0 22.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Sum 2/0 35.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** Compaction Stats [default] ** 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Uptime(secs): 0.1 total, 0.1 interval 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Flush(GB): cumulative 0.013, interval 0.013 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:49:02.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Cumulative compaction: 0.01 GB write, 258.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Interval compaction: 0.01 GB write, 258.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** Compaction Stats [default] ** 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: L0 1/0 13.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: L6 1/0 22.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:02.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Sum 2/0 35.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** Compaction Stats [default] ** 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 591.4 0.02 0.00 1 0.023 0 0 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Uptime(secs): 0.1 total, 0.0 interval 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Flush(GB): cumulative 0.013, interval 0.000 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:49:02.119 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Cumulative compaction: 0.01 GB write, 258.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.765+0000 7efc0f39c880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] at bind addrs [v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.769+0000 7efc0f39c880 1 mon.a@-1(???) e3 preinit fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.769+0000 7efc0f39c880 0 mon.a@-1(???).mds e8 new map 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.769+0000 7efc0f39c880 0 mon.a@-1(???).mds e8 print_map 2022-04-28T19:49:02.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: e8 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: enable_multiple, ever_enabled_multiple: 0,0 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: legacy client fscid: 1 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Filesystem 'foo' (1) 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: fs_name foo 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: epoch 5 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: flags 12 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: created 2022-04-28T19:42:27.245122+0000 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: modified 2022-04-28T19:42:37.174761+0000 2022-04-28T19:49:02.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: tableserver 0 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: root 0 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: session_timeout 60 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: session_autoclose 300 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: max_file_size 1099511627776 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: required_client_features {} 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: last_failure 0 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: last_failure_osd_epoch 0 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: max_mds 1 2022-04-28T19:49:02.122 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: in 0 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: up {0=34517} 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: failed 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: damaged 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: stopped 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: data_pools [4] 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: metadata_pool 3 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: inline_data disabled 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: balancer 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: standby_count_wanted 1 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: [mds.foo.smithi139.oefiza{0:34517} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] compat {c=[1],r=[1],i=[7ff]}] 2022-04-28T19:49:02.123 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: Standby daemons: 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: [mds.foo.smithi059.tnizcj{-1:14631} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] compat {c=[1],r=[1],i=[7ff]}] 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.769+0000 7efc0f39c880 0 mon.a@-1(???).osd e334 crush map has features 3314933069571702784, adjusting msgr requires 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.769+0000 7efc0f39c880 0 mon.a@-1(???).osd e334 crush map has features 288514119978713088, adjusting msgr requires 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: cluster 2022-04-28T19:48:58.621257+0000 mgr.y (mgr.14985) 28 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 288 KiB/s wr, 2 op/s 2022-04-28T19:49:02.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:01 smithi059 bash[59745]: debug 2022-04-28T19:49:01.773+0000 7efc0f39c880 1 mon.a@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2022-04-28T19:49:03.285 INFO:tasks.rados.rados.0.smithi059.stdout:1449: done (1 left) 2022-04-28T19:49:03.285 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 136 2022-04-28T19:49:03.286 INFO:tasks.rados.rados.0.smithi059.stdout:1450: seq_num 460 ranges {0=507904} 2022-04-28T19:49:03.291 INFO:tasks.rados.rados.0.smithi059.stdout:1450: writing smithi05955964-30 from 0 to 507904 tid 1 2022-04-28T19:49:03.291 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 136 2022-04-28T19:49:03.291 INFO:tasks.rados.rados.0.smithi059.stdout:1451: seq_num 461 ranges {6021120=524288} 2022-04-28T19:49:03.328 INFO:tasks.rados.rados.0.smithi059.stdout:1451: writing smithi05955964-20 from 6021120 to 6545408 tid 1 2022-04-28T19:49:03.450 INFO:tasks.rados.rados.0.smithi059.stdout:1452: rollback oid 45 current snap is 136 2022-04-28T19:49:03.452 INFO:tasks.rados.rados.0.smithi059.stdout:1452: done (3 left) 2022-04-28T19:49:03.452 INFO:tasks.rados.rados.0.smithi059.stdout:1453: read oid 37 snap -1 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1453: expect (ObjNum 443 snap 130 seq_num 443) 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1454: read oid 49 snap -1 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1454: expect deleted 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1455: rollback oid 43 current snap is 136 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1455: done (5 left) 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1456: read oid 14 snap -1 2022-04-28T19:49:03.453 INFO:tasks.rados.rados.0.smithi059.stdout:1456: expect deleted 2022-04-28T19:49:03.454 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 136 2022-04-28T19:49:03.454 INFO:tasks.rados.rados.0.smithi059.stdout:1457: seq_num 462 ranges {0=540672} 2022-04-28T19:49:03.455 INFO:tasks.rados.rados.0.smithi059.stdout:1457: writing smithi05955964-43 from 0 to 540672 tid 1 2022-04-28T19:49:03.455 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 136 2022-04-28T19:49:03.455 INFO:tasks.rados.rados.0.smithi059.stdout:1458: seq_num 463 ranges {0=557056} 2022-04-28T19:49:03.455 INFO:tasks.rados.rados.0.smithi059.stdout:1458: writing smithi05955964-42 from 0 to 557056 tid 1 2022-04-28T19:49:03.456 INFO:tasks.rados.rados.0.smithi059.stdout:1459: delete oid 45 current snap is 136 2022-04-28T19:49:03.456 INFO:tasks.rados.rados.0.smithi059.stdout:1447: expect (ObjNum 362 snap 108 seq_num 362) 2022-04-28T19:49:03.523 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:03.517Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:03.524 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:03.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:03.524 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:03.518Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:03.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:00.621824+0000 mgr.y (mgr.14985) 29 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 708 B/s rd, 266 KiB/s wr, 2 op/s 2022-04-28T19:49:03.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:01.985321+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:03.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:02.622558+0000 mgr.y (mgr.14985) 30 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 192 KiB/s wr, 1 op/s 2022-04-28T19:49:03.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.199859+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:03.525 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.214417+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:03.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.214691+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:03.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.214776+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2022-04-28T19:49:03.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.217177+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e25: y(active, since 42s), standbys: x 2022-04-28T19:49:03.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.230502+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e26: y(active, since 42s), standbys: x 2022-04-28T19:49:03.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.231876+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:03.526 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[59745]: audit 2022-04-28T19:49:03.240167+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.14985 ' entity='' 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[57771]: ignoring --setuser ceph since I am not root 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[57771]: ignoring --setgroup ceph since I am not root 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[57771]: debug 2022-04-28T19:49:03.445+0000 7f2d59607680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:00.621824+0000 mgr.y (mgr.14985) 29 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 708 B/s rd, 266 KiB/s wr, 2 op/s 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:01.985321+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:02.622558+0000 mgr.y (mgr.14985) 30 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 192 KiB/s wr, 1 op/s 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.199859+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:03.527 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.214417+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:03.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.214691+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:03.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.214776+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2022-04-28T19:49:03.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.217177+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e25: y(active, since 42s), standbys: x 2022-04-28T19:49:03.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.230502+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e26: y(active, since 42s), standbys: x 2022-04-28T19:49:03.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.231876+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:03.528 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[29114]: audit 2022-04-28T19:49:03.240167+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.14985 ' entity='' 2022-04-28T19:49:03.549 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:00.621824+0000 mgr.y (mgr.14985) 29 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 708 B/s rd, 266 KiB/s wr, 2 op/s 2022-04-28T19:49:03.549 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:01.985321+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:03.549 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:02.622558+0000 mgr.y (mgr.14985) 30 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 129 MiB data, 224 MiB used, 707 GiB / 715 GiB avail; 511 B/s rd, 192 KiB/s wr, 1 op/s 2022-04-28T19:49:03.549 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.199859+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:03.549 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.214417+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.214691+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.214776+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.217177+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e25: y(active, since 42s), standbys: x 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.230502+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e26: y(active, since 42s), standbys: x 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.231876+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[23439]: audit 2022-04-28T19:49:03.240167+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.14985 ' entity='' 2022-04-28T19:49:03.550 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[41342]: ignoring --setuser ceph since I am not root 2022-04-28T19:49:03.551 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[41342]: ignoring --setgroup ceph since I am not root 2022-04-28T19:49:03.551 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[41342]: debug 2022-04-28T19:49:03.283+0000 7f5c645d7700 1 -- 172.21.15.139:0/1434081484 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561ca815a4e0 con 0x561ca824e800 2022-04-28T19:49:03.551 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[41342]: debug 2022-04-28T19:49:03.439+0000 7f5c7477e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2022-04-28T19:49:03.551 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[41342]: debug 2022-04-28T19:49:03.543+0000 7f5c7477e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-04-28T19:49:03.724 INFO:tasks.rados.rados.0.smithi059.stdout:1450: finishing write tid 1 to smithi05955964-30 2022-04-28T19:49:03.724 INFO:tasks.rados.rados.0.smithi059.stdout:1450: finishing write tid 3 to smithi05955964-30 2022-04-28T19:49:03.725 INFO:tasks.rados.rados.0.smithi059.stdout:1450: finishing write tid 4 to smithi05955964-30 2022-04-28T19:49:03.725 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 728 (ObjNum 460 snap 136 seq_num 460) dirty exists 2022-04-28T19:49:03.725 INFO:tasks.rados.rados.0.smithi059.stdout:1450: left oid 30 (ObjNum 460 snap 136 seq_num 460) 2022-04-28T19:49:03.725 INFO:tasks.rados.rados.0.smithi059.stdout:1453: expect (ObjNum 443 snap 130 seq_num 443) 2022-04-28T19:49:03.729 INFO:tasks.rados.rados.0.smithi059.stdout:1457: finishing write tid 1 to smithi05955964-43 2022-04-28T19:49:03.729 INFO:tasks.rados.rados.0.smithi059.stdout:1457: finishing write tid 3 to smithi05955964-43 2022-04-28T19:49:03.729 INFO:tasks.rados.rados.0.smithi059.stdout:1457: finishing write tid 4 to smithi05955964-43 2022-04-28T19:49:03.729 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 726 (ObjNum 462 snap 136 seq_num 462) dirty exists 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1457: left oid 43 (ObjNum 462 snap 136 seq_num 462) 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1458: finishing write tid 1 to smithi05955964-42 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1458: finishing write tid 3 to smithi05955964-42 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1451: finishing write tid 1 to smithi05955964-20 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1451: finishing write tid 3 to smithi05955964-20 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1451: finishing write tid 4 to smithi05955964-20 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 650 (ObjNum 461 snap 136 seq_num 461) dirty exists 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1451: left oid 20 (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:03.730 INFO:tasks.rados.rados.0.smithi059.stdout:1458: finishing write tid 4 to smithi05955964-42 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 730 (ObjNum 463 snap 136 seq_num 463) dirty exists 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1458: left oid 42 (ObjNum 463 snap 136 seq_num 463) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1447: done (8 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1450: done (7 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1451: done (6 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1453: done (5 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1454: done (4 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1456: done (3 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1457: done (2 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1458: done (1 left) 2022-04-28T19:49:03.731 INFO:tasks.rados.rados.0.smithi059.stdout:1459: done (0 left) 2022-04-28T19:49:03.732 INFO:tasks.rados.rados.0.smithi059.stdout:1460: snap_create 2022-04-28T19:49:03.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[57771]: debug 2022-04-28T19:49:03.553+0000 7f2d59607680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2022-04-28T19:49:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:03.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:03.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:03.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:04.284 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:04 smithi059 bash[57771]: debug 2022-04-28T19:49:04.001+0000 7f2d59607680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-04-28T19:49:04.409 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:03 smithi139 bash[41342]: debug 2022-04-28T19:49:03.967+0000 7f5c7477e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1460: done (0 left) 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1461: delete oid 24 current snap is 137 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1461: done (0 left) 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1462: read oid 30 snap -1 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1462: expect (ObjNum 460 snap 136 seq_num 460) 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1463: read oid 43 snap -1 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1463: expect (ObjNum 462 snap 136 seq_num 462) 2022-04-28T19:49:04.411 INFO:tasks.rados.rados.0.smithi059.stdout:1464: delete oid 29 current snap is 137 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1464: done (2 left) 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1465: read oid 20 snap 136 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1465: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1466: read oid 2 snap -1 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1466: expect (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1467: delete oid 4 current snap is 137 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1463: expect (ObjNum 462 snap 136 seq_num 462) 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1462: expect (ObjNum 460 snap 136 seq_num 460) 2022-04-28T19:49:04.412 INFO:tasks.rados.rados.0.smithi059.stdout:1466: expect (ObjNum 293 snap 87 seq_num 293) 2022-04-28T19:49:04.508 INFO:tasks.rados.rados.0.smithi059.stdout:1465: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:04 smithi059 bash[29114]: cluster 2022-04-28T19:49:03.283411+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2022-04-28T19:49:04.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:04 smithi059 bash[59745]: cluster 2022-04-28T19:49:03.283411+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2022-04-28T19:49:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:04 smithi139 bash[23439]: cluster 2022-04-28T19:49:03.283411+0000 mon.a (mon.0) 10 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2022-04-28T19:49:05.040 INFO:tasks.rados.rados.0.smithi059.stdout:1462: done (4 left) 2022-04-28T19:49:05.040 INFO:tasks.rados.rados.0.smithi059.stdout:1463: done (3 left) 2022-04-28T19:49:05.040 INFO:tasks.rados.rados.0.smithi059.stdout:1465: done (2 left) 2022-04-28T19:49:05.040 INFO:tasks.rados.rados.0.smithi059.stdout:1466: done (1 left) 2022-04-28T19:49:05.040 INFO:tasks.rados.rados.0.smithi059.stdout:1467: done (0 left) 2022-04-28T19:49:05.040 INFO:tasks.rados.rados.0.smithi059.stdout:1468: snap_create 2022-04-28T19:49:05.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:04 smithi139 bash[41342]: debug 2022-04-28T19:49:04.863+0000 7f5c7477e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-04-28T19:49:05.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:04 smithi139 bash[41342]: debug 2022-04-28T19:49:04.951+0000 7f5c7477e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-04-28T19:49:05.293 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:04 smithi059 bash[57771]: debug 2022-04-28T19:49:04.957+0000 7f2d59607680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2022-04-28T19:49:05.293 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:05 smithi059 bash[57771]: debug 2022-04-28T19:49:05.057+0000 7f2d59607680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2022-04-28T19:49:05.310 INFO:tasks.rados.rados.0.smithi059.stdout:1468: done (0 left) 2022-04-28T19:49:05.311 INFO:tasks.rados.rados.0.smithi059.stdout:1469: snap_remove snap 136 2022-04-28T19:49:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:05 smithi059 bash[59745]: cluster 2022-04-28T19:49:04.295076+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2022-04-28T19:49:05.591 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:05 smithi059 bash[57771]: debug 2022-04-28T19:49:05.313+0000 7f2d59607680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-04-28T19:49:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:05 smithi059 bash[29114]: cluster 2022-04-28T19:49:04.295076+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2022-04-28T19:49:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:05 smithi139 bash[23439]: cluster 2022-04-28T19:49:04.295076+0000 mon.a (mon.0) 11 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2022-04-28T19:49:05.619 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:05 smithi139 bash[41342]: debug 2022-04-28T19:49:05.195+0000 7f5c7477e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2022-04-28T19:49:06.311 INFO:tasks.rados.rados.0.smithi059.stdout:1469: done (0 left) 2022-04-28T19:49:06.311 INFO:tasks.rados.rados.0.smithi059.stdout:1470: snap_create 2022-04-28T19:49:06.561 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:06 smithi139 bash[23439]: cluster 2022-04-28T19:49:05.309864+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2022-04-28T19:49:06.562 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:06 smithi139 bash[41342]: debug 2022-04-28T19:49:06.299+0000 7f5c7477e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-04-28T19:49:06.562 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:06 smithi139 bash[41342]: debug 2022-04-28T19:49:06.459+0000 7f5c7477e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:49:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:06 smithi059 bash[59745]: cluster 2022-04-28T19:49:05.309864+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2022-04-28T19:49:06.592 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:06 smithi059 bash[57771]: debug 2022-04-28T19:49:06.461+0000 7f2d59607680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2022-04-28T19:49:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:06 smithi059 bash[29114]: cluster 2022-04-28T19:49:05.309864+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2022-04-28T19:49:06.789 INFO:tasks.rados.rados.0.smithi059.stdout:1470: done (0 left) 2022-04-28T19:49:06.790 INFO:tasks.rados.rados.0.smithi059.stdout:1471: read oid 11 snap -1 2022-04-28T19:49:06.790 INFO:tasks.rados.rados.0.smithi059.stdout:1471: expect deleted 2022-04-28T19:49:06.790 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 139 2022-04-28T19:49:06.791 INFO:tasks.rados.rados.0.smithi059.stdout:1472: seq_num 468 ranges {0=647168} 2022-04-28T19:49:06.796 INFO:tasks.rados.rados.0.smithi059.stdout:1472: writing smithi05955964-24 from 0 to 647168 tid 1 2022-04-28T19:49:06.796 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 139 2022-04-28T19:49:06.796 INFO:tasks.rados.rados.0.smithi059.stdout:1473: seq_num 469 ranges {1982464=663552} 2022-04-28T19:49:06.812 INFO:tasks.rados.rados.0.smithi059.stdout:1473: writing smithi05955964-32 from 1982464 to 2646016 tid 1 2022-04-28T19:49:06.813 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 139 2022-04-28T19:49:06.813 INFO:tasks.rados.rados.0.smithi059.stdout:1474: seq_num 470 ranges {0=679936} 2022-04-28T19:49:06.816 INFO:tasks.rados.rados.0.smithi059.stdout:1474: writing smithi05955964-36 from 0 to 679936 tid 1 2022-04-28T19:49:06.817 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 139 2022-04-28T19:49:06.817 INFO:tasks.rados.rados.0.smithi059.stdout:1475: seq_num 471 ranges {172032=696320} 2022-04-28T19:49:06.822 INFO:tasks.rados.rados.0.smithi059.stdout:1475: writing smithi05955964-22 from 172032 to 868352 tid 1 2022-04-28T19:49:06.822 INFO:tasks.rados.rados.0.smithi059.stdout:1476: snap_remove snap 138 2022-04-28T19:49:06.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:06 smithi139 bash[41342]: debug 2022-04-28T19:49:06.555+0000 7f5c7477e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-04-28T19:49:06.868 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:06 smithi139 bash[41342]: debug 2022-04-28T19:49:06.667+0000 7f5c7477e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-04-28T19:49:07.076 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:06 smithi059 bash[57771]: debug 2022-04-28T19:49:06.625+0000 7f2d59607680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:49:07.076 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:06 smithi059 bash[57771]: debug 2022-04-28T19:49:06.725+0000 7f2d59607680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2022-04-28T19:49:07.076 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:06 smithi059 bash[57771]: debug 2022-04-28T19:49:06.849+0000 7f2d59607680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2022-04-28T19:49:07.260 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:06 smithi139 bash[41342]: debug 2022-04-28T19:49:06.887+0000 7f5c7477e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-04-28T19:49:07.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[57771]: debug 2022-04-28T19:49:07.069+0000 7f2d59607680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2022-04-28T19:49:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[29114]: cluster 2022-04-28T19:49:06.309903+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2022-04-28T19:49:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[29114]: cluster 2022-04-28T19:49:06.786645+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2022-04-28T19:49:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[59745]: cluster 2022-04-28T19:49:06.309903+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2022-04-28T19:49:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[59745]: cluster 2022-04-28T19:49:06.786645+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2022-04-28T19:49:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:07 smithi139 bash[23439]: cluster 2022-04-28T19:49:06.309903+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2022-04-28T19:49:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:07 smithi139 bash[23439]: cluster 2022-04-28T19:49:06.786645+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2022-04-28T19:49:07.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:07 smithi139 bash[41342]: debug 2022-04-28T19:49:07.255+0000 7f5c7477e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-04-28T19:49:07.619 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:07 smithi139 bash[41342]: debug 2022-04-28T19:49:07.379+0000 7f5c7477e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-04-28T19:49:07.802 INFO:tasks.rados.rados.0.smithi059.stdout:1476: done (5 left) 2022-04-28T19:49:07.802 INFO:tasks.rados.rados.0.smithi059.stdout:1477: rollback oid 34 current snap is 139 2022-04-28T19:49:07.802 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 34 to 137 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1478: delete oid 9 current snap is 139 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1472: finishing write tid 1 to smithi05955964-24 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1472: finishing write tid 3 to smithi05955964-24 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1472: finishing write tid 4 to smithi05955964-24 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 684 (ObjNum 468 snap 139 seq_num 468) dirty exists 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1472: left oid 24 (ObjNum 468 snap 139 seq_num 468) 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1473: finishing write tid 1 to smithi05955964-32 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1473: finishing write tid 3 to smithi05955964-32 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1474: finishing write tid 1 to smithi05955964-36 2022-04-28T19:49:07.803 INFO:tasks.rados.rados.0.smithi059.stdout:1474: finishing write tid 3 to smithi05955964-36 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1473: finishing write tid 4 to smithi05955964-32 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 729 (ObjNum 469 snap 139 seq_num 469) dirty exists 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1473: left oid 32 (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1474: finishing write tid 4 to smithi05955964-36 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 713 (ObjNum 470 snap 139 seq_num 470) dirty exists 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1474: left oid 36 (ObjNum 470 snap 139 seq_num 470) 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1475: finishing write tid 1 to smithi05955964-22 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1475: finishing write tid 3 to smithi05955964-22 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1475: finishing write tid 4 to smithi05955964-22 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 730 (ObjNum 471 snap 139 seq_num 471) dirty exists 2022-04-28T19:49:07.804 INFO:tasks.rados.rados.0.smithi059.stdout:1475: left oid 22 (ObjNum 471 snap 139 seq_num 471) 2022-04-28T19:49:07.805 INFO:tasks.rados.rados.0.smithi059.stdout:1477: finishing rollback tid 1 to smithi05955964-34 2022-04-28T19:49:07.805 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 0 (ObjNum 288 snap 0 seq_num 81) dirty dne 2022-04-28T19:49:07.810 INFO:tasks.rados.rados.0.smithi059.stdout:1471: done (6 left) 2022-04-28T19:49:07.810 INFO:tasks.rados.rados.0.smithi059.stdout:1472: done (5 left) 2022-04-28T19:49:07.810 INFO:tasks.rados.rados.0.smithi059.stdout:1473: done (4 left) 2022-04-28T19:49:07.811 INFO:tasks.rados.rados.0.smithi059.stdout:1474: done (3 left) 2022-04-28T19:49:07.811 INFO:tasks.rados.rados.0.smithi059.stdout:1475: done (2 left) 2022-04-28T19:49:07.811 INFO:tasks.rados.rados.0.smithi059.stdout:1477: done (1 left) 2022-04-28T19:49:07.811 INFO:tasks.rados.rados.0.smithi059.stdout:1478: done (0 left) 2022-04-28T19:49:07.811 INFO:tasks.rados.rados.0.smithi059.stdout:1479: read oid 3 snap 137 2022-04-28T19:49:07.812 INFO:tasks.rados.rados.0.smithi059.stdout:1479: expect (ObjNum 434 snap 129 seq_num 434) 2022-04-28T19:49:07.812 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 139 2022-04-28T19:49:07.812 INFO:tasks.rados.rados.0.smithi059.stdout:1480: seq_num 473 ranges {540672=729088} 2022-04-28T19:49:07.817 INFO:tasks.rados.rados.0.smithi059.stdout:1480: writing smithi05955964-43 from 540672 to 1269760 tid 1 2022-04-28T19:49:07.817 INFO:tasks.rados.rados.0.smithi059.stdout:1481: copy_from oid 48 from oid 42 current snap is 139 2022-04-28T19:49:07.818 INFO:tasks.rados.rados.0.smithi059.stdout:1482: setattr oid 1 current snap is 139 2022-04-28T19:49:07.818 INFO:tasks.rados.rados.0.smithi059.stdout:1479: expect (ObjNum 434 snap 129 seq_num 434) 2022-04-28T19:49:07.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[57771]: debug 2022-04-28T19:49:07.457+0000 7f2d59607680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2022-04-28T19:49:07.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:07 smithi059 bash[57771]: debug 2022-04-28T19:49:07.581+0000 7f2d59607680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2022-04-28T19:49:07.901 INFO:tasks.rados.rados.0.smithi059.stdout:1480: finishing write tid 1 to smithi05955964-43 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:1480: finishing write tid 3 to smithi05955964-43 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:1480: finishing write tid 4 to smithi05955964-43 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 734 (ObjNum 473 snap 139 seq_num 473) dirty exists 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:1480: left oid 43 (ObjNum 473 snap 139 seq_num 473) 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:1481: finishing copy_from to smithi05955964-48 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 730 (ObjNum 463 snap 136 seq_num 463) dirty exists 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:1481: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:49:07.902 INFO:tasks.rados.rados.0.smithi059.stdout:1479: done (3 left) 2022-04-28T19:49:07.903 INFO:tasks.rados.rados.0.smithi059.stdout:1480: done (2 left) 2022-04-28T19:49:07.903 INFO:tasks.rados.rados.0.smithi059.stdout:1481: done (1 left) 2022-04-28T19:49:07.903 INFO:tasks.rados.rados.0.smithi059.stdout:1483: read oid 27 snap -1 2022-04-28T19:49:07.903 INFO:tasks.rados.rados.0.smithi059.stdout:1483: expect (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:49:07.903 INFO:tasks.rados.rados.0.smithi059.stdout:1484: read oid 7 snap -1 2022-04-28T19:49:07.904 INFO:tasks.rados.rados.0.smithi059.stdout:1484: expect (ObjNum 452 snap 134 seq_num 452) 2022-04-28T19:49:07.904 INFO:tasks.rados.rados.0.smithi059.stdout:1485: read oid 37 snap -1 2022-04-28T19:49:07.904 INFO:tasks.rados.rados.0.smithi059.stdout:1485: expect (ObjNum 443 snap 130 seq_num 443) 2022-04-28T19:49:07.904 INFO:tasks.rados.rados.0.smithi059.stdout:1486: copy_from oid 35 from oid 14 current snap is 139 2022-04-28T19:49:07.904 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 139 2022-04-28T19:49:07.904 INFO:tasks.rados.rados.0.smithi059.stdout:1487: seq_num 474 ranges {868352=745472} 2022-04-28T19:49:07.917 INFO:tasks.rados.rados.0.smithi059.stdout:1487: writing smithi05955964-22 from 868352 to 1613824 tid 1 2022-04-28T19:49:07.917 INFO:tasks.rados.rados.0.smithi059.stdout:1488: read oid 24 snap 137 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:1488: expect deleted 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:1489: rollback oid 26 current snap is 139 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 137 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:1486: finishing copy_from to smithi05955964-35 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:1486: got expected ENOENT (src dne) 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:1486: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:49:07.918 INFO:tasks.rados.rados.0.smithi059.stdout:1483: expect (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:49:07.957 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 714 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:49:07.958 INFO:tasks.rados.rados.0.smithi059.stdout:1482: done (7 left) 2022-04-28T19:49:07.958 INFO:tasks.rados.rados.0.smithi059.stdout:1483: done (6 left) 2022-04-28T19:49:07.958 INFO:tasks.rados.rados.0.smithi059.stdout:1486: done (5 left) 2022-04-28T19:49:07.958 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 139 2022-04-28T19:49:07.958 INFO:tasks.rados.rados.0.smithi059.stdout:1490: seq_num 475 ranges {0=761856} 2022-04-28T19:49:07.962 INFO:tasks.rados.rados.0.smithi059.stdout:1490: writing smithi05955964-16 from 0 to 761856 tid 1 2022-04-28T19:49:07.962 INFO:tasks.rados.rados.0.smithi059.stdout:1491: read oid 3 snap 137 2022-04-28T19:49:07.962 INFO:tasks.rados.rados.0.smithi059.stdout:1491: expect (ObjNum 434 snap 129 seq_num 434) 2022-04-28T19:49:07.962 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 139 2022-04-28T19:49:07.962 INFO:tasks.rados.rados.0.smithi059.stdout:1492: seq_num 476 ranges {1654784=778240} 2022-04-28T19:49:07.979 INFO:tasks.rados.rados.0.smithi059.stdout:1492: writing smithi05955964-44 from 1654784 to 2433024 tid 1 2022-04-28T19:49:07.979 INFO:tasks.rados.rados.0.smithi059.stdout:1493: setattr oid 23 current snap is 139 2022-04-28T19:49:07.979 INFO:tasks.rados.rados.0.smithi059.stdout:1484: expect (ObjNum 452 snap 134 seq_num 452) 2022-04-28T19:49:08.106 INFO:tasks.rados.rados.0.smithi059.stdout:1485: expect (ObjNum 443 snap 130 seq_num 443) 2022-04-28T19:49:08.119 INFO:tasks.rados.rados.0.smithi059.stdout:1487: finishing write tid 1 to smithi05955964-22 2022-04-28T19:49:08.119 INFO:tasks.rados.rados.0.smithi059.stdout:1487: finishing write tid 3 to smithi05955964-22 2022-04-28T19:49:08.119 INFO:tasks.rados.rados.0.smithi059.stdout:1489: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:49:08.119 INFO:tasks.rados.rados.0.smithi059.stdout:1489: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:49:08.120 INFO:tasks.rados.rados.0.smithi059.stdout:1487: finishing write tid 4 to smithi05955964-22 2022-04-28T19:49:08.120 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 736 (ObjNum 474 snap 139 seq_num 474) dirty exists 2022-04-28T19:49:08.123 INFO:tasks.rados.rados.0.smithi059.stdout:1487: left oid 22 (ObjNum 474 snap 139 seq_num 474) 2022-04-28T19:49:08.123 INFO:tasks.rados.rados.0.smithi059.stdout:1489: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:49:08.123 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 740 (ObjNum 442 snap 130 seq_num 442) dirty exists 2022-04-28T19:49:08.123 INFO:tasks.rados.rados.0.smithi059.stdout:1490: finishing write tid 1 to smithi05955964-16 2022-04-28T19:49:08.124 INFO:tasks.rados.rados.0.smithi059.stdout:1490: finishing write tid 3 to smithi05955964-16 2022-04-28T19:49:08.124 INFO:tasks.rados.rados.0.smithi059.stdout:1491: expect (ObjNum 434 snap 129 seq_num 434) 2022-04-28T19:49:08.192 INFO:tasks.rados.rados.0.smithi059.stdout:1490: finishing write tid 4 to smithi05955964-16 2022-04-28T19:49:08.192 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 405 (ObjNum 475 snap 139 seq_num 475) dirty exists 2022-04-28T19:49:08.192 INFO:tasks.rados.rados.0.smithi059.stdout:1490: left oid 16 (ObjNum 475 snap 139 seq_num 475) 2022-04-28T19:49:08.192 INFO:tasks.rados.rados.0.smithi059.stdout:1492: finishing write tid 1 to smithi05955964-44 2022-04-28T19:49:08.192 INFO:tasks.rados.rados.0.smithi059.stdout:1492: finishing write tid 3 to smithi05955964-44 2022-04-28T19:49:08.192 INFO:tasks.rados.rados.0.smithi059.stdout:1492: finishing write tid 4 to smithi05955964-44 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 631 (ObjNum 476 snap 139 seq_num 476) dirty exists 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:1492: left oid 44 (ObjNum 476 snap 139 seq_num 476) 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:1484: done (8 left) 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:1485: done (7 left) 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:1487: done (6 left) 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:1488: done (5 left) 2022-04-28T19:49:08.193 INFO:tasks.rados.rados.0.smithi059.stdout:1489: done (4 left) 2022-04-28T19:49:08.194 INFO:tasks.rados.rados.0.smithi059.stdout:1490: done (3 left) 2022-04-28T19:49:08.194 INFO:tasks.rados.rados.0.smithi059.stdout:1491: done (2 left) 2022-04-28T19:49:08.194 INFO:tasks.rados.rados.0.smithi059.stdout:1492: done (1 left) 2022-04-28T19:49:08.194 INFO:tasks.rados.rados.0.smithi059.stdout:1494: delete oid 21 current snap is 139 2022-04-28T19:49:08.195 INFO:tasks.rados.rados.0.smithi059.stdout:1494: done (1 left) 2022-04-28T19:49:08.195 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 139 2022-04-28T19:49:08.195 INFO:tasks.rados.rados.0.smithi059.stdout:1495: seq_num 478 ranges {4939776=409600,5349376=401408} 2022-04-28T19:49:08.226 INFO:tasks.rados.rados.0.smithi059.stdout:1495: writing smithi05955964-6 from 4939776 to 5349376 tid 1 2022-04-28T19:49:08.229 INFO:tasks.rados.rados.0.smithi059.stdout:1495: writing smithi05955964-6 from 5349376 to 5750784 tid 2 2022-04-28T19:49:08.229 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 139 2022-04-28T19:49:08.229 INFO:tasks.rados.rados.0.smithi059.stdout:1496: seq_num 479 ranges {2883584=425984,3309568=401408} 2022-04-28T19:49:08.248 INFO:tasks.rados.rados.0.smithi059.stdout:1496: writing smithi05955964-2 from 2883584 to 3309568 tid 1 2022-04-28T19:49:08.250 INFO:tasks.rados.rados.0.smithi059.stdout:1496: writing smithi05955964-2 from 3309568 to 3710976 tid 2 2022-04-28T19:49:08.250 INFO:tasks.rados.rados.0.smithi059.stdout:1497: snap_remove snap 137 2022-04-28T19:49:08.250 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 711 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:49:08.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:08 smithi139 bash[41342]: debug 2022-04-28T19:49:08.239+0000 7f5c7477e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-04-28T19:49:08.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:08 smithi139 bash[41342]: debug 2022-04-28T19:49:08.343+0000 7f5c7477e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-04-28T19:49:08.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:08 smithi139 bash[41342]: debug 2022-04-28T19:49:08.459+0000 7f5c7477e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-04-28T19:49:08.801 INFO:tasks.rados.rados.0.smithi059.stdout:1493: done (3 left) 2022-04-28T19:49:08.801 INFO:tasks.rados.rados.0.smithi059.stdout:1497: done (2 left) 2022-04-28T19:49:08.801 INFO:tasks.rados.rados.0.smithi059.stdout:1498: delete oid 34 current snap is 139 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1495: finishing write tid 1 to smithi05955964-6 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1495: finishing write tid 2 to smithi05955964-6 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1495: finishing write tid 4 to smithi05955964-6 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1495: finishing write tid 5 to smithi05955964-6 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 655 (ObjNum 478 snap 139 seq_num 478) dirty exists 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1495: left oid 6 (ObjNum 478 snap 139 seq_num 478) 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1496: finishing write tid 1 to smithi05955964-2 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1496: finishing write tid 2 to smithi05955964-2 2022-04-28T19:49:08.802 INFO:tasks.rados.rados.0.smithi059.stdout:1496: finishing write tid 4 to smithi05955964-2 2022-04-28T19:49:08.803 INFO:tasks.rados.rados.0.smithi059.stdout:1496: finishing write tid 5 to smithi05955964-2 2022-04-28T19:49:08.803 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 388 (ObjNum 479 snap 139 seq_num 479) dirty exists 2022-04-28T19:49:08.803 INFO:tasks.rados.rados.0.smithi059.stdout:1496: left oid 2 (ObjNum 479 snap 139 seq_num 479) 2022-04-28T19:49:08.807 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:08 smithi059 bash[59745]: cluster 2022-04-28T19:49:07.798844+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2022-04-28T19:49:08.808 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:08 smithi059 bash[57771]: debug 2022-04-28T19:49:08.489+0000 7f2d59607680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2022-04-28T19:49:08.808 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:08 smithi059 bash[57771]: debug 2022-04-28T19:49:08.597+0000 7f2d59607680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2022-04-28T19:49:08.808 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:08 smithi059 bash[57771]: debug 2022-04-28T19:49:08.713+0000 7f2d59607680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2022-04-28T19:49:08.809 INFO:tasks.rados.rados.0.smithi059.stdout:1495: done (2 left) 2022-04-28T19:49:08.809 INFO:tasks.rados.rados.0.smithi059.stdout:1496: done (1 left) 2022-04-28T19:49:08.809 INFO:tasks.rados.rados.0.smithi059.stdout:1498: done (0 left) 2022-04-28T19:49:08.810 INFO:tasks.rados.rados.0.smithi059.stdout:1499: delete oid 25 current snap is 139 2022-04-28T19:49:08.815 INFO:tasks.rados.rados.0.smithi059.stdout:1499: done (0 left) 2022-04-28T19:49:08.815 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 139 2022-04-28T19:49:08.816 INFO:tasks.rados.rados.0.smithi059.stdout:1500: seq_num 482 ranges {4562944=475136,5038080=401408} 2022-04-28T19:49:08.844 INFO:tasks.rados.rados.0.smithi059.stdout:1500: writing smithi05955964-8 from 4562944 to 5038080 tid 1 2022-04-28T19:49:08.846 INFO:tasks.rados.rados.0.smithi059.stdout:1500: writing smithi05955964-8 from 5038080 to 5439488 tid 2 2022-04-28T19:49:08.846 INFO:tasks.rados.rados.0.smithi059.stdout:1501: delete oid 16 current snap is 139 2022-04-28T19:49:08.848 INFO:tasks.rados.rados.0.smithi059.stdout:1501: done (1 left) 2022-04-28T19:49:08.848 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 139 2022-04-28T19:49:08.848 INFO:tasks.rados.rados.0.smithi059.stdout:1502: seq_num 484 ranges {2457600=507904,2965504=401408} 2022-04-28T19:49:08.865 INFO:tasks.rados.rados.0.smithi059.stdout:1502: writing smithi05955964-7 from 2457600 to 2965504 tid 1 2022-04-28T19:49:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:1502: writing smithi05955964-7 from 2965504 to 3366912 tid 2 2022-04-28T19:49:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:1503: rollback oid 47 current snap is 139 2022-04-28T19:49:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:1503: done (2 left) 2022-04-28T19:49:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:1504: rollback oid 3 current snap is 139 2022-04-28T19:49:08.869 INFO:tasks.rados.rados.0.smithi059.stdout:1504: done (2 left) 2022-04-28T19:49:08.869 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 139 2022-04-28T19:49:08.869 INFO:tasks.rados.rados.0.smithi059.stdout:1505: seq_num 485 ranges {0=532480,532480=401408} 2022-04-28T19:49:08.871 INFO:tasks.rados.rados.0.smithi059.stdout:1505: writing smithi05955964-11 from 0 to 532480 tid 1 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1505: writing smithi05955964-11 from 532480 to 933888 tid 2 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1506: read oid 30 snap -1 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1506: expect (ObjNum 460 snap 136 seq_num 460) 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1507: rmattr oid 43 current snap is 139 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1507: done (4 left) 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1508: read oid 33 snap -1 2022-04-28T19:49:08.874 INFO:tasks.rados.rados.0.smithi059.stdout:1508: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:49:08.875 INFO:tasks.rados.rados.0.smithi059.stdout:1509: rmattr oid 47 current snap is 139 2022-04-28T19:49:08.875 INFO:tasks.rados.rados.0.smithi059.stdout:1500: finishing write tid 1 to smithi05955964-8 2022-04-28T19:49:08.875 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 139 2022-04-28T19:49:08.875 INFO:tasks.rados.rados.0.smithi059.stdout:1510: seq_num 486 ranges {2433024=548864,2981888=401408} 2022-04-28T19:49:08.891 INFO:tasks.rados.rados.0.smithi059.stdout:1510: writing smithi05955964-44 from 2433024 to 2981888 tid 1 2022-04-28T19:49:08.894 INFO:tasks.rados.rados.0.smithi059.stdout:1510: writing smithi05955964-44 from 2981888 to 3383296 tid 2 2022-04-28T19:49:08.894 INFO:tasks.rados.rados.0.smithi059.stdout:1511: read oid 10 snap -1 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1511: expect (ObjNum 457 snap 135 seq_num 457) 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1512: rollback oid 29 current snap is 139 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1512: done (8 left) 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1513: copy_from oid 40 from oid 38 current snap is 139 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1500: finishing write tid 2 to smithi05955964-8 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1500: finishing write tid 4 to smithi05955964-8 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1500: finishing write tid 5 to smithi05955964-8 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 680 (ObjNum 482 snap 139 seq_num 482) dirty exists 2022-04-28T19:49:08.895 INFO:tasks.rados.rados.0.smithi059.stdout:1500: left oid 8 (ObjNum 482 snap 139 seq_num 482) 2022-04-28T19:49:08.896 INFO:tasks.rados.rados.0.smithi059.stdout:1502: finishing write tid 1 to smithi05955964-7 2022-04-28T19:49:08.896 INFO:tasks.rados.rados.0.smithi059.stdout:1500: done (8 left) 2022-04-28T19:49:08.896 INFO:tasks.rados.rados.0.smithi059.stdout:1514: read oid 20 snap -1 2022-04-28T19:49:08.896 INFO:tasks.rados.rados.0.smithi059.stdout:1514: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:08.896 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 139 2022-04-28T19:49:08.896 INFO:tasks.rados.rados.0.smithi059.stdout:1515: seq_num 487 ranges {1269760=565248,1835008=401408} 2022-04-28T19:49:08.906 INFO:tasks.rados.rados.0.smithi059.stdout:1515: writing smithi05955964-43 from 1269760 to 1835008 tid 1 2022-04-28T19:49:08.908 INFO:tasks.rados.rados.0.smithi059.stdout:1515: writing smithi05955964-43 from 1835008 to 2236416 tid 2 2022-04-28T19:49:08.908 INFO:tasks.rados.rados.0.smithi059.stdout:1516: snap_create 2022-04-28T19:49:08.909 INFO:tasks.rados.rados.0.smithi059.stdout:1502: finishing write tid 2 to smithi05955964-7 2022-04-28T19:49:08.909 INFO:tasks.rados.rados.0.smithi059.stdout:1502: finishing write tid 4 to smithi05955964-7 2022-04-28T19:49:08.909 INFO:tasks.rados.rados.0.smithi059.stdout:1502: finishing write tid 5 to smithi05955964-7 2022-04-28T19:49:08.909 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 702 (ObjNum 484 snap 139 seq_num 484) dirty exists 2022-04-28T19:49:08.909 INFO:tasks.rados.rados.0.smithi059.stdout:1502: left oid 7 (ObjNum 484 snap 139 seq_num 484) 2022-04-28T19:49:08.909 INFO:tasks.rados.rados.0.smithi059.stdout:1505: finishing write tid 1 to smithi05955964-11 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:1505: finishing write tid 2 to smithi05955964-11 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:1505: finishing write tid 4 to smithi05955964-11 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 714 (ObjNum 414 snap 125 seq_num 414) dirty exists 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:1505: finishing write tid 5 to smithi05955964-11 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 691 (ObjNum 485 snap 139 seq_num 485) dirty exists 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:1505: left oid 11 (ObjNum 485 snap 139 seq_num 485) 2022-04-28T19:49:08.910 INFO:tasks.rados.rados.0.smithi059.stdout:1506: expect (ObjNum 460 snap 136 seq_num 460) 2022-04-28T19:49:08.922 INFO:tasks.rados.rados.0.smithi059.stdout:1508: expect (ObjNum 315 snap 93 seq_num 315) 2022-04-28T19:49:08.986 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:08 smithi139 bash[23439]: cluster 2022-04-28T19:49:07.798844+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2022-04-28T19:49:09.067 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:08 smithi139 bash[41342]: debug 2022-04-28T19:49:08.679+0000 7f5c7477e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-04-28T19:49:09.091 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:08 smithi059 bash[57771]: debug 2022-04-28T19:49:08.965+0000 7f2d59607680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2022-04-28T19:49:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:08 smithi059 bash[29114]: cluster 2022-04-28T19:49:07.798844+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2022-04-28T19:49:09.180 INFO:tasks.rados.rados.0.smithi059.stdout:1510: finishing write tid 1 to smithi05955964-44 2022-04-28T19:49:09.180 INFO:tasks.rados.rados.0.smithi059.stdout:1511: expect (ObjNum 457 snap 135 seq_num 457) 2022-04-28T19:49:09.291 INFO:tasks.rados.rados.0.smithi059.stdout:1515: finishing write tid 1 to smithi05955964-43 2022-04-28T19:49:09.347 INFO:tasks.rados.rados.0.smithi059.stdout:1515: finishing write tid 2 to smithi05955964-43 2022-04-28T19:49:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1515: finishing write tid 4 to smithi05955964-43 2022-04-28T19:49:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1515: finishing write tid 5 to smithi05955964-43 2022-04-28T19:49:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 741 (ObjNum 487 snap 139 seq_num 487) dirty exists 2022-04-28T19:49:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1515: left oid 43 (ObjNum 487 snap 139 seq_num 487) 2022-04-28T19:49:09.348 INFO:tasks.rados.rados.0.smithi059.stdout:1514: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:09.349 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:08 smithi139 bash[41342]: debug 2022-04-28T19:49:08.979+0000 7f5c7477e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-04-28T19:49:09.349 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[41342]: debug 2022-04-28T19:49:09.143+0000 7f5c7477e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-04-28T19:49:09.350 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[41342]: debug 2022-04-28T19:49:09.307+0000 7f5c7477e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:49:09.591 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[57771]: debug 2022-04-28T19:49:09.261+0000 7f2d59607680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2022-04-28T19:49:09.591 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[57771]: debug 2022-04-28T19:49:09.429+0000 7f2d59607680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2022-04-28T19:49:09.618 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[41342]: debug 2022-04-28T19:49:09.515+0000 7f5c7477e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-04-28T19:49:09.774 INFO:tasks.rados.rados.0.smithi059.stdout:1510: finishing write tid 2 to smithi05955964-44 2022-04-28T19:49:09.774 INFO:tasks.rados.rados.0.smithi059.stdout:1510: finishing write tid 4 to smithi05955964-44 2022-04-28T19:49:09.774 INFO:tasks.rados.rados.0.smithi059.stdout:1513: finishing copy_from to smithi05955964-40 2022-04-28T19:49:09.775 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 674 (ObjNum 450 snap 132 seq_num 450) dirty exists 2022-04-28T19:49:09.775 INFO:tasks.rados.rados.0.smithi059.stdout:1513: finishing copy_from racing read to smithi05955964-40 2022-04-28T19:49:09.775 INFO:tasks.rados.rados.0.smithi059.stdout:1510: finishing write tid 5 to smithi05955964-44 2022-04-28T19:49:09.775 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 634 (ObjNum 486 snap 139 seq_num 486) dirty exists 2022-04-28T19:49:09.775 INFO:tasks.rados.rados.0.smithi059.stdout:1510: left oid 44 (ObjNum 486 snap 139 seq_num 486) 2022-04-28T19:49:09.833 INFO:tasks.rados.rados.0.smithi059.stdout:1502: done (10 left) 2022-04-28T19:49:09.913 INFO:tasks.rados.rados.0.smithi059.stdout:1505: done (9 left) 2022-04-28T19:49:09.913 INFO:tasks.rados.rados.0.smithi059.stdout:1506: done (8 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1508: done (7 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1509: done (6 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1510: done (5 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1511: done (4 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1513: done (3 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1514: done (2 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1515: done (1 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1516: done (0 left) 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 140 2022-04-28T19:49:09.914 INFO:tasks.rados.rados.0.smithi059.stdout:1517: seq_num 488 ranges {10567680=581632,11149312=401408} 2022-04-28T19:49:09.917 INFO:tasks.rados.rados.0.smithi059.stdout:1517: writing smithi05955964-5 from 10567680 to 11149312 tid 1 2022-04-28T19:49:09.917 INFO:tasks.rados.rados.0.smithi059.stdout:1517: writing smithi05955964-5 from 11149312 to 11550720 tid 2 2022-04-28T19:49:09.917 INFO:tasks.rados.rados.0.smithi059.stdout:1518: rollback oid 11 current snap is 140 2022-04-28T19:49:09.917 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 139 2022-04-28T19:49:09.917 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 140 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1519: seq_num 489 ranges {0=598016,598016=401408} 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1519: writing smithi05955964-4 from 0 to 598016 tid 1 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1519: writing smithi05955964-4 from 598016 to 999424 tid 2 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1520: delete oid 30 current snap is 140 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1517: finishing write tid 1 to smithi05955964-5 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1517: finishing write tid 2 to smithi05955964-5 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1517: finishing write tid 4 to smithi05955964-5 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1517: finishing write tid 5 to smithi05955964-5 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 684 (ObjNum 488 snap 140 seq_num 488) dirty exists 2022-04-28T19:49:09.918 INFO:tasks.rados.rados.0.smithi059.stdout:1517: left oid 5 (ObjNum 488 snap 140 seq_num 488) 2022-04-28T19:49:09.919 INFO:tasks.rados.rados.0.smithi059.stdout:1518: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:49:09.919 INFO:tasks.rados.rados.0.smithi059.stdout:1518: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:49:09.919 INFO:tasks.rados.rados.0.smithi059.stdout:1517: done (3 left) 2022-04-28T19:49:09.919 INFO:tasks.rados.rados.0.smithi059.stdout:1520: done (2 left) 2022-04-28T19:49:09.919 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 140 2022-04-28T19:49:09.919 INFO:tasks.rados.rados.0.smithi059.stdout:1521: seq_num 491 ranges {2048000=630784,2678784=401408} 2022-04-28T19:49:09.927 INFO:tasks.rados.rados.0.smithi059.stdout:1521: writing smithi05955964-3 from 2048000 to 2678784 tid 1 2022-04-28T19:49:09.929 INFO:tasks.rados.rados.0.smithi059.stdout:1521: writing smithi05955964-3 from 2678784 to 3080192 tid 2 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1522: delete oid 30 current snap is 140 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1518: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 695 (ObjNum 485 snap 139 seq_num 485) dirty exists 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1519: finishing write tid 1 to smithi05955964-4 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1519: finishing write tid 2 to smithi05955964-4 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1519: finishing write tid 4 to smithi05955964-4 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1519: finishing write tid 5 to smithi05955964-4 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 698 (ObjNum 489 snap 140 seq_num 489) dirty exists 2022-04-28T19:49:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:1519: left oid 4 (ObjNum 489 snap 140 seq_num 489) 2022-04-28T19:49:09.932 INFO:tasks.rados.rados.0.smithi059.stdout:1518: done (3 left) 2022-04-28T19:49:09.932 INFO:tasks.rados.rados.0.smithi059.stdout:1519: done (2 left) 2022-04-28T19:49:09.932 INFO:tasks.rados.rados.0.smithi059.stdout:1522: done (1 left) 2022-04-28T19:49:09.933 INFO:tasks.rados.rados.0.smithi059.stdout:1523: rollback oid 13 current snap is 140 2022-04-28T19:49:09.933 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 13 to 139 2022-04-28T19:49:09.933 INFO:tasks.rados.rados.0.smithi059.stdout:1524: snap_create 2022-04-28T19:49:09.934 INFO:tasks.rados.rados.0.smithi059.stdout:1521: finishing write tid 1 to smithi05955964-3 2022-04-28T19:49:09.934 INFO:tasks.rados.rados.0.smithi059.stdout:1521: finishing write tid 2 to smithi05955964-3 2022-04-28T19:49:09.935 INFO:tasks.rados.rados.0.smithi059.stdout:1521: finishing write tid 4 to smithi05955964-3 2022-04-28T19:49:09.936 INFO:tasks.rados.rados.0.smithi059.stdout:1521: finishing write tid 5 to smithi05955964-3 2022-04-28T19:49:09.936 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 661 (ObjNum 491 snap 140 seq_num 491) dirty exists 2022-04-28T19:49:09.936 INFO:tasks.rados.rados.0.smithi059.stdout:1521: left oid 3 (ObjNum 491 snap 140 seq_num 491) 2022-04-28T19:49:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:1523: finishing rollback tid 0 to smithi05955964-13 2022-04-28T19:49:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:1523: finishing rollback tid 1 to smithi05955964-13 2022-04-28T19:49:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:1523: finishing rollback tid 2 to smithi05955964-13 2022-04-28T19:49:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 375 (ObjNum 398 snap 122 seq_num 398) dirty exists 2022-04-28T19:49:09.941 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[57771]: debug 2022-04-28T19:49:09.605+0000 7f2d59607680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2022-04-28T19:49:09.941 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[57771]: debug 2022-04-28T19:49:09.821+0000 7f2d59607680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: cluster 2022-04-28T19:49:08.798740+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.645177+0000 mon.a (mon.0) 17 : cluster [DBG] Standby manager daemon x restarted 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.645302+0000 mon.a (mon.0) 18 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: audit 2022-04-28T19:49:09.647430+0000 mon.b (mon.2) 140 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: audit 2022-04-28T19:49:09.648555+0000 mon.b (mon.2) 141 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: audit 2022-04-28T19:49:09.650583+0000 mon.b (mon.2) 142 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[29114]: audit 2022-04-28T19:49:09.651357+0000 mon.b (mon.2) 143 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:49:09.942 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: cluster 2022-04-28T19:49:08.798740+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2022-04-28T19:49:09.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.645177+0000 mon.a (mon.0) 17 : cluster [DBG] Standby manager daemon x restarted 2022-04-28T19:49:09.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.645302+0000 mon.a (mon.0) 18 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:49:09.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: audit 2022-04-28T19:49:09.647430+0000 mon.b (mon.2) 140 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:49:09.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: audit 2022-04-28T19:49:09.648555+0000 mon.b (mon.2) 141 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:49:09.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: audit 2022-04-28T19:49:09.650583+0000 mon.b (mon.2) 142 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:49:09.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[59745]: audit 2022-04-28T19:49:09.651357+0000 mon.b (mon.2) 143 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:49:09.974 INFO:tasks.rados.rados.0.smithi059.stdout:1521: done (2 left) 2022-04-28T19:49:09.974 INFO:tasks.rados.rados.0.smithi059.stdout:1523: done (1 left) 2022-04-28T19:49:09.975 INFO:tasks.rados.rados.0.smithi059.stdout:1524: done (0 left) 2022-04-28T19:49:09.975 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 141 2022-04-28T19:49:09.975 INFO:tasks.rados.rados.0.smithi059.stdout:1525: seq_num 493 ranges {344064=663552,1007616=401408} 2022-04-28T19:49:09.978 INFO:tasks.rados.rados.0.smithi059.stdout:1525: writing smithi05955964-38 from 344064 to 1007616 tid 1 2022-04-28T19:49:09.981 INFO:tasks.rados.rados.0.smithi059.stdout:1525: writing smithi05955964-38 from 1007616 to 1409024 tid 2 2022-04-28T19:49:09.981 INFO:tasks.rados.rados.0.smithi059.stdout:1526: rollback oid 50 current snap is 141 2022-04-28T19:49:09.981 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 140 2022-04-28T19:49:09.982 INFO:tasks.rados.rados.0.smithi059.stdout:1527: setattr oid 41 current snap is 141 2022-04-28T19:49:09.983 INFO:tasks.rados.rados.0.smithi059.stdout:1528: rollback oid 7 current snap is 141 2022-04-28T19:49:09.983 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 7 to 139 2022-04-28T19:49:09.983 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 141 2022-04-28T19:49:09.983 INFO:tasks.rados.rados.0.smithi059.stdout:1529: seq_num 494 ranges {4538368=679936,5218304=401408} 2022-04-28T19:49:10.016 INFO:tasks.rados.rados.0.smithi059.stdout:1529: writing smithi05955964-33 from 4538368 to 5218304 tid 1 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1529: writing smithi05955964-33 from 5218304 to 5619712 tid 2 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1530: setattr oid 49 current snap is 141 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1525: finishing write tid 1 to smithi05955964-38 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1525: finishing write tid 2 to smithi05955964-38 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1525: finishing write tid 4 to smithi05955964-38 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1526: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 611 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1526: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:1525: finishing write tid 5 to smithi05955964-38 2022-04-28T19:49:10.017 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 702 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:49:10.018 INFO:tasks.rados.rados.0.smithi059.stdout:1525: left oid 38 (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:49:10.018 INFO:tasks.rados.rados.0.smithi059.stdout:1526: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:49:10.018 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 745 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:49:10.018 INFO:tasks.rados.rados.0.smithi059.stdout:1528: finishing rollback tid 0 to smithi05955964-7 2022-04-28T19:49:10.018 INFO:tasks.rados.rados.0.smithi059.stdout:1528: finishing rollback tid 1 to smithi05955964-7 2022-04-28T19:49:10.019 INFO:tasks.rados.rados.0.smithi059.stdout:1528: finishing rollback tid 2 to smithi05955964-7 2022-04-28T19:49:10.019 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 706 (ObjNum 484 snap 139 seq_num 484) dirty exists 2022-04-28T19:49:10.019 INFO:tasks.rados.rados.0.smithi059.stdout:1525: done (5 left) 2022-04-28T19:49:10.019 INFO:tasks.rados.rados.0.smithi059.stdout:1526: done (4 left) 2022-04-28T19:49:10.019 INFO:tasks.rados.rados.0.smithi059.stdout:1527: done (3 left) 2022-04-28T19:49:10.020 INFO:tasks.rados.rados.0.smithi059.stdout:1528: done (2 left) 2022-04-28T19:49:10.020 INFO:tasks.rados.rados.0.smithi059.stdout:1531: delete oid 18 current snap is 141 2022-04-28T19:49:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:1529: finishing write tid 1 to smithi05955964-33 2022-04-28T19:49:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:1529: finishing write tid 2 to smithi05955964-33 2022-04-28T19:49:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 715 (ObjNum 2 snap 1808526322 seq_num 2686058498) dirty exists 2022-04-28T19:49:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:1529: finishing write tid 4 to smithi05955964-33 2022-04-28T19:49:10.022 INFO:tasks.rados.rados.0.smithi059.stdout:1530: done (2 left) 2022-04-28T19:49:10.022 INFO:tasks.rados.rados.0.smithi059.stdout:1531: done (1 left) 2022-04-28T19:49:10.022 INFO:tasks.rados.rados.0.smithi059.stdout:1532: snap_remove snap 139 2022-04-28T19:49:10.118 INFO:journalctl@ceph.mgr.x.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[41342]: debug 2022-04-28T19:49:09.619+0000 7f5c7477e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-04-28T19:49:10.159 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: cluster 2022-04-28T19:49:08.798740+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2022-04-28T19:49:10.159 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.645177+0000 mon.a (mon.0) 17 : cluster [DBG] Standby manager daemon x restarted 2022-04-28T19:49:10.160 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.645302+0000 mon.a (mon.0) 18 : cluster [DBG] Standby manager daemon x started 2022-04-28T19:49:10.160 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: audit 2022-04-28T19:49:09.647430+0000 mon.b (mon.2) 140 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2022-04-28T19:49:10.160 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: audit 2022-04-28T19:49:09.648555+0000 mon.b (mon.2) 141 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:49:10.160 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: audit 2022-04-28T19:49:09.650583+0000 mon.b (mon.2) 142 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2022-04-28T19:49:10.161 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:09 smithi139 bash[23439]: audit 2022-04-28T19:49:09.651357+0000 mon.b (mon.2) 143 : audit [DBG] from='mgr.? 172.21.15.139:0/463208285' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:49:10.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:09 smithi059 bash[57771]: debug 2022-04-28T19:49:09.937+0000 7f2d59607680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2022-04-28T19:49:10.976 INFO:tasks.rados.rados.0.smithi059.stdout:1532: done (1 left) 2022-04-28T19:49:10.977 INFO:tasks.rados.rados.0.smithi059.stdout:1533: read oid 22 snap 140 2022-04-28T19:49:10.977 INFO:tasks.rados.rados.0.smithi059.stdout:1533: expect (ObjNum 474 snap 139 seq_num 474) 2022-04-28T19:49:10.977 INFO:tasks.rados.rados.0.smithi059.stdout:1534: delete oid 19 current snap is 141 2022-04-28T19:49:10.977 INFO:tasks.rados.rados.0.smithi059.stdout:1529: finishing write tid 5 to smithi05955964-33 2022-04-28T19:49:10.977 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 749 (ObjNum 494 snap 141 seq_num 494) dirty exists 2022-04-28T19:49:10.977 INFO:tasks.rados.rados.0.smithi059.stdout:1529: left oid 33 (ObjNum 494 snap 141 seq_num 494) 2022-04-28T19:49:10.984 INFO:tasks.rados.rados.0.smithi059.stdout:1533: expect (ObjNum 474 snap 139 seq_num 474) 2022-04-28T19:49:11.041 INFO:tasks.rados.rados.0.smithi059.stdout:1529: done (2 left) 2022-04-28T19:49:11.041 INFO:tasks.rados.rados.0.smithi059.stdout:1533: done (1 left) 2022-04-28T19:49:11.041 INFO:tasks.rados.rados.0.smithi059.stdout:1534: done (0 left) 2022-04-28T19:49:11.041 INFO:tasks.rados.rados.0.smithi059.stdout:1535: read oid 32 snap -1 2022-04-28T19:49:11.041 INFO:tasks.rados.rados.0.smithi059.stdout:1535: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:11.042 INFO:tasks.rados.rados.0.smithi059.stdout:1536: read oid 7 snap -1 2022-04-28T19:49:11.042 INFO:tasks.rados.rados.0.smithi059.stdout:1536: expect (ObjNum 484 snap 139 seq_num 484) 2022-04-28T19:49:11.042 INFO:tasks.rados.rados.0.smithi059.stdout:1537: read oid 45 snap -1 2022-04-28T19:49:11.042 INFO:tasks.rados.rados.0.smithi059.stdout:1537: expect deleted 2022-04-28T19:49:11.042 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 141 2022-04-28T19:49:11.043 INFO:tasks.rados.rados.0.smithi059.stdout:1538: seq_num 497 ranges {5750784=729088,6479872=401408} 2022-04-28T19:49:11.080 INFO:tasks.rados.rados.0.smithi059.stdout:1538: writing smithi05955964-6 from 5750784 to 6479872 tid 1 2022-04-28T19:49:11.083 INFO:tasks.rados.rados.0.smithi059.stdout:1538: writing smithi05955964-6 from 6479872 to 6881280 tid 2 2022-04-28T19:49:11.083 INFO:tasks.rados.rados.0.smithi059.stdout:1539: rollback oid 37 current snap is 141 2022-04-28T19:49:11.084 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 37 to 140 2022-04-28T19:49:11.084 INFO:tasks.rados.rados.0.smithi059.stdout:1536: expect (ObjNum 484 snap 139 seq_num 484) 2022-04-28T19:49:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.827317+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2022-04-28T19:49:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.827984+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e27: y(active, since 49s), standbys: x 2022-04-28T19:49:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.961811+0000 mon.a (mon.0) 21 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:49:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.962475+0000 mon.a (mon.0) 22 : cluster [INF] Activating manager daemon y 2022-04-28T19:49:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[29114]: cluster 2022-04-28T19:49:09.966738+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2022-04-28T19:49:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.827317+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2022-04-28T19:49:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.827984+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e27: y(active, since 49s), standbys: x 2022-04-28T19:49:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.961811+0000 mon.a (mon.0) 21 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:49:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.962475+0000 mon.a (mon.0) 22 : cluster [INF] Activating manager daemon y 2022-04-28T19:49:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:10 smithi059 bash[59745]: cluster 2022-04-28T19:49:09.966738+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2022-04-28T19:49:11.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:10 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.827317+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2022-04-28T19:49:11.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:10 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.827984+0000 mon.a (mon.0) 20 : cluster [DBG] mgrmap e27: y(active, since 49s), standbys: x 2022-04-28T19:49:11.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:10 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.961811+0000 mon.a (mon.0) 21 : cluster [INF] Active manager daemon y restarted 2022-04-28T19:49:11.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:10 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.962475+0000 mon.a (mon.0) 22 : cluster [INF] Activating manager daemon y 2022-04-28T19:49:11.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:10 smithi139 bash[23439]: cluster 2022-04-28T19:49:09.966738+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2022-04-28T19:49:11.273 INFO:tasks.rados.rados.0.smithi059.stdout:1535: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:11.394 INFO:tasks.rados.rados.0.smithi059.stdout:1538: finishing write tid 1 to smithi05955964-6 2022-04-28T19:49:11.394 INFO:tasks.rados.rados.0.smithi059.stdout:1538: finishing write tid 2 to smithi05955964-6 2022-04-28T19:49:11.394 INFO:tasks.rados.rados.0.smithi059.stdout:1539: finishing rollback tid 0 to smithi05955964-37 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1538: finishing write tid 4 to smithi05955964-6 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1539: finishing rollback tid 1 to smithi05955964-37 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1539: finishing rollback tid 2 to smithi05955964-37 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 744 (ObjNum 443 snap 130 seq_num 443) dirty exists 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1538: finishing write tid 5 to smithi05955964-6 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 667 (ObjNum 497 snap 141 seq_num 497) dirty exists 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1538: left oid 6 (ObjNum 497 snap 141 seq_num 497) 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1535: done (4 left) 2022-04-28T19:49:11.395 INFO:tasks.rados.rados.0.smithi059.stdout:1536: done (3 left) 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1537: done (2 left) 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1538: done (1 left) 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1539: done (0 left) 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1540: read oid 14 snap 140 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1540: expect deleted 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1541: read oid 20 snap -1 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1541: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:11.396 INFO:tasks.rados.rados.0.smithi059.stdout:1542: setattr oid 40 current snap is 141 2022-04-28T19:49:11.401 INFO:tasks.rados.rados.0.smithi059.stdout:1540: done (2 left) 2022-04-28T19:49:11.401 INFO:tasks.rados.rados.0.smithi059.stdout:1543: read oid 8 snap -1 2022-04-28T19:49:11.402 INFO:tasks.rados.rados.0.smithi059.stdout:1543: expect (ObjNum 482 snap 139 seq_num 482) 2022-04-28T19:49:11.402 INFO:tasks.rados.rados.0.smithi059.stdout:1544: snap_create 2022-04-28T19:49:11.416 INFO:tasks.rados.rados.0.smithi059.stdout:1541: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:11.904 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 716 (ObjNum 450 snap 132 seq_num 450) dirty exists 2022-04-28T19:49:11.904 INFO:tasks.rados.rados.0.smithi059.stdout:1543: expect (ObjNum 482 snap 139 seq_num 482) 2022-04-28T19:49:12.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:11 smithi139 bash[23439]: cluster 2022-04-28T19:49:10.831171+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e28: y(active, starting, since 0.868783s), standbys: x 2022-04-28T19:49:12.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:11 smithi139 bash[23439]: cluster 2022-04-28T19:49:10.970086+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2022-04-28T19:49:12.332 INFO:tasks.rados.rados.0.smithi059.stdout:1541: done (3 left) 2022-04-28T19:49:12.333 INFO:tasks.rados.rados.0.smithi059.stdout:1542: done (2 left) 2022-04-28T19:49:12.333 INFO:tasks.rados.rados.0.smithi059.stdout:1543: done (1 left) 2022-04-28T19:49:12.333 INFO:tasks.rados.rados.0.smithi059.stdout:1544: done (0 left) 2022-04-28T19:49:12.333 INFO:tasks.rados.rados.0.smithi059.stdout:1545: snap_create 2022-04-28T19:49:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:11 smithi059 bash[59745]: cluster 2022-04-28T19:49:10.831171+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e28: y(active, starting, since 0.868783s), standbys: x 2022-04-28T19:49:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:11 smithi059 bash[59745]: cluster 2022-04-28T19:49:10.970086+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2022-04-28T19:49:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:11 smithi059 bash[29114]: cluster 2022-04-28T19:49:10.831171+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e28: y(active, starting, since 0.868783s), standbys: x 2022-04-28T19:49:12.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:11 smithi059 bash[29114]: cluster 2022-04-28T19:49:10.970086+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2022-04-28T19:49:12.979 INFO:tasks.rados.rados.0.smithi059.stdout:1545: done (0 left) 2022-04-28T19:49:12.979 INFO:tasks.rados.rados.0.smithi059.stdout:1546: rmattr oid 27 current snap is 143 2022-04-28T19:49:12.979 INFO:tasks.rados.rados.0.smithi059.stdout:1546: done (0 left) 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:1547: rollback oid 10 current snap is 143 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 141 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:1548: read oid 14 snap -1 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:1548: expect deleted 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:1549: read oid 42 snap -1 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:1549: expect (ObjNum 463 snap 136 seq_num 463) 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 143 2022-04-28T19:49:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:1550: seq_num 498 ranges {1654784=745472,2400256=401408} 2022-04-28T19:49:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:1550: writing smithi05955964-27 from 1654784 to 2400256 tid 1 2022-04-28T19:49:12.999 INFO:tasks.rados.rados.0.smithi059.stdout:1550: writing smithi05955964-27 from 2400256 to 2801664 tid 2 2022-04-28T19:49:12.999 INFO:tasks.rados.rados.0.smithi059.stdout:1551: rmattr oid 30 current snap is 143 2022-04-28T19:49:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:1551: done (4 left) 2022-04-28T19:49:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 143 2022-04-28T19:49:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:1552: seq_num 499 ranges {11550720=761856,12312576=401408} 2022-04-28T19:49:13.068 INFO:tasks.rados.rados.0.smithi059.stdout:1552: writing smithi05955964-5 from 11550720 to 12312576 tid 1 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:1552: writing smithi05955964-5 from 12312576 to 12713984 tid 2 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:1553: delete oid 6 current snap is 143 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:1547: finishing rollback tid 0 to smithi05955964-10 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:1547: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:1547: finishing rollback tid 2 to smithi05955964-10 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 718 (ObjNum 457 snap 135 seq_num 457) dirty exists 2022-04-28T19:49:13.071 INFO:tasks.rados.rados.0.smithi059.stdout:1549: expect (ObjNum 463 snap 136 seq_num 463) 2022-04-28T19:49:13.090 INFO:tasks.rados.rados.0.smithi059.stdout:1550: finishing write tid 1 to smithi05955964-27 2022-04-28T19:49:13.091 INFO:tasks.rados.rados.0.smithi059.stdout:1550: finishing write tid 2 to smithi05955964-27 2022-04-28T19:49:13.091 INFO:tasks.rados.rados.0.smithi059.stdout:1550: finishing write tid 4 to smithi05955964-27 2022-04-28T19:49:13.091 INFO:tasks.rados.rados.0.smithi059.stdout:1550: finishing write tid 5 to smithi05955964-27 2022-04-28T19:49:13.091 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 722 (ObjNum 498 snap 143 seq_num 498) dirty exists 2022-04-28T19:49:13.091 INFO:tasks.rados.rados.0.smithi059.stdout:1550: left oid 27 (ObjNum 498 snap 143 seq_num 498) 2022-04-28T19:49:13.091 INFO:tasks.rados.rados.0.smithi059.stdout:1552: finishing write tid 1 to smithi05955964-5 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1552: finishing write tid 2 to smithi05955964-5 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1552: finishing write tid 4 to smithi05955964-5 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1547: done (5 left) 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1548: done (4 left) 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1549: done (3 left) 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1550: done (2 left) 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1553: done (1 left) 2022-04-28T19:49:13.092 INFO:tasks.rados.rados.0.smithi059.stdout:1554: snap_remove snap 140 2022-04-28T19:49:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:12 smithi059 bash[29114]: cluster 2022-04-28T19:49:11.973523+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2022-04-28T19:49:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:12 smithi059 bash[59745]: cluster 2022-04-28T19:49:11.973523+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2022-04-28T19:49:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:12 smithi139 bash[23439]: cluster 2022-04-28T19:49:11.973523+0000 mon.a (mon.0) 26 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2022-04-28T19:49:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:13.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:49:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:13.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: dial tcp 172.21.15.59:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:49:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:13.518Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: dial tcp 172.21.15.139:8443: connect: connection refused; Post https://smithi059:8443/api/prometheus_receiver: dial tcp 172.21.15.59:8443: connect: connection refused" 2022-04-28T19:49:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:13.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: dial tcp 172.21.15.59:8443: connect: connection refused" context_err="context deadline exceeded" 2022-04-28T19:49:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:13.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:13.523Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: dial tcp 172.21.15.59:8443: connect: connection refused; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:14.005 INFO:tasks.rados.rados.0.smithi059.stdout:1554: done (1 left) 2022-04-28T19:49:14.005 INFO:tasks.rados.rados.0.smithi059.stdout:1555: setattr oid 23 current snap is 143 2022-04-28T19:49:14.006 INFO:tasks.rados.rados.0.smithi059.stdout:1552: finishing write tid 5 to smithi05955964-5 2022-04-28T19:49:14.006 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 690 (ObjNum 499 snap 143 seq_num 499) dirty exists 2022-04-28T19:49:14.006 INFO:tasks.rados.rados.0.smithi059.stdout:1552: left oid 5 (ObjNum 499 snap 143 seq_num 499) 2022-04-28T19:49:14.006 INFO:tasks.rados.rados.0.smithi059.stdout:1552: done (1 left) 2022-04-28T19:49:14.006 INFO:tasks.rados.rados.0.smithi059.stdout:1556: copy_from oid 42 from oid 28 current snap is 143 2022-04-28T19:49:14.006 INFO:tasks.rados.rados.0.smithi059.stdout:1557: read oid 26 snap -1 2022-04-28T19:49:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:1557: expect (ObjNum 442 snap 130 seq_num 442) 2022-04-28T19:49:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:1558: read oid 35 snap -1 2022-04-28T19:49:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:1558: expect (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:49:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:1559: delete oid 19 current snap is 143 2022-04-28T19:49:14.024 INFO:tasks.rados.rados.0.smithi059.stdout:1558: expect (ObjNum 265 snap 77 seq_num 265) 2022-04-28T19:49:14.118 INFO:tasks.rados.rados.0.smithi059.stdout:1557: expect (ObjNum 442 snap 130 seq_num 442) 2022-04-28T19:49:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[29114]: cluster 2022-04-28T19:49:12.975632+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2022-04-28T19:49:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:13 smithi059 bash[59745]: cluster 2022-04-28T19:49:12.975632+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2022-04-28T19:49:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:13 smithi139 bash[23439]: cluster 2022-04-28T19:49:12.975632+0000 mon.a (mon.0) 27 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2022-04-28T19:49:14.385 INFO:tasks.rados.rados.0.smithi059.stdout:1556: finishing copy_from to smithi05955964-42 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 384 (ObjNum 385 snap 118 seq_num 385) dirty exists 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:1556: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 724 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:1555: done (4 left) 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:1556: done (3 left) 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:1557: done (2 left) 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:1558: done (1 left) 2022-04-28T19:49:14.386 INFO:tasks.rados.rados.0.smithi059.stdout:1559: done (0 left) 2022-04-28T19:49:14.387 INFO:tasks.rados.rados.0.smithi059.stdout:1560: rollback oid 19 current snap is 143 2022-04-28T19:49:14.387 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 19 to 142 2022-04-28T19:49:14.387 INFO:tasks.rados.rados.0.smithi059.stdout:1561: rmattr oid 33 current snap is 143 2022-04-28T19:49:14.387 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 143 2022-04-28T19:49:14.387 INFO:tasks.rados.rados.0.smithi059.stdout:1562: seq_num 502 ranges {2768896=409600,3178496=417792,3596288=385024} 2022-04-28T19:49:14.410 INFO:tasks.rados.rados.0.smithi059.stdout:1562: writing smithi05955964-46 from 2768896 to 3178496 tid 1 2022-04-28T19:49:14.412 INFO:tasks.rados.rados.0.smithi059.stdout:1562: writing smithi05955964-46 from 3178496 to 3596288 tid 2 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:1562: writing smithi05955964-46 from 3596288 to 3981312 tid 3 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:1563: rmattr oid 40 current snap is 143 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:1560: finishing rollback tid 1 to smithi05955964-19 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 0 (ObjNum 1 snap 32762 seq_num 4162127248) dirty dne 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 759 (ObjNum 494 snap 141 seq_num 494) dirty exists 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:1560: done (3 left) 2022-04-28T19:49:14.415 INFO:tasks.rados.rados.0.smithi059.stdout:1561: done (2 left) 2022-04-28T19:49:14.416 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 143 2022-04-28T19:49:14.416 INFO:tasks.rados.rados.0.smithi059.stdout:1564: seq_num 503 ranges {0=425984,425984=704512,1130496=98304} 2022-04-28T19:49:14.417 INFO:tasks.rados.rados.0.smithi059.stdout:1564: writing smithi05955964-6 from 0 to 425984 tid 1 2022-04-28T19:49:14.422 INFO:tasks.rados.rados.0.smithi059.stdout:1564: writing smithi05955964-6 from 425984 to 1130496 tid 2 2022-04-28T19:49:14.422 INFO:tasks.rados.rados.0.smithi059.stdout:1564: writing smithi05955964-6 from 1130496 to 1228800 tid 3 2022-04-28T19:49:14.423 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 143 2022-04-28T19:49:14.423 INFO:tasks.rados.rados.0.smithi059.stdout:1565: seq_num 504 ranges {0=450560,450560=589824,1040384=212992} 2022-04-28T19:49:14.425 INFO:tasks.rados.rados.0.smithi059.stdout:1565: writing smithi05955964-18 from 0 to 450560 tid 1 2022-04-28T19:49:14.429 INFO:tasks.rados.rados.0.smithi059.stdout:1565: writing smithi05955964-18 from 450560 to 1040384 tid 2 2022-04-28T19:49:14.431 INFO:tasks.rados.rados.0.smithi059.stdout:1565: writing smithi05955964-18 from 1040384 to 1253376 tid 3 2022-04-28T19:49:14.431 INFO:tasks.rados.rados.0.smithi059.stdout:1566: read oid 25 snap -1 2022-04-28T19:49:14.431 INFO:tasks.rados.rados.0.smithi059.stdout:1566: expect deleted 2022-04-28T19:49:14.431 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 143 2022-04-28T19:49:14.431 INFO:tasks.rados.rados.0.smithi059.stdout:1567: seq_num 505 ranges {0=466944,466944=475136,942080=327680} 2022-04-28T19:49:14.433 INFO:tasks.rados.rados.0.smithi059.stdout:1567: writing smithi05955964-49 from 0 to 466944 tid 1 2022-04-28T19:49:14.436 INFO:tasks.rados.rados.0.smithi059.stdout:1567: writing smithi05955964-49 from 466944 to 942080 tid 2 2022-04-28T19:49:14.438 INFO:tasks.rados.rados.0.smithi059.stdout:1567: writing smithi05955964-49 from 942080 to 1269760 tid 3 2022-04-28T19:49:14.438 INFO:tasks.rados.rados.0.smithi059.stdout:1568: snap_remove snap 141 2022-04-28T19:49:15.000 INFO:tasks.rados.rados.0.smithi059.stdout:1568: done (6 left) 2022-04-28T19:49:15.001 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 143 2022-04-28T19:49:15.001 INFO:tasks.rados.rados.0.smithi059.stdout:1569: seq_num 506 ranges {3768320=483328,4251648=761856,5013504=40960} 2022-04-28T19:49:15.027 INFO:tasks.rados.rados.0.smithi059.stdout:1569: writing smithi05955964-17 from 3768320 to 4251648 tid 1 2022-04-28T19:49:15.040 INFO:tasks.rados.rados.0.smithi059.stdout:1569: writing smithi05955964-17 from 4251648 to 5013504 tid 2 2022-04-28T19:49:15.040 INFO:tasks.rados.rados.0.smithi059.stdout:1569: writing smithi05955964-17 from 5013504 to 5054464 tid 3 2022-04-28T19:49:15.040 INFO:tasks.rados.rados.0.smithi059.stdout:1570: snap_create 2022-04-28T19:49:15.040 INFO:tasks.rados.rados.0.smithi059.stdout:1562: finishing write tid 1 to smithi05955964-46 2022-04-28T19:49:15.040 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 717 (ObjNum 450 snap 132 seq_num 450) dirty exists 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:1562: finishing write tid 2 to smithi05955964-46 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:1562: finishing write tid 3 to smithi05955964-46 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:1562: finishing write tid 5 to smithi05955964-46 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:1564: finishing write tid 1 to smithi05955964-6 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:1562: finishing write tid 6 to smithi05955964-46 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 697 (ObjNum 502 snap 143 seq_num 502) dirty exists 2022-04-28T19:49:15.041 INFO:tasks.rados.rados.0.smithi059.stdout:1562: left oid 46 (ObjNum 502 snap 143 seq_num 502) 2022-04-28T19:49:15.042 INFO:tasks.rados.rados.0.smithi059.stdout:1564: finishing write tid 2 to smithi05955964-6 2022-04-28T19:49:15.042 INFO:tasks.rados.rados.0.smithi059.stdout:1564: finishing write tid 3 to smithi05955964-6 2022-04-28T19:49:15.042 INFO:tasks.rados.rados.0.smithi059.stdout:1564: finishing write tid 5 to smithi05955964-6 2022-04-28T19:49:15.042 INFO:tasks.rados.rados.0.smithi059.stdout:1564: finishing write tid 6 to smithi05955964-6 2022-04-28T19:49:15.042 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 675 (ObjNum 503 snap 143 seq_num 503) dirty exists 2022-04-28T19:49:15.042 INFO:tasks.rados.rados.0.smithi059.stdout:1564: left oid 6 (ObjNum 503 snap 143 seq_num 503) 2022-04-28T19:49:15.043 INFO:tasks.rados.rados.0.smithi059.stdout:1565: finishing write tid 1 to smithi05955964-18 2022-04-28T19:49:15.043 INFO:tasks.rados.rados.0.smithi059.stdout:1565: finishing write tid 2 to smithi05955964-18 2022-04-28T19:49:15.043 INFO:tasks.rados.rados.0.smithi059.stdout:1565: finishing write tid 3 to smithi05955964-18 2022-04-28T19:49:15.043 INFO:tasks.rados.rados.0.smithi059.stdout:1565: finishing write tid 5 to smithi05955964-18 2022-04-28T19:49:15.043 INFO:tasks.rados.rados.0.smithi059.stdout:1565: finishing write tid 6 to smithi05955964-18 2022-04-28T19:49:15.043 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 754 (ObjNum 504 snap 143 seq_num 504) dirty exists 2022-04-28T19:49:15.044 INFO:tasks.rados.rados.0.smithi059.stdout:1565: left oid 18 (ObjNum 504 snap 143 seq_num 504) 2022-04-28T19:49:15.044 INFO:tasks.rados.rados.0.smithi059.stdout:1567: finishing write tid 1 to smithi05955964-49 2022-04-28T19:49:15.044 INFO:tasks.rados.rados.0.smithi059.stdout:1567: finishing write tid 2 to smithi05955964-49 2022-04-28T19:49:15.044 INFO:tasks.rados.rados.0.smithi059.stdout:1567: finishing write tid 3 to smithi05955964-49 2022-04-28T19:49:15.044 INFO:tasks.rados.rados.0.smithi059.stdout:1567: finishing write tid 5 to smithi05955964-49 2022-04-28T19:49:15.044 INFO:tasks.rados.rados.0.smithi059.stdout:1567: finishing write tid 6 to smithi05955964-49 2022-04-28T19:49:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 721 (ObjNum 505 snap 143 seq_num 505) dirty exists 2022-04-28T19:49:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:1567: left oid 49 (ObjNum 505 snap 143 seq_num 505) 2022-04-28T19:49:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:1569: finishing write tid 1 to smithi05955964-17 2022-04-28T19:49:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:1569: finishing write tid 2 to smithi05955964-17 2022-04-28T19:49:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:1569: finishing write tid 3 to smithi05955964-17 2022-04-28T19:49:15.047 INFO:tasks.rados.rados.0.smithi059.stdout:1569: finishing write tid 5 to smithi05955964-17 2022-04-28T19:49:15.047 INFO:tasks.rados.rados.0.smithi059.stdout:1569: finishing write tid 6 to smithi05955964-17 2022-04-28T19:49:15.047 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 382 (ObjNum 506 snap 143 seq_num 506) dirty exists 2022-04-28T19:49:15.047 INFO:tasks.rados.rados.0.smithi059.stdout:1569: left oid 17 (ObjNum 506 snap 143 seq_num 506) 2022-04-28T19:49:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:15 smithi059 bash[29114]: cluster 2022-04-28T19:49:13.995155+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2022-04-28T19:49:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:15 smithi059 bash[59745]: cluster 2022-04-28T19:49:13.995155+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2022-04-28T19:49:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:15 smithi139 bash[23439]: cluster 2022-04-28T19:49:13.995155+0000 mon.a (mon.0) 28 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2022-04-28T19:49:16.013 INFO:tasks.rados.rados.0.smithi059.stdout:1562: done (7 left) 2022-04-28T19:49:16.013 INFO:tasks.rados.rados.0.smithi059.stdout:1563: done (6 left) 2022-04-28T19:49:16.013 INFO:tasks.rados.rados.0.smithi059.stdout:1564: done (5 left) 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:1565: done (4 left) 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:1566: done (3 left) 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:1567: done (2 left) 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:1569: done (1 left) 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:1570: done (0 left) 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 144 2022-04-28T19:49:16.014 INFO:tasks.rados.rados.0.smithi059.stdout:1571: seq_num 507 ranges {3383296=499712,3883008=638976,4521984=163840} 2022-04-28T19:49:16.034 INFO:tasks.rados.rados.0.smithi059.stdout:1571: writing smithi05955964-44 from 3383296 to 3883008 tid 1 2022-04-28T19:49:16.038 INFO:tasks.rados.rados.0.smithi059.stdout:1571: writing smithi05955964-44 from 3883008 to 4521984 tid 2 2022-04-28T19:49:16.039 INFO:tasks.rados.rados.0.smithi059.stdout:1571: writing smithi05955964-44 from 4521984 to 4685824 tid 3 2022-04-28T19:49:16.039 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 144 2022-04-28T19:49:16.040 INFO:tasks.rados.rados.0.smithi059.stdout:1572: seq_num 508 ranges {557056=516096,1073152=524288,1597440=278528} 2022-04-28T19:49:16.045 INFO:tasks.rados.rados.0.smithi059.stdout:1572: writing smithi05955964-48 from 557056 to 1073152 tid 1 2022-04-28T19:49:16.049 INFO:tasks.rados.rados.0.smithi059.stdout:1572: writing smithi05955964-48 from 1073152 to 1597440 tid 2 2022-04-28T19:49:16.051 INFO:tasks.rados.rados.0.smithi059.stdout:1572: writing smithi05955964-48 from 1597440 to 1875968 tid 3 2022-04-28T19:49:16.051 INFO:tasks.rados.rados.0.smithi059.stdout:1573: delete oid 25 current snap is 144 2022-04-28T19:49:16.051 INFO:tasks.rados.rados.0.smithi059.stdout:1571: finishing write tid 1 to smithi05955964-44 2022-04-28T19:49:16.051 INFO:tasks.rados.rados.0.smithi059.stdout:1571: finishing write tid 2 to smithi05955964-44 2022-04-28T19:49:16.051 INFO:tasks.rados.rados.0.smithi059.stdout:1571: finishing write tid 3 to smithi05955964-44 2022-04-28T19:49:16.051 INFO:tasks.rados.rados.0.smithi059.stdout:1571: finishing write tid 5 to smithi05955964-44 2022-04-28T19:49:16.052 INFO:tasks.rados.rados.0.smithi059.stdout:1571: finishing write tid 6 to smithi05955964-44 2022-04-28T19:49:16.052 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 638 (ObjNum 507 snap 144 seq_num 507) dirty exists 2022-04-28T19:49:16.052 INFO:tasks.rados.rados.0.smithi059.stdout:1571: left oid 44 (ObjNum 507 snap 144 seq_num 507) 2022-04-28T19:49:16.052 INFO:tasks.rados.rados.0.smithi059.stdout:1572: finishing write tid 1 to smithi05955964-48 2022-04-28T19:49:16.053 INFO:tasks.rados.rados.0.smithi059.stdout:1572: finishing write tid 2 to smithi05955964-48 2022-04-28T19:49:16.054 INFO:tasks.rados.rados.0.smithi059.stdout:1572: finishing write tid 3 to smithi05955964-48 2022-04-28T19:49:16.054 INFO:tasks.rados.rados.0.smithi059.stdout:1572: finishing write tid 5 to smithi05955964-48 2022-04-28T19:49:16.055 INFO:tasks.rados.rados.0.smithi059.stdout:1571: done (2 left) 2022-04-28T19:49:16.055 INFO:tasks.rados.rados.0.smithi059.stdout:1573: done (1 left) 2022-04-28T19:49:16.055 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 144 2022-04-28T19:49:16.056 INFO:tasks.rados.rados.0.smithi059.stdout:1574: seq_num 510 ranges {1613824=548864,2162688=745472,2908160=57344} 2022-04-28T19:49:16.067 INFO:tasks.rados.rados.0.smithi059.stdout:1574: writing smithi05955964-22 from 1613824 to 2162688 tid 1 2022-04-28T19:49:16.073 INFO:tasks.rados.rados.0.smithi059.stdout:1574: writing smithi05955964-22 from 2162688 to 2908160 tid 2 2022-04-28T19:49:16.073 INFO:tasks.rados.rados.0.smithi059.stdout:1574: writing smithi05955964-22 from 2908160 to 2965504 tid 3 2022-04-28T19:49:16.073 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 144 2022-04-28T19:49:16.073 INFO:tasks.rados.rados.0.smithi059.stdout:1575: seq_num 511 ranges {1269760=565248,1835008=630784,2465792=172032} 2022-04-28T19:49:16.083 INFO:tasks.rados.rados.0.smithi059.stdout:1575: writing smithi05955964-49 from 1269760 to 1835008 tid 1 2022-04-28T19:49:16.087 INFO:tasks.rados.rados.0.smithi059.stdout:1575: writing smithi05955964-49 from 1835008 to 2465792 tid 2 2022-04-28T19:49:16.091 INFO:tasks.rados.rados.0.smithi059.stdout:1575: writing smithi05955964-49 from 2465792 to 2637824 tid 3 2022-04-28T19:49:16.091 INFO:tasks.rados.rados.0.smithi059.stdout:1576: read oid 44 snap 142 2022-04-28T19:49:16.091 INFO:tasks.rados.rados.0.smithi059.stdout:1576: expect (ObjNum 486 snap 139 seq_num 486) 2022-04-28T19:49:16.092 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 144 2022-04-28T19:49:16.092 INFO:tasks.rados.rados.0.smithi059.stdout:1577: seq_num 512 ranges {3981312=581632,4562944=516096,5079040=286720} 2022-04-28T19:49:16.115 INFO:tasks.rados.rados.0.smithi059.stdout:1577: writing smithi05955964-46 from 3981312 to 4562944 tid 1 2022-04-28T19:49:16.118 INFO:tasks.rados.rados.0.smithi059.stdout:1577: writing smithi05955964-46 from 4562944 to 5079040 tid 2 2022-04-28T19:49:16.120 INFO:tasks.rados.rados.0.smithi059.stdout:1577: writing smithi05955964-46 from 5079040 to 5365760 tid 3 2022-04-28T19:49:16.120 INFO:tasks.rados.rados.0.smithi059.stdout:1578: delete oid 2 current snap is 144 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1572: finishing write tid 6 to smithi05955964-48 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 753 (ObjNum 508 snap 144 seq_num 508) dirty exists 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1572: left oid 48 (ObjNum 508 snap 144 seq_num 508) 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1574: finishing write tid 1 to smithi05955964-22 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1574: finishing write tid 2 to smithi05955964-22 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1574: finishing write tid 3 to smithi05955964-22 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1574: finishing write tid 5 to smithi05955964-22 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:1574: finishing write tid 6 to smithi05955964-22 2022-04-28T19:49:16.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 758 (ObjNum 510 snap 144 seq_num 510) dirty exists 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1574: left oid 22 (ObjNum 510 snap 144 seq_num 510) 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1575: finishing write tid 1 to smithi05955964-49 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1575: finishing write tid 2 to smithi05955964-49 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1575: finishing write tid 3 to smithi05955964-49 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1575: finishing write tid 5 to smithi05955964-49 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1575: finishing write tid 6 to smithi05955964-49 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 725 (ObjNum 511 snap 144 seq_num 511) dirty exists 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1575: left oid 49 (ObjNum 511 snap 144 seq_num 511) 2022-04-28T19:49:16.122 INFO:tasks.rados.rados.0.smithi059.stdout:1576: expect (ObjNum 486 snap 139 seq_num 486) 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1577: finishing write tid 1 to smithi05955964-46 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1577: finishing write tid 2 to smithi05955964-46 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1577: finishing write tid 3 to smithi05955964-46 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1577: finishing write tid 5 to smithi05955964-46 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1577: finishing write tid 6 to smithi05955964-46 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 707 (ObjNum 512 snap 144 seq_num 512) dirty exists 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1577: left oid 46 (ObjNum 512 snap 144 seq_num 512) 2022-04-28T19:49:16.273 INFO:tasks.rados.rados.0.smithi059.stdout:1572: done (5 left) 2022-04-28T19:49:16.274 INFO:tasks.rados.rados.0.smithi059.stdout:1574: done (4 left) 2022-04-28T19:49:16.274 INFO:tasks.rados.rados.0.smithi059.stdout:1575: done (3 left) 2022-04-28T19:49:16.274 INFO:tasks.rados.rados.0.smithi059.stdout:1576: done (2 left) 2022-04-28T19:49:16.274 INFO:tasks.rados.rados.0.smithi059.stdout:1577: done (1 left) 2022-04-28T19:49:16.274 INFO:tasks.rados.rados.0.smithi059.stdout:1578: done (0 left) 2022-04-28T19:49:16.274 INFO:tasks.rados.rados.0.smithi059.stdout:1579: setattr oid 28 current snap is 144 2022-04-28T19:49:16.276 INFO:tasks.rados.rados.0.smithi059.stdout:1580: snap_remove snap 143 2022-04-28T19:49:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:16 smithi059 bash[59745]: cluster 2022-04-28T19:49:14.997893+0000 mon.a (mon.0) 29 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2022-04-28T19:49:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:16 smithi059 bash[29114]: cluster 2022-04-28T19:49:14.997893+0000 mon.a (mon.0) 29 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2022-04-28T19:49:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:16 smithi139 bash[23439]: cluster 2022-04-28T19:49:14.997893+0000 mon.a (mon.0) 29 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2022-04-28T19:49:17.035 INFO:tasks.rados.rados.0.smithi059.stdout:1580: done (1 left) 2022-04-28T19:49:17.056 INFO:tasks.rados.rados.0.smithi059.stdout:1581: setattr oid 50 current snap is 144 2022-04-28T19:49:17.056 INFO:tasks.rados.rados.0.smithi059.stdout:1582: rollback oid 45 current snap is 144 2022-04-28T19:49:17.056 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 142 2022-04-28T19:49:17.057 INFO:tasks.rados.rados.0.smithi059.stdout:1583: copy_from oid 21 from oid 41 current snap is 144 2022-04-28T19:49:17.057 INFO:tasks.rados.rados.0.smithi059.stdout:1584: delete oid 34 current snap is 144 2022-04-28T19:49:17.057 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 410 (ObjNum 385 snap 118 seq_num 385) dirty exists 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:1579: done (4 left) 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:1584: done (3 left) 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:1585: read oid 12 snap -1 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:1585: expect (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:1586: setattr oid 10 current snap is 144 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 144 2022-04-28T19:49:17.058 INFO:tasks.rados.rados.0.smithi059.stdout:1587: seq_num 515 ranges {679936=630784,1310720=565248,1875968=237568} 2022-04-28T19:49:17.059 INFO:tasks.rados.rados.0.smithi059.stdout:1587: writing smithi05955964-36 from 679936 to 1310720 tid 1 2022-04-28T19:49:17.059 INFO:tasks.rados.rados.0.smithi059.stdout:1587: writing smithi05955964-36 from 1310720 to 1875968 tid 2 2022-04-28T19:49:17.059 INFO:tasks.rados.rados.0.smithi059.stdout:1587: writing smithi05955964-36 from 1875968 to 2113536 tid 3 2022-04-28T19:49:17.059 INFO:tasks.rados.rados.0.smithi059.stdout:1588: snap_create 2022-04-28T19:49:17.059 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 763 (ObjNum 315 snap 93 seq_num 315) dirty exists 2022-04-28T19:49:17.059 INFO:tasks.rados.rados.0.smithi059.stdout:1585: expect (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:1582: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 0 (ObjNum 1 snap 32762 seq_num 4161570800) dirty dne 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 735 (ObjNum 457 snap 135 seq_num 457) dirty exists 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:1587: finishing write tid 1 to smithi05955964-36 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:1587: finishing write tid 2 to smithi05955964-36 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:1587: finishing write tid 3 to smithi05955964-36 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:1587: finishing write tid 5 to smithi05955964-36 2022-04-28T19:49:17.066 INFO:tasks.rados.rados.0.smithi059.stdout:1583: finishing copy_from to smithi05955964-21 2022-04-28T19:49:17.067 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 611 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:17.067 INFO:tasks.rados.rados.0.smithi059.stdout:1587: finishing write tid 6 to smithi05955964-36 2022-04-28T19:49:17.067 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 729 (ObjNum 515 snap 144 seq_num 515) dirty exists 2022-04-28T19:49:17.067 INFO:tasks.rados.rados.0.smithi059.stdout:1587: left oid 36 (ObjNum 515 snap 144 seq_num 515) 2022-04-28T19:49:17.067 INFO:tasks.rados.rados.0.smithi059.stdout:1583: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:49:17.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[57771]: Warning: Permanently added '172.21.15.59' (ECDSA) to the list of known hosts. 2022-04-28T19:49:17.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[57771]: Warning: Permanently added '172.21.15.139' (ECDSA) to the list of known hosts. 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[57771]: debug 2022-04-28T19:49:17.153+0000 7f2cc8e26700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[57771]: debug 2022-04-28T19:49:17.153+0000 7f2cc6e22700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[57771]: debug 2022-04-28T19:49:17.153+0000 7f2cc6e22700 -1 client.0 error registering admin socket command: (17) File exists 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: cluster 2022-04-28T19:49:16.010555+0000 mon.a (mon.0) 30 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.840240+0000 mon.c (mon.1) 145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.840870+0000 mon.c (mon.1) 146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:49:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.841461+0000 mon.c (mon.1) 147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.842735+0000 mon.c (mon.1) 148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.843608+0000 mon.c (mon.1) 149 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.844433+0000 mon.c (mon.1) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.845171+0000 mon.c (mon.1) 151 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.845906+0000 mon.c (mon.1) 152 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.846844+0000 mon.c (mon.1) 153 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.847633+0000 mon.c (mon.1) 154 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.848483+0000 mon.c (mon.1) 155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.849208+0000 mon.c (mon.1) 156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:49:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.849907+0000 mon.c (mon.1) 157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:49:17.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.850670+0000 mon.c (mon.1) 158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:49:17.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.851381+0000 mon.c (mon.1) 159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:49:17.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.851746+0000 mon.c (mon.1) 160 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:49:17.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.852125+0000 mon.c (mon.1) 161 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:49:17.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: cluster 2022-04-28T19:49:16.010555+0000 mon.a (mon.0) 30 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.840240+0000 mon.c (mon.1) 145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.840870+0000 mon.c (mon.1) 146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.841461+0000 mon.c (mon.1) 147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.842735+0000 mon.c (mon.1) 148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.843608+0000 mon.c (mon.1) 149 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.844433+0000 mon.c (mon.1) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.845171+0000 mon.c (mon.1) 151 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.845906+0000 mon.c (mon.1) 152 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:49:17.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.846844+0000 mon.c (mon.1) 153 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.847633+0000 mon.c (mon.1) 154 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.848483+0000 mon.c (mon.1) 155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.849208+0000 mon.c (mon.1) 156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.849907+0000 mon.c (mon.1) 157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.850670+0000 mon.c (mon.1) 158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.851381+0000 mon.c (mon.1) 159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.851746+0000 mon.c (mon.1) 160 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:49:17.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.852125+0000 mon.c (mon.1) 161 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.852582+0000 mon.c (mon.1) 162 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.853890+0000 mon.c (mon.1) 163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.854966+0000 mon.c (mon.1) 164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/client_keyrings"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.855455+0000 mon.c (mon.1) 165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/config_checks"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.855958+0000 mon.c (mon.1) 166 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.856479+0000 mon.c (mon.1) 167 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.857008+0000 mon.c (mon.1) 168 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:49:17.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.857620+0000 mon.c (mon.1) 169 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.858246+0000 mon.c (mon.1) 170 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.858719+0000 mon.c (mon.1) 171 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.859184+0000 mon.c (mon.1) 172 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.859671+0000 mon.c (mon.1) 173 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.860133+0000 mon.c (mon.1) 174 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.860628+0000 mon.c (mon.1) 175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:49:17.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.861058+0000 mon.c (mon.1) 176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:49:17.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.861496+0000 mon.c (mon.1) 177 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:49:17.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.861926+0000 mon.c (mon.1) 178 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:49:17.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.852582+0000 mon.c (mon.1) 162 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:49:17.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.853890+0000 mon.c (mon.1) 163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:49:17.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.854966+0000 mon.c (mon.1) 164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/client_keyrings"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.855455+0000 mon.c (mon.1) 165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/config_checks"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.855958+0000 mon.c (mon.1) 166 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.856479+0000 mon.c (mon.1) 167 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.857008+0000 mon.c (mon.1) 168 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.857620+0000 mon.c (mon.1) 169 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.858246+0000 mon.c (mon.1) 170 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.858719+0000 mon.c (mon.1) 171 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.859184+0000 mon.c (mon.1) 172 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:49:17.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.859671+0000 mon.c (mon.1) 173 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.860133+0000 mon.c (mon.1) 174 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.860628+0000 mon.c (mon.1) 175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.861058+0000 mon.c (mon.1) 176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.861496+0000 mon.c (mon.1) 177 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.861926+0000 mon.c (mon.1) 178 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.862353+0000 mon.c (mon.1) 179 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.862832+0000 mon.c (mon.1) 180 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:49:17.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.863395+0000 mon.c (mon.1) 181 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.863954+0000 mon.c (mon.1) 182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.864584+0000 mon.c (mon.1) 183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.865119+0000 mon.c (mon.1) 184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.865602+0000 mon.c (mon.1) 185 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.866112+0000 mon.c (mon.1) 186 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.866628+0000 mon.c (mon.1) 187 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: audit 2022-04-28T19:49:16.867198+0000 mon.c (mon.1) 188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:49:17.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[59745]: cluster 2022-04-28T19:49:16.872397+0000 mon.a (mon.0) 31 : cluster [INF] Manager daemon y is now available 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.862353+0000 mon.c (mon.1) 179 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.862832+0000 mon.c (mon.1) 180 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.863395+0000 mon.c (mon.1) 181 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.863954+0000 mon.c (mon.1) 182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.864584+0000 mon.c (mon.1) 183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.865119+0000 mon.c (mon.1) 184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:49:17.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.865602+0000 mon.c (mon.1) 185 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:49:17.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.866112+0000 mon.c (mon.1) 186 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:49:17.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.866628+0000 mon.c (mon.1) 187 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:49:17.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: audit 2022-04-28T19:49:16.867198+0000 mon.c (mon.1) 188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:49:17.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:17 smithi059 bash[29114]: cluster 2022-04-28T19:49:16.872397+0000 mon.a (mon.0) 31 : cluster [INF] Manager daemon y is now available 2022-04-28T19:49:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: cluster 2022-04-28T19:49:16.010555+0000 mon.a (mon.0) 30 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2022-04-28T19:49:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.840240+0000 mon.c (mon.1) 145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2022-04-28T19:49:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.840870+0000 mon.c (mon.1) 146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2022-04-28T19:49:17.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.841461+0000 mon.c (mon.1) 147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2022-04-28T19:49:17.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.842735+0000 mon.c (mon.1) 148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:49:17.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.843608+0000 mon.c (mon.1) 149 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:49:17.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.844433+0000 mon.c (mon.1) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2022-04-28T19:49:17.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.845171+0000 mon.c (mon.1) 151 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2022-04-28T19:49:17.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.845906+0000 mon.c (mon.1) 152 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:49:17.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.846844+0000 mon.c (mon.1) 153 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:49:17.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.847633+0000 mon.c (mon.1) 154 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:49:17.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.848483+0000 mon.c (mon.1) 155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:49:17.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.849208+0000 mon.c (mon.1) 156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:49:17.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.849907+0000 mon.c (mon.1) 157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:49:17.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.850670+0000 mon.c (mon.1) 158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.851381+0000 mon.c (mon.1) 159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.851746+0000 mon.c (mon.1) 160 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.852125+0000 mon.c (mon.1) 161 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.852582+0000 mon.c (mon.1) 162 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.853890+0000 mon.c (mon.1) 163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key ls"}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.854966+0000 mon.c (mon.1) 164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/client_keyrings"}]: dispatch 2022-04-28T19:49:17.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.855455+0000 mon.c (mon.1) 165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/config_checks"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.855958+0000 mon.c (mon.1) 166 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_crt"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.856479+0000 mon.c (mon.1) 167 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/grafana_key"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.857008+0000 mon.c (mon.1) 168 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi059"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.857620+0000 mon.c (mon.1) 169 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/host.smithi139"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.858246+0000 mon.c (mon.1) 170 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/inventory"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.858719+0000 mon.c (mon.1) 171 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/osd_remove_queue"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.859184+0000 mon.c (mon.1) 172 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.alertmanager"}]: dispatch 2022-04-28T19:49:17.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.859671+0000 mon.c (mon.1) 173 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.grafana"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.860133+0000 mon.c (mon.1) 174 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mds.foo"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.860628+0000 mon.c (mon.1) 175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.mgr"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.861058+0000 mon.c (mon.1) 176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.node-exporter"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.861496+0000 mon.c (mon.1) 177 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/spec.prometheus"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.861926+0000 mon.c (mon.1) 178 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_key"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.862353+0000 mon.c (mon.1) 179 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_identity_pub"}]: dispatch 2022-04-28T19:49:17.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.862832+0000 mon.c (mon.1) 180 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/ssh_user"}]: dispatch 2022-04-28T19:49:17.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.863395+0000 mon.c (mon.1) 181 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:49:17.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.863954+0000 mon.c (mon.1) 182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/accessdb_v2"}]: dispatch 2022-04-28T19:49:17.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.864584+0000 mon.c (mon.1) 183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2022-04-28T19:49:17.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.865119+0000 mon.c (mon.1) 184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2022-04-28T19:49:17.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.865602+0000 mon.c (mon.1) 185 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/devicehealth/last_scrape"}]: dispatch 2022-04-28T19:49:17.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.866112+0000 mon.c (mon.1) 186 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/progress/completed"}]: dispatch 2022-04-28T19:49:17.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.866628+0000 mon.c (mon.1) 187 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/report_id"}]: dispatch 2022-04-28T19:49:17.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: audit 2022-04-28T19:49:16.867198+0000 mon.c (mon.1) 188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/telemetry/salt"}]: dispatch 2022-04-28T19:49:17.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:17 smithi139 bash[23439]: cluster 2022-04-28T19:49:16.872397+0000 mon.a (mon.0) 31 : cluster [INF] Manager daemon y is now available 2022-04-28T19:49:17.827 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:49:18.047 INFO:tasks.rados.rados.0.smithi059.stdout:1581: done (6 left) 2022-04-28T19:49:18.047 INFO:tasks.rados.rados.0.smithi059.stdout:1582: done (5 left) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1583: done (4 left) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1585: done (3 left) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1586: done (2 left) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1587: done (1 left) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1588: done (0 left) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1589: read oid 44 snap 144 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1589: expect (ObjNum 507 snap 144 seq_num 507) 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 145 2022-04-28T19:49:18.048 INFO:tasks.rados.rados.0.smithi059.stdout:1590: seq_num 516 ranges {4538368=647168,5185536=450560,5636096=352256} 2022-04-28T19:49:18.076 INFO:tasks.rados.rados.0.smithi059.stdout:1590: writing smithi05955964-50 from 4538368 to 5185536 tid 1 2022-04-28T19:49:18.079 INFO:tasks.rados.rados.0.smithi059.stdout:1590: writing smithi05955964-50 from 5185536 to 5636096 tid 2 2022-04-28T19:49:18.081 INFO:tasks.rados.rados.0.smithi059.stdout:1590: writing smithi05955964-50 from 5636096 to 5988352 tid 3 2022-04-28T19:49:18.081 INFO:tasks.rados.rados.0.smithi059.stdout:1591: read oid 12 snap -1 2022-04-28T19:49:18.081 INFO:tasks.rados.rados.0.smithi059.stdout:1591: expect (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:49:18.081 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 145 2022-04-28T19:49:18.082 INFO:tasks.rados.rados.0.smithi059.stdout:1592: seq_num 517 ranges {2236416=663552,2899968=786432,3686400=16384} 2022-04-28T19:49:18.097 INFO:tasks.rados.rados.0.smithi059.stdout:1592: writing smithi05955964-43 from 2236416 to 2899968 tid 1 2022-04-28T19:49:18.102 INFO:tasks.rados.rados.0.smithi059.stdout:1592: writing smithi05955964-43 from 2899968 to 3686400 tid 2 2022-04-28T19:49:18.102 INFO:tasks.rados.rados.0.smithi059.stdout:1592: writing smithi05955964-43 from 3686400 to 3702784 tid 3 2022-04-28T19:49:18.103 INFO:tasks.rados.rados.0.smithi059.stdout:1593: snap_remove snap 142 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: cluster 2022-04-28T19:49:17.030215+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.043841+0000 mon.c (mon.1) 189 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.047028+0000 mon.c (mon.1) 190 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.051977+0000 mon.c (mon.1) 191 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.057985+0000 mon.c (mon.1) 192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: cephadm 2022-04-28T19:49:17.064351+0000 mgr.y (mgr.24961) 1 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.101618+0000 mon.c (mon.1) 193 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.102300+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.115330+0000 mon.c (mon.1) 194 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.116104+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.382071+0000 mon.c (mon.1) 195 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: audit 2022-04-28T19:49:17.385808+0000 mon.c (mon.1) 196 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[59745]: cluster 2022-04-28T19:49:17.800180+0000 mon.a (mon.0) 35 : cluster [DBG] mgrmap e29: y(active, since 7s), standbys: x 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: cluster 2022-04-28T19:49:17.030215+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.043841+0000 mon.c (mon.1) 189 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.047028+0000 mon.c (mon.1) 190 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.051977+0000 mon.c (mon.1) 191 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.057985+0000 mon.c (mon.1) 192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: cephadm 2022-04-28T19:49:17.064351+0000 mgr.y (mgr.24961) 1 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.101618+0000 mon.c (mon.1) 193 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.102300+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.115330+0000 mon.c (mon.1) 194 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.116104+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.382071+0000 mon.c (mon.1) 195 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: audit 2022-04-28T19:49:17.385808+0000 mon.c (mon.1) 196 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:18 smithi059 bash[29114]: cluster 2022-04-28T19:49:17.800180+0000 mon.a (mon.0) 35 : cluster [DBG] mgrmap e29: y(active, since 7s), standbys: x 2022-04-28T19:49:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: cluster 2022-04-28T19:49:17.030215+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2022-04-28T19:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.043841+0000 mon.c (mon.1) 189 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.047028+0000 mon.c (mon.1) 190 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.051977+0000 mon.c (mon.1) 191 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.057985+0000 mon.c (mon.1) 192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: cephadm 2022-04-28T19:49:17.064351+0000 mgr.y (mgr.24961) 1 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.101618+0000 mon.c (mon.1) 193 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:49:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.102300+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:49:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.115330+0000 mon.c (mon.1) 194 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:49:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.116104+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:49:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.382071+0000 mon.c (mon.1) 195 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:18.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: audit 2022-04-28T19:49:17.385808+0000 mon.c (mon.1) 196 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:18.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:18 smithi139 bash[23439]: cluster 2022-04-28T19:49:17.800180+0000 mon.a (mon.0) 35 : cluster [DBG] mgrmap e29: y(active, since 7s), standbys: x 2022-04-28T19:49:19.063 INFO:tasks.rados.rados.0.smithi059.stdout:1589: expect (ObjNum 507 snap 144 seq_num 507) 2022-04-28T19:49:19.092 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (8m) 47s ago 8m 14.0M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (8m) 24s ago 8m 25.7M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (6m) 47s ago 6m 13.7M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (6m) 24s ago 6m 16.8M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (30s) 24s ago 12m 131M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (66s) 47s ago 14m 402M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 starting - - - 2048M 2022-04-28T19:49:19.093 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (12m) 24s ago 12m 63.2M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 0e12092e0bc3 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (13m) 47s ago 13m 57.8M 2048M 15.2.16-79-gb51d8665 6d41e73600e1 a5fa501e4aaf 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (8m) 47s ago 8m 10.7M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (8m) 24s ago 8m 10.6M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (12m) 47s ago 12m 185M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 547b5580e073 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (11m) 47s ago 11m 151M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (11m) 47s ago 11m 195M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (10m) 47s ago 10m 160M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (10m) 24s ago 10m 183M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (10m) 24s ago 10m 190M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:49:19.094 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (9m) 24s ago 9m 152M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:49:19.095 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (9m) 24s ago 9m 128M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:49:19.095 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (82s) 24s ago 8m 33.4M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:49:19.300 INFO:tasks.rados.rados.0.smithi059.stdout:1591: expect (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:1590: finishing write tid 1 to smithi05955964-50 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:1590: finishing write tid 2 to smithi05955964-50 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:1590: finishing write tid 3 to smithi05955964-50 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:1590: finishing write tid 5 to smithi05955964-50 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:1590: finishing write tid 6 to smithi05955964-50 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 768 (ObjNum 516 snap 145 seq_num 516) dirty exists 2022-04-28T19:49:19.309 INFO:tasks.rados.rados.0.smithi059.stdout:1590: left oid 50 (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:49:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:1589: done (4 left) 2022-04-28T19:49:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:1590: done (3 left) 2022-04-28T19:49:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:1591: done (2 left) 2022-04-28T19:49:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:1593: done (1 left) 2022-04-28T19:49:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 145 2022-04-28T19:49:19.310 INFO:tasks.rados.rados.0.smithi059.stdout:1594: seq_num 518 ranges {2637824=679936,3317760=671744,3989504=131072} 2022-04-28T19:49:19.335 INFO:tasks.rados.rados.0.smithi059.stdout:1594: writing smithi05955964-49 from 2637824 to 3317760 tid 1 2022-04-28T19:49:19.339 INFO:tasks.rados.rados.0.smithi059.stdout:1594: writing smithi05955964-49 from 3317760 to 3989504 tid 2 2022-04-28T19:49:19.341 INFO:tasks.rados.rados.0.smithi059.stdout:1594: writing smithi05955964-49 from 3989504 to 4120576 tid 3 2022-04-28T19:49:19.341 INFO:tasks.rados.rados.0.smithi059.stdout:1595: copy_from oid 35 from oid 47 current snap is 145 2022-04-28T19:49:19.341 INFO:tasks.rados.rados.0.smithi059.stdout:1592: finishing write tid 1 to smithi05955964-43 2022-04-28T19:49:19.341 INFO:tasks.rados.rados.0.smithi059.stdout:1592: finishing write tid 2 to smithi05955964-43 2022-04-28T19:49:19.342 INFO:tasks.rados.rados.0.smithi059.stdout:1592: finishing write tid 3 to smithi05955964-43 2022-04-28T19:49:19.342 INFO:tasks.rados.rados.0.smithi059.stdout:1592: finishing write tid 5 to smithi05955964-43 2022-04-28T19:49:19.342 INFO:tasks.rados.rados.0.smithi059.stdout:1592: finishing write tid 6 to smithi05955964-43 2022-04-28T19:49:19.342 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 773 (ObjNum 517 snap 145 seq_num 517) dirty exists 2022-04-28T19:49:19.343 INFO:tasks.rados.rados.0.smithi059.stdout:1592: left oid 43 (ObjNum 517 snap 145 seq_num 517) 2022-04-28T19:49:19.343 INFO:tasks.rados.rados.0.smithi059.stdout:1592: done (2 left) 2022-04-28T19:49:19.343 INFO:tasks.rados.rados.0.smithi059.stdout:1596: rollback oid 30 current snap is 145 2022-04-28T19:49:19.343 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 30 to 144 2022-04-28T19:49:19.343 INFO:tasks.rados.rados.0.smithi059.stdout:1597: rollback oid 16 current snap is 145 2022-04-28T19:49:19.344 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 144 2022-04-28T19:49:19.344 INFO:tasks.rados.rados.0.smithi059.stdout:1598: delete oid 37 current snap is 145 2022-04-28T19:49:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: audit 2022-04-28T19:49:17.811315+0000 mgr.y (mgr.24961) 2 : audit [DBG] from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: cluster 2022-04-28T19:49:17.813275+0000 mgr.y (mgr.24961) 3 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: cluster 2022-04-28T19:49:18.044749+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2022-04-28T19:49:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: audit 2022-04-28T19:49:18.045117+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:49:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: audit 2022-04-28T19:49:18.046428+0000 mon.c (mon.1) 197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:49:19.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: audit 2022-04-28T19:49:18.046886+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:49:19.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: audit 2022-04-28T19:49:18.140608+0000 mgr.y (mgr.24961) 4 : audit [DBG] from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:19.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[59745]: cluster 2022-04-28T19:49:18.809010+0000 mon.a (mon.0) 39 : cluster [DBG] mgrmap e30: y(active, since 8s), standbys: x 2022-04-28T19:49:19.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: audit 2022-04-28T19:49:17.811315+0000 mgr.y (mgr.24961) 2 : audit [DBG] from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:19.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: cluster 2022-04-28T19:49:17.813275+0000 mgr.y (mgr.24961) 3 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:19.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: cluster 2022-04-28T19:49:18.044749+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2022-04-28T19:49:19.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: audit 2022-04-28T19:49:18.045117+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:49:19.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: audit 2022-04-28T19:49:18.046428+0000 mon.c (mon.1) 197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:49:19.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: audit 2022-04-28T19:49:18.046886+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:49:19.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: audit 2022-04-28T19:49:18.140608+0000 mgr.y (mgr.24961) 4 : audit [DBG] from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:19.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:19 smithi059 bash[29114]: cluster 2022-04-28T19:49:18.809010+0000 mon.a (mon.0) 39 : cluster [DBG] mgrmap e30: y(active, since 8s), standbys: x 2022-04-28T19:49:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:1596: finishing rollback tid 1 to smithi05955964-30 2022-04-28T19:49:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 0 (ObjNum -133716704 snap 21893 seq_num 4162036160) dirty dne 2022-04-28T19:49:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:1594: finishing write tid 1 to smithi05955964-49 2022-04-28T19:49:19.351 INFO:tasks.rados.rados.0.smithi059.stdout:1596: done (4 left) 2022-04-28T19:49:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:1598: done (3 left) 2022-04-28T19:49:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 145 2022-04-28T19:49:19.352 INFO:tasks.rados.rados.0.smithi059.stdout:1599: seq_num 520 ranges {5054464=712704,5767168=434176,6201344=368640} 2022-04-28T19:49:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: audit 2022-04-28T19:49:17.811315+0000 mgr.y (mgr.24961) 2 : audit [DBG] from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: cluster 2022-04-28T19:49:17.813275+0000 mgr.y (mgr.24961) 3 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: cluster 2022-04-28T19:49:18.044749+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2022-04-28T19:49:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: audit 2022-04-28T19:49:18.045117+0000 mon.a (mon.0) 37 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2022-04-28T19:49:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: audit 2022-04-28T19:49:18.046428+0000 mon.c (mon.1) 197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:49:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: audit 2022-04-28T19:49:18.046886+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2022-04-28T19:49:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: audit 2022-04-28T19:49:18.140608+0000 mgr.y (mgr.24961) 4 : audit [DBG] from='client.44142 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:19 smithi139 bash[23439]: cluster 2022-04-28T19:49:18.809010+0000 mon.a (mon.0) 39 : cluster [DBG] mgrmap e30: y(active, since 8s), standbys: x 2022-04-28T19:49:19.380 INFO:tasks.rados.rados.0.smithi059.stdout:1599: writing smithi05955964-1 from 5054464 to 5767168 tid 1 2022-04-28T19:49:19.383 INFO:tasks.rados.rados.0.smithi059.stdout:1599: writing smithi05955964-1 from 5767168 to 6201344 tid 2 2022-04-28T19:49:19.387 INFO:tasks.rados.rados.0.smithi059.stdout:1599: writing smithi05955964-1 from 6201344 to 6569984 tid 3 2022-04-28T19:49:19.387 INFO:tasks.rados.rados.0.smithi059.stdout:1600: read oid 18 snap -1 2022-04-28T19:49:19.387 INFO:tasks.rados.rados.0.smithi059.stdout:1600: expect (ObjNum 504 snap 143 seq_num 504) 2022-04-28T19:49:19.387 INFO:tasks.rados.rados.0.smithi059.stdout:1601: read oid 31 snap 144 2022-04-28T19:49:19.387 INFO:tasks.rados.rados.0.smithi059.stdout:1601: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:49:19.387 INFO:tasks.rados.rados.0.smithi059.stdout:1602: snap_create 2022-04-28T19:49:19.388 INFO:tasks.rados.rados.0.smithi059.stdout:1594: finishing write tid 2 to smithi05955964-49 2022-04-28T19:49:19.388 INFO:tasks.rados.rados.0.smithi059.stdout:1594: finishing write tid 3 to smithi05955964-49 2022-04-28T19:49:19.388 INFO:tasks.rados.rados.0.smithi059.stdout:1594: finishing write tid 5 to smithi05955964-49 2022-04-28T19:49:19.388 INFO:tasks.rados.rados.0.smithi059.stdout:1597: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:49:19.388 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 0 (ObjNum 1649369203 snap 1498636389 seq_num 1600082798) dirty dne 2022-04-28T19:49:19.388 INFO:tasks.rados.rados.0.smithi059.stdout:1594: finishing write tid 6 to smithi05955964-49 2022-04-28T19:49:19.389 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 733 (ObjNum 518 snap 145 seq_num 518) dirty exists 2022-04-28T19:49:19.389 INFO:tasks.rados.rados.0.smithi059.stdout:1594: left oid 49 (ObjNum 518 snap 145 seq_num 518) 2022-04-28T19:49:19.389 INFO:tasks.rados.rados.0.smithi059.stdout:1595: finishing copy_from to smithi05955964-35 2022-04-28T19:49:19.389 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 714 (ObjNum 414 snap 125 seq_num 414) dirty exists 2022-04-28T19:49:19.389 INFO:tasks.rados.rados.0.smithi059.stdout:1595: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:49:19.390 INFO:tasks.rados.rados.0.smithi059.stdout:1599: finishing write tid 1 to smithi05955964-1 2022-04-28T19:49:19.391 INFO:tasks.rados.rados.0.smithi059.stdout:1599: finishing write tid 2 to smithi05955964-1 2022-04-28T19:49:19.391 INFO:tasks.rados.rados.0.smithi059.stdout:1599: finishing write tid 3 to smithi05955964-1 2022-04-28T19:49:19.391 INFO:tasks.rados.rados.0.smithi059.stdout:1599: finishing write tid 5 to smithi05955964-1 2022-04-28T19:49:19.392 INFO:tasks.rados.rados.0.smithi059.stdout:1599: finishing write tid 6 to smithi05955964-1 2022-04-28T19:49:19.392 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 737 (ObjNum 520 snap 145 seq_num 520) dirty exists 2022-04-28T19:49:19.392 INFO:tasks.rados.rados.0.smithi059.stdout:1599: left oid 1 (ObjNum 520 snap 145 seq_num 520) 2022-04-28T19:49:19.399 INFO:tasks.rados.rados.0.smithi059.stdout:1600: expect (ObjNum 504 snap 143 seq_num 504) 2022-04-28T19:49:19.447 INFO:tasks.rados.rados.0.smithi059.stdout:1601: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2, 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:19.487 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 12, 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:49:19.488 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:49:20.069 INFO:tasks.rados.rados.0.smithi059.stdout:1594: done (6 left) 2022-04-28T19:49:20.069 INFO:tasks.rados.rados.0.smithi059.stdout:1595: done (5 left) 2022-04-28T19:49:20.069 INFO:tasks.rados.rados.0.smithi059.stdout:1597: done (4 left) 2022-04-28T19:49:20.069 INFO:tasks.rados.rados.0.smithi059.stdout:1599: done (3 left) 2022-04-28T19:49:20.069 INFO:tasks.rados.rados.0.smithi059.stdout:1600: done (2 left) 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:1601: done (1 left) 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:1602: done (0 left) 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:1603: rollback oid 40 current snap is 146 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 144 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:1604: read oid 42 snap -1 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:1604: expect (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 146 2022-04-28T19:49:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:1605: seq_num 521 ranges {4915200=729088,5644288=720896,6365184=81920} 2022-04-28T19:49:20.102 INFO:tasks.rados.rados.0.smithi059.stdout:1605: writing smithi05955964-47 from 4915200 to 5644288 tid 1 2022-04-28T19:49:20.107 INFO:tasks.rados.rados.0.smithi059.stdout:1605: writing smithi05955964-47 from 5644288 to 6365184 tid 2 2022-04-28T19:49:20.108 INFO:tasks.rados.rados.0.smithi059.stdout:1605: writing smithi05955964-47 from 6365184 to 6447104 tid 3 2022-04-28T19:49:20.108 INFO:tasks.rados.rados.0.smithi059.stdout:1606: read oid 9 snap -1 2022-04-28T19:49:20.108 INFO:tasks.rados.rados.0.smithi059.stdout:1606: expect deleted 2022-04-28T19:49:20.108 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 146 2022-04-28T19:49:20.108 INFO:tasks.rados.rados.0.smithi059.stdout:1607: seq_num 522 ranges {0=745472,745472=606208,1351680=196608} 2022-04-28T19:49:20.112 INFO:tasks.rados.rados.0.smithi059.stdout:1607: writing smithi05955964-15 from 0 to 745472 tid 1 2022-04-28T19:49:20.116 INFO:tasks.rados.rados.0.smithi059.stdout:1607: writing smithi05955964-15 from 745472 to 1351680 tid 2 2022-04-28T19:49:20.117 INFO:tasks.rados.rados.0.smithi059.stdout:1607: writing smithi05955964-15 from 1351680 to 1548288 tid 3 2022-04-28T19:49:20.117 INFO:tasks.rados.rados.0.smithi059.stdout:1608: setattr oid 39 current snap is 146 2022-04-28T19:49:20.118 INFO:tasks.rados.rados.0.smithi059.stdout:1603: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:49:20.118 INFO:tasks.rados.rados.0.smithi059.stdout:1603: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:49:20.118 INFO:tasks.rados.rados.0.smithi059.stdout:1603: finishing rollback tid 2 to smithi05955964-40 2022-04-28T19:49:20.118 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 740 (ObjNum 450 snap 132 seq_num 450) dirty exists 2022-04-28T19:49:20.118 INFO:tasks.rados.rados.0.smithi059.stdout:1604: expect (ObjNum 385 snap 118 seq_num 385) 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:1605: finishing write tid 1 to smithi05955964-47 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:1605: finishing write tid 2 to smithi05955964-47 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:1605: finishing write tid 3 to smithi05955964-47 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:1605: finishing write tid 5 to smithi05955964-47 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:1605: finishing write tid 6 to smithi05955964-47 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 746 (ObjNum 521 snap 146 seq_num 521) dirty exists 2022-04-28T19:49:20.157 INFO:tasks.rados.rados.0.smithi059.stdout:1605: left oid 47 (ObjNum 521 snap 146 seq_num 521) 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1607: finishing write tid 1 to smithi05955964-15 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1607: finishing write tid 2 to smithi05955964-15 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1607: finishing write tid 3 to smithi05955964-15 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1607: finishing write tid 5 to smithi05955964-15 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1607: finishing write tid 6 to smithi05955964-15 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 714 (ObjNum 522 snap 146 seq_num 522) dirty exists 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1607: left oid 15 (ObjNum 522 snap 146 seq_num 522) 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1603: done (5 left) 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1604: done (4 left) 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1605: done (3 left) 2022-04-28T19:49:20.158 INFO:tasks.rados.rados.0.smithi059.stdout:1606: done (2 left) 2022-04-28T19:49:20.159 INFO:tasks.rados.rados.0.smithi059.stdout:1607: done (1 left) 2022-04-28T19:49:20.159 INFO:tasks.rados.rados.0.smithi059.stdout:1609: snap_remove snap 145 2022-04-28T19:49:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: cluster 2022-04-28T19:49:18.847597+0000 mgr.y (mgr.24961) 5 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.055906+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:49:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: cluster 2022-04-28T19:49:19.056078+0000 mon.a (mon.0) 41 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2022-04-28T19:49:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.068146+0000 mon.c (mon.1) 198 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.068906+0000 mon.c (mon.1) 199 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.069590+0000 mon.c (mon.1) 200 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.072610+0000 mon.c (mon.1) 201 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.073392+0000 mon.c (mon.1) 202 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.073919+0000 mon.c (mon.1) 203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.077492+0000 mon.c (mon.1) 204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.078253+0000 mon.c (mon.1) 205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.079052+0000 mon.c (mon.1) 206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.080574+0000 mon.c (mon.1) 207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.081367+0000 mon.c (mon.1) 208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[59745]: audit 2022-04-28T19:49:19.485841+0000 mon.c (mon.1) 209 : audit [DBG] from='client.? 172.21.15.59:0/2724797457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:20.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: cluster 2022-04-28T19:49:18.847597+0000 mgr.y (mgr.24961) 5 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:20.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.055906+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:49:20.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: cluster 2022-04-28T19:49:19.056078+0000 mon.a (mon.0) 41 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2022-04-28T19:49:20.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.068146+0000 mon.c (mon.1) 198 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.068906+0000 mon.c (mon.1) 199 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.069590+0000 mon.c (mon.1) 200 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.072610+0000 mon.c (mon.1) 201 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.073392+0000 mon.c (mon.1) 202 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.073919+0000 mon.c (mon.1) 203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.077492+0000 mon.c (mon.1) 204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.078253+0000 mon.c (mon.1) 205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.079052+0000 mon.c (mon.1) 206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.080574+0000 mon.c (mon.1) 207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.081367+0000 mon.c (mon.1) 208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:20 smithi059 bash[29114]: audit 2022-04-28T19:49:19.485841+0000 mon.c (mon.1) 209 : audit [DBG] from='client.? 172.21.15.59:0/2724797457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: cluster 2022-04-28T19:49:18.847597+0000 mgr.y (mgr.24961) 5 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.055906+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2022-04-28T19:49:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: cluster 2022-04-28T19:49:19.056078+0000 mon.a (mon.0) 41 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2022-04-28T19:49:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.068146+0000 mon.c (mon.1) 198 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.068906+0000 mon.c (mon.1) 199 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.069590+0000 mon.c (mon.1) 200 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.072610+0000 mon.c (mon.1) 201 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.073392+0000 mon.c (mon.1) 202 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.073919+0000 mon.c (mon.1) 203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.077492+0000 mon.c (mon.1) 204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.078253+0000 mon.c (mon.1) 205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.079052+0000 mon.c (mon.1) 206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.080574+0000 mon.c (mon.1) 207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.081367+0000 mon.c (mon.1) 208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:20 smithi139 bash[23439]: audit 2022-04-28T19:49:19.485841+0000 mon.c (mon.1) 209 : audit [DBG] from='client.? 172.21.15.59:0/2724797457' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:21.083 INFO:tasks.rados.rados.0.smithi059.stdout:1609: done (1 left) 2022-04-28T19:49:21.083 INFO:tasks.rados.rados.0.smithi059.stdout:1610: rmattr oid 26 current snap is 146 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:1610: done (1 left) 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:1611: rollback oid 16 current snap is 146 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 144 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 781 (ObjNum 449 snap 132 seq_num 449) dirty exists 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:1608: done (1 left) 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:1612: rollback oid 40 current snap is 146 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 144 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:1613: read oid 2 snap -1 2022-04-28T19:49:21.084 INFO:tasks.rados.rados.0.smithi059.stdout:1613: expect deleted 2022-04-28T19:49:21.085 INFO:tasks.rados.rados.0.smithi059.stdout:1614: read oid 31 snap -1 2022-04-28T19:49:21.085 INFO:tasks.rados.rados.0.smithi059.stdout:1614: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:49:21.085 INFO:tasks.rados.rados.0.smithi059.stdout:1615: rmattr oid 35 current snap is 146 2022-04-28T19:49:21.085 INFO:tasks.rados.rados.0.smithi059.stdout:1616: copy_from oid 42 from oid 23 current snap is 146 2022-04-28T19:49:21.085 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 146 2022-04-28T19:49:21.085 INFO:tasks.rados.rados.0.smithi059.stdout:1617: seq_num 523 ranges {6447104=770048,7217152=491520,7708672=311296} 2022-04-28T19:49:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:1617: writing smithi05955964-47 from 6447104 to 7217152 tid 1 2022-04-28T19:49:21.130 INFO:tasks.rados.rados.0.smithi059.stdout:1617: writing smithi05955964-47 from 7217152 to 7708672 tid 2 2022-04-28T19:49:21.136 INFO:tasks.rados.rados.0.smithi059.stdout:1617: writing smithi05955964-47 from 7708672 to 8019968 tid 3 2022-04-28T19:49:21.136 INFO:tasks.rados.rados.0.smithi059.stdout:1618: snap_create 2022-04-28T19:49:21.136 INFO:tasks.rados.rados.0.smithi059.stdout:1611: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 0 (ObjNum 0 snap 21893 seq_num 4161632096) dirty dne 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 715 (ObjNum 414 snap 125 seq_num 414) dirty exists 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:1612: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:1612: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:1612: finishing rollback tid 2 to smithi05955964-40 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 743 (ObjNum 450 snap 132 seq_num 450) dirty exists 2022-04-28T19:49:21.137 INFO:tasks.rados.rados.0.smithi059.stdout:1614: expect (ObjNum 226 snap 60 seq_num 226) 2022-04-28T19:49:21.181 INFO:tasks.rados.rados.0.smithi059.stdout:1616: finishing copy_from to smithi05955964-42 2022-04-28T19:49:21.182 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 724 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:49:21.182 INFO:tasks.rados.rados.0.smithi059.stdout:1616: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:49:21.182 INFO:tasks.rados.rados.0.smithi059.stdout:1617: finishing write tid 1 to smithi05955964-47 2022-04-28T19:49:21.182 INFO:tasks.rados.rados.0.smithi059.stdout:1617: finishing write tid 2 to smithi05955964-47 2022-04-28T19:49:21.182 INFO:tasks.rados.rados.0.smithi059.stdout:1617: finishing write tid 3 to smithi05955964-47 2022-04-28T19:49:21.183 INFO:tasks.rados.rados.0.smithi059.stdout:1617: finishing write tid 5 to smithi05955964-47 2022-04-28T19:49:21.183 INFO:tasks.rados.rados.0.smithi059.stdout:1617: finishing write tid 6 to smithi05955964-47 2022-04-28T19:49:21.183 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 752 (ObjNum 523 snap 146 seq_num 523) dirty exists 2022-04-28T19:49:21.183 INFO:tasks.rados.rados.0.smithi059.stdout:1617: left oid 47 (ObjNum 523 snap 146 seq_num 523) 2022-04-28T19:49:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:21 smithi059 bash[29114]: cluster 2022-04-28T19:49:20.067186+0000 mon.a (mon.0) 42 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2022-04-28T19:49:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:21 smithi059 bash[59745]: cluster 2022-04-28T19:49:20.067186+0000 mon.a (mon.0) 42 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2022-04-28T19:49:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:21 smithi139 bash[23439]: cluster 2022-04-28T19:49:20.067186+0000 mon.a (mon.0) 42 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2022-04-28T19:49:21.798 INFO:tasks.rados.rados.0.smithi059.stdout:1611: done (7 left) 2022-04-28T19:49:21.798 INFO:tasks.rados.rados.0.smithi059.stdout:1612: done (6 left) 2022-04-28T19:49:21.798 INFO:tasks.rados.rados.0.smithi059.stdout:1613: done (5 left) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1614: done (4 left) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1615: done (3 left) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1616: done (2 left) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1617: done (1 left) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1618: done (0 left) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1619: read oid 9 snap -1 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1619: expect deleted 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1620: read oid 50 snap -1 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1620: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:49:21.799 INFO:tasks.rados.rados.0.smithi059.stdout:1621: snap_remove snap 144 2022-04-28T19:49:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:22 smithi059 bash[59745]: cluster 2022-04-28T19:49:20.848361+0000 mgr.y (mgr.24961) 6 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:22 smithi059 bash[59745]: cluster 2022-04-28T19:49:21.077898+0000 mon.a (mon.0) 43 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2022-04-28T19:49:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:22 smithi059 bash[59745]: cluster 2022-04-28T19:49:21.794930+0000 mon.a (mon.0) 44 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2022-04-28T19:49:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:22 smithi059 bash[29114]: cluster 2022-04-28T19:49:20.848361+0000 mgr.y (mgr.24961) 6 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:22 smithi059 bash[29114]: cluster 2022-04-28T19:49:21.077898+0000 mon.a (mon.0) 43 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2022-04-28T19:49:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:22 smithi059 bash[29114]: cluster 2022-04-28T19:49:21.794930+0000 mon.a (mon.0) 44 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2022-04-28T19:49:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:22 smithi139 bash[23439]: cluster 2022-04-28T19:49:20.848361+0000 mgr.y (mgr.24961) 6 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 122 MiB data, 212 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:22 smithi139 bash[23439]: cluster 2022-04-28T19:49:21.077898+0000 mon.a (mon.0) 43 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2022-04-28T19:49:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:22 smithi139 bash[23439]: cluster 2022-04-28T19:49:21.794930+0000 mon.a (mon.0) 44 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2022-04-28T19:49:22.812 INFO:tasks.rados.rados.0.smithi059.stdout:1621: done (2 left) 2022-04-28T19:49:22.911 INFO:tasks.rados.rados.0.smithi059.stdout:1622: delete oid 22 current snap is 147 2022-04-28T19:49:22.911 INFO:tasks.rados.rados.0.smithi059.stdout:1620: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:49:23.193 INFO:tasks.rados.rados.0.smithi059.stdout:1619: done (2 left) 2022-04-28T19:49:23.193 INFO:tasks.rados.rados.0.smithi059.stdout:1620: done (1 left) 2022-04-28T19:49:23.193 INFO:tasks.rados.rados.0.smithi059.stdout:1622: done (0 left) 2022-04-28T19:49:23.194 INFO:tasks.rados.rados.0.smithi059.stdout:1623: rollback oid 38 current snap is 147 2022-04-28T19:49:23.194 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 146 2022-04-28T19:49:23.194 INFO:tasks.rados.rados.0.smithi059.stdout:1624: snap_create 2022-04-28T19:49:23.199 INFO:tasks.rados.rados.0.smithi059.stdout:1623: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:49:23.199 INFO:tasks.rados.rados.0.smithi059.stdout:1623: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:49:23.200 INFO:tasks.rados.rados.0.smithi059.stdout:1623: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:49:23.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 712 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:49:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.313199+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.390303+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.593639+0000 mon.c (mon.1) 210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.594783+0000 mon.c (mon.1) 211 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.595681+0000 mon.c (mon.1) 212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.596488+0000 mon.c (mon.1) 213 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.597220+0000 mon.c (mon.1) 214 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.598086+0000 mon.c (mon.1) 215 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.598823+0000 mon.c (mon.1) 216 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.599690+0000 mon.c (mon.1) 217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.600451+0000 mon.c (mon.1) 218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.601211+0000 mon.c (mon.1) 219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.601959+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.676580+0000 mon.c (mon.1) 220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.677843+0000 mon.c (mon.1) 221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.678889+0000 mon.c (mon.1) 222 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:23.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:23.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:23.518Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:23.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:23.518Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:23.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:23.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:23.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:23.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:23.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:23.523Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.679888+0000 mon.c (mon.1) 223 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.680892+0000 mon.c (mon.1) 224 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.681893+0000 mon.c (mon.1) 225 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.682795+0000 mon.c (mon.1) 226 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.683612+0000 mon.c (mon.1) 227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.684464+0000 mon.c (mon.1) 228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.685309+0000 mon.c (mon.1) 229 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.685885+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.694219+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.709863+0000 mon.c (mon.1) 230 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.713907+0000 mon.c (mon.1) 231 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.715609+0000 mon.c (mon.1) 232 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:49:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: audit 2022-04-28T19:49:22.716993+0000 mon.c (mon.1) 233 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-04-28T19:49:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: cephadm 2022-04-28T19:49:22.717948+0000 mgr.y (mgr.24961) 7 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2022-04-28T19:49:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: cluster 2022-04-28T19:49:22.807997+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2022-04-28T19:49:23.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[59745]: cluster 2022-04-28T19:49:22.849173+0000 mgr.y (mgr.24961) 8 : cluster [DBG] pgmap v13: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 131 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 5.2 MiB/s wr, 25 op/s 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.313199+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.390303+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.593639+0000 mon.c (mon.1) 210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.594783+0000 mon.c (mon.1) 211 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.595681+0000 mon.c (mon.1) 212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.596488+0000 mon.c (mon.1) 213 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.597220+0000 mon.c (mon.1) 214 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.598086+0000 mon.c (mon.1) 215 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.598823+0000 mon.c (mon.1) 216 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.599690+0000 mon.c (mon.1) 217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.600451+0000 mon.c (mon.1) 218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:49:23.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.601211+0000 mon.c (mon.1) 219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.601959+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.676580+0000 mon.c (mon.1) 220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.677843+0000 mon.c (mon.1) 221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.678889+0000 mon.c (mon.1) 222 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.679888+0000 mon.c (mon.1) 223 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.680892+0000 mon.c (mon.1) 224 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.681893+0000 mon.c (mon.1) 225 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.682795+0000 mon.c (mon.1) 226 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.683612+0000 mon.c (mon.1) 227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.684464+0000 mon.c (mon.1) 228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:49:23.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.685309+0000 mon.c (mon.1) 229 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.685885+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.694219+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.709863+0000 mon.c (mon.1) 230 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:23.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.713907+0000 mon.c (mon.1) 231 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:23.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.715609+0000 mon.c (mon.1) 232 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:49:23.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: audit 2022-04-28T19:49:22.716993+0000 mon.c (mon.1) 233 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-04-28T19:49:23.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: cephadm 2022-04-28T19:49:22.717948+0000 mgr.y (mgr.24961) 7 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2022-04-28T19:49:23.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: cluster 2022-04-28T19:49:22.807997+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2022-04-28T19:49:23.603 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:23 smithi059 bash[29114]: cluster 2022-04-28T19:49:22.849173+0000 mgr.y (mgr.24961) 8 : cluster [DBG] pgmap v13: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 131 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 5.2 MiB/s wr, 25 op/s 2022-04-28T19:49:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.313199+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.390303+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.593639+0000 mon.c (mon.1) 210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.594783+0000 mon.c (mon.1) 211 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.595681+0000 mon.c (mon.1) 212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.596488+0000 mon.c (mon.1) 213 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.597220+0000 mon.c (mon.1) 214 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.598086+0000 mon.c (mon.1) 215 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.598823+0000 mon.c (mon.1) 216 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.599690+0000 mon.c (mon.1) 217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.600451+0000 mon.c (mon.1) 218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.601211+0000 mon.c (mon.1) 219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.601959+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.676580+0000 mon.c (mon.1) 220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.677843+0000 mon.c (mon.1) 221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.678889+0000 mon.c (mon.1) 222 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.679888+0000 mon.c (mon.1) 223 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.680892+0000 mon.c (mon.1) 224 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.681893+0000 mon.c (mon.1) 225 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.682795+0000 mon.c (mon.1) 226 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:49:23.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.683612+0000 mon.c (mon.1) 227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.684464+0000 mon.c (mon.1) 228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.685309+0000 mon.c (mon.1) 229 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.685885+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.24961 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.694219+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.709863+0000 mon.c (mon.1) 230 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.713907+0000 mon.c (mon.1) 231 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:23.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.715609+0000 mon.c (mon.1) 232 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:49:23.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: audit 2022-04-28T19:49:22.716993+0000 mon.c (mon.1) 233 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2022-04-28T19:49:23.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: cephadm 2022-04-28T19:49:22.717948+0000 mgr.y (mgr.24961) 7 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2022-04-28T19:49:23.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: cluster 2022-04-28T19:49:22.807997+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2022-04-28T19:49:23.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:23 smithi139 bash[23439]: cluster 2022-04-28T19:49:22.849173+0000 mgr.y (mgr.24961) 8 : cluster [DBG] pgmap v13: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 131 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 5.2 MiB/s wr, 25 op/s 2022-04-28T19:49:23.814 INFO:tasks.rados.rados.0.smithi059.stdout:1623: done (1 left) 2022-04-28T19:49:23.814 INFO:tasks.rados.rados.0.smithi059.stdout:1624: done (0 left) 2022-04-28T19:49:23.814 INFO:tasks.rados.rados.0.smithi059.stdout:1625: snap_create 2022-04-28T19:49:24.825 INFO:tasks.rados.rados.0.smithi059.stdout:1625: done (0 left) 2022-04-28T19:49:24.826 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 149 2022-04-28T19:49:24.826 INFO:tasks.rados.rados.0.smithi059.stdout:1626: seq_num 525 ranges {0=802816,802816=712704,1515520=90112} 2022-04-28T19:49:24.835 INFO:tasks.rados.rados.0.smithi059.stdout:1626: writing smithi05955964-14 from 0 to 802816 tid 1 2022-04-28T19:49:24.840 INFO:tasks.rados.rados.0.smithi059.stdout:1626: writing smithi05955964-14 from 802816 to 1515520 tid 2 2022-04-28T19:49:24.841 INFO:tasks.rados.rados.0.smithi059.stdout:1626: writing smithi05955964-14 from 1515520 to 1605632 tid 3 2022-04-28T19:49:24.841 INFO:tasks.rados.rados.0.smithi059.stdout:1627: setattr oid 16 current snap is 149 2022-04-28T19:49:24.842 INFO:tasks.rados.rados.0.smithi059.stdout:1628: snap_remove snap 148 2022-04-28T19:49:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:24 smithi059 bash[29114]: cluster 2022-04-28T19:49:23.812193+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2022-04-28T19:49:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:24 smithi059 bash[59745]: cluster 2022-04-28T19:49:23.812193+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2022-04-28T19:49:25.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:24 smithi139 bash[23439]: cluster 2022-04-28T19:49:23.812193+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2022-04-28T19:49:25.836 INFO:tasks.rados.rados.0.smithi059.stdout:1628: done (2 left) 2022-04-28T19:49:25.836 INFO:tasks.rados.rados.0.smithi059.stdout:1629: rmattr oid 8 current snap is 149 2022-04-28T19:49:25.836 INFO:tasks.rados.rados.0.smithi059.stdout:1629: done (2 left) 2022-04-28T19:49:25.836 INFO:tasks.rados.rados.0.smithi059.stdout:1630: read oid 46 snap 146 2022-04-28T19:49:25.837 INFO:tasks.rados.rados.0.smithi059.stdout:1630: expect (ObjNum 512 snap 144 seq_num 512) 2022-04-28T19:49:25.837 INFO:tasks.rados.rados.0.smithi059.stdout:1631: snap_remove snap 147 2022-04-28T19:49:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: cluster 2022-04-28T19:49:24.824657+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2022-04-28T19:49:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: cluster 2022-04-28T19:49:24.849780+0000 mgr.y (mgr.24961) 9 : cluster [DBG] pgmap v16: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 131 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 5.2 MiB/s wr, 25 op/s 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: cephadm 2022-04-28T19:49:25.095570+0000 mgr.y (mgr.24961) 10 : cephadm [INF] Upgrade: Updating mon.c 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: audit 2022-04-28T19:49:25.104491+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: audit 2022-04-28T19:49:25.106566+0000 mon.c (mon.1) 234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: audit 2022-04-28T19:49:25.108136+0000 mon.c (mon.1) 235 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: audit 2022-04-28T19:49:25.109665+0000 mon.c (mon.1) 236 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: cephadm 2022-04-28T19:49:25.111158+0000 mgr.y (mgr.24961) 11 : cephadm [INF] Deploying daemon mon.c on smithi059 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[59745]: audit 2022-04-28T19:49:25.112102+0000 mon.c (mon.1) 237 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "container_image"}]: dispatch 2022-04-28T19:49:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: cluster 2022-04-28T19:49:24.824657+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2022-04-28T19:49:26.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: cluster 2022-04-28T19:49:24.849780+0000 mgr.y (mgr.24961) 9 : cluster [DBG] pgmap v16: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 131 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 5.2 MiB/s wr, 25 op/s 2022-04-28T19:49:26.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: cephadm 2022-04-28T19:49:25.095570+0000 mgr.y (mgr.24961) 10 : cephadm [INF] Upgrade: Updating mon.c 2022-04-28T19:49:26.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: audit 2022-04-28T19:49:25.104491+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:26.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: audit 2022-04-28T19:49:25.106566+0000 mon.c (mon.1) 234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:26.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: audit 2022-04-28T19:49:25.108136+0000 mon.c (mon.1) 235 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:26.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: audit 2022-04-28T19:49:25.109665+0000 mon.c (mon.1) 236 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:26.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: cephadm 2022-04-28T19:49:25.111158+0000 mgr.y (mgr.24961) 11 : cephadm [INF] Deploying daemon mon.c on smithi059 2022-04-28T19:49:26.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:25 smithi059 bash[29114]: audit 2022-04-28T19:49:25.112102+0000 mon.c (mon.1) 237 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "container_image"}]: dispatch 2022-04-28T19:49:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: cluster 2022-04-28T19:49:24.824657+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2022-04-28T19:49:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: cluster 2022-04-28T19:49:24.849780+0000 mgr.y (mgr.24961) 9 : cluster [DBG] pgmap v16: 89 pgs: 3 active+clean+snaptrim, 86 active+clean; 131 MiB data, 229 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 5.2 MiB/s wr, 25 op/s 2022-04-28T19:49:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: cephadm 2022-04-28T19:49:25.095570+0000 mgr.y (mgr.24961) 10 : cephadm [INF] Upgrade: Updating mon.c 2022-04-28T19:49:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: audit 2022-04-28T19:49:25.104491+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: audit 2022-04-28T19:49:25.106566+0000 mon.c (mon.1) 234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: audit 2022-04-28T19:49:25.108136+0000 mon.c (mon.1) 235 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: audit 2022-04-28T19:49:25.109665+0000 mon.c (mon.1) 236 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: cephadm 2022-04-28T19:49:25.111158+0000 mgr.y (mgr.24961) 11 : cephadm [INF] Deploying daemon mon.c on smithi059 2022-04-28T19:49:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:25 smithi139 bash[23439]: audit 2022-04-28T19:49:25.112102+0000 mon.c (mon.1) 237 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "container_image"}]: dispatch 2022-04-28T19:49:26.799 INFO:tasks.rados.rados.0.smithi059.stdout:1631: done (3 left) 2022-04-28T19:49:26.799 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 149 2022-04-28T19:49:26.799 INFO:tasks.rados.rados.0.smithi059.stdout:1632: seq_num 526 ranges {1875968=417792,2293760=589824,2883584=614400} 2022-04-28T19:49:26.812 INFO:tasks.rados.rados.0.smithi059.stdout:1632: writing smithi05955964-48 from 1875968 to 2293760 tid 1 2022-04-28T19:49:26.815 INFO:tasks.rados.rados.0.smithi059.stdout:1632: writing smithi05955964-48 from 2293760 to 2883584 tid 2 2022-04-28T19:49:26.819 INFO:tasks.rados.rados.0.smithi059.stdout:1632: writing smithi05955964-48 from 2883584 to 3497984 tid 3 2022-04-28T19:49:26.819 INFO:tasks.rados.rados.0.smithi059.stdout:1633: setattr oid 1 current snap is 149 2022-04-28T19:49:26.819 INFO:tasks.rados.rados.0.smithi059.stdout:1626: finishing write tid 1 to smithi05955964-14 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:1626: finishing write tid 2 to smithi05955964-14 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:1626: finishing write tid 3 to smithi05955964-14 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:1626: finishing write tid 5 to smithi05955964-14 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 417 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:1626: finishing write tid 6 to smithi05955964-14 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 772 (ObjNum 525 snap 149 seq_num 525) dirty exists 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:1626: left oid 14 (ObjNum 525 snap 149 seq_num 525) 2022-04-28T19:49:26.820 INFO:tasks.rados.rados.0.smithi059.stdout:1630: expect (ObjNum 512 snap 144 seq_num 512) 2022-04-28T19:49:27.083 INFO:tasks.rados.rados.0.smithi059.stdout:1632: finishing write tid 1 to smithi05955964-48 2022-04-28T19:49:27.083 INFO:tasks.rados.rados.0.smithi059.stdout:1632: finishing write tid 2 to smithi05955964-48 2022-04-28T19:49:27.083 INFO:tasks.rados.rados.0.smithi059.stdout:1632: finishing write tid 3 to smithi05955964-48 2022-04-28T19:49:27.083 INFO:tasks.rados.rados.0.smithi059.stdout:1632: finishing write tid 5 to smithi05955964-48 2022-04-28T19:49:27.083 INFO:tasks.rados.rados.0.smithi059.stdout:1632: finishing write tid 6 to smithi05955964-48 2022-04-28T19:49:27.083 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 777 (ObjNum 526 snap 149 seq_num 526) dirty exists 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1632: left oid 48 (ObjNum 526 snap 149 seq_num 526) 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1626: done (4 left) 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1627: done (3 left) 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1630: done (2 left) 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1632: done (1 left) 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1634: read oid 21 snap -1 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1634: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:49:27.084 INFO:tasks.rados.rados.0.smithi059.stdout:1635: snap_create 2022-04-28T19:49:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:26 smithi059 bash[29114]: cluster 2022-04-28T19:49:25.834565+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2022-04-28T19:49:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:26 smithi059 bash[29114]: cluster 2022-04-28T19:49:26.794400+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:27.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:26 smithi059 bash[59745]: cluster 2022-04-28T19:49:25.834565+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2022-04-28T19:49:27.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:26 smithi059 bash[59745]: cluster 2022-04-28T19:49:26.794400+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:27.093 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 744 (ObjNum 520 snap 145 seq_num 520) dirty exists 2022-04-28T19:49:27.098 INFO:tasks.rados.rados.0.smithi059.stdout:1634: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:49:27.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:26 smithi139 bash[23439]: cluster 2022-04-28T19:49:25.834565+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2022-04-28T19:49:27.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:26 smithi139 bash[23439]: cluster 2022-04-28T19:49:26.794400+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:27.573 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:27 smithi059 systemd[1]: Stopping Ceph mon.c for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:49:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:27 smithi059 bash[29114]: debug 2022-04-28T19:49:27.569+0000 7f517d956700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:49:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:27 smithi059 bash[29114]: debug 2022-04-28T19:49:27.569+0000 7f517d956700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2022-04-28T19:49:28.525 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:28 smithi059 bash[61041]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.c 2022-04-28T19:49:28.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:28 smithi059 bash[61041]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon-c 2022-04-28T19:49:28.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:28 smithi059 bash[61041]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.c 2022-04-28T19:49:28.526 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:28 smithi059 systemd[1]: Stopped Ceph mon.c for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:49:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:28 smithi059 systemd[1]: Started Ceph mon.c for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:49:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.721+0000 7f9f9be9f880 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.721+0000 7f9f9be9f880 0 ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable), process ceph-mon, pid 1 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.721+0000 7f9f9be9f880 0 pidfile_write: ignore empty --pid-file 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.725+0000 7f9f9be9f880 0 load: jerasure load: lrc load: isa 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: RocksDB version: 6.8.1 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Compile date Apr 26 2022 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: DB SUMMARY 2022-04-28T19:49:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: CURRENT file: CURRENT 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: IDENTITY file: IDENTITY 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 658 Bytes 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000020.sst 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000018.log size: 26565622 ; 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.create_if_missing: 0 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.env: 0x562116ad1000 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.fs: Posix File System 2022-04-28T19:49:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.info_log: 0x562118b02440 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:49:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.db_log_dir: 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:49:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.write_buffer_manager: 0x562119802090 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:49:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:49:29.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:49:29.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:49:29.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:49:29.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:49:29.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:49:29.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.unordered_write: 0 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.row_cache: None 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.wal_filter: None 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:49:29.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.persist_stats_to_disk: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.write_dbid_to_manifest: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.log_readahead_size: 0 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:49:29.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:49:29.851 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.strict_bytes_per_sync: 0 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Compression algorithms supported: 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kZSTD supported: 0 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:49:29.852 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.merge_operator: 2022-04-28T19:49:29.853 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562118aa5d10) 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: cache_index_and_filter_blocks: 1 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: cache_index_and_filter_blocks_with_high_priority: 0 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: pin_top_level_index_and_filter: 1 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: index_type: 0 2022-04-28T19:49:29.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: data_block_index_type: 0 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: index_shortening: 1 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: hash_index_allow_collision: 1 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: checksum: 1 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: no_block_cache: 0 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_cache: 0x562118adcf10 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_cache_name: BinnedLRUCache 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_cache_options: 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: capacity : 536870912 2022-04-28T19:49:29.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: num_shard_bits : 4 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: strict_capacity_limit : 0 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: high_pri_pool_ratio: 0.000 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_cache_compressed: (nil) 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: persistent_cache: (nil) 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_size: 4096 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_size_deviation: 10 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_restart_interval: 16 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: index_block_restart_interval: 1 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: metadata_block_size: 4096 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: partition_filters: 0 2022-04-28T19:49:29.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: use_delta_encoding: 1 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: whole_key_filtering: 1 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: verify_compression: 0 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: read_amp_bytes_per_bit: 0 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: format_version: 2 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: enable_index_compression: 1 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: block_align: 0 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:49:29.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.num_levels: 7 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-04-28T19:49:29.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:49:29.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:49:29.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:49:29.861 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:49:29.862 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:49:29.863 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.ttl: 2592000 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: Options.periodic_compaction_seconds: 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 22, last_sequence is 11909, log_number is 18,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:49:29.864 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 18 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175369733622, "job": 1, "event": "recovery_started", "log_files": [18]} 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.729+0000 7f9f9be9f880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #18 mode 2 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.757+0000 7f9f9be9f880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.785+0000 7f9f9be9f880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175369789102, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 22, "file_size": 17337224, "table_properties": {"data_size": 17301018, "index_size": 30835, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 4549, "raw_key_size": 39004, "raw_average_key_size": 21, "raw_value_size": 17254043, "raw_average_value_size": 9698, "num_data_blocks": 942, "num_entries": 1779, "num_deletions": 377, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1651175369, "oldest_key_time": 3, "file_creation_time": 0}} 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.785+0000 7f9f9be9f880 4 rocksdb: [version_set.cc:3826] Creating manifest 23 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.785+0000 7f9f9be9f880 4 rocksdb: [version_set.cc:3291] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.785+0000 7f9f9be9f880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175369790131, "job": 1, "event": "recovery_finished"} 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.793+0000 7f9f9be9f880 4 rocksdb: DB pointer 0x562118ba5800 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.793+0000 7f9f82ed1700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2022-04-28T19:49:29.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.793+0000 7f9f82ed1700 4 rocksdb: [db_impl/db_impl.cc:851] 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** DB Stats ** 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Uptime(secs): 0.1 total, 0.1 interval 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:49:29.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** Compaction Stats [default] ** 2022-04-28T19:49:29.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:29.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:29.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: L0 1/0 16.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: L6 1/0 22.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:29.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Sum 2/0 38.82 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** Compaction Stats [default] ** 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Uptime(secs): 0.1 total, 0.1 interval 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Flush(GB): cumulative 0.016, interval 0.016 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:29.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:49:29.869 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:49:29.869 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:49:29.869 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Cumulative compaction: 0.02 GB write, 253.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:29.869 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Interval compaction: 0.02 GB write, 253.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:29.869 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:49:29.870 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:49:29.870 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** Compaction Stats [default] ** 2022-04-28T19:49:29.870 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:29.870 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:29.871 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: L0 1/0 16.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.871 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: L6 1/0 22.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:29.871 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Sum 2/0 38.82 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.871 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:29.871 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** Compaction Stats [default] ** 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 588.7 0.03 0.00 1 0.028 0 0 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Uptime(secs): 0.1 total, 0.0 interval 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Flush(GB): cumulative 0.016, interval 0.000 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:29.872 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:49:29.873 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:49:29.873 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:49:29.873 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Cumulative compaction: 0.02 GB write, 253.18 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:29.873 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.793+0000 7f9f9be9f880 0 starting mon.c rank 1 at public addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] at bind addrs [v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.797+0000 7f9f9be9f880 1 mon.c@-1(???) e3 preinit fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.797+0000 7f9f9be9f880 0 mon.c@-1(???).mds e8 new map 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.797+0000 7f9f9be9f880 0 mon.c@-1(???).mds e8 print_map 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: e8 2022-04-28T19:49:29.874 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: enable_multiple, ever_enabled_multiple: 0,0 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: legacy client fscid: 1 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Filesystem 'foo' (1) 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: fs_name foo 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: epoch 5 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: flags 12 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: created 2022-04-28T19:42:27.245122+0000 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: modified 2022-04-28T19:42:37.174761+0000 2022-04-28T19:49:29.875 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: tableserver 0 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: root 0 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: session_timeout 60 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: session_autoclose 300 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: max_file_size 1099511627776 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: required_client_features {} 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: last_failure 0 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: last_failure_osd_epoch 0 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: max_mds 1 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: in 0 2022-04-28T19:49:29.876 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: up {0=34517} 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: failed 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: damaged 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: stopped 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: data_pools [4] 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: metadata_pool 3 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: inline_data disabled 2022-04-28T19:49:29.877 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: balancer 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: standby_count_wanted 1 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: [mds.foo.smithi139.oefiza{0:34517} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] compat {c=[1],r=[1],i=[7ff]}] 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: Standby daemons: 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: [mds.foo.smithi059.tnizcj{-1:14631} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] compat {c=[1],r=[1],i=[7ff]}] 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.797+0000 7f9f9be9f880 0 mon.c@-1(???).osd e360 crush map has features 3314933069571702784, adjusting msgr requires 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.797+0000 7f9f9be9f880 0 mon.c@-1(???).osd e360 crush map has features 288514119978713088, adjusting msgr requires 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: cluster 2022-04-28T19:49:25.834565+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: cluster 2022-04-28T19:49:26.794400+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:29.878 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:29 smithi059 bash[61152]: debug 2022-04-28T19:49:29.797+0000 7f9f9be9f880 1 mon.c@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2022-04-28T19:49:31.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:31.625Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:31.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:31.625Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:31.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:31.625Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:31.878 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:28.851726+0000 mgr.y (mgr.24961) 13 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 129 MiB data, 227 MiB used, 707 GiB / 715 GiB avail; 4.2 MiB/s rd, 625 KiB/s wr, 12 op/s 2022-04-28T19:49:31.997 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.013633+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:32.004 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.846517+0000 mon.a (mon.0) 56 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:32.004 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.850695+0000 mon.a (mon.0) 57 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:32.004 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.852446+0000 mgr.y (mgr.24961) 14 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 129 MiB data, 227 MiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 523 KiB/s wr, 10 op/s 2022-04-28T19:49:32.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.866216+0000 mon.a (mon.0) 58 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:32.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.866443+0000 mon.a (mon.0) 59 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:32.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.866536+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:32.005 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.867223+0000 mon.a (mon.0) 61 : cluster [DBG] mgrmap e30: y(active, since 20s), standbys: x 2022-04-28T19:49:32.006 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cluster 2022-04-28T19:49:30.874889+0000 mon.a (mon.0) 62 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:32.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: audit 2022-04-28T19:49:30.882100+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.24961 ' entity='' 2022-04-28T19:49:32.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: audit 2022-04-28T19:49:30.883775+0000 mon.b (mon.2) 147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:32.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: audit 2022-04-28T19:49:30.886214+0000 mon.b (mon.2) 148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:32.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: audit 2022-04-28T19:49:30.888039+0000 mon.b (mon.2) 149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:32.007 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: cephadm 2022-04-28T19:49:30.890553+0000 mgr.y (mgr.24961) 15 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:32.008 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[59745]: audit 2022-04-28T19:49:30.891446+0000 mon.b (mon.2) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:32.018 INFO:tasks.rados.rados.0.smithi059.stdout:1633: done (2 left) 2022-04-28T19:49:32.019 INFO:tasks.rados.rados.0.smithi059.stdout:1634: done (1 left) 2022-04-28T19:49:32.019 INFO:tasks.rados.rados.0.smithi059.stdout:1635: done (0 left) 2022-04-28T19:49:32.019 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 150 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1636: seq_num 527 ranges {3080192=434176,3514368=475136,3989504=565248,4554752=163840} 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1636: writing smithi05955964-3 from 3080192 to 3514368 tid 1 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1636: writing smithi05955964-3 from 3514368 to 3989504 tid 2 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1636: writing smithi05955964-3 from 3989504 to 4554752 tid 3 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1636: writing smithi05955964-3 from 4554752 to 4718592 tid 4 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1637: read oid 22 snap 149 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1637: expect deleted 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1638: read oid 9 snap 146 2022-04-28T19:49:32.020 INFO:tasks.rados.rados.0.smithi059.stdout:1638: expect deleted 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 150 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1639: seq_num 528 ranges {1654784=450560,2105344=761856,2867200=442368} 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1639: writing smithi05955964-28 from 1654784 to 2105344 tid 1 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1639: writing smithi05955964-28 from 2105344 to 2867200 tid 2 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1639: writing smithi05955964-28 from 2867200 to 3309568 tid 3 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1640: setattr oid 33 current snap is 150 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1636: finishing write tid 1 to smithi05955964-3 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1636: finishing write tid 2 to smithi05955964-3 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1636: finishing write tid 3 to smithi05955964-3 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1636: finishing write tid 4 to smithi05955964-3 2022-04-28T19:49:32.021 INFO:tasks.rados.rados.0.smithi059.stdout:1636: finishing write tid 6 to smithi05955964-3 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1636: finishing write tid 7 to smithi05955964-3 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 686 (ObjNum 527 snap 150 seq_num 527) dirty exists 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1636: left oid 3 (ObjNum 527 snap 150 seq_num 527) 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1636: done (4 left) 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1637: done (3 left) 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1638: done (2 left) 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 150 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1641: seq_num 529 ranges {0=466944,466944=647168,1114112=532480,1646592=24576} 2022-04-28T19:49:32.022 INFO:tasks.rados.rados.0.smithi059.stdout:1641: writing smithi05955964-30 from 0 to 466944 tid 1 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1641: writing smithi05955964-30 from 466944 to 1114112 tid 2 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1641: writing smithi05955964-30 from 1114112 to 1646592 tid 3 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1641: writing smithi05955964-30 from 1646592 to 1671168 tid 4 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1642: copy_from oid 20 from oid 25 current snap is 150 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1639: finishing write tid 1 to smithi05955964-28 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1639: finishing write tid 2 to smithi05955964-28 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1639: finishing write tid 3 to smithi05955964-28 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1643: delete oid 5 current snap is 150 2022-04-28T19:49:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:1639: finishing write tid 5 to smithi05955964-28 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 791 (ObjNum 494 snap 141 seq_num 494) dirty exists 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1639: finishing write tid 6 to smithi05955964-28 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 422 (ObjNum 528 snap 150 seq_num 528) dirty exists 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1639: left oid 28 (ObjNum 528 snap 150 seq_num 528) 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1641: finishing write tid 1 to smithi05955964-30 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1642: finishing copy_from to smithi05955964-20 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1642: got expected ENOENT (src dne) 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1642: finishing copy_from racing read to smithi05955964-20 2022-04-28T19:49:32.024 INFO:tasks.rados.rados.0.smithi059.stdout:1639: done (4 left) 2022-04-28T19:49:32.025 INFO:tasks.rados.rados.0.smithi059.stdout:1640: done (3 left) 2022-04-28T19:49:32.025 INFO:tasks.rados.rados.0.smithi059.stdout:1642: done (2 left) 2022-04-28T19:49:32.025 INFO:tasks.rados.rados.0.smithi059.stdout:1643: done (1 left) 2022-04-28T19:49:32.025 INFO:tasks.rados.rados.0.smithi059.stdout:1644: read oid 35 snap -1 2022-04-28T19:49:32.025 INFO:tasks.rados.rados.0.smithi059.stdout:1644: expect (ObjNum 414 snap 125 seq_num 414) 2022-04-28T19:49:32.025 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 150 2022-04-28T19:49:32.026 INFO:tasks.rados.rados.0.smithi059.stdout:1645: seq_num 531 ranges {6569984=499712,7069696=417792,7487488=548864,8036352=237568} 2022-04-28T19:49:32.026 INFO:tasks.rados.rados.0.smithi059.stdout:1645: writing smithi05955964-1 from 6569984 to 7069696 tid 1 2022-04-28T19:49:32.026 INFO:tasks.rados.rados.0.smithi059.stdout:1645: writing smithi05955964-1 from 7069696 to 7487488 tid 2 2022-04-28T19:49:32.026 INFO:tasks.rados.rados.0.smithi059.stdout:1645: writing smithi05955964-1 from 7487488 to 8036352 tid 3 2022-04-28T19:49:32.027 INFO:tasks.rados.rados.0.smithi059.stdout:1645: writing smithi05955964-1 from 8036352 to 8273920 tid 4 2022-04-28T19:49:32.027 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 150 2022-04-28T19:49:32.027 INFO:tasks.rados.rados.0.smithi059.stdout:1646: seq_num 532 ranges {647168=516096,1163264=696320,1859584=507904} 2022-04-28T19:49:32.027 INFO:tasks.rados.rados.0.smithi059.stdout:1646: writing smithi05955964-24 from 647168 to 1163264 tid 1 2022-04-28T19:49:32.027 INFO:tasks.rados.rados.0.smithi059.stdout:1646: writing smithi05955964-24 from 1163264 to 1859584 tid 2 2022-04-28T19:49:32.028 INFO:tasks.rados.rados.0.smithi059.stdout:1646: writing smithi05955964-24 from 1859584 to 2367488 tid 3 2022-04-28T19:49:32.028 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 150 2022-04-28T19:49:32.028 INFO:tasks.rados.rados.0.smithi059.stdout:1647: seq_num 533 ranges {1228800=532480,1761280=630784,2392064=516096,2908160=57344} 2022-04-28T19:49:32.028 INFO:tasks.rados.rados.0.smithi059.stdout:1647: writing smithi05955964-6 from 1228800 to 1761280 tid 1 2022-04-28T19:49:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:1647: writing smithi05955964-6 from 1761280 to 2392064 tid 2 2022-04-28T19:49:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:1647: writing smithi05955964-6 from 2392064 to 2908160 tid 3 2022-04-28T19:49:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:1647: writing smithi05955964-6 from 2908160 to 2965504 tid 4 2022-04-28T19:49:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:1648: read oid 28 snap -1 2022-04-28T19:49:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:1648: expect (ObjNum 528 snap 150 seq_num 528) 2022-04-28T19:49:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:1649: read oid 32 snap -1 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1649: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1650: copy_from oid 2 from oid 27 current snap is 150 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1641: finishing write tid 2 to smithi05955964-30 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1641: finishing write tid 3 to smithi05955964-30 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1641: finishing write tid 4 to smithi05955964-30 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1641: finishing write tid 6 to smithi05955964-30 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1641: finishing write tid 7 to smithi05955964-30 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 772 (ObjNum 529 snap 150 seq_num 529) dirty exists 2022-04-28T19:49:32.033 INFO:tasks.rados.rados.0.smithi059.stdout:1641: left oid 30 (ObjNum 529 snap 150 seq_num 529) 2022-04-28T19:49:32.034 INFO:tasks.rados.rados.0.smithi059.stdout:1644: expect (ObjNum 414 snap 125 seq_num 414) 2022-04-28T19:49:32.216 INFO:tasks.rados.rados.0.smithi059.stdout:1645: finishing write tid 1 to smithi05955964-1 2022-04-28T19:49:32.217 INFO:tasks.rados.rados.0.smithi059.stdout:1645: finishing write tid 2 to smithi05955964-1 2022-04-28T19:49:32.217 INFO:tasks.rados.rados.0.smithi059.stdout:1645: finishing write tid 3 to smithi05955964-1 2022-04-28T19:49:32.217 INFO:tasks.rados.rados.0.smithi059.stdout:1645: finishing write tid 4 to smithi05955964-1 2022-04-28T19:49:32.217 INFO:tasks.rados.rados.0.smithi059.stdout:1645: finishing write tid 6 to smithi05955964-1 2022-04-28T19:49:32.217 INFO:tasks.rados.rados.0.smithi059.stdout:1645: finishing write tid 7 to smithi05955964-1 2022-04-28T19:49:32.217 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 749 (ObjNum 531 snap 150 seq_num 531) dirty exists 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1645: left oid 1 (ObjNum 531 snap 150 seq_num 531) 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1646: finishing write tid 1 to smithi05955964-24 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1646: finishing write tid 2 to smithi05955964-24 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1646: finishing write tid 3 to smithi05955964-24 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1646: finishing write tid 5 to smithi05955964-24 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1646: finishing write tid 6 to smithi05955964-24 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 717 (ObjNum 532 snap 150 seq_num 532) dirty exists 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1646: left oid 24 (ObjNum 532 snap 150 seq_num 532) 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1647: finishing write tid 1 to smithi05955964-6 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1647: finishing write tid 2 to smithi05955964-6 2022-04-28T19:49:32.218 INFO:tasks.rados.rados.0.smithi059.stdout:1641: done (7 left) 2022-04-28T19:49:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:1644: done (6 left) 2022-04-28T19:49:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:1645: done (5 left) 2022-04-28T19:49:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:1646: done (4 left) 2022-04-28T19:49:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 150 2022-04-28T19:49:32.219 INFO:tasks.rados.rados.0.smithi059.stdout:1651: seq_num 534 ranges {5054464=548864,5603328=516096,6119424=688128} 2022-04-28T19:49:32.256 INFO:tasks.rados.rados.0.smithi059.stdout:1651: writing smithi05955964-17 from 5054464 to 5603328 tid 1 2022-04-28T19:49:32.259 INFO:tasks.rados.rados.0.smithi059.stdout:1651: writing smithi05955964-17 from 5603328 to 6119424 tid 2 2022-04-28T19:49:32.263 INFO:tasks.rados.rados.0.smithi059.stdout:1651: writing smithi05955964-17 from 6119424 to 6807552 tid 3 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1652: copy_from oid 39 from oid 3 current snap is 150 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1647: finishing write tid 3 to smithi05955964-6 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1647: finishing write tid 4 to smithi05955964-6 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1647: finishing write tid 6 to smithi05955964-6 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1647: finishing write tid 7 to smithi05955964-6 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 692 (ObjNum 533 snap 150 seq_num 533) dirty exists 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1647: left oid 6 (ObjNum 533 snap 150 seq_num 533) 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1647: done (5 left) 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1653: delete oid 19 current snap is 150 2022-04-28T19:49:32.264 INFO:tasks.rados.rados.0.smithi059.stdout:1649: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:28.851726+0000 mgr.y (mgr.24961) 13 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 129 MiB data, 227 MiB used, 707 GiB / 715 GiB avail; 4.2 MiB/s rd, 625 KiB/s wr, 12 op/s 2022-04-28T19:49:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.013633+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.846517+0000 mon.a (mon.0) 56 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.850695+0000 mon.a (mon.0) 57 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.852446+0000 mgr.y (mgr.24961) 14 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 129 MiB data, 227 MiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 523 KiB/s wr, 10 op/s 2022-04-28T19:49:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.866216+0000 mon.a (mon.0) 58 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.866443+0000 mon.a (mon.0) 59 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.866536+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.867223+0000 mon.a (mon.0) 61 : cluster [DBG] mgrmap e30: y(active, since 20s), standbys: x 2022-04-28T19:49:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cluster 2022-04-28T19:49:30.874889+0000 mon.a (mon.0) 62 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: audit 2022-04-28T19:49:30.882100+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.24961 ' entity='' 2022-04-28T19:49:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: audit 2022-04-28T19:49:30.883775+0000 mon.b (mon.2) 147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:32.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: audit 2022-04-28T19:49:30.886214+0000 mon.b (mon.2) 148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:32.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: audit 2022-04-28T19:49:30.888039+0000 mon.b (mon.2) 149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:32.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: cephadm 2022-04-28T19:49:30.890553+0000 mgr.y (mgr.24961) 15 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:32.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:31 smithi059 bash[61152]: audit 2022-04-28T19:49:30.891446+0000 mon.b (mon.2) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:28.851726+0000 mgr.y (mgr.24961) 13 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 129 MiB data, 227 MiB used, 707 GiB / 715 GiB avail; 4.2 MiB/s rd, 625 KiB/s wr, 12 op/s 2022-04-28T19:49:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.013633+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.846517+0000 mon.a (mon.0) 56 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.850695+0000 mon.a (mon.0) 57 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.852446+0000 mgr.y (mgr.24961) 14 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 129 MiB data, 227 MiB used, 707 GiB / 715 GiB avail; 3.5 MiB/s rd, 523 KiB/s wr, 10 op/s 2022-04-28T19:49:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.866216+0000 mon.a (mon.0) 58 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.866443+0000 mon.a (mon.0) 59 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.866536+0000 mon.a (mon.0) 60 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2022-04-28T19:49:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.867223+0000 mon.a (mon.0) 61 : cluster [DBG] mgrmap e30: y(active, since 20s), standbys: x 2022-04-28T19:49:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cluster 2022-04-28T19:49:30.874889+0000 mon.a (mon.0) 62 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: audit 2022-04-28T19:49:30.882100+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.24961 ' entity='' 2022-04-28T19:49:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: audit 2022-04-28T19:49:30.883775+0000 mon.b (mon.2) 147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: audit 2022-04-28T19:49:30.886214+0000 mon.b (mon.2) 148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: audit 2022-04-28T19:49:30.888039+0000 mon.b (mon.2) 149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:32.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: cephadm 2022-04-28T19:49:30.890553+0000 mgr.y (mgr.24961) 15 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:32.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:31 smithi139 bash[23439]: audit 2022-04-28T19:49:30.891446+0000 mon.b (mon.2) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:32.396 INFO:tasks.rados.rados.0.smithi059.stdout:1650: finishing copy_from to smithi05955964-2 2022-04-28T19:49:32.396 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 722 (ObjNum 498 snap 143 seq_num 498) dirty exists 2022-04-28T19:49:32.396 INFO:tasks.rados.rados.0.smithi059.stdout:1650: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:49:32.396 INFO:tasks.rados.rados.0.smithi059.stdout:1648: expect (ObjNum 528 snap 150 seq_num 528) 2022-04-28T19:49:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1651: finishing write tid 1 to smithi05955964-17 2022-04-28T19:49:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1651: finishing write tid 2 to smithi05955964-17 2022-04-28T19:49:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1651: finishing write tid 3 to smithi05955964-17 2022-04-28T19:49:32.500 INFO:tasks.rados.rados.0.smithi059.stdout:1651: finishing write tid 5 to smithi05955964-17 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1651: finishing write tid 6 to smithi05955964-17 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 389 (ObjNum 534 snap 150 seq_num 534) dirty exists 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1651: left oid 17 (ObjNum 534 snap 150 seq_num 534) 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1652: finishing copy_from to smithi05955964-39 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 686 (ObjNum 527 snap 150 seq_num 527) dirty exists 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1652: finishing copy_from racing read to smithi05955964-39 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1648: done (5 left) 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1649: done (4 left) 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1650: done (3 left) 2022-04-28T19:49:32.501 INFO:tasks.rados.rados.0.smithi059.stdout:1651: done (2 left) 2022-04-28T19:49:32.502 INFO:tasks.rados.rados.0.smithi059.stdout:1652: done (1 left) 2022-04-28T19:49:32.502 INFO:tasks.rados.rados.0.smithi059.stdout:1653: done (0 left) 2022-04-28T19:49:32.502 INFO:tasks.rados.rados.0.smithi059.stdout:1654: copy_from oid 25 from oid 27 current snap is 150 2022-04-28T19:49:32.502 INFO:tasks.rados.rados.0.smithi059.stdout:1655: delete oid 11 current snap is 150 2022-04-28T19:49:32.504 INFO:tasks.rados.rados.0.smithi059.stdout:1655: done (1 left) 2022-04-28T19:49:32.504 INFO:tasks.rados.rados.0.smithi059.stdout:1656: read oid 18 snap -1 2022-04-28T19:49:32.504 INFO:tasks.rados.rados.0.smithi059.stdout:1656: expect (ObjNum 504 snap 143 seq_num 504) 2022-04-28T19:49:32.505 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 150 2022-04-28T19:49:32.505 INFO:tasks.rados.rados.0.smithi059.stdout:1657: seq_num 537 ranges {2367488=598016,2965504=573440,3538944=491520,4030464=139264} 2022-04-28T19:49:32.525 INFO:tasks.rados.rados.0.smithi059.stdout:1657: writing smithi05955964-24 from 2367488 to 2965504 tid 1 2022-04-28T19:49:32.528 INFO:tasks.rados.rados.0.smithi059.stdout:1657: writing smithi05955964-24 from 2965504 to 3538944 tid 2 2022-04-28T19:49:32.531 INFO:tasks.rados.rados.0.smithi059.stdout:1657: writing smithi05955964-24 from 3538944 to 4030464 tid 3 2022-04-28T19:49:32.532 INFO:tasks.rados.rados.0.smithi059.stdout:1657: writing smithi05955964-24 from 4030464 to 4169728 tid 4 2022-04-28T19:49:32.532 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 150 2022-04-28T19:49:32.533 INFO:tasks.rados.rados.0.smithi059.stdout:1658: seq_num 538 ranges {4120576=614400,4734976=450560,5185536=704512,5890048=49152} 2022-04-28T19:49:32.559 INFO:tasks.rados.rados.0.smithi059.stdout:1658: writing smithi05955964-49 from 4120576 to 4734976 tid 1 2022-04-28T19:49:32.562 INFO:tasks.rados.rados.0.smithi059.stdout:1658: writing smithi05955964-49 from 4734976 to 5185536 tid 2 2022-04-28T19:49:32.567 INFO:tasks.rados.rados.0.smithi059.stdout:1658: writing smithi05955964-49 from 5185536 to 5890048 tid 3 2022-04-28T19:49:32.567 INFO:tasks.rados.rados.0.smithi059.stdout:1658: writing smithi05955964-49 from 5890048 to 5939200 tid 4 2022-04-28T19:49:32.567 INFO:tasks.rados.rados.0.smithi059.stdout:1659: read oid 12 snap -1 2022-04-28T19:49:32.567 INFO:tasks.rados.rados.0.smithi059.stdout:1659: expect (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:49:32.568 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 150 2022-04-28T19:49:32.568 INFO:tasks.rados.rados.0.smithi059.stdout:1660: seq_num 539 ranges {5365760=630784,5996544=737280,6733824=466944} 2022-04-28T19:49:32.601 INFO:tasks.rados.rados.0.smithi059.stdout:1660: writing smithi05955964-46 from 5365760 to 5996544 tid 1 2022-04-28T19:49:32.605 INFO:tasks.rados.rados.0.smithi059.stdout:1660: writing smithi05955964-46 from 5996544 to 6733824 tid 2 2022-04-28T19:49:32.608 INFO:tasks.rados.rados.0.smithi059.stdout:1660: writing smithi05955964-46 from 6733824 to 7200768 tid 3 2022-04-28T19:49:32.608 INFO:tasks.rados.rados.0.smithi059.stdout:1661: snap_create 2022-04-28T19:49:32.608 INFO:tasks.rados.rados.0.smithi059.stdout:1656: expect (ObjNum 504 snap 143 seq_num 504) 2022-04-28T19:49:32.637 INFO:tasks.rados.rados.0.smithi059.stdout:1654: finishing copy_from to smithi05955964-25 2022-04-28T19:49:32.637 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 722 (ObjNum 498 snap 143 seq_num 498) dirty exists 2022-04-28T19:49:32.637 INFO:tasks.rados.rados.0.smithi059.stdout:1654: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:49:32.637 INFO:tasks.rados.rados.0.smithi059.stdout:1657: finishing write tid 1 to smithi05955964-24 2022-04-28T19:49:32.637 INFO:tasks.rados.rados.0.smithi059.stdout:1657: finishing write tid 2 to smithi05955964-24 2022-04-28T19:49:32.637 INFO:tasks.rados.rados.0.smithi059.stdout:1657: finishing write tid 3 to smithi05955964-24 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1657: finishing write tid 4 to smithi05955964-24 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1657: finishing write tid 6 to smithi05955964-24 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1657: finishing write tid 7 to smithi05955964-24 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 724 (ObjNum 537 snap 150 seq_num 537) dirty exists 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1657: left oid 24 (ObjNum 537 snap 150 seq_num 537) 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1658: finishing write tid 1 to smithi05955964-49 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1658: finishing write tid 2 to smithi05955964-49 2022-04-28T19:49:32.638 INFO:tasks.rados.rados.0.smithi059.stdout:1659: expect (ObjNum 445 snap 132 seq_num 445) 2022-04-28T19:49:32.645 INFO:tasks.rados.rados.0.smithi059.stdout:1658: finishing write tid 3 to smithi05955964-49 2022-04-28T19:49:32.645 INFO:tasks.rados.rados.0.smithi059.stdout:1658: finishing write tid 4 to smithi05955964-49 2022-04-28T19:49:32.645 INFO:tasks.rados.rados.0.smithi059.stdout:1658: finishing write tid 6 to smithi05955964-49 2022-04-28T19:49:32.645 INFO:tasks.rados.rados.0.smithi059.stdout:1658: finishing write tid 7 to smithi05955964-49 2022-04-28T19:49:32.645 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 754 (ObjNum 538 snap 150 seq_num 538) dirty exists 2022-04-28T19:49:32.645 INFO:tasks.rados.rados.0.smithi059.stdout:1658: left oid 49 (ObjNum 538 snap 150 seq_num 538) 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:1660: finishing write tid 1 to smithi05955964-46 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:1660: finishing write tid 2 to smithi05955964-46 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:1660: finishing write tid 3 to smithi05955964-46 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:1660: finishing write tid 5 to smithi05955964-46 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:1660: finishing write tid 6 to smithi05955964-46 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 726 (ObjNum 539 snap 150 seq_num 539) dirty exists 2022-04-28T19:49:32.646 INFO:tasks.rados.rados.0.smithi059.stdout:1660: left oid 46 (ObjNum 539 snap 150 seq_num 539) 2022-04-28T19:49:32.891 INFO:tasks.rados.rados.0.smithi059.stdout:1654: done (6 left) 2022-04-28T19:49:32.891 INFO:tasks.rados.rados.0.smithi059.stdout:1656: done (5 left) 2022-04-28T19:49:32.891 INFO:tasks.rados.rados.0.smithi059.stdout:1657: done (4 left) 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:1658: done (3 left) 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:1659: done (2 left) 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:1660: done (1 left) 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:1661: done (0 left) 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:1662: rollback oid 13 current snap is 151 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 13 to 146 2022-04-28T19:49:32.892 INFO:tasks.rados.rados.0.smithi059.stdout:1663: read oid 46 snap -1 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1663: expect (ObjNum 539 snap 150 seq_num 539) 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1664: read oid 41 snap 146 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1664: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1665: copy_from oid 4 from oid 21 current snap is 151 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1666: rmattr oid 32 current snap is 151 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1666: done (4 left) 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 151 2022-04-28T19:49:32.893 INFO:tasks.rados.rados.0.smithi059.stdout:1667: seq_num 540 ranges {0=647168,647168=671744,1318912=532480} 2022-04-28T19:49:32.895 INFO:tasks.rados.rados.0.smithi059.stdout:1667: writing smithi05955964-34 from 0 to 647168 tid 1 2022-04-28T19:49:32.899 INFO:tasks.rados.rados.0.smithi059.stdout:1667: writing smithi05955964-34 from 647168 to 1318912 tid 2 2022-04-28T19:49:32.904 INFO:tasks.rados.rados.0.smithi059.stdout:1667: writing smithi05955964-34 from 1318912 to 1851392 tid 3 2022-04-28T19:49:32.904 INFO:tasks.rados.rados.0.smithi059.stdout:1668: delete oid 24 current snap is 151 2022-04-28T19:49:32.904 INFO:tasks.rados.rados.0.smithi059.stdout:1664: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:49:32.953 INFO:tasks.rados.rados.0.smithi059.stdout:1662: finishing rollback tid 0 to smithi05955964-13 2022-04-28T19:49:32.953 INFO:tasks.rados.rados.0.smithi059.stdout:1662: finishing rollback tid 1 to smithi05955964-13 2022-04-28T19:49:32.953 INFO:tasks.rados.rados.0.smithi059.stdout:1662: finishing rollback tid 2 to smithi05955964-13 2022-04-28T19:49:32.953 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 393 (ObjNum 398 snap 122 seq_num 398) dirty exists 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:1665: finishing copy_from to smithi05955964-4 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 611 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:1665: finishing copy_from racing read to smithi05955964-4 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:1667: finishing write tid 1 to smithi05955964-34 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:1662: done (5 left) 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:1664: done (4 left) 2022-04-28T19:49:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:1665: done (3 left) 2022-04-28T19:49:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:1668: done (2 left) 2022-04-28T19:49:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 151 2022-04-28T19:49:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:1669: seq_num 542 ranges {1974272=679936,2654208=442368,3096576=679936,3776512=81920} 2022-04-28T19:49:32.975 INFO:tasks.rados.rados.0.smithi059.stdout:1669: writing smithi05955964-41 from 1974272 to 2654208 tid 1 2022-04-28T19:49:32.978 INFO:tasks.rados.rados.0.smithi059.stdout:1669: writing smithi05955964-41 from 2654208 to 3096576 tid 2 2022-04-28T19:49:32.981 INFO:tasks.rados.rados.0.smithi059.stdout:1669: writing smithi05955964-41 from 3096576 to 3776512 tid 3 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1669: writing smithi05955964-41 from 3776512 to 3858432 tid 4 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1670: snap_create 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1667: finishing write tid 2 to smithi05955964-34 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1667: finishing write tid 3 to smithi05955964-34 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1667: finishing write tid 5 to smithi05955964-34 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1667: finishing write tid 6 to smithi05955964-34 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 776 (ObjNum 540 snap 151 seq_num 540) dirty exists 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1667: left oid 34 (ObjNum 540 snap 151 seq_num 540) 2022-04-28T19:49:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:1663: expect (ObjNum 539 snap 150 seq_num 539) 2022-04-28T19:49:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:32 smithi059 bash[59745]: cluster 2022-04-28T19:49:31.880762+0000 mon.a (mon.0) 64 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2022-04-28T19:49:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:32 smithi059 bash[61152]: cluster 2022-04-28T19:49:31.880762+0000 mon.a (mon.0) 64 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2022-04-28T19:49:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:32 smithi139 bash[23439]: cluster 2022-04-28T19:49:31.880762+0000 mon.a (mon.0) 64 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:1669: finishing write tid 1 to smithi05955964-41 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:1669: finishing write tid 2 to smithi05955964-41 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:1669: finishing write tid 3 to smithi05955964-41 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:1669: finishing write tid 4 to smithi05955964-41 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:1669: finishing write tid 6 to smithi05955964-41 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:1669: finishing write tid 7 to smithi05955964-41 2022-04-28T19:49:33.373 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 616 (ObjNum 542 snap 151 seq_num 542) dirty exists 2022-04-28T19:49:33.374 INFO:tasks.rados.rados.0.smithi059.stdout:1669: left oid 41 (ObjNum 542 snap 151 seq_num 542) 2022-04-28T19:49:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:33.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:33.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:33.519Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:33.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:33.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:33.523Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:33.893 INFO:tasks.rados.rados.0.smithi059.stdout:1663: done (3 left) 2022-04-28T19:49:33.893 INFO:tasks.rados.rados.0.smithi059.stdout:1667: done (2 left) 2022-04-28T19:49:33.894 INFO:tasks.rados.rados.0.smithi059.stdout:1669: done (1 left) 2022-04-28T19:49:33.894 INFO:tasks.rados.rados.0.smithi059.stdout:1670: done (0 left) 2022-04-28T19:49:33.894 INFO:tasks.rados.rados.0.smithi059.stdout:1671: rollback oid 21 current snap is 152 2022-04-28T19:49:33.894 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 151 2022-04-28T19:49:33.895 INFO:tasks.rados.rados.0.smithi059.stdout:1672: read oid 13 snap 149 2022-04-28T19:49:33.895 INFO:tasks.rados.rados.0.smithi059.stdout:1672: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:49:33.895 INFO:tasks.rados.rados.0.smithi059.stdout:1673: rollback oid 15 current snap is 152 2022-04-28T19:49:33.895 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 150 2022-04-28T19:49:33.895 INFO:tasks.rados.rados.0.smithi059.stdout:1674: read oid 19 snap -1 2022-04-28T19:49:33.895 INFO:tasks.rados.rados.0.smithi059.stdout:1674: expect deleted 2022-04-28T19:49:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1675: copy_from oid 36 from oid 6 current snap is 152 2022-04-28T19:49:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1676: snap_create 2022-04-28T19:49:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:1671: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:49:33.901 INFO:tasks.rados.rados.0.smithi059.stdout:1673: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:49:33.901 INFO:tasks.rados.rados.0.smithi059.stdout:1673: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:49:33.902 INFO:tasks.rados.rados.0.smithi059.stdout:1673: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:49:33.902 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 718 (ObjNum 522 snap 146 seq_num 522) dirty exists 2022-04-28T19:49:33.905 INFO:tasks.rados.rados.0.smithi059.stdout:1671: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:49:33.905 INFO:tasks.rados.rados.0.smithi059.stdout:1671: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:49:33.905 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 695 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:33.908 INFO:tasks.rados.rados.0.smithi059.stdout:1672: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:49:33.980 INFO:tasks.rados.rados.0.smithi059.stdout:1675: finishing copy_from to smithi05955964-36 2022-04-28T19:49:33.980 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 692 (ObjNum 533 snap 150 seq_num 533) dirty exists 2022-04-28T19:49:33.980 INFO:tasks.rados.rados.0.smithi059.stdout:1675: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:49:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[61152]: cluster 2022-04-28T19:49:32.853656+0000 mgr.y (mgr.24961) 16 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.2 MiB/s wr, 23 op/s 2022-04-28T19:49:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[61152]: cluster 2022-04-28T19:49:32.887855+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2022-04-28T19:49:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[59745]: cluster 2022-04-28T19:49:32.853656+0000 mgr.y (mgr.24961) 16 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.2 MiB/s wr, 23 op/s 2022-04-28T19:49:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:33 smithi059 bash[59745]: cluster 2022-04-28T19:49:32.887855+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2022-04-28T19:49:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:33 smithi139 bash[23439]: cluster 2022-04-28T19:49:32.853656+0000 mgr.y (mgr.24961) 16 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.2 MiB/s wr, 23 op/s 2022-04-28T19:49:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:33 smithi139 bash[23439]: cluster 2022-04-28T19:49:32.887855+0000 mon.a (mon.0) 65 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1671: done (5 left) 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1672: done (4 left) 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1673: done (3 left) 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1674: done (2 left) 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1675: done (1 left) 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1676: done (0 left) 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1677: rollback oid 1 current snap is 153 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 150 2022-04-28T19:49:34.904 INFO:tasks.rados.rados.0.smithi059.stdout:1678: read oid 38 snap -1 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:1678: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:1679: rollback oid 18 current snap is 153 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 146 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:1680: rollback oid 27 current snap is 153 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 152 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 153 2022-04-28T19:49:34.905 INFO:tasks.rados.rados.0.smithi059.stdout:1681: seq_num 543 ranges {4915200=704512,5619712=729088,6348800=475136} 2022-04-28T19:49:34.944 INFO:tasks.rados.rados.0.smithi059.stdout:1681: writing smithi05955964-35 from 4915200 to 5619712 tid 1 2022-04-28T19:49:34.948 INFO:tasks.rados.rados.0.smithi059.stdout:1681: writing smithi05955964-35 from 5619712 to 6348800 tid 2 2022-04-28T19:49:34.951 INFO:tasks.rados.rados.0.smithi059.stdout:1681: writing smithi05955964-35 from 6348800 to 6823936 tid 3 2022-04-28T19:49:34.951 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 153 2022-04-28T19:49:34.952 INFO:tasks.rados.rados.0.smithi059.stdout:1682: seq_num 544 ranges {5439488=720896,6160384=614400,6774784=589824} 2022-04-28T19:49:34.987 INFO:tasks.rados.rados.0.smithi059.stdout:1682: writing smithi05955964-8 from 5439488 to 6160384 tid 1 2022-04-28T19:49:34.990 INFO:tasks.rados.rados.0.smithi059.stdout:1682: writing smithi05955964-8 from 6160384 to 6774784 tid 2 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1682: writing smithi05955964-8 from 6774784 to 7364608 tid 3 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1683: copy_from oid 14 from oid 21 current snap is 153 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1680: finishing rollback tid 0 to smithi05955964-27 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1680: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1680: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 758 (ObjNum 498 snap 143 seq_num 498) dirty exists 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1680: done (6 left) 2022-04-28T19:49:34.994 INFO:tasks.rados.rados.0.smithi059.stdout:1684: snap_remove snap 152 2022-04-28T19:49:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:34 smithi059 bash[59745]: cluster 2022-04-28T19:49:33.888910+0000 mon.a (mon.0) 66 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2022-04-28T19:49:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:34 smithi059 bash[61152]: cluster 2022-04-28T19:49:33.888910+0000 mon.a (mon.0) 66 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2022-04-28T19:49:35.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:34 smithi139 bash[23439]: cluster 2022-04-28T19:49:33.888910+0000 mon.a (mon.0) 66 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1684: done (6 left) 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1685: snap_create 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1679: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1679: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1679: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 780 (ObjNum 504 snap 143 seq_num 504) dirty exists 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1677: finishing rollback tid 0 to smithi05955964-1 2022-04-28T19:49:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:1678: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:49:35.975 INFO:tasks.rados.rados.0.smithi059.stdout:1677: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:49:35.975 INFO:tasks.rados.rados.0.smithi059.stdout:1677: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:49:35.975 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 757 (ObjNum 531 snap 150 seq_num 531) dirty exists 2022-04-28T19:49:35.975 INFO:tasks.rados.rados.0.smithi059.stdout:1681: finishing write tid 1 to smithi05955964-35 2022-04-28T19:49:35.975 INFO:tasks.rados.rados.0.smithi059.stdout:1681: finishing write tid 2 to smithi05955964-35 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1681: finishing write tid 3 to smithi05955964-35 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1681: finishing write tid 5 to smithi05955964-35 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1681: finishing write tid 6 to smithi05955964-35 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 719 (ObjNum 543 snap 153 seq_num 543) dirty exists 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1681: left oid 35 (ObjNum 543 snap 153 seq_num 543) 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1682: finishing write tid 1 to smithi05955964-8 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1682: finishing write tid 2 to smithi05955964-8 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1682: finishing write tid 3 to smithi05955964-8 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1682: finishing write tid 5 to smithi05955964-8 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:1682: finishing write tid 6 to smithi05955964-8 2022-04-28T19:49:35.976 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 730 (ObjNum 544 snap 153 seq_num 544) dirty exists 2022-04-28T19:49:35.977 INFO:tasks.rados.rados.0.smithi059.stdout:1682: left oid 8 (ObjNum 544 snap 153 seq_num 544) 2022-04-28T19:49:35.977 INFO:tasks.rados.rados.0.smithi059.stdout:1683: finishing copy_from to smithi05955964-14 2022-04-28T19:49:35.977 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 695 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:35.977 INFO:tasks.rados.rados.0.smithi059.stdout:1683: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:49:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: cluster 2022-04-28T19:49:34.854326+0000 mgr.y (mgr.24961) 17 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.4 MiB/s wr, 17 op/s 2022-04-28T19:49:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: cluster 2022-04-28T19:49:34.899134+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2022-04-28T19:49:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: audit 2022-04-28T19:49:35.582331+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: audit 2022-04-28T19:49:35.594757+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: audit 2022-04-28T19:49:35.606283+0000 mon.b (mon.2) 151 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: audit 2022-04-28T19:49:35.611641+0000 mon.b (mon.2) 152 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: audit 2022-04-28T19:49:35.614051+0000 mon.b (mon.2) 153 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:49:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[61152]: audit 2022-04-28T19:49:35.616056+0000 mon.b (mon.2) 154 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: cluster 2022-04-28T19:49:34.854326+0000 mgr.y (mgr.24961) 17 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.4 MiB/s wr, 17 op/s 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: cluster 2022-04-28T19:49:34.899134+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: audit 2022-04-28T19:49:35.582331+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: audit 2022-04-28T19:49:35.594757+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: audit 2022-04-28T19:49:35.606283+0000 mon.b (mon.2) 151 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: audit 2022-04-28T19:49:35.611641+0000 mon.b (mon.2) 152 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: audit 2022-04-28T19:49:35.614051+0000 mon.b (mon.2) 153 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:49:36.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:35 smithi059 bash[59745]: audit 2022-04-28T19:49:35.616056+0000 mon.b (mon.2) 154 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-04-28T19:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: cluster 2022-04-28T19:49:34.854326+0000 mgr.y (mgr.24961) 17 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail; 5.9 MiB/s rd, 4.4 MiB/s wr, 17 op/s 2022-04-28T19:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: cluster 2022-04-28T19:49:34.899134+0000 mon.a (mon.0) 67 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2022-04-28T19:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: audit 2022-04-28T19:49:35.582331+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: audit 2022-04-28T19:49:35.594757+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: audit 2022-04-28T19:49:35.606283+0000 mon.b (mon.2) 151 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: audit 2022-04-28T19:49:35.611641+0000 mon.b (mon.2) 152 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: audit 2022-04-28T19:49:35.614051+0000 mon.b (mon.2) 153 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2022-04-28T19:49:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:35 smithi139 bash[23439]: audit 2022-04-28T19:49:35.616056+0000 mon.b (mon.2) 154 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2022-04-28T19:49:36.807 INFO:tasks.rados.rados.0.smithi059.stdout:1677: done (6 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1678: done (5 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1679: done (4 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1681: done (3 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1682: done (2 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1683: done (1 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1685: done (0 left) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1686: read oid 21 snap 149 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1686: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:49:36.808 INFO:tasks.rados.rados.0.smithi059.stdout:1687: copy_from oid 11 from oid 42 current snap is 154 2022-04-28T19:49:36.809 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 154 2022-04-28T19:49:36.809 INFO:tasks.rados.rados.0.smithi059.stdout:1688: seq_num 545 ranges {7200768=737280,7938048=491520,8429568=450560,8880128=262144} 2022-04-28T19:49:36.848 INFO:tasks.rados.rados.0.smithi059.stdout:1688: writing smithi05955964-46 from 7200768 to 7938048 tid 1 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1688: writing smithi05955964-46 from 7938048 to 8429568 tid 2 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1688: writing smithi05955964-46 from 8429568 to 8880128 tid 3 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1688: writing smithi05955964-46 from 8880128 to 9142272 tid 4 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1689: read oid 20 snap -1 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1689: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1690: copy_from oid 2 from oid 38 current snap is 154 2022-04-28T19:49:36.861 INFO:tasks.rados.rados.0.smithi059.stdout:1686: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:49:36.912 INFO:tasks.rados.rados.0.smithi059.stdout:1687: finishing copy_from to smithi05955964-11 2022-04-28T19:49:36.912 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 724 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:49:36.912 INFO:tasks.rados.rados.0.smithi059.stdout:1687: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:49:36.912 INFO:tasks.rados.rados.0.smithi059.stdout:1686: done (4 left) 2022-04-28T19:49:36.912 INFO:tasks.rados.rados.0.smithi059.stdout:1687: done (3 left) 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1691: read oid 1 snap -1 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1691: expect (ObjNum 531 snap 150 seq_num 531) 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1692: copy_from oid 11 from oid 45 current snap is 154 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: finishing write tid 1 to smithi05955964-46 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: finishing write tid 2 to smithi05955964-46 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: finishing write tid 3 to smithi05955964-46 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: finishing write tid 4 to smithi05955964-46 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: finishing write tid 6 to smithi05955964-46 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: finishing write tid 7 to smithi05955964-46 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 735 (ObjNum 545 snap 154 seq_num 545) dirty exists 2022-04-28T19:49:36.913 INFO:tasks.rados.rados.0.smithi059.stdout:1688: left oid 46 (ObjNum 545 snap 154 seq_num 545) 2022-04-28T19:49:36.914 INFO:tasks.rados.rados.0.smithi059.stdout:1690: finishing copy_from to smithi05955964-2 2022-04-28T19:49:36.914 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 712 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:49:36.914 INFO:tasks.rados.rados.0.smithi059.stdout:1690: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:49:36.914 INFO:tasks.rados.rados.0.smithi059.stdout:1689: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:49:37.260 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:36 smithi139 bash[23439]: cephadm 2022-04-28T19:49:35.618408+0000 mgr.y (mgr.24961) 18 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2022-04-28T19:49:37.261 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:36 smithi139 bash[23439]: cluster 2022-04-28T19:49:35.936124+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2022-04-28T19:49:37.261 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:36 smithi139 bash[23439]: cluster 2022-04-28T19:49:36.801168+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2022-04-28T19:49:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:36 smithi059 bash[61152]: cephadm 2022-04-28T19:49:35.618408+0000 mgr.y (mgr.24961) 18 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2022-04-28T19:49:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:36 smithi059 bash[61152]: cluster 2022-04-28T19:49:35.936124+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2022-04-28T19:49:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:36 smithi059 bash[61152]: cluster 2022-04-28T19:49:36.801168+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2022-04-28T19:49:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:36 smithi059 bash[59745]: cephadm 2022-04-28T19:49:35.618408+0000 mgr.y (mgr.24961) 18 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2022-04-28T19:49:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:36 smithi059 bash[59745]: cluster 2022-04-28T19:49:35.936124+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2022-04-28T19:49:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:36 smithi059 bash[59745]: cluster 2022-04-28T19:49:36.801168+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2022-04-28T19:49:37.400 INFO:tasks.rados.rados.0.smithi059.stdout:1692: finishing copy_from to smithi05955964-11 2022-04-28T19:49:37.400 INFO:tasks.rados.rados.0.smithi059.stdout:1692: got expected ENOENT (src dne) 2022-04-28T19:49:37.400 INFO:tasks.rados.rados.0.smithi059.stdout:1692: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:49:37.401 INFO:tasks.rados.rados.0.smithi059.stdout:1691: expect (ObjNum 531 snap 150 seq_num 531) 2022-04-28T19:49:37.841 INFO:tasks.rados.rados.0.smithi059.stdout:1688: done (4 left) 2022-04-28T19:49:37.841 INFO:tasks.rados.rados.0.smithi059.stdout:1689: done (3 left) 2022-04-28T19:49:37.841 INFO:tasks.rados.rados.0.smithi059.stdout:1690: done (2 left) 2022-04-28T19:49:37.841 INFO:tasks.rados.rados.0.smithi059.stdout:1691: done (1 left) 2022-04-28T19:49:37.843 INFO:tasks.rados.rados.0.smithi059.stdout:1692: done (0 left) 2022-04-28T19:49:37.843 INFO:tasks.rados.rados.0.smithi059.stdout:1693: read oid 13 snap -1 2022-04-28T19:49:37.843 INFO:tasks.rados.rados.0.smithi059.stdout:1693: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:49:37.843 INFO:tasks.rados.rados.0.smithi059.stdout:1694: copy_from oid 27 from oid 1 current snap is 154 2022-04-28T19:49:37.843 INFO:tasks.rados.rados.0.smithi059.stdout:1695: snap_create 2022-04-28T19:49:37.858 INFO:tasks.rados.rados.0.smithi059.stdout:1693: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:49:37.931 INFO:tasks.rados.rados.0.smithi059.stdout:1694: finishing copy_from to smithi05955964-27 2022-04-28T19:49:37.931 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 757 (ObjNum 531 snap 150 seq_num 531) dirty exists 2022-04-28T19:49:37.931 INFO:tasks.rados.rados.0.smithi059.stdout:1694: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:49:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[61152]: cluster 2022-04-28T19:49:36.854809+0000 mgr.y (mgr.24961) 19 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[61152]: audit 2022-04-28T19:49:37.852512+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[61152]: audit 2022-04-28T19:49:37.858080+0000 mon.b (mon.2) 155 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[61152]: audit 2022-04-28T19:49:37.858972+0000 mon.b (mon.2) 156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[61152]: audit 2022-04-28T19:49:37.859741+0000 mon.b (mon.2) 157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[61152]: audit 2022-04-28T19:49:37.860978+0000 mon.b (mon.2) 158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "container_image"}]: dispatch 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[59745]: cluster 2022-04-28T19:49:36.854809+0000 mgr.y (mgr.24961) 19 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[59745]: audit 2022-04-28T19:49:37.852512+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[59745]: audit 2022-04-28T19:49:37.858080+0000 mon.b (mon.2) 155 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[59745]: audit 2022-04-28T19:49:37.858972+0000 mon.b (mon.2) 156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:38.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[59745]: audit 2022-04-28T19:49:37.859741+0000 mon.b (mon.2) 157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:38.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:37 smithi059 bash[59745]: audit 2022-04-28T19:49:37.860978+0000 mon.b (mon.2) 158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "container_image"}]: dispatch 2022-04-28T19:49:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:37 smithi139 bash[23439]: cluster 2022-04-28T19:49:36.854809+0000 mgr.y (mgr.24961) 19 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 156 MiB data, 243 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:37 smithi139 bash[23439]: audit 2022-04-28T19:49:37.852512+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.24961 ' entity='mgr.y' 2022-04-28T19:49:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:37 smithi139 bash[23439]: audit 2022-04-28T19:49:37.858080+0000 mon.b (mon.2) 155 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:37 smithi139 bash[23439]: audit 2022-04-28T19:49:37.858972+0000 mon.b (mon.2) 156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:37 smithi139 bash[23439]: audit 2022-04-28T19:49:37.859741+0000 mon.b (mon.2) 157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:37 smithi139 bash[23439]: audit 2022-04-28T19:49:37.860978+0000 mon.b (mon.2) 158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "container_image"}]: dispatch 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:1693: done (2 left) 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:1694: done (1 left) 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:1695: done (0 left) 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:1696: read oid 44 snap -1 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:1696: expect (ObjNum 507 snap 144 seq_num 507) 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 155 2022-04-28T19:49:38.821 INFO:tasks.rados.rados.0.smithi059.stdout:1697: seq_num 546 ranges {3366912=753664,4120576=778240,4898816=425984} 2022-04-28T19:49:38.843 INFO:tasks.rados.rados.0.smithi059.stdout:1697: writing smithi05955964-7 from 3366912 to 4120576 tid 1 2022-04-28T19:49:38.848 INFO:tasks.rados.rados.0.smithi059.stdout:1697: writing smithi05955964-7 from 4120576 to 4898816 tid 2 2022-04-28T19:49:38.850 INFO:tasks.rados.rados.0.smithi059.stdout:1697: writing smithi05955964-7 from 4898816 to 5324800 tid 3 2022-04-28T19:49:38.851 INFO:tasks.rados.rados.0.smithi059.stdout:1698: delete oid 12 current snap is 155 2022-04-28T19:49:38.851 INFO:tasks.rados.rados.0.smithi059.stdout:1696: expect (ObjNum 507 snap 144 seq_num 507) 2022-04-28T19:49:39.090 INFO:tasks.rados.rados.0.smithi059.stdout:1697: finishing write tid 1 to smithi05955964-7 2022-04-28T19:49:39.090 INFO:tasks.rados.rados.0.smithi059.stdout:1697: finishing write tid 2 to smithi05955964-7 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1697: finishing write tid 3 to smithi05955964-7 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1697: finishing write tid 5 to smithi05955964-7 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1697: finishing write tid 6 to smithi05955964-7 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 723 (ObjNum 546 snap 155 seq_num 546) dirty exists 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1697: left oid 7 (ObjNum 546 snap 155 seq_num 546) 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1696: done (2 left) 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1697: done (1 left) 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1698: done (0 left) 2022-04-28T19:49:39.091 INFO:tasks.rados.rados.0.smithi059.stdout:1699: rollback oid 10 current snap is 155 2022-04-28T19:49:39.092 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 153 2022-04-28T19:49:39.092 INFO:tasks.rados.rados.0.smithi059.stdout:1700: snap_remove snap 150 2022-04-28T19:49:39.234 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:38 smithi139 bash[23439]: cephadm 2022-04-28T19:49:37.843708+0000 mgr.y (mgr.24961) 20 : cephadm [INF] Upgrade: Updating mon.b 2022-04-28T19:49:39.235 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:38 smithi139 bash[23439]: cephadm 2022-04-28T19:49:37.861290+0000 mgr.y (mgr.24961) 21 : cephadm [INF] Deploying daemon mon.b on smithi139 2022-04-28T19:49:39.235 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:38 smithi139 bash[23439]: cluster 2022-04-28T19:49:38.818538+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2022-04-28T19:49:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:38 smithi059 bash[61152]: cephadm 2022-04-28T19:49:37.843708+0000 mgr.y (mgr.24961) 20 : cephadm [INF] Upgrade: Updating mon.b 2022-04-28T19:49:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:38 smithi059 bash[61152]: cephadm 2022-04-28T19:49:37.861290+0000 mgr.y (mgr.24961) 21 : cephadm [INF] Deploying daemon mon.b on smithi139 2022-04-28T19:49:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:38 smithi059 bash[61152]: cluster 2022-04-28T19:49:38.818538+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2022-04-28T19:49:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:38 smithi059 bash[59745]: cephadm 2022-04-28T19:49:37.843708+0000 mgr.y (mgr.24961) 20 : cephadm [INF] Upgrade: Updating mon.b 2022-04-28T19:49:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:38 smithi059 bash[59745]: cephadm 2022-04-28T19:49:37.861290+0000 mgr.y (mgr.24961) 21 : cephadm [INF] Deploying daemon mon.b on smithi139 2022-04-28T19:49:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:38 smithi059 bash[59745]: cluster 2022-04-28T19:49:38.818538+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2022-04-28T19:49:39.951 INFO:tasks.rados.rados.0.smithi059.stdout:1700: done (1 left) 2022-04-28T19:49:39.951 INFO:tasks.rados.rados.0.smithi059.stdout:1701: copy_from oid 28 from oid 21 current snap is 155 2022-04-28T19:49:39.951 INFO:tasks.rados.rados.0.smithi059.stdout:1702: snap_remove snap 151 2022-04-28T19:49:40.117 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:39 smithi139 systemd[1]: Stopping Ceph mon.b for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:49:40.284 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:39 smithi139 bash[23439]: cluster 2022-04-28T19:49:38.855520+0000 mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 177 MiB data, 296 MiB used, 707 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 19 op/s 2022-04-28T19:49:40.285 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 bash[23439]: debug 2022-04-28T19:49:40.038+0000 7f25e4aa3700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:49:40.288 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 bash[23439]: debug 2022-04-28T19:49:40.038+0000 7f25e4aa3700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2022-04-28T19:49:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:39 smithi059 bash[61152]: cluster 2022-04-28T19:49:38.855520+0000 mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 177 MiB data, 296 MiB used, 707 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 19 op/s 2022-04-28T19:49:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:39 smithi059 bash[59745]: cluster 2022-04-28T19:49:38.855520+0000 mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 177 MiB data, 296 MiB used, 707 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 19 op/s 2022-04-28T19:49:41.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 bash[43196]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.b 2022-04-28T19:49:41.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 bash[43196]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon-b 2022-04-28T19:49:41.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 bash[43196]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-mon.b 2022-04-28T19:49:41.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 systemd[1]: Stopped Ceph mon.b for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:49:41.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:40 smithi139 systemd[1]: Started Ceph mon.b for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:49:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:41.625Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:41.625Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:41.625Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.082+0000 7fb5817fa880 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.082+0000 7fb5817fa880 0 ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable), process ceph-mon, pid 1 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.082+0000 7fb5817fa880 0 pidfile_write: ignore empty --pid-file 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 0 load: jerasure load: lrc load: isa 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: RocksDB version: 6.8.1 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2022-04-28T19:49:42.101 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Compile date Apr 26 2022 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: DB SUMMARY 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: CURRENT file: CURRENT 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: IDENTITY file: IDENTITY 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 796 Bytes 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000023.sst 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000021.log size: 626963 ; 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.error_if_exists: 0 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.create_if_missing: 0 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.paranoid_checks: 1 2022-04-28T19:49:42.102 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.env: 0x557ce9077000 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.fs: Posix File System 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.info_log: 0x557cea924440 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_file_opening_threads: 16 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.statistics: (nil) 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.use_fsync: 0 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_log_file_size: 0 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.log_file_time_to_roll: 0 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.keep_log_file_num: 1000 2022-04-28T19:49:42.103 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.recycle_log_file_num: 0 2022-04-28T19:49:42.104 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.allow_fallocate: 1 2022-04-28T19:49:42.104 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.allow_mmap_reads: 0 2022-04-28T19:49:42.104 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.allow_mmap_writes: 0 2022-04-28T19:49:42.104 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.use_direct_reads: 0 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.create_missing_column_families: 0 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.db_log_dir: 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.table_cache_numshardbits: 6 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_subcompactions: 1 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_background_flushes: -1 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.WAL_ttl_seconds: 0 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.WAL_size_limit_MB: 0 2022-04-28T19:49:42.105 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.is_fd_close_on_exec: 1 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.advise_random_on_open: 1 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.db_write_buffer_size: 0 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.write_buffer_manager: 0x557ceb624090 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.use_adaptive_mutex: 0 2022-04-28T19:49:42.106 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.rate_limiter: (nil) 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.wal_recovery_mode: 2 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.enable_thread_tracking: 0 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.enable_pipelined_write: 0 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.unordered_write: 0 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.row_cache: None 2022-04-28T19:49:42.107 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.wal_filter: None 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.allow_ingest_behind: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.preserve_deletes: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.two_write_queues: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.manual_wal_flush: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.atomic_flush: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.persist_stats_to_disk: 0 2022-04-28T19:49:42.108 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.write_dbid_to_manifest: 0 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.log_readahead_size: 0 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_background_jobs: 2 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_background_compactions: -1 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2022-04-28T19:49:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.delayed_write_rate : 16777216 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_total_wal_size: 0 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.stats_dump_period_sec: 600 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.stats_persist_period_sec: 600 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_open_files: -1 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bytes_per_sync: 0 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.wal_bytes_per_sync: 0 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.strict_bytes_per_sync: 0 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_readahead_size: 0 2022-04-28T19:49:42.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Compression algorithms supported: 2022-04-28T19:49:42.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kZSTD supported: 0 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kXpressCompression supported: 0 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kLZ4HCCompression supported: 1 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kLZ4Compression supported: 1 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kBZip2Compression supported: 0 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kZlibCompression supported: 1 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: kSnappyCompression supported: 1 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Fast CRC32 supported: Supported on x86 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2022-04-28T19:49:42.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.merge_operator: 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_filter: None 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_filter_factory: None 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.memtable_factory: SkipListFactory 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.table_factory: BlockBasedTable 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557cea8c7d10) 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: cache_index_and_filter_blocks: 1 2022-04-28T19:49:42.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: cache_index_and_filter_blocks_with_high_priority: 0 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: pin_l0_filter_and_index_blocks_in_cache: 0 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: pin_top_level_index_and_filter: 1 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: index_type: 0 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: data_block_index_type: 0 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: index_shortening: 1 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: data_block_hash_table_util_ratio: 0.750000 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: hash_index_allow_collision: 1 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: checksum: 1 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: no_block_cache: 0 2022-04-28T19:49:42.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_cache: 0x557cea8fef10 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_cache_name: BinnedLRUCache 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_cache_options: 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: capacity : 536870912 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: num_shard_bits : 4 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: strict_capacity_limit : 0 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: high_pri_pool_ratio: 0.000 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_cache_compressed: (nil) 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: persistent_cache: (nil) 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_size: 4096 2022-04-28T19:49:42.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_size_deviation: 10 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_restart_interval: 16 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: index_block_restart_interval: 1 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: metadata_block_size: 4096 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: partition_filters: 0 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: use_delta_encoding: 1 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: filter_policy: rocksdb.BuiltinBloomFilter 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: whole_key_filtering: 1 2022-04-28T19:49:42.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: verify_compression: 0 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: read_amp_bytes_per_bit: 0 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: format_version: 2 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: enable_index_compression: 1 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: block_align: 0 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.write_buffer_size: 33554432 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_write_buffer_number: 2 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression: NoCompression 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression: Disabled 2022-04-28T19:49:42.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.prefix_extractor: nullptr 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.num_levels: 7 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2022-04-28T19:49:42.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression_opts.window_bits: -14 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression_opts.level: 32767 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression_opts.strategy: 0 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compression_opts.enabled: false 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2022-04-28T19:49:42.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.target_file_size_base: 67108864 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.target_file_size_multiplier: 1 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2022-04-28T19:49:42.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.arena_block_size: 4194304 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2022-04-28T19:49:42.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.disable_auto_compactions: 0 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2022-04-28T19:49:42.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2022-04-28T19:49:42.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2022-04-28T19:49:42.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.table_properties_collectors: 2022-04-28T19:49:42.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.inplace_update_support: 0 2022-04-28T19:49:42.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.inplace_update_num_locks: 10000 2022-04-28T19:49:42.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2022-04-28T19:49:42.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2022-04-28T19:49:42.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.memtable_huge_page_size: 0 2022-04-28T19:49:42.385 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.bloom_locality: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.max_successive_merges: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.optimize_filters_for_hits: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.paranoid_file_checks: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.force_consistency_checks: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.report_bg_io_stats: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.ttl: 2592000 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: Options.periodic_compaction_seconds: 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 25, last_sequence is 17647, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 21 2022-04-28T19:49:42.386 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175382095693, "job": 1, "event": "recovery_started", "log_files": [21]} 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #21 mode 2 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.090+0000 7fb5817fa880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.094+0000 7fb5817fa880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175382098654, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 25, "file_size": 417684, "table_properties": {"data_size": 415910, "index_size": 764, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 197, "raw_key_size": 1561, "raw_average_key_size": 25, "raw_value_size": 414220, "raw_average_value_size": 6680, "num_data_blocks": 24, "num_entries": 62, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1651175382, "oldest_key_time": 3, "file_creation_time": 0}} 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.094+0000 7fb5817fa880 4 rocksdb: [version_set.cc:3826] Creating manifest 26 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.094+0000 7fb5817fa880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1651175382101179, "job": 1, "event": "recovery_finished"} 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.098+0000 7fb5817fa880 4 rocksdb: DB pointer 0x557cea9c7800 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.098+0000 7fb56882c700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.098+0000 7fb56882c700 4 rocksdb: [db_impl/db_impl.cc:851] 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** DB Stats ** 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:49:42.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** Compaction Stats [default] ** 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: L0 1/0 407.89 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: L6 1/0 31.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Sum 2/0 32.08 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** Compaction Stats [default] ** 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:42.389 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Cumulative compaction: 0.00 GB write, 45.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Interval compaction: 0.00 GB write, 45.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** Compaction Stats [default] ** 2022-04-28T19:49:42.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: L0 1/0 407.89 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: L6 1/0 31.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Sum 2/0 32.08 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** Compaction Stats [default] ** 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 215.9 0.00 0.00 1 0.002 0 0 2022-04-28T19:49:42.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Uptime(secs): 0.0 total, 0.0 interval 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Flush(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(GB): cumulative 0.000, interval 0.000 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(Total Files): cumulative 0, interval 0 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(L0 Files): cumulative 0, interval 0 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: AddFile(Keys): cumulative 0, interval 0 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Cumulative compaction: 0.00 GB write, 45.29 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2022-04-28T19:49:42.392 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: ** File Read Latency Histogram By Level [default] ** 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.098+0000 7fb5817fa880 0 starting mon.b rank 2 at public addrs [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] at bind addrs [v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.102+0000 7fb5817fa880 1 mon.b@-1(???) e3 preinit fsid 20a1061c-c72a-11ec-8c39-001a4aab830c 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.102+0000 7fb5817fa880 0 mon.b@-1(???).mds e8 new map 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.102+0000 7fb5817fa880 0 mon.b@-1(???).mds e8 print_map 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: e8 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: enable_multiple, ever_enabled_multiple: 0,0 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: legacy client fscid: 1 2022-04-28T19:49:42.393 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Filesystem 'foo' (1) 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: fs_name foo 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: epoch 5 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: flags 12 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: created 2022-04-28T19:42:27.245122+0000 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: modified 2022-04-28T19:42:37.174761+0000 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: tableserver 0 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: root 0 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: session_timeout 60 2022-04-28T19:49:42.394 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: session_autoclose 300 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: max_file_size 1099511627776 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: required_client_features {} 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: last_failure 0 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: last_failure_osd_epoch 0 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: max_mds 1 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: in 0 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: up {0=34517} 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: failed 2022-04-28T19:49:42.395 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: damaged 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: stopped 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: data_pools [4] 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: metadata_pool 3 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: inline_data disabled 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: balancer 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: standby_count_wanted 1 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: [mds.foo.smithi139.oefiza{0:34517} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.139:6832/1765843380,v1:172.21.15.139:6833/1765843380] compat {c=[1],r=[1],i=[7ff]}] 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: Standby daemons: 2022-04-28T19:49:42.396 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: 2022-04-28T19:49:42.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: [mds.foo.smithi059.tnizcj{-1:14631} state up:standby seq 3 join_fscid=1 addr [v2:172.21.15.59:6834/2953447171,v1:172.21.15.59:6835/2953447171] compat {c=[1],r=[1],i=[7ff]}] 2022-04-28T19:49:42.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.102+0000 7fb5817fa880 0 mon.b@-1(???).osd e368 crush map has features 3314933069571702784, adjusting msgr requires 2022-04-28T19:49:42.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.102+0000 7fb5817fa880 0 mon.b@-1(???).osd e368 crush map has features 288514119978713088, adjusting msgr requires 2022-04-28T19:49:42.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: cluster 2022-04-28T19:49:38.855520+0000 mgr.y (mgr.24961) 22 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 177 MiB data, 296 MiB used, 707 GiB / 715 GiB avail; 21 MiB/s rd, 5.5 MiB/s wr, 19 op/s 2022-04-28T19:49:42.397 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:42 smithi139 bash[43301]: debug 2022-04-28T19:49:42.102+0000 7fb5817fa880 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1702: done (2 left) 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1703: read oid 50 snap -1 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1703: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1704: delete oid 29 current snap is 155 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1699: finishing rollback tid 0 to smithi05955964-10 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1699: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:49:42.398 INFO:tasks.rados.rados.0.smithi059.stdout:1699: finishing rollback tid 2 to smithi05955964-10 2022-04-28T19:49:42.399 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 766 (ObjNum 457 snap 135 seq_num 457) dirty exists 2022-04-28T19:49:42.399 INFO:tasks.rados.rados.0.smithi059.stdout:1701: finishing copy_from to smithi05955964-28 2022-04-28T19:49:42.399 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 695 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:42.399 INFO:tasks.rados.rados.0.smithi059.stdout:1701: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:49:42.400 INFO:tasks.rados.rados.0.smithi059.stdout:1699: done (3 left) 2022-04-28T19:49:42.400 INFO:tasks.rados.rados.0.smithi059.stdout:1701: done (2 left) 2022-04-28T19:49:42.400 INFO:tasks.rados.rados.0.smithi059.stdout:1704: done (1 left) 2022-04-28T19:49:42.400 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 155 2022-04-28T19:49:42.400 INFO:tasks.rados.rados.0.smithi059.stdout:1705: seq_num 549 ranges {0=802816,802816=483328,1286144=434176,1720320=286720} 2022-04-28T19:49:42.404 INFO:tasks.rados.rados.0.smithi059.stdout:1705: writing smithi05955964-45 from 0 to 802816 tid 1 2022-04-28T19:49:42.408 INFO:tasks.rados.rados.0.smithi059.stdout:1705: writing smithi05955964-45 from 802816 to 1286144 tid 2 2022-04-28T19:49:42.410 INFO:tasks.rados.rados.0.smithi059.stdout:1705: writing smithi05955964-45 from 1286144 to 1720320 tid 3 2022-04-28T19:49:42.412 INFO:tasks.rados.rados.0.smithi059.stdout:1705: writing smithi05955964-45 from 1720320 to 2007040 tid 4 2022-04-28T19:49:42.412 INFO:tasks.rados.rados.0.smithi059.stdout:1706: rollback oid 4 current snap is 155 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 154 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1707: read oid 16 snap -1 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1707: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1708: read oid 48 snap 153 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1708: expect (ObjNum 526 snap 149 seq_num 526) 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1709: read oid 32 snap -1 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1709: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 155 2022-04-28T19:49:42.413 INFO:tasks.rados.rados.0.smithi059.stdout:1710: seq_num 550 ranges {0=417792,417792=770048,1187840=638976,1826816=196608} 2022-04-28T19:49:42.415 INFO:tasks.rados.rados.0.smithi059.stdout:1710: writing smithi05955964-29 from 0 to 417792 tid 1 2022-04-28T19:49:42.420 INFO:tasks.rados.rados.0.smithi059.stdout:1710: writing smithi05955964-29 from 417792 to 1187840 tid 2 2022-04-28T19:49:42.424 INFO:tasks.rados.rados.0.smithi059.stdout:1710: writing smithi05955964-29 from 1187840 to 1826816 tid 3 2022-04-28T19:49:42.426 INFO:tasks.rados.rados.0.smithi059.stdout:1710: writing smithi05955964-29 from 1826816 to 2023424 tid 4 2022-04-28T19:49:42.426 INFO:tasks.rados.rados.0.smithi059.stdout:1711: rollback oid 3 current snap is 155 2022-04-28T19:49:42.426 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 146 2022-04-28T19:49:42.427 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 155 2022-04-28T19:49:42.427 INFO:tasks.rados.rados.0.smithi059.stdout:1712: seq_num 551 ranges {344064=434176,778240=647168,1425408=450560,1875968=507904} 2022-04-28T19:49:42.431 INFO:tasks.rados.rados.0.smithi059.stdout:1712: writing smithi05955964-40 from 344064 to 778240 tid 1 2022-04-28T19:49:42.436 INFO:tasks.rados.rados.0.smithi059.stdout:1712: writing smithi05955964-40 from 778240 to 1425408 tid 2 2022-04-28T19:49:42.438 INFO:tasks.rados.rados.0.smithi059.stdout:1712: writing smithi05955964-40 from 1425408 to 1875968 tid 3 2022-04-28T19:49:42.441 INFO:tasks.rados.rados.0.smithi059.stdout:1712: writing smithi05955964-40 from 1875968 to 2383872 tid 4 2022-04-28T19:49:42.441 INFO:tasks.rados.rados.0.smithi059.stdout:1713: snap_create 2022-04-28T19:49:42.442 INFO:tasks.rados.rados.0.smithi059.stdout:1707: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:1705: finishing write tid 1 to smithi05955964-45 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:1705: finishing write tid 2 to smithi05955964-45 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:1706: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:1706: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:1706: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 737 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:49:42.444 INFO:tasks.rados.rados.0.smithi059.stdout:1703: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:49:42.820 INFO:tasks.rados.rados.0.smithi059.stdout:1705: finishing write tid 3 to smithi05955964-45 2022-04-28T19:49:42.820 INFO:tasks.rados.rados.0.smithi059.stdout:1705: finishing write tid 4 to smithi05955964-45 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1705: finishing write tid 6 to smithi05955964-45 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1705: finishing write tid 7 to smithi05955964-45 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 697 (ObjNum 549 snap 155 seq_num 549) dirty exists 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1705: left oid 45 (ObjNum 549 snap 155 seq_num 549) 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1711: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1711: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1711: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 701 (ObjNum 491 snap 140 seq_num 491) dirty exists 2022-04-28T19:49:42.821 INFO:tasks.rados.rados.0.smithi059.stdout:1708: expect (ObjNum 526 snap 149 seq_num 526) 2022-04-28T19:49:42.938 INFO:tasks.rados.rados.0.smithi059.stdout:1712: finishing write tid 1 to smithi05955964-40 2022-04-28T19:49:42.938 INFO:tasks.rados.rados.0.smithi059.stdout:1709: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:43.055 INFO:tasks.rados.rados.0.smithi059.stdout:1710: finishing write tid 1 to smithi05955964-29 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1710: finishing write tid 2 to smithi05955964-29 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1710: finishing write tid 3 to smithi05955964-29 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1710: finishing write tid 4 to smithi05955964-29 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1710: finishing write tid 6 to smithi05955964-29 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1710: finishing write tid 7 to smithi05955964-29 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 748 (ObjNum 550 snap 155 seq_num 550) dirty exists 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1710: left oid 29 (ObjNum 550 snap 155 seq_num 550) 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1712: finishing write tid 2 to smithi05955964-40 2022-04-28T19:49:43.056 INFO:tasks.rados.rados.0.smithi059.stdout:1712: finishing write tid 3 to smithi05955964-40 2022-04-28T19:49:43.057 INFO:tasks.rados.rados.0.smithi059.stdout:1712: finishing write tid 4 to smithi05955964-40 2022-04-28T19:49:43.057 INFO:tasks.rados.rados.0.smithi059.stdout:1712: finishing write tid 6 to smithi05955964-40 2022-04-28T19:49:43.057 INFO:tasks.rados.rados.0.smithi059.stdout:1712: finishing write tid 7 to smithi05955964-40 2022-04-28T19:49:43.057 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 762 (ObjNum 551 snap 155 seq_num 551) dirty exists 2022-04-28T19:49:43.057 INFO:tasks.rados.rados.0.smithi059.stdout:1712: left oid 40 (ObjNum 551 snap 155 seq_num 551) 2022-04-28T19:49:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:43.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:43.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:43.519Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:43.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:43.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:43.524Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:48.445 INFO:tasks.rados.rados.0.smithi059.stdout:1703: done (9 left) 2022-04-28T19:49:48.445 INFO:tasks.rados.rados.0.smithi059.stdout:1705: done (8 left) 2022-04-28T19:49:48.445 INFO:tasks.rados.rados.0.smithi059.stdout:1706: done (7 left) 2022-04-28T19:49:48.445 INFO:tasks.rados.rados.0.smithi059.stdout:1707: done (6 left) 2022-04-28T19:49:48.445 INFO:tasks.rados.rados.0.smithi059.stdout:1708: done (5 left) 2022-04-28T19:49:48.446 INFO:tasks.rados.rados.0.smithi059.stdout:1709: done (4 left) 2022-04-28T19:49:48.446 INFO:tasks.rados.rados.0.smithi059.stdout:1710: done (3 left) 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1711: done (2 left) 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1712: done (1 left) 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1713: done (0 left) 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1714: read oid 12 snap -1 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1714: expect deleted 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1715: rollback oid 6 current snap is 156 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 153 2022-04-28T19:49:48.447 INFO:tasks.rados.rados.0.smithi059.stdout:1716: delete oid 36 current snap is 156 2022-04-28T19:49:48.458 INFO:tasks.rados.rados.0.smithi059.stdout:1715: finishing rollback tid 0 to smithi05955964-6 2022-04-28T19:49:48.459 INFO:tasks.rados.rados.0.smithi059.stdout:1715: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:49:48.459 INFO:tasks.rados.rados.0.smithi059.stdout:1715: finishing rollback tid 2 to smithi05955964-6 2022-04-28T19:49:48.459 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 705 (ObjNum 533 snap 150 seq_num 533) dirty exists 2022-04-28T19:49:48.460 INFO:tasks.rados.rados.0.smithi059.stdout:1714: done (2 left) 2022-04-28T19:49:48.460 INFO:tasks.rados.rados.0.smithi059.stdout:1715: done (1 left) 2022-04-28T19:49:48.460 INFO:tasks.rados.rados.0.smithi059.stdout:1716: done (0 left) 2022-04-28T19:49:48.460 INFO:tasks.rados.rados.0.smithi059.stdout:1717: snap_remove snap 149 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:42.391766+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:42.394927+0000 mon.b (mon.2) 2 : cluster [INF] mon.b calling monitor election 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.399105+0000 mon.c (mon.1) 3 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.401309+0000 mon.a (mon.0) 90 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.401455+0000 mon.a (mon.0) 91 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.401520+0000 mon.a (mon.0) 92 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2022-04-28T19:49:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.402257+0000 mon.a (mon.0) 93 : cluster [DBG] mgrmap e30: y(active, since 37s), standbys: x 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.402627+0000 mon.a (mon.0) 94 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.407673+0000 mon.a (mon.0) 95 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.408317+0000 mon.a (mon.0) 96 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.426403+0000 mon.a (mon.0) 97 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.439300+0000 mon.a (mon.0) 98 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.439467+0000 mon.a (mon.0) 99 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.439583+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.440217+0000 mon.a (mon.0) 101 : cluster [DBG] mgrmap e30: y(active, since 37s), standbys: x 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.440541+0000 mon.a (mon.0) 102 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.440606+0000 mon.a (mon.0) 103 : cluster [INF] Cluster is now healthy 2022-04-28T19:49:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cluster 2022-04-28T19:49:47.447461+0000 mon.a (mon.0) 104 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: audit 2022-04-28T19:49:47.453975+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: audit 2022-04-28T19:49:47.455843+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: audit 2022-04-28T19:49:47.458231+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: audit 2022-04-28T19:49:47.459627+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: cephadm 2022-04-28T19:49:47.460985+0000 mgr.y (mgr.24961) 27 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[59745]: audit 2022-04-28T19:49:47.462576+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:48.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:42.391766+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:42.394927+0000 mon.b (mon.2) 2 : cluster [INF] mon.b calling monitor election 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.399105+0000 mon.c (mon.1) 3 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.401309+0000 mon.a (mon.0) 90 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.401455+0000 mon.a (mon.0) 91 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.401520+0000 mon.a (mon.0) 92 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.402257+0000 mon.a (mon.0) 93 : cluster [DBG] mgrmap e30: y(active, since 37s), standbys: x 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.402627+0000 mon.a (mon.0) 94 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.407673+0000 mon.a (mon.0) 95 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.408317+0000 mon.a (mon.0) 96 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:48.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.426403+0000 mon.a (mon.0) 97 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.439300+0000 mon.a (mon.0) 98 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.439467+0000 mon.a (mon.0) 99 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.439583+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.440217+0000 mon.a (mon.0) 101 : cluster [DBG] mgrmap e30: y(active, since 37s), standbys: x 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.440541+0000 mon.a (mon.0) 102 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.440606+0000 mon.a (mon.0) 103 : cluster [INF] Cluster is now healthy 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cluster 2022-04-28T19:49:47.447461+0000 mon.a (mon.0) 104 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: audit 2022-04-28T19:49:47.453975+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: audit 2022-04-28T19:49:47.455843+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:48.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: audit 2022-04-28T19:49:47.458231+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:48.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: audit 2022-04-28T19:49:47.459627+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:48.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: cephadm 2022-04-28T19:49:47.460985+0000 mgr.y (mgr.24961) 27 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:48.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:48 smithi059 bash[61152]: audit 2022-04-28T19:49:47.462576+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:48.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:42.391766+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:42.394927+0000 mon.b (mon.2) 2 : cluster [INF] mon.b calling monitor election 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.399105+0000 mon.c (mon.1) 3 : cluster [INF] mon.c calling monitor election 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.401309+0000 mon.a (mon.0) 90 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.401455+0000 mon.a (mon.0) 91 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.401520+0000 mon.a (mon.0) 92 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.402257+0000 mon.a (mon.0) 93 : cluster [DBG] mgrmap e30: y(active, since 37s), standbys: x 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.402627+0000 mon.a (mon.0) 94 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.407673+0000 mon.a (mon.0) 95 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.408317+0000 mon.a (mon.0) 96 : cluster [INF] mon.a calling monitor election 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.426403+0000 mon.a (mon.0) 97 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.439300+0000 mon.a (mon.0) 98 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.59:3300/0,v1:172.21.15.59:6789/0],b=[v2:172.21.15.139:3300/0,v1:172.21.15.139:6789/0],c=[v2:172.21.15.59:3301/0,v1:172.21.15.59:6790/0]} 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.439467+0000 mon.a (mon.0) 99 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.439583+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.440217+0000 mon.a (mon.0) 101 : cluster [DBG] mgrmap e30: y(active, since 37s), standbys: x 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.440541+0000 mon.a (mon.0) 102 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.440606+0000 mon.a (mon.0) 103 : cluster [INF] Cluster is now healthy 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cluster 2022-04-28T19:49:47.447461+0000 mon.a (mon.0) 104 : cluster [INF] overall HEALTH_OK 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: audit 2022-04-28T19:49:47.453975+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: audit 2022-04-28T19:49:47.455843+0000 mon.a (mon.0) 106 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:49:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: audit 2022-04-28T19:49:47.458231+0000 mon.a (mon.0) 107 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: audit 2022-04-28T19:49:47.459627+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:49:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: cephadm 2022-04-28T19:49:47.460985+0000 mgr.y (mgr.24961) 27 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:49:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:48 smithi139 bash[43301]: audit 2022-04-28T19:49:47.462576+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:49:49.465 INFO:tasks.rados.rados.0.smithi059.stdout:1717: done (0 left) 2022-04-28T19:49:49.465 INFO:tasks.rados.rados.0.smithi059.stdout:1718: read oid 13 snap 154 2022-04-28T19:49:49.466 INFO:tasks.rados.rados.0.smithi059.stdout:1718: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:49:49.466 INFO:tasks.rados.rados.0.smithi059.stdout:1719: delete oid 8 current snap is 156 2022-04-28T19:49:49.480 INFO:tasks.rados.rados.0.smithi059.stdout:1719: done (1 left) 2022-04-28T19:49:49.480 INFO:tasks.rados.rados.0.smithi059.stdout:1720: rollback oid 19 current snap is 156 2022-04-28T19:49:49.480 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 19 to 155 2022-04-28T19:49:49.481 INFO:tasks.rados.rados.0.smithi059.stdout:1721: snap_create 2022-04-28T19:49:49.484 INFO:tasks.rados.rados.0.smithi059.stdout:1720: finishing rollback tid 1 to smithi05955964-19 2022-04-28T19:49:49.484 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 0 (ObjNum -133263744 snap 21893 seq_num 4160798464) dirty dne 2022-04-28T19:49:49.487 INFO:tasks.rados.rados.0.smithi059.stdout:1718: expect (ObjNum 398 snap 122 seq_num 398) 2022-04-28T19:49:49.837 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:49:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:49 smithi059 bash[59745]: cluster 2022-04-28T19:49:48.443251+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2022-04-28T19:49:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:49 smithi059 bash[59745]: cluster 2022-04-28T19:49:48.859088+0000 mgr.y (mgr.24961) 28 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 192 MiB data, 321 MiB used, 707 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2022-04-28T19:49:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:49 smithi059 bash[61152]: cluster 2022-04-28T19:49:48.443251+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2022-04-28T19:49:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:49 smithi059 bash[61152]: cluster 2022-04-28T19:49:48.859088+0000 mgr.y (mgr.24961) 28 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 192 MiB data, 321 MiB used, 707 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2022-04-28T19:49:49.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:49 smithi139 bash[43301]: cluster 2022-04-28T19:49:48.443251+0000 mon.a (mon.0) 110 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2022-04-28T19:49:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:49 smithi139 bash[43301]: cluster 2022-04-28T19:49:48.859088+0000 mgr.y (mgr.24961) 28 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 192 MiB data, 321 MiB used, 707 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.2 MiB/s wr, 7 op/s 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (8m) 14s ago 9m 14.4M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (8m) 27s ago 8m 25.8M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (7m) 14s ago 7m 14.4M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (7m) 27s ago 7m 17.2M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (61s) 27s ago 12m 374M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (97s) 14s ago 15m 408M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:49:50.165 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (49s) 14s ago 15m 42.1M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:49:50.166 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 starting - - - 2048M 2022-04-28T19:49:50.166 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (21s) 14s ago 13m 26.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (9m) 14s ago 9m 10.8M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (9m) 27s ago 9m 10.6M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (12m) 14s ago 12m 219M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 547b5580e073 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (12m) 14s ago 12m 159M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (11m) 14s ago 11m 230M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (11m) 14s ago 11m 176M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (11m) 27s ago 11m 194M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (10m) 27s ago 10m 200M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (10m) 27s ago 10m 154M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:49:50.167 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (9m) 27s ago 9m 133M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:49:50.168 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (113s) 27s ago 9m 33.8M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:49:50.468 INFO:tasks.rados.rados.0.smithi059.stdout:1718: done (2 left) 2022-04-28T19:49:50.468 INFO:tasks.rados.rados.0.smithi059.stdout:1720: done (1 left) 2022-04-28T19:49:50.468 INFO:tasks.rados.rados.0.smithi059.stdout:1721: done (0 left) 2022-04-28T19:49:50.468 INFO:tasks.rados.rados.0.smithi059.stdout:1722: rmattr oid 18 current snap is 157 2022-04-28T19:49:50.468 INFO:tasks.rados.rados.0.smithi059.stdout:1722: done (0 left) 2022-04-28T19:49:50.468 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 157 2022-04-28T19:49:50.469 INFO:tasks.rados.rados.0.smithi059.stdout:1723: seq_num 554 ranges {5054464=483328,5537792=704512,6242304=622592,6864896=278528} 2022-04-28T19:49:50.503 INFO:tasks.rados.rados.0.smithi059.stdout:1723: writing smithi05955964-11 from 5054464 to 5537792 tid 1 2022-04-28T19:49:50.507 INFO:tasks.rados.rados.0.smithi059.stdout:1723: writing smithi05955964-11 from 5537792 to 6242304 tid 2 2022-04-28T19:49:50.511 INFO:tasks.rados.rados.0.smithi059.stdout:1723: writing smithi05955964-11 from 6242304 to 6864896 tid 3 2022-04-28T19:49:50.513 INFO:tasks.rados.rados.0.smithi059.stdout:1723: writing smithi05955964-11 from 6864896 to 7143424 tid 4 2022-04-28T19:49:50.513 INFO:tasks.rados.rados.0.smithi059.stdout:1724: snap_create 2022-04-28T19:49:50.518 INFO:tasks.rados.rados.0.smithi059.stdout:1723: finishing write tid 1 to smithi05955964-11 2022-04-28T19:49:50.518 INFO:tasks.rados.rados.0.smithi059.stdout:1723: finishing write tid 2 to smithi05955964-11 2022-04-28T19:49:50.521 INFO:tasks.rados.rados.0.smithi059.stdout:1723: finishing write tid 3 to smithi05955964-11 2022-04-28T19:49:50.523 INFO:tasks.rados.rados.0.smithi059.stdout:1723: finishing write tid 4 to smithi05955964-11 2022-04-28T19:49:50.523 INFO:tasks.rados.rados.0.smithi059.stdout:1723: finishing write tid 6 to smithi05955964-11 2022-04-28T19:49:50.525 INFO:tasks.rados.rados.0.smithi059.stdout:1723: finishing write tid 7 to smithi05955964-11 2022-04-28T19:49:50.525 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 749 (ObjNum 554 snap 157 seq_num 554) dirty exists 2022-04-28T19:49:50.526 INFO:tasks.rados.rados.0.smithi059.stdout:1723: left oid 11 (ObjNum 554 snap 157 seq_num 554) 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:49:50.548 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 10, 2022-04-28T19:49:50.549 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 5 2022-04-28T19:49:50.550 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:49:50.550 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:49:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: cluster 2022-04-28T19:49:49.463940+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2022-04-28T19:49:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:49.822614+0000 mgr.y (mgr.24961) 29 : audit [DBG] from='client.44154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.141639+0000 mgr.y (mgr.24961) 30 : audit [DBG] from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.144583+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.145358+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.146014+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.146688+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.147345+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.147963+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.150324+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.150934+0000 mon.a (mon.0) 119 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.151618+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.152231+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[59745]: audit 2022-04-28T19:49:50.152934+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: cluster 2022-04-28T19:49:49.463940+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2022-04-28T19:49:50.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:49.822614+0000 mgr.y (mgr.24961) 29 : audit [DBG] from='client.44154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:50.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.141639+0000 mgr.y (mgr.24961) 30 : audit [DBG] from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:50.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.144583+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.145358+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.146014+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.146688+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.147345+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.147963+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.150324+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.150934+0000 mon.a (mon.0) 119 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.151618+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.152231+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:50 smithi059 bash[61152]: audit 2022-04-28T19:49:50.152934+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: cluster 2022-04-28T19:49:49.463940+0000 mon.a (mon.0) 111 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:49.822614+0000 mgr.y (mgr.24961) 29 : audit [DBG] from='client.44154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.141639+0000 mgr.y (mgr.24961) 30 : audit [DBG] from='client.54109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.144583+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.145358+0000 mon.a (mon.0) 113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.146014+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.146688+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.147345+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.147963+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.150324+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.150934+0000 mon.a (mon.0) 119 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.151618+0000 mon.a (mon.0) 120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.152231+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:50 smithi139 bash[43301]: audit 2022-04-28T19:49:50.152934+0000 mon.a (mon.0) 122 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1723: done (1 left) 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1724: done (0 left) 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1725: copy_from oid 34 from oid 41 current snap is 158 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1726: rollback oid 2 current snap is 158 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 2 to 146 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1727: rollback oid 26 current snap is 158 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 155 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1728: read oid 9 snap -1 2022-04-28T19:49:51.485 INFO:tasks.rados.rados.0.smithi059.stdout:1728: expect deleted 2022-04-28T19:49:51.486 INFO:tasks.rados.rados.0.smithi059.stdout:1729: read oid 16 snap -1 2022-04-28T19:49:51.486 INFO:tasks.rados.rados.0.smithi059.stdout:1729: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:49:51.486 INFO:tasks.rados.rados.0.smithi059.stdout:1730: delete oid 4 current snap is 158 2022-04-28T19:49:51.487 INFO:tasks.rados.rados.0.smithi059.stdout:1729: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:49:51.495 INFO:tasks.rados.rados.0.smithi059.stdout:1727: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:49:51.495 INFO:tasks.rados.rados.0.smithi059.stdout:1727: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:49:51.495 INFO:tasks.rados.rados.0.smithi059.stdout:1727: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:49:51.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 793 (ObjNum 442 snap 130 seq_num 442) dirty exists 2022-04-28T19:49:51.496 INFO:tasks.rados.rados.0.smithi059.stdout:1726: finishing rollback tid 0 to smithi05955964-2 2022-04-28T19:49:51.496 INFO:tasks.rados.rados.0.smithi059.stdout:1727: done (5 left) 2022-04-28T19:49:51.497 INFO:tasks.rados.rados.0.smithi059.stdout:1728: done (4 left) 2022-04-28T19:49:51.497 INFO:tasks.rados.rados.0.smithi059.stdout:1729: done (3 left) 2022-04-28T19:49:51.497 INFO:tasks.rados.rados.0.smithi059.stdout:1730: done (2 left) 2022-04-28T19:49:51.497 INFO:tasks.rados.rados.0.smithi059.stdout:1731: snap_create 2022-04-28T19:49:51.497 INFO:tasks.rados.rados.0.smithi059.stdout:1726: finishing rollback tid 1 to smithi05955964-2 2022-04-28T19:49:51.497 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 724 (ObjNum -133182864 snap 21893 seq_num 4161854576) dirty dne 2022-04-28T19:49:51.515 INFO:tasks.rados.rados.0.smithi059.stdout:1725: finishing copy_from to smithi05955964-34 2022-04-28T19:49:51.515 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 616 (ObjNum 542 snap 151 seq_num 542) dirty exists 2022-04-28T19:49:51.520 INFO:tasks.rados.rados.0.smithi059.stdout:1725: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:49:51.809 INFO:tasks.rados.rados.0.smithi059.stdout:1725: done (2 left) 2022-04-28T19:49:51.809 INFO:tasks.rados.rados.0.smithi059.stdout:1726: done (1 left) 2022-04-28T19:49:51.809 INFO:tasks.rados.rados.0.smithi059.stdout:1731: done (0 left) 2022-04-28T19:49:51.810 INFO:tasks.rados.rados.0.smithi059.stdout:1732: copy_from oid 2 from oid 47 current snap is 159 2022-04-28T19:49:51.810 INFO:tasks.rados.rados.0.smithi059.stdout:1733: snap_remove snap 157 2022-04-28T19:49:51.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[61152]: cluster 2022-04-28T19:49:50.467060+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2022-04-28T19:49:51.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[61152]: audit 2022-04-28T19:49:50.547077+0000 mon.a (mon.0) 124 : audit [DBG] from='client.? 172.21.15.59:0/3382161460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:51.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[61152]: cluster 2022-04-28T19:49:50.859873+0000 mgr.y (mgr.24961) 31 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 192 MiB data, 321 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:51.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:51.626Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:51.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:51.626Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:51.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:51.626Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:49:51.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[59745]: cluster 2022-04-28T19:49:50.467060+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2022-04-28T19:49:51.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[59745]: audit 2022-04-28T19:49:50.547077+0000 mon.a (mon.0) 124 : audit [DBG] from='client.? 172.21.15.59:0/3382161460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:51.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:51 smithi059 bash[59745]: cluster 2022-04-28T19:49:50.859873+0000 mgr.y (mgr.24961) 31 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 192 MiB data, 321 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:51.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:51 smithi139 bash[43301]: cluster 2022-04-28T19:49:50.467060+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2022-04-28T19:49:51.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:51 smithi139 bash[43301]: audit 2022-04-28T19:49:50.547077+0000 mon.a (mon.0) 124 : audit [DBG] from='client.? 172.21.15.59:0/3382161460' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:49:51.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:51 smithi139 bash[43301]: cluster 2022-04-28T19:49:50.859873+0000 mgr.y (mgr.24961) 31 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 192 MiB data, 321 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1733: done (1 left) 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1734: rollback oid 25 current snap is 159 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 25 to 158 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1735: rmattr oid 5 current snap is 159 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1735: done (2 left) 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1736: read oid 48 snap 153 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1736: expect (ObjNum 526 snap 149 seq_num 526) 2022-04-28T19:49:52.821 INFO:tasks.rados.rados.0.smithi059.stdout:1737: delete oid 43 current snap is 159 2022-04-28T19:49:52.822 INFO:tasks.rados.rados.0.smithi059.stdout:1732: finishing copy_from to smithi05955964-2 2022-04-28T19:49:52.822 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 752 (ObjNum 523 snap 146 seq_num 523) dirty exists 2022-04-28T19:49:52.822 INFO:tasks.rados.rados.0.smithi059.stdout:1732: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:49:52.832 INFO:tasks.rados.rados.0.smithi059.stdout:1734: finishing rollback tid 0 to smithi05955964-25 2022-04-28T19:49:52.832 INFO:tasks.rados.rados.0.smithi059.stdout:1734: finishing rollback tid 1 to smithi05955964-25 2022-04-28T19:49:52.832 INFO:tasks.rados.rados.0.smithi059.stdout:1732: done (3 left) 2022-04-28T19:49:52.832 INFO:tasks.rados.rados.0.smithi059.stdout:1737: done (2 left) 2022-04-28T19:49:52.833 INFO:tasks.rados.rados.0.smithi059.stdout:1738: read oid 41 snap -1 2022-04-28T19:49:52.833 INFO:tasks.rados.rados.0.smithi059.stdout:1738: expect (ObjNum 542 snap 151 seq_num 542) 2022-04-28T19:49:52.833 INFO:tasks.rados.rados.0.smithi059.stdout:1739: snap_create 2022-04-28T19:49:52.833 INFO:tasks.rados.rados.0.smithi059.stdout:1734: finishing rollback tid 2 to smithi05955964-25 2022-04-28T19:49:52.833 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 760 (ObjNum 498 snap 143 seq_num 498) dirty exists 2022-04-28T19:49:52.843 INFO:tasks.rados.rados.0.smithi059.stdout:1736: expect (ObjNum 526 snap 149 seq_num 526) 2022-04-28T19:49:52.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: cluster 2022-04-28T19:49:51.483317+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: cluster 2022-04-28T19:49:51.806187+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: audit 2022-04-28T19:49:52.408476+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: audit 2022-04-28T19:49:52.417845+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: audit 2022-04-28T19:49:52.426388+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: audit 2022-04-28T19:49:52.427352+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: audit 2022-04-28T19:49:52.428393+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:52.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[61152]: audit 2022-04-28T19:49:52.429820+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "container_image"}]: dispatch 2022-04-28T19:49:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: cluster 2022-04-28T19:49:51.483317+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2022-04-28T19:49:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: cluster 2022-04-28T19:49:51.806187+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2022-04-28T19:49:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: audit 2022-04-28T19:49:52.408476+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:52.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: audit 2022-04-28T19:49:52.417845+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: audit 2022-04-28T19:49:52.426388+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: audit 2022-04-28T19:49:52.427352+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: audit 2022-04-28T19:49:52.428393+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:52.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:52 smithi059 bash[59745]: audit 2022-04-28T19:49:52.429820+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "container_image"}]: dispatch 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: cluster 2022-04-28T19:49:51.483317+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: cluster 2022-04-28T19:49:51.806187+0000 mon.a (mon.0) 126 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: audit 2022-04-28T19:49:52.408476+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: audit 2022-04-28T19:49:52.417845+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: audit 2022-04-28T19:49:52.426388+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: audit 2022-04-28T19:49:52.427352+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: audit 2022-04-28T19:49:52.428393+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:52 smithi139 bash[43301]: audit 2022-04-28T19:49:52.429820+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "container_image"}]: dispatch 2022-04-28T19:49:52.964 INFO:tasks.rados.rados.0.smithi059.stdout:1738: expect (ObjNum 542 snap 151 seq_num 542) 2022-04-28T19:49:53.824 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:53.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:53.825 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:53.519Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:53.825 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:53.520Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:53.825 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:53.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:49:53.825 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:53.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:49:53.825 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:49:53.524Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:49:53.827 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[59745]: cephadm 2022-04-28T19:49:52.425870+0000 mgr.y (mgr.24961) 32 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:49:53.827 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[59745]: cephadm 2022-04-28T19:49:52.429280+0000 mgr.y (mgr.24961) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:49:53.827 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[59745]: cluster 2022-04-28T19:49:52.818690+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2022-04-28T19:49:53.828 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[59745]: cluster 2022-04-28T19:49:52.860429+0000 mgr.y (mgr.24961) 34 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.9 MiB/s wr, 7 op/s 2022-04-28T19:49:53.830 INFO:tasks.rados.rados.0.smithi059.stdout:1734: done (3 left) 2022-04-28T19:49:53.830 INFO:tasks.rados.rados.0.smithi059.stdout:1736: done (2 left) 2022-04-28T19:49:53.830 INFO:tasks.rados.rados.0.smithi059.stdout:1738: done (1 left) 2022-04-28T19:49:53.830 INFO:tasks.rados.rados.0.smithi059.stdout:1739: done (0 left) 2022-04-28T19:49:53.830 INFO:tasks.rados.rados.0.smithi059.stdout:1740: snap_remove snap 155 2022-04-28T19:49:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[61152]: cephadm 2022-04-28T19:49:52.425870+0000 mgr.y (mgr.24961) 32 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:49:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[61152]: cephadm 2022-04-28T19:49:52.429280+0000 mgr.y (mgr.24961) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:49:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[61152]: cluster 2022-04-28T19:49:52.818690+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2022-04-28T19:49:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:53 smithi059 bash[61152]: cluster 2022-04-28T19:49:52.860429+0000 mgr.y (mgr.24961) 34 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.9 MiB/s wr, 7 op/s 2022-04-28T19:49:54.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:53 smithi139 bash[43301]: cephadm 2022-04-28T19:49:52.425870+0000 mgr.y (mgr.24961) 32 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2022-04-28T19:49:54.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:53 smithi139 bash[43301]: cephadm 2022-04-28T19:49:52.429280+0000 mgr.y (mgr.24961) 33 : cephadm [INF] Reconfiguring daemon mon.a on smithi059 2022-04-28T19:49:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:53 smithi139 bash[43301]: cluster 2022-04-28T19:49:52.818690+0000 mon.a (mon.0) 133 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2022-04-28T19:49:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:53 smithi139 bash[43301]: cluster 2022-04-28T19:49:52.860429+0000 mgr.y (mgr.24961) 34 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.9 MiB/s wr, 7 op/s 2022-04-28T19:49:54.830 INFO:tasks.rados.rados.0.smithi059.stdout:1740: done (0 left) 2022-04-28T19:49:54.831 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 160 2022-04-28T19:49:54.831 INFO:tasks.rados.rados.0.smithi059.stdout:1741: seq_num 557 ranges {0=532480,532480=802816,1335296=794624,2129920=8192} 2022-04-28T19:49:54.835 INFO:tasks.rados.rados.0.smithi059.stdout:1741: writing smithi05955964-8 from 0 to 532480 tid 1 2022-04-28T19:49:54.842 INFO:tasks.rados.rados.0.smithi059.stdout:1741: writing smithi05955964-8 from 532480 to 1335296 tid 2 2022-04-28T19:49:54.847 INFO:tasks.rados.rados.0.smithi059.stdout:1741: writing smithi05955964-8 from 1335296 to 2129920 tid 3 2022-04-28T19:49:54.847 INFO:tasks.rados.rados.0.smithi059.stdout:1741: writing smithi05955964-8 from 2129920 to 2138112 tid 4 2022-04-28T19:49:54.848 INFO:tasks.rados.rados.0.smithi059.stdout:1742: delete oid 12 current snap is 160 2022-04-28T19:49:54.851 INFO:tasks.rados.rados.0.smithi059.stdout:1741: finishing write tid 1 to smithi05955964-8 2022-04-28T19:49:54.856 INFO:tasks.rados.rados.0.smithi059.stdout:1742: done (1 left) 2022-04-28T19:49:54.856 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 160 2022-04-28T19:49:54.856 INFO:tasks.rados.rados.0.smithi059.stdout:1743: seq_num 559 ranges {6823936=565248,7389184=573440,7962624=409600,8372224=622592} 2022-04-28T19:49:54.898 INFO:tasks.rados.rados.0.smithi059.stdout:1743: writing smithi05955964-35 from 6823936 to 7389184 tid 1 2022-04-28T19:49:54.902 INFO:tasks.rados.rados.0.smithi059.stdout:1743: writing smithi05955964-35 from 7389184 to 7962624 tid 2 2022-04-28T19:49:54.904 INFO:tasks.rados.rados.0.smithi059.stdout:1743: writing smithi05955964-35 from 7962624 to 8372224 tid 3 2022-04-28T19:49:54.908 INFO:tasks.rados.rados.0.smithi059.stdout:1743: writing smithi05955964-35 from 8372224 to 8994816 tid 4 2022-04-28T19:49:54.908 INFO:tasks.rados.rados.0.smithi059.stdout:1744: delete oid 4 current snap is 160 2022-04-28T19:49:54.908 INFO:tasks.rados.rados.0.smithi059.stdout:1741: finishing write tid 2 to smithi05955964-8 2022-04-28T19:49:54.908 INFO:tasks.rados.rados.0.smithi059.stdout:1741: finishing write tid 3 to smithi05955964-8 2022-04-28T19:49:54.909 INFO:tasks.rados.rados.0.smithi059.stdout:1741: finishing write tid 4 to smithi05955964-8 2022-04-28T19:49:54.909 INFO:tasks.rados.rados.0.smithi059.stdout:1741: finishing write tid 6 to smithi05955964-8 2022-04-28T19:49:54.909 INFO:tasks.rados.rados.0.smithi059.stdout:1741: finishing write tid 7 to smithi05955964-8 2022-04-28T19:49:54.909 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 769 (ObjNum 557 snap 160 seq_num 557) dirty exists 2022-04-28T19:49:54.910 INFO:tasks.rados.rados.0.smithi059.stdout:1741: left oid 8 (ObjNum 557 snap 160 seq_num 557) 2022-04-28T19:49:54.910 INFO:tasks.rados.rados.0.smithi059.stdout:1743: finishing write tid 1 to smithi05955964-35 2022-04-28T19:49:54.910 INFO:tasks.rados.rados.0.smithi059.stdout:1743: finishing write tid 2 to smithi05955964-35 2022-04-28T19:49:54.910 INFO:tasks.rados.rados.0.smithi059.stdout:1743: finishing write tid 3 to smithi05955964-35 2022-04-28T19:49:54.911 INFO:tasks.rados.rados.0.smithi059.stdout:1741: done (2 left) 2022-04-28T19:49:54.911 INFO:tasks.rados.rados.0.smithi059.stdout:1744: done (1 left) 2022-04-28T19:49:54.912 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 160 2022-04-28T19:49:54.912 INFO:tasks.rados.rados.0.smithi059.stdout:1745: seq_num 561 ranges {8019968=598016,8617984=745472,9363456=778240,10141696=81920} 2022-04-28T19:49:54.959 INFO:tasks.rados.rados.0.smithi059.stdout:1745: writing smithi05955964-47 from 8019968 to 8617984 tid 1 2022-04-28T19:49:54.963 INFO:tasks.rados.rados.0.smithi059.stdout:1745: writing smithi05955964-47 from 8617984 to 9363456 tid 2 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1745: writing smithi05955964-47 from 9363456 to 10141696 tid 3 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1745: writing smithi05955964-47 from 10141696 to 10223616 tid 4 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1746: rmattr oid 23 current snap is 160 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1743: finishing write tid 4 to smithi05955964-35 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1743: finishing write tid 6 to smithi05955964-35 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1743: finishing write tid 7 to smithi05955964-35 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 724 (ObjNum 559 snap 160 seq_num 559) dirty exists 2022-04-28T19:49:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:1743: left oid 35 (ObjNum 559 snap 160 seq_num 559) 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:1745: finishing write tid 1 to smithi05955964-47 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:1745: finishing write tid 2 to smithi05955964-47 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:1743: done (2 left) 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:1747: copy_from oid 31 from oid 2 current snap is 160 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:1748: copy_from oid 7 from oid 46 current snap is 160 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 160 2022-04-28T19:49:54.970 INFO:tasks.rados.rados.0.smithi059.stdout:1749: seq_num 562 ranges {0=622592,622592=630784,1253376=581632,1835008=393216} 2022-04-28T19:49:54.972 INFO:tasks.rados.rados.0.smithi059.stdout:1749: writing smithi05955964-43 from 0 to 622592 tid 1 2022-04-28T19:49:54.976 INFO:tasks.rados.rados.0.smithi059.stdout:1749: writing smithi05955964-43 from 622592 to 1253376 tid 2 2022-04-28T19:49:54.980 INFO:tasks.rados.rados.0.smithi059.stdout:1749: writing smithi05955964-43 from 1253376 to 1835008 tid 3 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:1749: writing smithi05955964-43 from 1835008 to 2228224 tid 4 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:1750: delete oid 48 current snap is 160 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:1745: finishing write tid 3 to smithi05955964-47 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:1745: finishing write tid 4 to smithi05955964-47 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:1745: finishing write tid 6 to smithi05955964-47 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:1745: finishing write tid 7 to smithi05955964-47 2022-04-28T19:49:54.983 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 784 (ObjNum 561 snap 160 seq_num 561) dirty exists 2022-04-28T19:49:54.984 INFO:tasks.rados.rados.0.smithi059.stdout:1745: left oid 47 (ObjNum 561 snap 160 seq_num 561) 2022-04-28T19:49:54.984 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 786 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:49:54.984 INFO:tasks.rados.rados.0.smithi059.stdout:1749: finishing write tid 1 to smithi05955964-43 2022-04-28T19:49:54.985 INFO:tasks.rados.rados.0.smithi059.stdout:1745: done (5 left) 2022-04-28T19:49:54.986 INFO:tasks.rados.rados.0.smithi059.stdout:1746: done (4 left) 2022-04-28T19:49:54.986 INFO:tasks.rados.rados.0.smithi059.stdout:1750: done (3 left) 2022-04-28T19:49:54.986 INFO:tasks.rados.rados.0.smithi059.stdout:1751: read oid 36 snap -1 2022-04-28T19:49:54.986 INFO:tasks.rados.rados.0.smithi059.stdout:1751: expect deleted 2022-04-28T19:49:54.986 INFO:tasks.rados.rados.0.smithi059.stdout:1752: delete oid 48 current snap is 160 2022-04-28T19:49:54.987 INFO:tasks.rados.rados.0.smithi059.stdout:1749: finishing write tid 2 to smithi05955964-43 2022-04-28T19:49:54.987 INFO:tasks.rados.rados.0.smithi059.stdout:1749: finishing write tid 3 to smithi05955964-43 2022-04-28T19:49:54.988 INFO:tasks.rados.rados.0.smithi059.stdout:1751: done (4 left) 2022-04-28T19:49:54.988 INFO:tasks.rados.rados.0.smithi059.stdout:1752: done (3 left) 2022-04-28T19:49:54.988 INFO:tasks.rados.rados.0.smithi059.stdout:1753: snap_create 2022-04-28T19:49:54.991 INFO:tasks.rados.rados.0.smithi059.stdout:1749: finishing write tid 4 to smithi05955964-43 2022-04-28T19:49:54.991 INFO:tasks.rados.rados.0.smithi059.stdout:1749: finishing write tid 6 to smithi05955964-43 2022-04-28T19:49:54.992 INFO:tasks.rados.rados.0.smithi059.stdout:1749: finishing write tid 7 to smithi05955964-43 2022-04-28T19:49:54.992 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 806 (ObjNum 562 snap 160 seq_num 562) dirty exists 2022-04-28T19:49:54.992 INFO:tasks.rados.rados.0.smithi059.stdout:1749: left oid 43 (ObjNum 562 snap 160 seq_num 562) 2022-04-28T19:49:55.029 INFO:tasks.rados.rados.0.smithi059.stdout:1747: finishing copy_from to smithi05955964-31 2022-04-28T19:49:55.029 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 752 (ObjNum 523 snap 146 seq_num 523) dirty exists 2022-04-28T19:49:55.029 INFO:tasks.rados.rados.0.smithi059.stdout:1747: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:49:55.029 INFO:tasks.rados.rados.0.smithi059.stdout:1748: finishing copy_from to smithi05955964-7 2022-04-28T19:49:55.029 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 735 (ObjNum 545 snap 154 seq_num 545) dirty exists 2022-04-28T19:49:55.030 INFO:tasks.rados.rados.0.smithi059.stdout:1748: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:49:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[59745]: cluster 2022-04-28T19:49:53.829150+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2022-04-28T19:49:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[59745]: audit 2022-04-28T19:49:54.537070+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[59745]: audit 2022-04-28T19:49:54.538537+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[59745]: audit 2022-04-28T19:49:54.540166+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[59745]: audit 2022-04-28T19:49:54.541918+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[59745]: audit 2022-04-28T19:49:54.544338+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.y","key": "container_image"}]: dispatch 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[61152]: cluster 2022-04-28T19:49:53.829150+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[61152]: audit 2022-04-28T19:49:54.537070+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[61152]: audit 2022-04-28T19:49:54.538537+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[61152]: audit 2022-04-28T19:49:54.540166+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:49:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[61152]: audit 2022-04-28T19:49:54.541918+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:55.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:54 smithi059 bash[61152]: audit 2022-04-28T19:49:54.544338+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.y","key": "container_image"}]: dispatch 2022-04-28T19:49:55.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:54 smithi139 bash[43301]: cluster 2022-04-28T19:49:53.829150+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2022-04-28T19:49:55.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:54 smithi139 bash[43301]: audit 2022-04-28T19:49:54.537070+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:55.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:54 smithi139 bash[43301]: audit 2022-04-28T19:49:54.538537+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:49:55.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:54 smithi139 bash[43301]: audit 2022-04-28T19:49:54.540166+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:49:55.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:54 smithi139 bash[43301]: audit 2022-04-28T19:49:54.541918+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:54 smithi139 bash[43301]: audit 2022-04-28T19:49:54.544338+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.y","key": "container_image"}]: dispatch 2022-04-28T19:49:55.848 INFO:tasks.rados.rados.0.smithi059.stdout:1747: done (3 left) 2022-04-28T19:49:55.849 INFO:tasks.rados.rados.0.smithi059.stdout:1748: done (2 left) 2022-04-28T19:49:55.849 INFO:tasks.rados.rados.0.smithi059.stdout:1749: done (1 left) 2022-04-28T19:49:55.849 INFO:tasks.rados.rados.0.smithi059.stdout:1753: done (0 left) 2022-04-28T19:49:55.849 INFO:tasks.rados.rados.0.smithi059.stdout:1754: delete oid 9 current snap is 161 2022-04-28T19:49:55.851 INFO:tasks.rados.rados.0.smithi059.stdout:1754: done (0 left) 2022-04-28T19:49:55.851 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 161 2022-04-28T19:49:55.851 INFO:tasks.rados.rados.0.smithi059.stdout:1755: seq_num 566 ranges {4718592=688128,5406720=614400,6021120=565248,6586368=425984} 2022-04-28T19:49:55.881 INFO:tasks.rados.rados.0.smithi059.stdout:1755: writing smithi05955964-39 from 4718592 to 5406720 tid 1 2022-04-28T19:49:55.885 INFO:tasks.rados.rados.0.smithi059.stdout:1755: writing smithi05955964-39 from 5406720 to 6021120 tid 2 2022-04-28T19:49:55.888 INFO:tasks.rados.rados.0.smithi059.stdout:1755: writing smithi05955964-39 from 6021120 to 6586368 tid 3 2022-04-28T19:49:55.891 INFO:tasks.rados.rados.0.smithi059.stdout:1755: writing smithi05955964-39 from 6586368 to 7012352 tid 4 2022-04-28T19:49:55.891 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 161 2022-04-28T19:49:55.891 INFO:tasks.rados.rados.0.smithi059.stdout:1756: seq_num 567 ranges {10223616=704512,10928128=499712,11427840=770048,12197888=335872} 2022-04-28T19:49:55.953 INFO:tasks.rados.rados.0.smithi059.stdout:1756: writing smithi05955964-47 from 10223616 to 10928128 tid 1 2022-04-28T19:49:55.955 INFO:tasks.rados.rados.0.smithi059.stdout:1756: writing smithi05955964-47 from 10928128 to 11427840 tid 2 2022-04-28T19:49:55.961 INFO:tasks.rados.rados.0.smithi059.stdout:1756: writing smithi05955964-47 from 11427840 to 12197888 tid 3 2022-04-28T19:49:55.963 INFO:tasks.rados.rados.0.smithi059.stdout:1756: writing smithi05955964-47 from 12197888 to 12533760 tid 4 2022-04-28T19:49:55.963 INFO:tasks.rados.rados.0.smithi059.stdout:1757: read oid 32 snap -1 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1757: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1758: snap_create 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: finishing write tid 1 to smithi05955964-39 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: finishing write tid 2 to smithi05955964-39 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: finishing write tid 3 to smithi05955964-39 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: finishing write tid 4 to smithi05955964-39 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: finishing write tid 6 to smithi05955964-39 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: finishing write tid 7 to smithi05955964-39 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 812 (ObjNum 566 snap 161 seq_num 566) dirty exists 2022-04-28T19:49:55.964 INFO:tasks.rados.rados.0.smithi059.stdout:1755: left oid 39 (ObjNum 566 snap 161 seq_num 566) 2022-04-28T19:49:55.965 INFO:tasks.rados.rados.0.smithi059.stdout:1756: finishing write tid 1 to smithi05955964-47 2022-04-28T19:49:55.965 INFO:tasks.rados.rados.0.smithi059.stdout:1756: finishing write tid 2 to smithi05955964-47 2022-04-28T19:49:55.968 INFO:tasks.rados.rados.0.smithi059.stdout:1756: finishing write tid 3 to smithi05955964-47 2022-04-28T19:49:55.968 INFO:tasks.rados.rados.0.smithi059.stdout:1756: finishing write tid 4 to smithi05955964-47 2022-04-28T19:49:55.968 INFO:tasks.rados.rados.0.smithi059.stdout:1756: finishing write tid 6 to smithi05955964-47 2022-04-28T19:49:55.970 INFO:tasks.rados.rados.0.smithi059.stdout:1756: finishing write tid 7 to smithi05955964-47 2022-04-28T19:49:55.970 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 792 (ObjNum 567 snap 161 seq_num 567) dirty exists 2022-04-28T19:49:55.970 INFO:tasks.rados.rados.0.smithi059.stdout:1756: left oid 47 (ObjNum 567 snap 161 seq_num 567) 2022-04-28T19:49:55.977 INFO:tasks.rados.rados.0.smithi059.stdout:1757: expect (ObjNum 469 snap 139 seq_num 469) 2022-04-28T19:49:56.035 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[59745]: cephadm 2022-04-28T19:49:54.537997+0000 mgr.y (mgr.24961) 35 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2022-04-28T19:49:56.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[59745]: cephadm 2022-04-28T19:49:54.543355+0000 mgr.y (mgr.24961) 36 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:49:56.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[59745]: cluster 2022-04-28T19:49:54.827731+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2022-04-28T19:49:56.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[59745]: cluster 2022-04-28T19:49:54.860949+0000 mgr.y (mgr.24961) 37 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.9 MiB/s wr, 7 op/s 2022-04-28T19:49:56.038 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[61152]: cephadm 2022-04-28T19:49:54.537997+0000 mgr.y (mgr.24961) 35 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2022-04-28T19:49:56.038 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[61152]: cephadm 2022-04-28T19:49:54.543355+0000 mgr.y (mgr.24961) 36 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:49:56.038 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[61152]: cluster 2022-04-28T19:49:54.827731+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2022-04-28T19:49:56.039 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:55 smithi059 bash[61152]: cluster 2022-04-28T19:49:54.860949+0000 mgr.y (mgr.24961) 37 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.9 MiB/s wr, 7 op/s 2022-04-28T19:49:56.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:55 smithi139 bash[43301]: cephadm 2022-04-28T19:49:54.537997+0000 mgr.y (mgr.24961) 35 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2022-04-28T19:49:56.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:55 smithi139 bash[43301]: cephadm 2022-04-28T19:49:54.543355+0000 mgr.y (mgr.24961) 36 : cephadm [INF] Reconfiguring daemon mgr.y on smithi059 2022-04-28T19:49:56.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:55 smithi139 bash[43301]: cluster 2022-04-28T19:49:54.827731+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2022-04-28T19:49:56.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:55 smithi139 bash[43301]: cluster 2022-04-28T19:49:54.860949+0000 mgr.y (mgr.24961) 37 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.9 MiB/s wr, 7 op/s 2022-04-28T19:49:56.809 INFO:tasks.rados.rados.0.smithi059.stdout:1755: done (3 left) 2022-04-28T19:49:56.809 INFO:tasks.rados.rados.0.smithi059.stdout:1756: done (2 left) 2022-04-28T19:49:56.810 INFO:tasks.rados.rados.0.smithi059.stdout:1757: done (1 left) 2022-04-28T19:49:56.810 INFO:tasks.rados.rados.0.smithi059.stdout:1758: done (0 left) 2022-04-28T19:49:56.810 INFO:tasks.rados.rados.0.smithi059.stdout:1759: copy_from oid 23 from oid 48 current snap is 162 2022-04-28T19:49:56.810 INFO:tasks.rados.rados.0.smithi059.stdout:1760: delete oid 40 current snap is 162 2022-04-28T19:49:56.817 INFO:tasks.rados.rados.0.smithi059.stdout:1760: done (1 left) 2022-04-28T19:49:56.817 INFO:tasks.rados.rados.0.smithi059.stdout:1761: snap_remove snap 158 2022-04-28T19:49:57.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: cluster 2022-04-28T19:49:55.846274+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2022-04-28T19:49:57.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: audit 2022-04-28T19:49:56.630181+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:57.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: audit 2022-04-28T19:49:56.632139+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:57.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: audit 2022-04-28T19:49:56.633553+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: audit 2022-04-28T19:49:56.634785+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: audit 2022-04-28T19:49:56.636688+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "container_image"}]: dispatch 2022-04-28T19:49:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:56 smithi139 bash[43301]: cluster 2022-04-28T19:49:56.806937+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2022-04-28T19:49:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: cluster 2022-04-28T19:49:55.846274+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2022-04-28T19:49:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: audit 2022-04-28T19:49:56.630181+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: audit 2022-04-28T19:49:56.632139+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: audit 2022-04-28T19:49:56.633553+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: audit 2022-04-28T19:49:56.634785+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: audit 2022-04-28T19:49:56.636688+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "container_image"}]: dispatch 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[61152]: cluster 2022-04-28T19:49:56.806937+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: cluster 2022-04-28T19:49:55.846274+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: audit 2022-04-28T19:49:56.630181+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:49:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: audit 2022-04-28T19:49:56.632139+0000 mon.a (mon.0) 143 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:49:57.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: audit 2022-04-28T19:49:56.633553+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:49:57.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: audit 2022-04-28T19:49:56.634785+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:49:57.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: audit 2022-04-28T19:49:56.636688+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "container_image"}]: dispatch 2022-04-28T19:49:57.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:56 smithi059 bash[59745]: cluster 2022-04-28T19:49:56.806937+0000 mon.a (mon.0) 147 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2022-04-28T19:49:57.823 INFO:tasks.rados.rados.0.smithi059.stdout:1761: done (1 left) 2022-04-28T19:49:57.823 INFO:tasks.rados.rados.0.smithi059.stdout:1762: setattr oid 37 current snap is 162 2022-04-28T19:49:57.824 INFO:tasks.rados.rados.0.smithi059.stdout:1759: finishing copy_from to smithi05955964-23 2022-04-28T19:49:57.824 INFO:tasks.rados.rados.0.smithi059.stdout:1759: got expected ENOENT (src dne) 2022-04-28T19:49:57.824 INFO:tasks.rados.rados.0.smithi059.stdout:1759: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:49:57.826 INFO:tasks.rados.rados.0.smithi059.stdout:1759: done (1 left) 2022-04-28T19:49:57.826 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 162 2022-04-28T19:49:57.826 INFO:tasks.rados.rados.0.smithi059.stdout:1763: seq_num 569 ranges {9142272=737280,9879552=671744,10551296=737280,11288576=196608} 2022-04-28T19:49:57.884 INFO:tasks.rados.rados.0.smithi059.stdout:1763: writing smithi05955964-46 from 9142272 to 9879552 tid 1 2022-04-28T19:49:57.887 INFO:tasks.rados.rados.0.smithi059.stdout:1763: writing smithi05955964-46 from 9879552 to 10551296 tid 2 2022-04-28T19:49:57.892 INFO:tasks.rados.rados.0.smithi059.stdout:1763: writing smithi05955964-46 from 10551296 to 11288576 tid 3 2022-04-28T19:49:57.894 INFO:tasks.rados.rados.0.smithi059.stdout:1763: writing smithi05955964-46 from 11288576 to 11485184 tid 4 2022-04-28T19:49:57.894 INFO:tasks.rados.rados.0.smithi059.stdout:1764: delete oid 23 current snap is 162 2022-04-28T19:49:57.894 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 795 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:49:57.894 INFO:tasks.rados.rados.0.smithi059.stdout:1763: finishing write tid 1 to smithi05955964-46 2022-04-28T19:49:57.895 INFO:tasks.rados.rados.0.smithi059.stdout:1763: finishing write tid 2 to smithi05955964-46 2022-04-28T19:49:57.898 INFO:tasks.rados.rados.0.smithi059.stdout:1762: done (2 left) 2022-04-28T19:49:57.898 INFO:tasks.rados.rados.0.smithi059.stdout:1764: done (1 left) 2022-04-28T19:49:57.898 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 162 2022-04-28T19:49:57.898 INFO:tasks.rados.rados.0.smithi059.stdout:1765: seq_num 571 ranges {3956736=770048,4726784=483328,5210112=753664,5963776=368640} 2022-04-28T19:49:57.924 INFO:tasks.rados.rados.0.smithi059.stdout:1765: writing smithi05955964-26 from 3956736 to 4726784 tid 1 2022-04-28T19:49:57.927 INFO:tasks.rados.rados.0.smithi059.stdout:1765: writing smithi05955964-26 from 4726784 to 5210112 tid 2 2022-04-28T19:49:57.932 INFO:tasks.rados.rados.0.smithi059.stdout:1765: writing smithi05955964-26 from 5210112 to 5963776 tid 3 2022-04-28T19:49:57.935 INFO:tasks.rados.rados.0.smithi059.stdout:1765: writing smithi05955964-26 from 5963776 to 6332416 tid 4 2022-04-28T19:49:57.935 INFO:tasks.rados.rados.0.smithi059.stdout:1766: snap_remove snap 160 2022-04-28T19:49:58.078 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[61152]: cephadm 2022-04-28T19:49:56.631649+0000 mgr.y (mgr.24961) 38 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-04-28T19:49:58.078 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[61152]: cephadm 2022-04-28T19:49:56.635939+0000 mgr.y (mgr.24961) 39 : cephadm [INF] Reconfiguring daemon mon.c on smithi059 2022-04-28T19:49:58.078 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[61152]: cluster 2022-04-28T19:49:56.861486+0000 mgr.y (mgr.24961) 40 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:58.078 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[61152]: cluster 2022-04-28T19:49:57.820075+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2022-04-28T19:49:58.079 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[59745]: cephadm 2022-04-28T19:49:56.631649+0000 mgr.y (mgr.24961) 38 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-04-28T19:49:58.079 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[59745]: cephadm 2022-04-28T19:49:56.635939+0000 mgr.y (mgr.24961) 39 : cephadm [INF] Reconfiguring daemon mon.c on smithi059 2022-04-28T19:49:58.079 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[59745]: cluster 2022-04-28T19:49:56.861486+0000 mgr.y (mgr.24961) 40 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:58.079 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:57 smithi059 bash[59745]: cluster 2022-04-28T19:49:57.820075+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2022-04-28T19:49:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:57 smithi139 bash[43301]: cephadm 2022-04-28T19:49:56.631649+0000 mgr.y (mgr.24961) 38 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2022-04-28T19:49:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:57 smithi139 bash[43301]: cephadm 2022-04-28T19:49:56.635939+0000 mgr.y (mgr.24961) 39 : cephadm [INF] Reconfiguring daemon mon.c on smithi059 2022-04-28T19:49:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:57 smithi139 bash[43301]: cluster 2022-04-28T19:49:56.861486+0000 mgr.y (mgr.24961) 40 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 203 MiB data, 339 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:49:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:57 smithi139 bash[43301]: cluster 2022-04-28T19:49:57.820075+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2022-04-28T19:49:58.859 INFO:tasks.rados.rados.0.smithi059.stdout:1766: done (2 left) 2022-04-28T19:49:58.859 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 162 2022-04-28T19:49:58.859 INFO:tasks.rados.rados.0.smithi059.stdout:1767: seq_num 572 ranges {0=786432,786432=770048,1556480=557056,2113536=278528} 2022-04-28T19:49:58.863 INFO:tasks.rados.rados.0.smithi059.stdout:1767: writing smithi05955964-9 from 0 to 786432 tid 1 2022-04-28T19:49:58.868 INFO:tasks.rados.rados.0.smithi059.stdout:1767: writing smithi05955964-9 from 786432 to 1556480 tid 2 2022-04-28T19:49:58.871 INFO:tasks.rados.rados.0.smithi059.stdout:1767: writing smithi05955964-9 from 1556480 to 2113536 tid 3 2022-04-28T19:49:58.874 INFO:tasks.rados.rados.0.smithi059.stdout:1767: writing smithi05955964-9 from 2113536 to 2392064 tid 4 2022-04-28T19:49:58.875 INFO:tasks.rados.rados.0.smithi059.stdout:1768: snap_create 2022-04-28T19:49:58.875 INFO:tasks.rados.rados.0.smithi059.stdout:1763: finishing write tid 3 to smithi05955964-46 2022-04-28T19:49:58.875 INFO:tasks.rados.rados.0.smithi059.stdout:1763: finishing write tid 4 to smithi05955964-46 2022-04-28T19:49:58.875 INFO:tasks.rados.rados.0.smithi059.stdout:1763: finishing write tid 6 to smithi05955964-46 2022-04-28T19:49:58.875 INFO:tasks.rados.rados.0.smithi059.stdout:1763: finishing write tid 7 to smithi05955964-46 2022-04-28T19:49:58.875 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 777 (ObjNum 569 snap 162 seq_num 569) dirty exists 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1763: left oid 46 (ObjNum 569 snap 162 seq_num 569) 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: finishing write tid 1 to smithi05955964-26 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: finishing write tid 2 to smithi05955964-26 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: finishing write tid 3 to smithi05955964-26 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: finishing write tid 4 to smithi05955964-26 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: finishing write tid 6 to smithi05955964-26 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: finishing write tid 7 to smithi05955964-26 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 808 (ObjNum 571 snap 162 seq_num 571) dirty exists 2022-04-28T19:49:58.876 INFO:tasks.rados.rados.0.smithi059.stdout:1765: left oid 26 (ObjNum 571 snap 162 seq_num 571) 2022-04-28T19:49:58.877 INFO:tasks.rados.rados.0.smithi059.stdout:1767: finishing write tid 1 to smithi05955964-9 2022-04-28T19:49:58.878 INFO:tasks.rados.rados.0.smithi059.stdout:1767: finishing write tid 2 to smithi05955964-9 2022-04-28T19:49:58.879 INFO:tasks.rados.rados.0.smithi059.stdout:1767: finishing write tid 3 to smithi05955964-9 2022-04-28T19:49:58.880 INFO:tasks.rados.rados.0.smithi059.stdout:1767: finishing write tid 4 to smithi05955964-9 2022-04-28T19:49:58.880 INFO:tasks.rados.rados.0.smithi059.stdout:1767: finishing write tid 6 to smithi05955964-9 2022-04-28T19:49:58.881 INFO:tasks.rados.rados.0.smithi059.stdout:1767: finishing write tid 7 to smithi05955964-9 2022-04-28T19:49:58.881 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 643 (ObjNum 572 snap 162 seq_num 572) dirty exists 2022-04-28T19:49:58.881 INFO:tasks.rados.rados.0.smithi059.stdout:1767: left oid 9 (ObjNum 572 snap 162 seq_num 572) 2022-04-28T19:49:59.869 INFO:tasks.rados.rados.0.smithi059.stdout:1763: done (3 left) 2022-04-28T19:49:59.869 INFO:tasks.rados.rados.0.smithi059.stdout:1765: done (2 left) 2022-04-28T19:49:59.869 INFO:tasks.rados.rados.0.smithi059.stdout:1767: done (1 left) 2022-04-28T19:49:59.870 INFO:tasks.rados.rados.0.smithi059.stdout:1768: done (0 left) 2022-04-28T19:49:59.870 INFO:tasks.rados.rados.0.smithi059.stdout:1769: rollback oid 36 current snap is 163 2022-04-28T19:49:59.870 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 161 2022-04-28T19:49:59.870 INFO:tasks.rados.rados.0.smithi059.stdout:1770: snap_create 2022-04-28T19:49:59.874 INFO:tasks.rados.rados.0.smithi059.stdout:1769: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:49:59.874 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 763 (ObjNum 1667788387 snap 1567185266 seq_num 1500080483) dirty dne 2022-04-28T19:50:00.056 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[61152]: audit 2022-04-28T19:49:58.686526+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[61152]: cephadm 2022-04-28T19:49:58.688115+0000 mgr.y (mgr.24961) 41 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[61152]: cephadm 2022-04-28T19:49:58.689832+0000 mgr.y (mgr.24961) 42 : cephadm [INF] Reconfiguring daemon osd.0 on smithi059 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[61152]: audit 2022-04-28T19:49:58.691104+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "container_image"}]: dispatch 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[61152]: cluster 2022-04-28T19:49:58.856862+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[61152]: cluster 2022-04-28T19:49:58.861962+0000 mgr.y (mgr.24961) 43 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 232 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.3 MiB/s wr, 20 op/s 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[59745]: audit 2022-04-28T19:49:58.686526+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[59745]: cephadm 2022-04-28T19:49:58.688115+0000 mgr.y (mgr.24961) 41 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2022-04-28T19:50:00.057 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[59745]: cephadm 2022-04-28T19:49:58.689832+0000 mgr.y (mgr.24961) 42 : cephadm [INF] Reconfiguring daemon osd.0 on smithi059 2022-04-28T19:50:00.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[59745]: audit 2022-04-28T19:49:58.691104+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "container_image"}]: dispatch 2022-04-28T19:50:00.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[59745]: cluster 2022-04-28T19:49:58.856862+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2022-04-28T19:50:00.058 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:49:59 smithi059 bash[59745]: cluster 2022-04-28T19:49:58.861962+0000 mgr.y (mgr.24961) 43 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 232 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.3 MiB/s wr, 20 op/s 2022-04-28T19:50:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:59 smithi139 bash[43301]: audit 2022-04-28T19:49:58.686526+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:59 smithi139 bash[43301]: cephadm 2022-04-28T19:49:58.688115+0000 mgr.y (mgr.24961) 41 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2022-04-28T19:50:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:59 smithi139 bash[43301]: cephadm 2022-04-28T19:49:58.689832+0000 mgr.y (mgr.24961) 42 : cephadm [INF] Reconfiguring daemon osd.0 on smithi059 2022-04-28T19:50:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:59 smithi139 bash[43301]: audit 2022-04-28T19:49:58.691104+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "container_image"}]: dispatch 2022-04-28T19:50:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:59 smithi139 bash[43301]: cluster 2022-04-28T19:49:58.856862+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2022-04-28T19:50:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:49:59 smithi139 bash[43301]: cluster 2022-04-28T19:49:58.861962+0000 mgr.y (mgr.24961) 43 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 232 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.3 MiB/s wr, 20 op/s 2022-04-28T19:50:00.872 INFO:tasks.rados.rados.0.smithi059.stdout:1769: done (1 left) 2022-04-28T19:50:00.873 INFO:tasks.rados.rados.0.smithi059.stdout:1770: done (0 left) 2022-04-28T19:50:00.873 INFO:tasks.rados.rados.0.smithi059.stdout:1771: read oid 4 snap 162 2022-04-28T19:50:00.873 INFO:tasks.rados.rados.0.smithi059.stdout:1771: expect deleted 2022-04-28T19:50:00.873 INFO:tasks.rados.rados.0.smithi059.stdout:1772: snap_remove snap 161 2022-04-28T19:50:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[59745]: cluster 2022-04-28T19:49:59.868492+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2022-04-28T19:50:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:00.000150+0000 mon.a (mon.0) 153 : cluster [INF] overall HEALTH_OK 2022-04-28T19:50:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[59745]: audit 2022-04-28T19:50:00.661285+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[59745]: audit 2022-04-28T19:50:00.664366+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "container_image"}]: dispatch 2022-04-28T19:50:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[61152]: cluster 2022-04-28T19:49:59.868492+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2022-04-28T19:50:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:00.000150+0000 mon.a (mon.0) 153 : cluster [INF] overall HEALTH_OK 2022-04-28T19:50:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[61152]: audit 2022-04-28T19:50:00.661285+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:00 smithi059 bash[61152]: audit 2022-04-28T19:50:00.664366+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "container_image"}]: dispatch 2022-04-28T19:50:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:00 smithi139 bash[43301]: cluster 2022-04-28T19:49:59.868492+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2022-04-28T19:50:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:00.000150+0000 mon.a (mon.0) 153 : cluster [INF] overall HEALTH_OK 2022-04-28T19:50:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:00 smithi139 bash[43301]: audit 2022-04-28T19:50:00.661285+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:00 smithi139 bash[43301]: audit 2022-04-28T19:50:00.664366+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "container_image"}]: dispatch 2022-04-28T19:50:01.809 INFO:tasks.rados.rados.0.smithi059.stdout:1772: done (1 left) 2022-04-28T19:50:01.809 INFO:tasks.rados.rados.0.smithi059.stdout:1773: setattr oid 49 current snap is 164 2022-04-28T19:50:01.811 INFO:tasks.rados.rados.0.smithi059.stdout:1771: done (1 left) 2022-04-28T19:50:01.811 INFO:tasks.rados.rados.0.smithi059.stdout:1774: snap_remove snap 146 2022-04-28T19:50:01.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:01.626Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:01.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:01.626Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:01.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:01.626Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:02.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[61152]: cephadm 2022-04-28T19:50:00.662431+0000 mgr.y (mgr.24961) 44 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2022-04-28T19:50:02.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[61152]: cephadm 2022-04-28T19:50:00.663614+0000 mgr.y (mgr.24961) 45 : cephadm [INF] Reconfiguring daemon osd.1 on smithi059 2022-04-28T19:50:02.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[61152]: cluster 2022-04-28T19:50:00.862796+0000 mgr.y (mgr.24961) 46 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 232 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 7.1 MiB/s rd, 7.2 MiB/s wr, 20 op/s 2022-04-28T19:50:02.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[61152]: cluster 2022-04-28T19:50:00.868266+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2022-04-28T19:50:02.144 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[61152]: cluster 2022-04-28T19:50:01.805405+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2022-04-28T19:50:02.145 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[59745]: cephadm 2022-04-28T19:50:00.662431+0000 mgr.y (mgr.24961) 44 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2022-04-28T19:50:02.145 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[59745]: cephadm 2022-04-28T19:50:00.663614+0000 mgr.y (mgr.24961) 45 : cephadm [INF] Reconfiguring daemon osd.1 on smithi059 2022-04-28T19:50:02.145 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[59745]: cluster 2022-04-28T19:50:00.862796+0000 mgr.y (mgr.24961) 46 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 232 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 7.1 MiB/s rd, 7.2 MiB/s wr, 20 op/s 2022-04-28T19:50:02.145 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[59745]: cluster 2022-04-28T19:50:00.868266+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2022-04-28T19:50:02.145 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:01 smithi059 bash[59745]: cluster 2022-04-28T19:50:01.805405+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2022-04-28T19:50:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:01 smithi139 bash[43301]: cephadm 2022-04-28T19:50:00.662431+0000 mgr.y (mgr.24961) 44 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2022-04-28T19:50:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:01 smithi139 bash[43301]: cephadm 2022-04-28T19:50:00.663614+0000 mgr.y (mgr.24961) 45 : cephadm [INF] Reconfiguring daemon osd.1 on smithi059 2022-04-28T19:50:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:01 smithi139 bash[43301]: cluster 2022-04-28T19:50:00.862796+0000 mgr.y (mgr.24961) 46 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 232 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 7.1 MiB/s rd, 7.2 MiB/s wr, 20 op/s 2022-04-28T19:50:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:01 smithi139 bash[43301]: cluster 2022-04-28T19:50:00.868266+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2022-04-28T19:50:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:01 smithi139 bash[43301]: cluster 2022-04-28T19:50:01.805405+0000 mon.a (mon.0) 157 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2022-04-28T19:50:02.823 INFO:tasks.rados.rados.0.smithi059.stdout:1774: done (1 left) 2022-04-28T19:50:02.843 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 164 2022-04-28T19:50:02.843 INFO:tasks.rados.rados.0.smithi059.stdout:1775: seq_num 573 ranges {3858432=802816,4661248=655360,5316608=720896,6037504=229376} 2022-04-28T19:50:02.853 INFO:tasks.rados.rados.0.smithi059.stdout:1775: writing smithi05955964-41 from 3858432 to 4661248 tid 1 2022-04-28T19:50:02.857 INFO:tasks.rados.rados.0.smithi059.stdout:1775: writing smithi05955964-41 from 4661248 to 5316608 tid 2 2022-04-28T19:50:02.862 INFO:tasks.rados.rados.0.smithi059.stdout:1775: writing smithi05955964-41 from 5316608 to 6037504 tid 3 2022-04-28T19:50:02.865 INFO:tasks.rados.rados.0.smithi059.stdout:1775: writing smithi05955964-41 from 6037504 to 6266880 tid 4 2022-04-28T19:50:02.865 INFO:tasks.rados.rados.0.smithi059.stdout:1776: copy_from oid 47 from oid 44 current snap is 164 2022-04-28T19:50:02.865 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 765 (ObjNum 538 snap 150 seq_num 538) dirty exists 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1775: finishing write tid 1 to smithi05955964-41 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1775: finishing write tid 2 to smithi05955964-41 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1773: done (2 left) 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1777: rollback oid 8 current snap is 164 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 8 to 159 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 164 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1778: seq_num 574 ranges {0=16384} 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1778: writing smithi05955964-5 from 0 to 16384 tid 1 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1779: read oid 7 snap -1 2022-04-28T19:50:02.866 INFO:tasks.rados.rados.0.smithi059.stdout:1779: expect (ObjNum 545 snap 154 seq_num 545) 2022-04-28T19:50:02.867 INFO:tasks.rados.rados.0.smithi059.stdout:1780: rmattr oid 10 current snap is 164 2022-04-28T19:50:02.867 INFO:tasks.rados.rados.0.smithi059.stdout:1781: copy_from oid 31 from oid 36 current snap is 164 2022-04-28T19:50:02.867 INFO:tasks.rados.rados.0.smithi059.stdout:1782: delete oid 6 current snap is 164 2022-04-28T19:50:02.867 INFO:tasks.rados.rados.0.smithi059.stdout:1775: finishing write tid 3 to smithi05955964-41 2022-04-28T19:50:02.871 INFO:tasks.rados.rados.0.smithi059.stdout:1777: finishing rollback tid 0 to smithi05955964-8 2022-04-28T19:50:02.871 INFO:tasks.rados.rados.0.smithi059.stdout:1782: done (7 left) 2022-04-28T19:50:02.871 INFO:tasks.rados.rados.0.smithi059.stdout:1783: rollback oid 22 current snap is 164 2022-04-28T19:50:02.871 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 159 2022-04-28T19:50:02.871 INFO:tasks.rados.rados.0.smithi059.stdout:1784: snap_create 2022-04-28T19:50:02.871 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 814 (ObjNum 457 snap 135 seq_num 457) dirty exists 2022-04-28T19:50:02.872 INFO:tasks.rados.rados.0.smithi059.stdout:1783: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:50:02.872 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 0 (ObjNum -132987600 snap 21893 seq_num 188341456) dirty dne 2022-04-28T19:50:02.872 INFO:tasks.rados.rados.0.smithi059.stdout:1777: finishing rollback tid 1 to smithi05955964-8 2022-04-28T19:50:02.872 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 790 (ObjNum 1 snap 21893 seq_num 4161725296) dirty dne 2022-04-28T19:50:02.872 INFO:tasks.rados.rados.0.smithi059.stdout:1778: finishing write tid 1 to smithi05955964-5 2022-04-28T19:50:02.873 INFO:tasks.rados.rados.0.smithi059.stdout:1778: finishing write tid 3 to smithi05955964-5 2022-04-28T19:50:02.874 INFO:tasks.rados.rados.0.smithi059.stdout:1775: finishing write tid 4 to smithi05955964-41 2022-04-28T19:50:02.874 INFO:tasks.rados.rados.0.smithi059.stdout:1775: finishing write tid 6 to smithi05955964-41 2022-04-28T19:50:02.874 INFO:tasks.rados.rados.0.smithi059.stdout:1775: finishing write tid 7 to smithi05955964-41 2022-04-28T19:50:02.874 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 621 (ObjNum 573 snap 164 seq_num 573) dirty exists 2022-04-28T19:50:02.875 INFO:tasks.rados.rados.0.smithi059.stdout:1775: left oid 41 (ObjNum 573 snap 164 seq_num 573) 2022-04-28T19:50:02.875 INFO:tasks.rados.rados.0.smithi059.stdout:1778: finishing write tid 4 to smithi05955964-5 2022-04-28T19:50:02.875 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 792 (ObjNum 574 snap 164 seq_num 574) dirty exists 2022-04-28T19:50:02.875 INFO:tasks.rados.rados.0.smithi059.stdout:1778: left oid 5 (ObjNum 574 snap 164 seq_num 574) 2022-04-28T19:50:02.876 INFO:tasks.rados.rados.0.smithi059.stdout:1781: finishing copy_from to smithi05955964-31 2022-04-28T19:50:02.876 INFO:tasks.rados.rados.0.smithi059.stdout:1781: got expected ENOENT (src dne) 2022-04-28T19:50:02.876 INFO:tasks.rados.rados.0.smithi059.stdout:1781: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:50:02.888 INFO:tasks.rados.rados.0.smithi059.stdout:1776: finishing copy_from to smithi05955964-47 2022-04-28T19:50:02.888 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 638 (ObjNum 507 snap 144 seq_num 507) dirty exists 2022-04-28T19:50:02.889 INFO:tasks.rados.rados.0.smithi059.stdout:1776: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:50:02.914 INFO:tasks.rados.rados.0.smithi059.stdout:1779: expect (ObjNum 545 snap 154 seq_num 545) 2022-04-28T19:50:03.831 INFO:tasks.rados.rados.0.smithi059.stdout:1775: done (8 left) 2022-04-28T19:50:03.831 INFO:tasks.rados.rados.0.smithi059.stdout:1776: done (7 left) 2022-04-28T19:50:03.831 INFO:tasks.rados.rados.0.smithi059.stdout:1777: done (6 left) 2022-04-28T19:50:03.832 INFO:tasks.rados.rados.0.smithi059.stdout:1778: done (5 left) 2022-04-28T19:50:03.832 INFO:tasks.rados.rados.0.smithi059.stdout:1779: done (4 left) 2022-04-28T19:50:03.834 INFO:tasks.rados.rados.0.smithi059.stdout:1780: done (3 left) 2022-04-28T19:50:03.834 INFO:tasks.rados.rados.0.smithi059.stdout:1781: done (2 left) 2022-04-28T19:50:03.834 INFO:tasks.rados.rados.0.smithi059.stdout:1783: done (1 left) 2022-04-28T19:50:03.834 INFO:tasks.rados.rados.0.smithi059.stdout:1784: done (0 left) 2022-04-28T19:50:03.834 INFO:tasks.rados.rados.0.smithi059.stdout:1785: read oid 30 snap 153 2022-04-28T19:50:03.835 INFO:tasks.rados.rados.0.smithi059.stdout:1785: expect (ObjNum 529 snap 150 seq_num 529) 2022-04-28T19:50:03.835 INFO:tasks.rados.rados.0.smithi059.stdout:1786: rollback oid 17 current snap is 165 2022-04-28T19:50:03.835 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 164 2022-04-28T19:50:03.835 INFO:tasks.rados.rados.0.smithi059.stdout:1787: read oid 14 snap 163 2022-04-28T19:50:03.835 INFO:tasks.rados.rados.0.smithi059.stdout:1787: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:50:03.835 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 165 2022-04-28T19:50:03.836 INFO:tasks.rados.rados.0.smithi059.stdout:1788: seq_num 576 ranges {2007040=49152} 2022-04-28T19:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[61152]: audit 2022-04-28T19:50:02.761914+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[61152]: cephadm 2022-04-28T19:50:02.763657+0000 mgr.y (mgr.24961) 47 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2022-04-28T19:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[61152]: cephadm 2022-04-28T19:50:02.765441+0000 mgr.y (mgr.24961) 48 : cephadm [INF] Reconfiguring daemon osd.2 on smithi059 2022-04-28T19:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[61152]: audit 2022-04-28T19:50:02.766442+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "container_image"}]: dispatch 2022-04-28T19:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[61152]: cluster 2022-04-28T19:50:02.818053+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2022-04-28T19:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[61152]: cluster 2022-04-28T19:50:02.863650+0000 mgr.y (mgr.24961) 49 : cluster [DBG] pgmap v62: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 1.7 MiB/s wr, 14 op/s 2022-04-28T19:50:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:03.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:03.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:03.520Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:50:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:03.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:03.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:03.524Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[59745]: audit 2022-04-28T19:50:02.761914+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[59745]: cephadm 2022-04-28T19:50:02.763657+0000 mgr.y (mgr.24961) 47 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2022-04-28T19:50:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[59745]: cephadm 2022-04-28T19:50:02.765441+0000 mgr.y (mgr.24961) 48 : cephadm [INF] Reconfiguring daemon osd.2 on smithi059 2022-04-28T19:50:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[59745]: audit 2022-04-28T19:50:02.766442+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "container_image"}]: dispatch 2022-04-28T19:50:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[59745]: cluster 2022-04-28T19:50:02.818053+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2022-04-28T19:50:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:03 smithi059 bash[59745]: cluster 2022-04-28T19:50:02.863650+0000 mgr.y (mgr.24961) 49 : cluster [DBG] pgmap v62: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 1.7 MiB/s wr, 14 op/s 2022-04-28T19:50:03.845 INFO:tasks.rados.rados.0.smithi059.stdout:1788: writing smithi05955964-45 from 2007040 to 2056192 tid 1 2022-04-28T19:50:03.845 INFO:tasks.rados.rados.0.smithi059.stdout:1789: rmattr oid 7 current snap is 165 2022-04-28T19:50:03.846 INFO:tasks.rados.rados.0.smithi059.stdout:1786: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:50:03.846 INFO:tasks.rados.rados.0.smithi059.stdout:1786: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:50:03.846 INFO:tasks.rados.rados.0.smithi059.stdout:1790: copy_from oid 6 from oid 20 current snap is 165 2022-04-28T19:50:03.846 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 165 2022-04-28T19:50:03.846 INFO:tasks.rados.rados.0.smithi059.stdout:1791: seq_num 577 ranges {2646016=65536} 2022-04-28T19:50:03.861 INFO:tasks.rados.rados.0.smithi059.stdout:1791: writing smithi05955964-32 from 2646016 to 2711552 tid 1 2022-04-28T19:50:03.861 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 165 2022-04-28T19:50:03.861 INFO:tasks.rados.rados.0.smithi059.stdout:1792: seq_num 578 ranges {2228224=81920} 2022-04-28T19:50:03.875 INFO:tasks.rados.rados.0.smithi059.stdout:1792: writing smithi05955964-43 from 2228224 to 2310144 tid 1 2022-04-28T19:50:03.875 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 165 2022-04-28T19:50:03.876 INFO:tasks.rados.rados.0.smithi059.stdout:1793: seq_num 579 ranges {0=98304} 2022-04-28T19:50:03.876 INFO:tasks.rados.rados.0.smithi059.stdout:1793: writing smithi05955964-36 from 0 to 98304 tid 1 2022-04-28T19:50:03.876 INFO:tasks.rados.rados.0.smithi059.stdout:1794: snap_create 2022-04-28T19:50:03.876 INFO:tasks.rados.rados.0.smithi059.stdout:1787: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:50:03.930 INFO:tasks.rados.rados.0.smithi059.stdout:1786: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:50:03.931 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 407 (ObjNum 534 snap 150 seq_num 534) dirty exists 2022-04-28T19:50:03.931 INFO:tasks.rados.rados.0.smithi059.stdout:1785: expect (ObjNum 529 snap 150 seq_num 529) 2022-04-28T19:50:03.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 762 (ObjNum 545 snap 154 seq_num 545) dirty exists 2022-04-28T19:50:03.969 INFO:tasks.rados.rados.0.smithi059.stdout:1788: finishing write tid 1 to smithi05955964-45 2022-04-28T19:50:03.969 INFO:tasks.rados.rados.0.smithi059.stdout:1788: finishing write tid 3 to smithi05955964-45 2022-04-28T19:50:03.969 INFO:tasks.rados.rados.0.smithi059.stdout:1791: finishing write tid 1 to smithi05955964-32 2022-04-28T19:50:03.969 INFO:tasks.rados.rados.0.smithi059.stdout:1791: finishing write tid 3 to smithi05955964-32 2022-04-28T19:50:03.970 INFO:tasks.rados.rados.0.smithi059.stdout:1791: finishing write tid 4 to smithi05955964-32 2022-04-28T19:50:03.970 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 827 (ObjNum 577 snap 165 seq_num 577) dirty exists 2022-04-28T19:50:03.970 INFO:tasks.rados.rados.0.smithi059.stdout:1791: left oid 32 (ObjNum 577 snap 165 seq_num 577) 2022-04-28T19:50:03.970 INFO:tasks.rados.rados.0.smithi059.stdout:1788: finishing write tid 4 to smithi05955964-45 2022-04-28T19:50:03.970 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 720 (ObjNum 576 snap 165 seq_num 576) dirty exists 2022-04-28T19:50:03.970 INFO:tasks.rados.rados.0.smithi059.stdout:1788: left oid 45 (ObjNum 576 snap 165 seq_num 576) 2022-04-28T19:50:03.971 INFO:tasks.rados.rados.0.smithi059.stdout:1790: finishing copy_from to smithi05955964-6 2022-04-28T19:50:03.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 650 (ObjNum 461 snap 136 seq_num 461) dirty exists 2022-04-28T19:50:03.971 INFO:tasks.rados.rados.0.smithi059.stdout:1793: finishing write tid 1 to smithi05955964-36 2022-04-28T19:50:03.971 INFO:tasks.rados.rados.0.smithi059.stdout:1792: finishing write tid 1 to smithi05955964-43 2022-04-28T19:50:03.971 INFO:tasks.rados.rados.0.smithi059.stdout:1793: finishing write tid 3 to smithi05955964-36 2022-04-28T19:50:03.971 INFO:tasks.rados.rados.0.smithi059.stdout:1792: finishing write tid 3 to smithi05955964-43 2022-04-28T19:50:03.972 INFO:tasks.rados.rados.0.smithi059.stdout:1790: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:50:03.972 INFO:tasks.rados.rados.0.smithi059.stdout:1792: finishing write tid 4 to smithi05955964-43 2022-04-28T19:50:03.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 830 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:50:03.972 INFO:tasks.rados.rados.0.smithi059.stdout:1792: left oid 43 (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:50:03.972 INFO:tasks.rados.rados.0.smithi059.stdout:1793: finishing write tid 4 to smithi05955964-36 2022-04-28T19:50:03.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 767 (ObjNum 579 snap 165 seq_num 579) dirty exists 2022-04-28T19:50:03.973 INFO:tasks.rados.rados.0.smithi059.stdout:1793: left oid 36 (ObjNum 579 snap 165 seq_num 579) 2022-04-28T19:50:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:03 smithi139 bash[43301]: audit 2022-04-28T19:50:02.761914+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:03 smithi139 bash[43301]: cephadm 2022-04-28T19:50:02.763657+0000 mgr.y (mgr.24961) 47 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2022-04-28T19:50:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:03 smithi139 bash[43301]: cephadm 2022-04-28T19:50:02.765441+0000 mgr.y (mgr.24961) 48 : cephadm [INF] Reconfiguring daemon osd.2 on smithi059 2022-04-28T19:50:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:03 smithi139 bash[43301]: audit 2022-04-28T19:50:02.766442+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "container_image"}]: dispatch 2022-04-28T19:50:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:03 smithi139 bash[43301]: cluster 2022-04-28T19:50:02.818053+0000 mon.a (mon.0) 160 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2022-04-28T19:50:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:03 smithi139 bash[43301]: cluster 2022-04-28T19:50:02.863650+0000 mgr.y (mgr.24961) 49 : cluster [DBG] pgmap v62: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail; 767 B/s rd, 1.7 MiB/s wr, 14 op/s 2022-04-28T19:50:04.838 INFO:tasks.rados.rados.0.smithi059.stdout:1785: done (9 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1786: done (8 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1787: done (7 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1788: done (6 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1789: done (5 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1790: done (4 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1791: done (3 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1792: done (2 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1793: done (1 left) 2022-04-28T19:50:04.839 INFO:tasks.rados.rados.0.smithi059.stdout:1794: done (0 left) 2022-04-28T19:50:04.840 INFO:tasks.rados.rados.0.smithi059.stdout:1795: delete oid 40 current snap is 166 2022-04-28T19:50:04.844 INFO:tasks.rados.rados.0.smithi059.stdout:1795: done (0 left) 2022-04-28T19:50:04.844 INFO:tasks.rados.rados.0.smithi059.stdout:1796: read oid 17 snap 165 2022-04-28T19:50:04.844 INFO:tasks.rados.rados.0.smithi059.stdout:1796: expect (ObjNum 534 snap 150 seq_num 534) 2022-04-28T19:50:04.844 INFO:tasks.rados.rados.0.smithi059.stdout:1797: setattr oid 7 current snap is 166 2022-04-28T19:50:04.847 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 166 2022-04-28T19:50:04.847 INFO:tasks.rados.rados.0.smithi059.stdout:1798: seq_num 581 ranges {1974272=139264} 2022-04-28T19:50:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:1798: writing smithi05955964-14 from 1974272 to 2113536 tid 1 2022-04-28T19:50:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 166 2022-04-28T19:50:04.861 INFO:tasks.rados.rados.0.smithi059.stdout:1799: seq_num 582 ranges {2801664=155648} 2022-04-28T19:50:04.878 INFO:tasks.rados.rados.0.smithi059.stdout:1799: writing smithi05955964-25 from 2801664 to 2957312 tid 1 2022-04-28T19:50:04.878 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 166 2022-04-28T19:50:04.878 INFO:tasks.rados.rados.0.smithi059.stdout:1800: seq_num 583 ranges {2326528=172032} 2022-04-28T19:50:04.904 INFO:tasks.rados.rados.0.smithi059.stdout:1800: writing smithi05955964-10 from 2326528 to 2498560 tid 1 2022-04-28T19:50:04.904 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 166 2022-04-28T19:50:04.904 INFO:tasks.rados.rados.0.smithi059.stdout:1801: seq_num 584 ranges {0=188416} 2022-04-28T19:50:04.904 INFO:tasks.rados.rados.0.smithi059.stdout:1801: writing smithi05955964-48 from 0 to 188416 tid 1 2022-04-28T19:50:04.904 INFO:tasks.rados.rados.0.smithi059.stdout:1802: rmattr oid 9 current snap is 166 2022-04-28T19:50:04.904 INFO:tasks.rados.rados.0.smithi059.stdout:1802: done (6 left) 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:1803: setattr oid 41 current snap is 166 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 764 (ObjNum 545 snap 154 seq_num 545) dirty exists 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:1798: finishing write tid 1 to smithi05955964-14 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:1798: finishing write tid 3 to smithi05955964-14 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:1798: finishing write tid 4 to smithi05955964-14 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 821 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:04.905 INFO:tasks.rados.rados.0.smithi059.stdout:1798: left oid 14 (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:50:04.906 INFO:tasks.rados.rados.0.smithi059.stdout:1796: expect (ObjNum 534 snap 150 seq_num 534) 2022-04-28T19:50:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:04 smithi059 bash[61152]: cluster 2022-04-28T19:50:03.826462+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2022-04-28T19:50:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:04 smithi059 bash[61152]: audit 2022-04-28T19:50:04.761619+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:04 smithi059 bash[61152]: audit 2022-04-28T19:50:04.766389+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "container_image"}]: dispatch 2022-04-28T19:50:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:04 smithi059 bash[59745]: cluster 2022-04-28T19:50:03.826462+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2022-04-28T19:50:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:04 smithi059 bash[59745]: audit 2022-04-28T19:50:04.761619+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:04 smithi059 bash[59745]: audit 2022-04-28T19:50:04.766389+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "container_image"}]: dispatch 2022-04-28T19:50:05.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:04 smithi139 bash[43301]: cluster 2022-04-28T19:50:03.826462+0000 mon.a (mon.0) 161 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2022-04-28T19:50:05.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:04 smithi139 bash[43301]: audit 2022-04-28T19:50:04.761619+0000 mon.a (mon.0) 162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:05.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:04 smithi139 bash[43301]: audit 2022-04-28T19:50:04.766389+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "container_image"}]: dispatch 2022-04-28T19:50:05.288 INFO:tasks.rados.rados.0.smithi059.stdout:1799: finishing write tid 1 to smithi05955964-25 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1799: finishing write tid 3 to smithi05955964-25 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1799: finishing write tid 4 to smithi05955964-25 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 795 (ObjNum 582 snap 166 seq_num 582) dirty exists 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1799: left oid 25 (ObjNum 582 snap 166 seq_num 582) 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1801: finishing write tid 1 to smithi05955964-48 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1800: finishing write tid 1 to smithi05955964-10 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1800: finishing write tid 3 to smithi05955964-10 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1801: finishing write tid 3 to smithi05955964-48 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:1800: finishing write tid 4 to smithi05955964-10 2022-04-28T19:50:05.289 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 819 (ObjNum 583 snap 166 seq_num 583) dirty exists 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1800: left oid 10 (ObjNum 583 snap 166 seq_num 583) 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1801: finishing write tid 4 to smithi05955964-48 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 823 (ObjNum 584 snap 166 seq_num 584) dirty exists 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1801: left oid 48 (ObjNum 584 snap 166 seq_num 584) 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1796: done (6 left) 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1797: done (5 left) 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1798: done (4 left) 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1799: done (3 left) 2022-04-28T19:50:05.290 INFO:tasks.rados.rados.0.smithi059.stdout:1800: done (2 left) 2022-04-28T19:50:05.291 INFO:tasks.rados.rados.0.smithi059.stdout:1801: done (1 left) 2022-04-28T19:50:05.291 INFO:tasks.rados.rados.0.smithi059.stdout:1804: rollback oid 18 current snap is 166 2022-04-28T19:50:05.291 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 165 2022-04-28T19:50:05.291 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 166 2022-04-28T19:50:05.291 INFO:tasks.rados.rados.0.smithi059.stdout:1805: seq_num 585 ranges {2400256=204800} 2022-04-28T19:50:05.308 INFO:tasks.rados.rados.0.smithi059.stdout:1805: writing smithi05955964-13 from 2400256 to 2605056 tid 1 2022-04-28T19:50:05.308 INFO:tasks.rados.rados.0.smithi059.stdout:1806: rollback oid 35 current snap is 166 2022-04-28T19:50:05.308 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 35 to 164 2022-04-28T19:50:05.308 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 622 (ObjNum 573 snap 164 seq_num 573) dirty exists 2022-04-28T19:50:05.308 INFO:tasks.rados.rados.0.smithi059.stdout:1804: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:1804: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:1804: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 801 (ObjNum 504 snap 143 seq_num 504) dirty exists 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:1803: done (3 left) 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:1804: done (2 left) 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:1807: copy_from oid 36 from oid 6 current snap is 166 2022-04-28T19:50:05.309 INFO:tasks.rados.rados.0.smithi059.stdout:1808: delete oid 22 current snap is 166 2022-04-28T19:50:05.311 INFO:tasks.rados.rados.0.smithi059.stdout:1808: done (3 left) 2022-04-28T19:50:05.311 INFO:tasks.rados.rados.0.smithi059.stdout:1809: rmattr oid 27 current snap is 166 2022-04-28T19:50:05.311 INFO:tasks.rados.rados.0.smithi059.stdout:1810: copy_from oid 18 from oid 9 current snap is 166 2022-04-28T19:50:05.311 INFO:tasks.rados.rados.0.smithi059.stdout:1811: setattr oid 3 current snap is 166 2022-04-28T19:50:05.312 INFO:tasks.rados.rados.0.smithi059.stdout:1812: snap_create 2022-04-28T19:50:05.314 INFO:tasks.rados.rados.0.smithi059.stdout:1805: finishing write tid 1 to smithi05955964-13 2022-04-28T19:50:05.314 INFO:tasks.rados.rados.0.smithi059.stdout:1806: finishing rollback tid 0 to smithi05955964-35 2022-04-28T19:50:05.315 INFO:tasks.rados.rados.0.smithi059.stdout:1806: finishing rollback tid 1 to smithi05955964-35 2022-04-28T19:50:05.315 INFO:tasks.rados.rados.0.smithi059.stdout:1806: finishing rollback tid 2 to smithi05955964-35 2022-04-28T19:50:05.315 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 727 (ObjNum 559 snap 160 seq_num 559) dirty exists 2022-04-28T19:50:05.316 INFO:tasks.rados.rados.0.smithi059.stdout:1805: finishing write tid 3 to smithi05955964-13 2022-04-28T19:50:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:1805: finishing write tid 4 to smithi05955964-13 2022-04-28T19:50:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 410 (ObjNum 585 snap 166 seq_num 585) dirty exists 2022-04-28T19:50:05.318 INFO:tasks.rados.rados.0.smithi059.stdout:1805: left oid 13 (ObjNum 585 snap 166 seq_num 585) 2022-04-28T19:50:05.319 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 821 (ObjNum 531 snap 150 seq_num 531) dirty exists 2022-04-28T19:50:05.322 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 723 (ObjNum 491 snap 140 seq_num 491) dirty exists 2022-04-28T19:50:05.338 INFO:tasks.rados.rados.0.smithi059.stdout:1810: finishing copy_from to smithi05955964-18 2022-04-28T19:50:05.339 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 643 (ObjNum 572 snap 162 seq_num 572) dirty exists 2022-04-28T19:50:05.339 INFO:tasks.rados.rados.0.smithi059.stdout:1807: finishing copy_from to smithi05955964-36 2022-04-28T19:50:05.339 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 650 (ObjNum 461 snap 136 seq_num 461) dirty exists 2022-04-28T19:50:05.339 INFO:tasks.rados.rados.0.smithi059.stdout:1810: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:50:05.339 INFO:tasks.rados.rados.0.smithi059.stdout:1807: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:50:05.849 INFO:tasks.rados.rados.0.smithi059.stdout:1805: done (6 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1806: done (5 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1807: done (4 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1809: done (3 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1810: done (2 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1811: done (1 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1812: done (0 left) 2022-04-28T19:50:05.850 INFO:tasks.rados.rados.0.smithi059.stdout:1813: delete oid 24 current snap is 167 2022-04-28T19:50:05.855 INFO:tasks.rados.rados.0.smithi059.stdout:1813: done (0 left) 2022-04-28T19:50:05.855 INFO:tasks.rados.rados.0.smithi059.stdout:1814: rmattr oid 14 current snap is 167 2022-04-28T19:50:05.855 INFO:tasks.rados.rados.0.smithi059.stdout:1815: delete oid 7 current snap is 167 2022-04-28T19:50:05.858 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 826 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:05.862 INFO:tasks.rados.rados.0.smithi059.stdout:1814: done (1 left) 2022-04-28T19:50:05.862 INFO:tasks.rados.rados.0.smithi059.stdout:1815: done (0 left) 2022-04-28T19:50:05.862 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 167 2022-04-28T19:50:05.862 INFO:tasks.rados.rados.0.smithi059.stdout:1816: seq_num 589 ranges {0=270336} 2022-04-28T19:50:05.863 INFO:tasks.rados.rados.0.smithi059.stdout:1816: writing smithi05955964-4 from 0 to 270336 tid 1 2022-04-28T19:50:05.864 INFO:tasks.rados.rados.0.smithi059.stdout:1817: snap_create 2022-04-28T19:50:05.867 INFO:tasks.rados.rados.0.smithi059.stdout:1816: finishing write tid 1 to smithi05955964-4 2022-04-28T19:50:05.867 INFO:tasks.rados.rados.0.smithi059.stdout:1816: finishing write tid 3 to smithi05955964-4 2022-04-28T19:50:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:1816: finishing write tid 4 to smithi05955964-4 2022-04-28T19:50:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 769 (ObjNum 589 snap 167 seq_num 589) dirty exists 2022-04-28T19:50:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:1816: left oid 4 (ObjNum 589 snap 167 seq_num 589) 2022-04-28T19:50:06.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:05 smithi139 bash[43301]: cephadm 2022-04-28T19:50:04.763469+0000 mgr.y (mgr.24961) 50 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2022-04-28T19:50:06.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:05 smithi139 bash[43301]: cephadm 2022-04-28T19:50:04.765370+0000 mgr.y (mgr.24961) 51 : cephadm [INF] Reconfiguring daemon osd.3 on smithi059 2022-04-28T19:50:06.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:05 smithi139 bash[43301]: cluster 2022-04-28T19:50:04.835622+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2022-04-28T19:50:06.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:05 smithi139 bash[43301]: cluster 2022-04-28T19:50:04.864150+0000 mgr.y (mgr.24961) 52 : cluster [DBG] pgmap v65: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail; 768 B/s rd, 1.7 MiB/s wr, 15 op/s 2022-04-28T19:50:06.219 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[61152]: cephadm 2022-04-28T19:50:04.763469+0000 mgr.y (mgr.24961) 50 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2022-04-28T19:50:06.219 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[61152]: cephadm 2022-04-28T19:50:04.765370+0000 mgr.y (mgr.24961) 51 : cephadm [INF] Reconfiguring daemon osd.3 on smithi059 2022-04-28T19:50:06.219 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[61152]: cluster 2022-04-28T19:50:04.835622+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2022-04-28T19:50:06.219 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[61152]: cluster 2022-04-28T19:50:04.864150+0000 mgr.y (mgr.24961) 52 : cluster [DBG] pgmap v65: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail; 768 B/s rd, 1.7 MiB/s wr, 15 op/s 2022-04-28T19:50:06.220 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[59745]: cephadm 2022-04-28T19:50:04.763469+0000 mgr.y (mgr.24961) 50 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2022-04-28T19:50:06.220 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[59745]: cephadm 2022-04-28T19:50:04.765370+0000 mgr.y (mgr.24961) 51 : cephadm [INF] Reconfiguring daemon osd.3 on smithi059 2022-04-28T19:50:06.220 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[59745]: cluster 2022-04-28T19:50:04.835622+0000 mon.a (mon.0) 164 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2022-04-28T19:50:06.220 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:05 smithi059 bash[59745]: cluster 2022-04-28T19:50:04.864150+0000 mgr.y (mgr.24961) 52 : cluster [DBG] pgmap v65: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail; 768 B/s rd, 1.7 MiB/s wr, 15 op/s 2022-04-28T19:50:06.814 INFO:tasks.rados.rados.0.smithi059.stdout:1816: done (1 left) 2022-04-28T19:50:06.814 INFO:tasks.rados.rados.0.smithi059.stdout:1817: done (0 left) 2022-04-28T19:50:06.814 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 168 2022-04-28T19:50:06.814 INFO:tasks.rados.rados.0.smithi059.stdout:1818: seq_num 590 ranges {2056192=286720} 2022-04-28T19:50:06.828 INFO:tasks.rados.rados.0.smithi059.stdout:1818: writing smithi05955964-45 from 2056192 to 2342912 tid 1 2022-04-28T19:50:06.828 INFO:tasks.rados.rados.0.smithi059.stdout:1819: snap_remove snap 165 2022-04-28T19:50:07.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:06 smithi139 bash[43301]: cluster 2022-04-28T19:50:05.846182+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2022-04-28T19:50:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:06 smithi139 bash[43301]: cluster 2022-04-28T19:50:06.813194+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2022-04-28T19:50:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:06 smithi139 bash[43301]: audit 2022-04-28T19:50:06.838453+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:06 smithi139 bash[43301]: audit 2022-04-28T19:50:06.840339+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:50:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[59745]: cluster 2022-04-28T19:50:05.846182+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[59745]: cluster 2022-04-28T19:50:06.813194+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[59745]: audit 2022-04-28T19:50:06.838453+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[59745]: audit 2022-04-28T19:50:06.840339+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[61152]: cluster 2022-04-28T19:50:05.846182+0000 mon.a (mon.0) 165 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[61152]: cluster 2022-04-28T19:50:06.813194+0000 mon.a (mon.0) 166 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[61152]: audit 2022-04-28T19:50:06.838453+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:06 smithi059 bash[61152]: audit 2022-04-28T19:50:06.840339+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:50:07.824 INFO:tasks.rados.rados.0.smithi059.stdout:1819: done (1 left) 2022-04-28T19:50:07.824 INFO:tasks.rados.rados.0.smithi059.stdout:1820: copy_from oid 25 from oid 40 current snap is 168 2022-04-28T19:50:07.824 INFO:tasks.rados.rados.0.smithi059.stdout:1818: finishing write tid 1 to smithi05955964-45 2022-04-28T19:50:07.824 INFO:tasks.rados.rados.0.smithi059.stdout:1818: finishing write tid 3 to smithi05955964-45 2022-04-28T19:50:07.824 INFO:tasks.rados.rados.0.smithi059.stdout:1818: finishing write tid 4 to smithi05955964-45 2022-04-28T19:50:07.824 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 726 (ObjNum 590 snap 168 seq_num 590) dirty exists 2022-04-28T19:50:07.825 INFO:tasks.rados.rados.0.smithi059.stdout:1818: left oid 45 (ObjNum 590 snap 168 seq_num 590) 2022-04-28T19:50:07.825 INFO:tasks.rados.rados.0.smithi059.stdout:1818: done (1 left) 2022-04-28T19:50:07.825 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 168 2022-04-28T19:50:07.825 INFO:tasks.rados.rados.0.smithi059.stdout:1821: seq_num 591 ranges {2711552=303104} 2022-04-28T19:50:07.845 INFO:tasks.rados.rados.0.smithi059.stdout:1821: writing smithi05955964-32 from 2711552 to 3014656 tid 1 2022-04-28T19:50:07.845 INFO:tasks.rados.rados.0.smithi059.stdout:1822: snap_remove snap 164 2022-04-28T19:50:08.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:07 smithi139 bash[43301]: cephadm 2022-04-28T19:50:06.839968+0000 mgr.y (mgr.24961) 53 : cephadm [INF] Reconfiguring mds.foo.smithi059.tnizcj (monmap changed)... 2022-04-28T19:50:08.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:07 smithi139 bash[43301]: audit 2022-04-28T19:50:06.841483+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:08.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:07 smithi139 bash[43301]: cephadm 2022-04-28T19:50:06.842145+0000 mgr.y (mgr.24961) 54 : cephadm [INF] Reconfiguring daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:50:08.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:07 smithi139 bash[43301]: audit 2022-04-28T19:50:06.842558+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "container_image"}]: dispatch 2022-04-28T19:50:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:07 smithi139 bash[43301]: cluster 2022-04-28T19:50:06.864551+0000 mgr.y (mgr.24961) 55 : cluster [DBG] pgmap v68: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:07 smithi139 bash[43301]: cluster 2022-04-28T19:50:07.821274+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2022-04-28T19:50:08.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[59745]: cephadm 2022-04-28T19:50:06.839968+0000 mgr.y (mgr.24961) 53 : cephadm [INF] Reconfiguring mds.foo.smithi059.tnizcj (monmap changed)... 2022-04-28T19:50:08.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[59745]: audit 2022-04-28T19:50:06.841483+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:08.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[59745]: cephadm 2022-04-28T19:50:06.842145+0000 mgr.y (mgr.24961) 54 : cephadm [INF] Reconfiguring daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:50:08.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[59745]: audit 2022-04-28T19:50:06.842558+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "container_image"}]: dispatch 2022-04-28T19:50:08.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[59745]: cluster 2022-04-28T19:50:06.864551+0000 mgr.y (mgr.24961) 55 : cluster [DBG] pgmap v68: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:08.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[59745]: cluster 2022-04-28T19:50:07.821274+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2022-04-28T19:50:08.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[61152]: cephadm 2022-04-28T19:50:06.839968+0000 mgr.y (mgr.24961) 53 : cephadm [INF] Reconfiguring mds.foo.smithi059.tnizcj (monmap changed)... 2022-04-28T19:50:08.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[61152]: audit 2022-04-28T19:50:06.841483+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:08.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[61152]: cephadm 2022-04-28T19:50:06.842145+0000 mgr.y (mgr.24961) 54 : cephadm [INF] Reconfiguring daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:50:08.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[61152]: audit 2022-04-28T19:50:06.842558+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "container_image"}]: dispatch 2022-04-28T19:50:08.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[61152]: cluster 2022-04-28T19:50:06.864551+0000 mgr.y (mgr.24961) 55 : cluster [DBG] pgmap v68: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 239 MiB data, 394 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:08.125 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:07 smithi059 bash[61152]: cluster 2022-04-28T19:50:07.821274+0000 mon.a (mon.0) 171 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2022-04-28T19:50:08.856 INFO:tasks.rados.rados.0.smithi059.stdout:1822: done (2 left) 2022-04-28T19:50:08.856 INFO:tasks.rados.rados.0.smithi059.stdout:1823: delete oid 19 current snap is 168 2022-04-28T19:50:08.856 INFO:tasks.rados.rados.0.smithi059.stdout:1820: finishing copy_from to smithi05955964-25 2022-04-28T19:50:08.856 INFO:tasks.rados.rados.0.smithi059.stdout:1820: got expected ENOENT (src dne) 2022-04-28T19:50:08.856 INFO:tasks.rados.rados.0.smithi059.stdout:1820: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:50:08.857 INFO:tasks.rados.rados.0.smithi059.stdout:1821: finishing write tid 1 to smithi05955964-32 2022-04-28T19:50:08.857 INFO:tasks.rados.rados.0.smithi059.stdout:1821: finishing write tid 3 to smithi05955964-32 2022-04-28T19:50:08.857 INFO:tasks.rados.rados.0.smithi059.stdout:1821: finishing write tid 4 to smithi05955964-32 2022-04-28T19:50:08.857 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 833 (ObjNum 591 snap 168 seq_num 591) dirty exists 2022-04-28T19:50:08.857 INFO:tasks.rados.rados.0.smithi059.stdout:1821: left oid 32 (ObjNum 591 snap 168 seq_num 591) 2022-04-28T19:50:08.867 INFO:tasks.rados.rados.0.smithi059.stdout:1820: done (2 left) 2022-04-28T19:50:08.867 INFO:tasks.rados.rados.0.smithi059.stdout:1821: done (1 left) 2022-04-28T19:50:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:1823: done (0 left) 2022-04-28T19:50:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 168 2022-04-28T19:50:08.868 INFO:tasks.rados.rados.0.smithi059.stdout:1824: seq_num 593 ranges {0=335872} 2022-04-28T19:50:08.869 INFO:tasks.rados.rados.0.smithi059.stdout:1824: writing smithi05955964-23 from 0 to 335872 tid 1 2022-04-28T19:50:08.870 INFO:tasks.rados.rados.0.smithi059.stdout:1825: snap_create 2022-04-28T19:50:08.877 INFO:tasks.rados.rados.0.smithi059.stdout:1824: finishing write tid 1 to smithi05955964-23 2022-04-28T19:50:08.878 INFO:tasks.rados.rados.0.smithi059.stdout:1824: finishing write tid 3 to smithi05955964-23 2022-04-28T19:50:08.880 INFO:tasks.rados.rados.0.smithi059.stdout:1824: finishing write tid 4 to smithi05955964-23 2022-04-28T19:50:08.880 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 831 (ObjNum 593 snap 168 seq_num 593) dirty exists 2022-04-28T19:50:08.880 INFO:tasks.rados.rados.0.smithi059.stdout:1824: left oid 23 (ObjNum 593 snap 168 seq_num 593) 2022-04-28T19:50:09.869 INFO:tasks.rados.rados.0.smithi059.stdout:1824: done (1 left) 2022-04-28T19:50:09.869 INFO:tasks.rados.rados.0.smithi059.stdout:1825: done (0 left) 2022-04-28T19:50:09.869 INFO:tasks.rados.rados.0.smithi059.stdout:1826: read oid 33 snap -1 2022-04-28T19:50:09.870 INFO:tasks.rados.rados.0.smithi059.stdout:1826: expect (ObjNum 494 snap 141 seq_num 494) 2022-04-28T19:50:09.870 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 169 2022-04-28T19:50:09.870 INFO:tasks.rados.rados.0.smithi059.stdout:1827: seq_num 594 ranges {1974272=352256} 2022-04-28T19:50:09.882 INFO:tasks.rados.rados.0.smithi059.stdout:1827: writing smithi05955964-28 from 1974272 to 2326528 tid 1 2022-04-28T19:50:09.883 INFO:tasks.rados.rados.0.smithi059.stdout:1828: delete oid 36 current snap is 169 2022-04-28T19:50:09.888 INFO:tasks.rados.rados.0.smithi059.stdout:1828: done (2 left) 2022-04-28T19:50:09.888 INFO:tasks.rados.rados.0.smithi059.stdout:1829: copy_from oid 26 from oid 43 current snap is 169 2022-04-28T19:50:09.888 INFO:tasks.rados.rados.0.smithi059.stdout:1830: read oid 30 snap -1 2022-04-28T19:50:09.888 INFO:tasks.rados.rados.0.smithi059.stdout:1830: expect (ObjNum 529 snap 150 seq_num 529) 2022-04-28T19:50:09.888 INFO:tasks.rados.rados.0.smithi059.stdout:1831: rmattr oid 17 current snap is 169 2022-04-28T19:50:09.888 INFO:tasks.rados.rados.0.smithi059.stdout:1832: delete oid 34 current snap is 169 2022-04-28T19:50:09.895 INFO:tasks.rados.rados.0.smithi059.stdout:1827: finishing write tid 1 to smithi05955964-28 2022-04-28T19:50:09.896 INFO:tasks.rados.rados.0.smithi059.stdout:1827: finishing write tid 3 to smithi05955964-28 2022-04-28T19:50:09.896 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 418 (ObjNum 534 snap 150 seq_num 534) dirty exists 2022-04-28T19:50:09.904 INFO:tasks.rados.rados.0.smithi059.stdout:1827: finishing write tid 4 to smithi05955964-28 2022-04-28T19:50:09.904 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 697 (ObjNum 594 snap 169 seq_num 594) dirty exists 2022-04-28T19:50:09.904 INFO:tasks.rados.rados.0.smithi059.stdout:1827: left oid 28 (ObjNum 594 snap 169 seq_num 594) 2022-04-28T19:50:09.907 INFO:tasks.rados.rados.0.smithi059.stdout:1826: expect (ObjNum 494 snap 141 seq_num 494) 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: cluster 2022-04-28T19:50:08.854843+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: cluster 2022-04-28T19:50:08.865087+0000 mgr.y (mgr.24961) 56 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 243 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: audit 2022-04-28T19:50:08.899958+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: cephadm 2022-04-28T19:50:08.900961+0000 mgr.y (mgr.24961) 57 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: audit 2022-04-28T19:50:08.901331+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: audit 2022-04-28T19:50:08.902336+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:50:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: audit 2022-04-28T19:50:08.903274+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: cephadm 2022-04-28T19:50:08.904135+0000 mgr.y (mgr.24961) 58 : cephadm [INF] Reconfiguring daemon mon.b on smithi139 2022-04-28T19:50:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:09 smithi139 bash[43301]: audit 2022-04-28T19:50:08.904696+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "container_image"}]: dispatch 2022-04-28T19:50:10.276 INFO:tasks.rados.rados.0.smithi059.stdout:1830: expect (ObjNum 529 snap 150 seq_num 529) 2022-04-28T19:50:10.313 INFO:tasks.rados.rados.0.smithi059.stdout:1829: finishing copy_from to smithi05955964-26 2022-04-28T19:50:10.313 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 830 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1829: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1826: done (5 left) 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1827: done (4 left) 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1829: done (3 left) 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1830: done (2 left) 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1831: done (1 left) 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1832: done (0 left) 2022-04-28T19:50:10.314 INFO:tasks.rados.rados.0.smithi059.stdout:1833: snap_remove snap 156 2022-04-28T19:50:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: cluster 2022-04-28T19:50:08.854843+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2022-04-28T19:50:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: cluster 2022-04-28T19:50:08.865087+0000 mgr.y (mgr.24961) 56 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 243 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2022-04-28T19:50:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: audit 2022-04-28T19:50:08.899958+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: cephadm 2022-04-28T19:50:08.900961+0000 mgr.y (mgr.24961) 57 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-04-28T19:50:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: audit 2022-04-28T19:50:08.901331+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: audit 2022-04-28T19:50:08.902336+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: audit 2022-04-28T19:50:08.903274+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: cephadm 2022-04-28T19:50:08.904135+0000 mgr.y (mgr.24961) 58 : cephadm [INF] Reconfiguring daemon mon.b on smithi139 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[61152]: audit 2022-04-28T19:50:08.904696+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "container_image"}]: dispatch 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: cluster 2022-04-28T19:50:08.854843+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: cluster 2022-04-28T19:50:08.865087+0000 mgr.y (mgr.24961) 56 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 243 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2022-04-28T19:50:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: audit 2022-04-28T19:50:08.899958+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: cephadm 2022-04-28T19:50:08.900961+0000 mgr.y (mgr.24961) 57 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2022-04-28T19:50:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: audit 2022-04-28T19:50:08.901331+0000 mon.a (mon.0) 174 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2022-04-28T19:50:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: audit 2022-04-28T19:50:08.902336+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2022-04-28T19:50:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: audit 2022-04-28T19:50:08.903274+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: cephadm 2022-04-28T19:50:08.904135+0000 mgr.y (mgr.24961) 58 : cephadm [INF] Reconfiguring daemon mon.b on smithi139 2022-04-28T19:50:10.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:09 smithi059 bash[59745]: audit 2022-04-28T19:50:08.904696+0000 mon.a (mon.0) 177 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "container_image"}]: dispatch 2022-04-28T19:50:10.869 INFO:tasks.rados.rados.0.smithi059.stdout:1833: done (0 left) 2022-04-28T19:50:10.869 INFO:tasks.rados.rados.0.smithi059.stdout:1834: read oid 9 snap -1 2022-04-28T19:50:10.869 INFO:tasks.rados.rados.0.smithi059.stdout:1834: expect (ObjNum 572 snap 162 seq_num 572) 2022-04-28T19:50:10.869 INFO:tasks.rados.rados.0.smithi059.stdout:1835: read oid 28 snap 168 2022-04-28T19:50:10.869 INFO:tasks.rados.rados.0.smithi059.stdout:1835: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:50:10.869 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 169 2022-04-28T19:50:10.870 INFO:tasks.rados.rados.0.smithi059.stdout:1836: seq_num 597 ranges {0=401408} 2022-04-28T19:50:10.873 INFO:tasks.rados.rados.0.smithi059.stdout:1836: writing smithi05955964-16 from 0 to 401408 tid 1 2022-04-28T19:50:10.873 INFO:tasks.rados.rados.0.smithi059.stdout:1837: delete oid 2 current snap is 169 2022-04-28T19:50:10.897 INFO:tasks.rados.rados.0.smithi059.stdout:1837: done (3 left) 2022-04-28T19:50:10.897 INFO:tasks.rados.rados.0.smithi059.stdout:1838: rollback oid 42 current snap is 169 2022-04-28T19:50:10.897 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 163 2022-04-28T19:50:10.897 INFO:tasks.rados.rados.0.smithi059.stdout:1839: delete oid 44 current snap is 169 2022-04-28T19:50:10.897 INFO:tasks.rados.rados.0.smithi059.stdout:1835: expect (ObjNum 404 snap 124 seq_num 404) 2022-04-28T19:50:10.946 INFO:tasks.rados.rados.0.smithi059.stdout:1836: finishing write tid 1 to smithi05955964-16 2022-04-28T19:50:10.946 INFO:tasks.rados.rados.0.smithi059.stdout:1836: finishing write tid 3 to smithi05955964-16 2022-04-28T19:50:10.946 INFO:tasks.rados.rados.0.smithi059.stdout:1836: finishing write tid 4 to smithi05955964-16 2022-04-28T19:50:10.946 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 699 (ObjNum 597 snap 169 seq_num 597) dirty exists 2022-04-28T19:50:10.946 INFO:tasks.rados.rados.0.smithi059.stdout:1836: left oid 16 (ObjNum 597 snap 169 seq_num 597) 2022-04-28T19:50:10.947 INFO:tasks.rados.rados.0.smithi059.stdout:1834: expect (ObjNum 572 snap 162 seq_num 572) 2022-04-28T19:50:10.999 INFO:tasks.rados.rados.0.smithi059.stdout:1838: finishing rollback tid 0 to smithi05955964-42 2022-04-28T19:50:10.999 INFO:tasks.rados.rados.0.smithi059.stdout:1838: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1838: finishing rollback tid 2 to smithi05955964-42 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 816 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1834: done (4 left) 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1835: done (3 left) 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1836: done (2 left) 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1838: done (1 left) 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1839: done (0 left) 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 169 2022-04-28T19:50:11.000 INFO:tasks.rados.rados.0.smithi059.stdout:1840: seq_num 600 ranges {2023424=450560} 2022-04-28T19:50:11.019 INFO:tasks.rados.rados.0.smithi059.stdout:1840: writing smithi05955964-29 from 2023424 to 2473984 tid 1 2022-04-28T19:50:11.020 INFO:tasks.rados.rados.0.smithi059.stdout:1841: snap_remove snap 154 2022-04-28T19:50:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:10 smithi059 bash[59745]: cluster 2022-04-28T19:50:09.868380+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2022-04-28T19:50:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:10 smithi059 bash[61152]: cluster 2022-04-28T19:50:09.868380+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2022-04-28T19:50:11.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:10 smithi139 bash[43301]: cluster 2022-04-28T19:50:09.868380+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2022-04-28T19:50:11.816 INFO:tasks.rados.rados.0.smithi059.stdout:1841: done (1 left) 2022-04-28T19:50:11.874 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 169 2022-04-28T19:50:11.875 INFO:tasks.rados.rados.0.smithi059.stdout:1842: seq_num 601 ranges {0=475136} 2022-04-28T19:50:11.876 INFO:tasks.rados.rados.0.smithi059.stdout:1842: writing smithi05955964-19 from 0 to 475136 tid 1 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1843: read oid 13 snap -1 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1843: expect (ObjNum 585 snap 166 seq_num 585) 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 169 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1844: seq_num 602 ranges {8273920=491520} 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1844: writing smithi05955964-1 from 8273920 to 8765440 tid 1 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1845: setattr oid 42 current snap is 169 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1840: finishing write tid 1 to smithi05955964-29 2022-04-28T19:50:11.877 INFO:tasks.rados.rados.0.smithi059.stdout:1840: finishing write tid 3 to smithi05955964-29 2022-04-28T19:50:11.878 INFO:tasks.rados.rados.0.smithi059.stdout:1840: finishing write tid 4 to smithi05955964-29 2022-04-28T19:50:11.878 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 806 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:50:11.878 INFO:tasks.rados.rados.0.smithi059.stdout:1840: left oid 29 (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:50:11.878 INFO:tasks.rados.rados.0.smithi059.stdout:1842: finishing write tid 1 to smithi05955964-19 2022-04-28T19:50:11.878 INFO:tasks.rados.rados.0.smithi059.stdout:1842: finishing write tid 3 to smithi05955964-19 2022-04-28T19:50:11.878 INFO:tasks.rados.rados.0.smithi059.stdout:1842: finishing write tid 4 to smithi05955964-19 2022-04-28T19:50:11.879 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 845 (ObjNum 601 snap 169 seq_num 601) dirty exists 2022-04-28T19:50:11.879 INFO:tasks.rados.rados.0.smithi059.stdout:1842: left oid 19 (ObjNum 601 snap 169 seq_num 601) 2022-04-28T19:50:11.879 INFO:tasks.rados.rados.0.smithi059.stdout:1843: expect (ObjNum 585 snap 166 seq_num 585) 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1844: finishing write tid 1 to smithi05955964-1 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1844: finishing write tid 3 to smithi05955964-1 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1844: finishing write tid 4 to smithi05955964-1 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 770 (ObjNum 602 snap 169 seq_num 602) dirty exists 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1844: left oid 1 (ObjNum 602 snap 169 seq_num 602) 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1840: done (4 left) 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1842: done (3 left) 2022-04-28T19:50:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:1843: done (2 left) 2022-04-28T19:50:11.993 INFO:tasks.rados.rados.0.smithi059.stdout:1844: done (1 left) 2022-04-28T19:50:11.993 INFO:tasks.rados.rados.0.smithi059.stdout:1846: snap_remove snap 163 2022-04-28T19:50:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: cluster 2022-04-28T19:50:10.865758+0000 mgr.y (mgr.24961) 59 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 243 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2022-04-28T19:50:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: cluster 2022-04-28T19:50:10.866618+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2022-04-28T19:50:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: audit 2022-04-28T19:50:10.920733+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: cephadm 2022-04-28T19:50:10.921588+0000 mgr.y (mgr.24961) 60 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2022-04-28T19:50:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: audit 2022-04-28T19:50:10.921883+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:50:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: audit 2022-04-28T19:50:10.922689+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:50:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: audit 2022-04-28T19:50:10.923363+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:12.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: cephadm 2022-04-28T19:50:10.924042+0000 mgr.y (mgr.24961) 61 : cephadm [INF] Reconfiguring daemon mgr.x on smithi139 2022-04-28T19:50:12.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: audit 2022-04-28T19:50:10.924532+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.x","key": "container_image"}]: dispatch 2022-04-28T19:50:12.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[61152]: cluster 2022-04-28T19:50:11.812450+0000 mon.a (mon.0) 185 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2022-04-28T19:50:12.094 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:11.627Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:12.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:11.627Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:12.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:11.627Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:12.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: cluster 2022-04-28T19:50:10.865758+0000 mgr.y (mgr.24961) 59 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 243 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2022-04-28T19:50:12.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: cluster 2022-04-28T19:50:10.866618+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2022-04-28T19:50:12.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: audit 2022-04-28T19:50:10.920733+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:12.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: cephadm 2022-04-28T19:50:10.921588+0000 mgr.y (mgr.24961) 60 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2022-04-28T19:50:12.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: audit 2022-04-28T19:50:10.921883+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:50:12.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: audit 2022-04-28T19:50:10.922689+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:50:12.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: audit 2022-04-28T19:50:10.923363+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:12.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: cephadm 2022-04-28T19:50:10.924042+0000 mgr.y (mgr.24961) 61 : cephadm [INF] Reconfiguring daemon mgr.x on smithi139 2022-04-28T19:50:12.098 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: audit 2022-04-28T19:50:10.924532+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.x","key": "container_image"}]: dispatch 2022-04-28T19:50:12.098 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:11 smithi059 bash[59745]: cluster 2022-04-28T19:50:11.812450+0000 mon.a (mon.0) 185 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2022-04-28T19:50:12.175 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: cluster 2022-04-28T19:50:10.865758+0000 mgr.y (mgr.24961) 59 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 243 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 46 op/s 2022-04-28T19:50:12.175 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: cluster 2022-04-28T19:50:10.866618+0000 mon.a (mon.0) 179 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2022-04-28T19:50:12.175 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: audit 2022-04-28T19:50:10.920733+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:12.175 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: cephadm 2022-04-28T19:50:10.921588+0000 mgr.y (mgr.24961) 60 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2022-04-28T19:50:12.175 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: audit 2022-04-28T19:50:10.921883+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2022-04-28T19:50:12.176 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: audit 2022-04-28T19:50:10.922689+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2022-04-28T19:50:12.176 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: audit 2022-04-28T19:50:10.923363+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:12.176 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: cephadm 2022-04-28T19:50:10.924042+0000 mgr.y (mgr.24961) 61 : cephadm [INF] Reconfiguring daemon mgr.x on smithi139 2022-04-28T19:50:12.176 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: audit 2022-04-28T19:50:10.924532+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mgr.x","key": "container_image"}]: dispatch 2022-04-28T19:50:12.176 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:11 smithi139 bash[43301]: cluster 2022-04-28T19:50:11.812450+0000 mon.a (mon.0) 185 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2022-04-28T19:50:12.882 INFO:tasks.rados.rados.0.smithi059.stdout:1846: done (1 left) 2022-04-28T19:50:12.882 INFO:tasks.rados.rados.0.smithi059.stdout:1847: copy_from oid 49 from oid 5 current snap is 169 2022-04-28T19:50:12.882 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 169 2022-04-28T19:50:12.882 INFO:tasks.rados.rados.0.smithi059.stdout:1848: seq_num 603 ranges {8019968=507904} 2022-04-28T19:50:12.932 INFO:tasks.rados.rados.0.smithi059.stdout:1848: writing smithi05955964-31 from 8019968 to 8527872 tid 1 2022-04-28T19:50:12.932 INFO:tasks.rados.rados.0.smithi059.stdout:1849: copy_from oid 27 from oid 26 current snap is 169 2022-04-28T19:50:12.932 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 823 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:50:12.933 INFO:tasks.rados.rados.0.smithi059.stdout:1845: done (3 left) 2022-04-28T19:50:12.933 INFO:tasks.rados.rados.0.smithi059.stdout:1850: delete oid 28 current snap is 169 2022-04-28T19:50:12.933 INFO:tasks.rados.rados.0.smithi059.stdout:1847: finishing copy_from to smithi05955964-49 2022-04-28T19:50:12.933 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 792 (ObjNum 574 snap 164 seq_num 574) dirty exists 2022-04-28T19:50:12.933 INFO:tasks.rados.rados.0.smithi059.stdout:1847: finishing copy_from racing read to smithi05955964-49 2022-04-28T19:50:12.935 INFO:tasks.rados.rados.0.smithi059.stdout:1847: done (3 left) 2022-04-28T19:50:12.936 INFO:tasks.rados.rados.0.smithi059.stdout:1850: done (2 left) 2022-04-28T19:50:12.936 INFO:tasks.rados.rados.0.smithi059.stdout:1851: copy_from oid 47 from oid 14 current snap is 169 2022-04-28T19:50:12.936 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 169 2022-04-28T19:50:12.936 INFO:tasks.rados.rados.0.smithi059.stdout:1852: seq_num 605 ranges {2957312=540672} 2022-04-28T19:50:12.956 INFO:tasks.rados.rados.0.smithi059.stdout:1852: writing smithi05955964-25 from 2957312 to 3497984 tid 1 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1853: delete oid 30 current snap is 169 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1848: finishing write tid 1 to smithi05955964-31 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1848: finishing write tid 3 to smithi05955964-31 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1848: finishing write tid 4 to smithi05955964-31 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 785 (ObjNum 603 snap 169 seq_num 603) dirty exists 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1848: left oid 31 (ObjNum 603 snap 169 seq_num 603) 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1849: finishing copy_from to smithi05955964-27 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 830 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:50:12.957 INFO:tasks.rados.rados.0.smithi059.stdout:1849: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:50:12.958 INFO:tasks.rados.rados.0.smithi059.stdout:1851: finishing copy_from to smithi05955964-47 2022-04-28T19:50:12.958 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 826 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:12.958 INFO:tasks.rados.rados.0.smithi059.stdout:1851: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:50:12.958 INFO:tasks.rados.rados.0.smithi059.stdout:1848: done (4 left) 2022-04-28T19:50:12.959 INFO:tasks.rados.rados.0.smithi059.stdout:1849: done (3 left) 2022-04-28T19:50:12.959 INFO:tasks.rados.rados.0.smithi059.stdout:1851: done (2 left) 2022-04-28T19:50:12.959 INFO:tasks.rados.rados.0.smithi059.stdout:1853: done (1 left) 2022-04-28T19:50:12.959 INFO:tasks.rados.rados.0.smithi059.stdout:1854: snap_remove snap 153 2022-04-28T19:50:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:13.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:13.520Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:13.520Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:50:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:13.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:13.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:13.525Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:50:13.892 INFO:tasks.rados.rados.0.smithi059.stdout:1854: done (1 left) 2022-04-28T19:50:13.892 INFO:tasks.rados.rados.0.smithi059.stdout:1855: read oid 11 snap -1 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:1855: expect (ObjNum 554 snap 157 seq_num 554) 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:1856: snap_create 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:1852: finishing write tid 1 to smithi05955964-25 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:1852: finishing write tid 3 to smithi05955964-25 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:1852: finishing write tid 4 to smithi05955964-25 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 821 (ObjNum 605 snap 169 seq_num 605) dirty exists 2022-04-28T19:50:13.893 INFO:tasks.rados.rados.0.smithi059.stdout:1852: left oid 25 (ObjNum 605 snap 169 seq_num 605) 2022-04-28T19:50:13.941 INFO:tasks.rados.rados.0.smithi059.stdout:1855: expect (ObjNum 554 snap 157 seq_num 554) 2022-04-28T19:50:14.259 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:13 smithi139 bash[43301]: cluster 2022-04-28T19:50:12.867132+0000 mgr.y (mgr.24961) 62 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2022-04-28T19:50:14.259 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:13 smithi139 bash[43301]: cluster 2022-04-28T19:50:12.879646+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2022-04-28T19:50:14.259 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:13 smithi139 bash[43301]: audit 2022-04-28T19:50:12.975500+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:14.259 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:13 smithi139 bash[43301]: cephadm 2022-04-28T19:50:12.976552+0000 mgr.y (mgr.24961) 63 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2022-04-28T19:50:14.259 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:13 smithi139 bash[43301]: cephadm 2022-04-28T19:50:12.977630+0000 mgr.y (mgr.24961) 64 : cephadm [INF] Reconfiguring daemon osd.4 on smithi139 2022-04-28T19:50:14.259 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:13 smithi139 bash[43301]: audit 2022-04-28T19:50:12.978239+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "container_image"}]: dispatch 2022-04-28T19:50:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[61152]: cluster 2022-04-28T19:50:12.867132+0000 mgr.y (mgr.24961) 62 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2022-04-28T19:50:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[61152]: cluster 2022-04-28T19:50:12.879646+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2022-04-28T19:50:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[61152]: audit 2022-04-28T19:50:12.975500+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[61152]: cephadm 2022-04-28T19:50:12.976552+0000 mgr.y (mgr.24961) 63 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[61152]: cephadm 2022-04-28T19:50:12.977630+0000 mgr.y (mgr.24961) 64 : cephadm [INF] Reconfiguring daemon osd.4 on smithi139 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[61152]: audit 2022-04-28T19:50:12.978239+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "container_image"}]: dispatch 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[59745]: cluster 2022-04-28T19:50:12.867132+0000 mgr.y (mgr.24961) 62 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[59745]: cluster 2022-04-28T19:50:12.879646+0000 mon.a (mon.0) 186 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[59745]: audit 2022-04-28T19:50:12.975500+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[59745]: cephadm 2022-04-28T19:50:12.976552+0000 mgr.y (mgr.24961) 63 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[59745]: cephadm 2022-04-28T19:50:12.977630+0000 mgr.y (mgr.24961) 64 : cephadm [INF] Reconfiguring daemon osd.4 on smithi139 2022-04-28T19:50:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:13 smithi059 bash[59745]: audit 2022-04-28T19:50:12.978239+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "container_image"}]: dispatch 2022-04-28T19:50:14.903 INFO:tasks.rados.rados.0.smithi059.stdout:1852: done (2 left) 2022-04-28T19:50:14.903 INFO:tasks.rados.rados.0.smithi059.stdout:1855: done (1 left) 2022-04-28T19:50:14.903 INFO:tasks.rados.rados.0.smithi059.stdout:1856: done (0 left) 2022-04-28T19:50:14.903 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 170 2022-04-28T19:50:14.903 INFO:tasks.rados.rados.0.smithi059.stdout:1857: seq_num 607 ranges {2392064=573440} 2022-04-28T19:50:14.919 INFO:tasks.rados.rados.0.smithi059.stdout:1857: writing smithi05955964-9 from 2392064 to 2965504 tid 1 2022-04-28T19:50:14.919 INFO:tasks.rados.rados.0.smithi059.stdout:1858: snap_remove snap 169 2022-04-28T19:50:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:14 smithi059 bash[61152]: cluster 2022-04-28T19:50:13.891031+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2022-04-28T19:50:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:14 smithi059 bash[61152]: audit 2022-04-28T19:50:14.880193+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:14 smithi059 bash[61152]: audit 2022-04-28T19:50:14.884447+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "container_image"}]: dispatch 2022-04-28T19:50:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:14 smithi059 bash[59745]: cluster 2022-04-28T19:50:13.891031+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2022-04-28T19:50:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:14 smithi059 bash[59745]: audit 2022-04-28T19:50:14.880193+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:14 smithi059 bash[59745]: audit 2022-04-28T19:50:14.884447+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "container_image"}]: dispatch 2022-04-28T19:50:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:14 smithi139 bash[43301]: cluster 2022-04-28T19:50:13.891031+0000 mon.a (mon.0) 189 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2022-04-28T19:50:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:14 smithi139 bash[43301]: audit 2022-04-28T19:50:14.880193+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:14 smithi139 bash[43301]: audit 2022-04-28T19:50:14.884447+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "container_image"}]: dispatch 2022-04-28T19:50:15.906 INFO:tasks.rados.rados.0.smithi059.stdout:1858: done (1 left) 2022-04-28T19:50:15.906 INFO:tasks.rados.rados.0.smithi059.stdout:1859: snap_remove snap 168 2022-04-28T19:50:16.183 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:15 smithi139 bash[43301]: cluster 2022-04-28T19:50:14.867913+0000 mgr.y (mgr.24961) 65 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2022-04-28T19:50:16.183 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:15 smithi139 bash[43301]: cephadm 2022-04-28T19:50:14.881750+0000 mgr.y (mgr.24961) 66 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2022-04-28T19:50:16.183 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:15 smithi139 bash[43301]: cephadm 2022-04-28T19:50:14.883424+0000 mgr.y (mgr.24961) 67 : cephadm [INF] Reconfiguring daemon osd.5 on smithi139 2022-04-28T19:50:16.183 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:15 smithi139 bash[43301]: cluster 2022-04-28T19:50:14.900480+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2022-04-28T19:50:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[61152]: cluster 2022-04-28T19:50:14.867913+0000 mgr.y (mgr.24961) 65 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2022-04-28T19:50:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[61152]: cephadm 2022-04-28T19:50:14.881750+0000 mgr.y (mgr.24961) 66 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2022-04-28T19:50:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[61152]: cephadm 2022-04-28T19:50:14.883424+0000 mgr.y (mgr.24961) 67 : cephadm [INF] Reconfiguring daemon osd.5 on smithi139 2022-04-28T19:50:16.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[61152]: cluster 2022-04-28T19:50:14.900480+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2022-04-28T19:50:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[59745]: cluster 2022-04-28T19:50:14.867913+0000 mgr.y (mgr.24961) 65 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail; 10 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2022-04-28T19:50:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[59745]: cephadm 2022-04-28T19:50:14.881750+0000 mgr.y (mgr.24961) 66 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2022-04-28T19:50:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[59745]: cephadm 2022-04-28T19:50:14.883424+0000 mgr.y (mgr.24961) 67 : cephadm [INF] Reconfiguring daemon osd.5 on smithi139 2022-04-28T19:50:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:15 smithi059 bash[59745]: cluster 2022-04-28T19:50:14.900480+0000 mon.a (mon.0) 192 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2022-04-28T19:50:16.821 INFO:tasks.rados.rados.0.smithi059.stdout:1859: done (1 left) 2022-04-28T19:50:16.821 INFO:tasks.rados.rados.0.smithi059.stdout:1860: copy_from oid 15 from oid 42 current snap is 170 2022-04-28T19:50:16.821 INFO:tasks.rados.rados.0.smithi059.stdout:1857: finishing write tid 1 to smithi05955964-9 2022-04-28T19:50:16.821 INFO:tasks.rados.rados.0.smithi059.stdout:1857: finishing write tid 3 to smithi05955964-9 2022-04-28T19:50:16.821 INFO:tasks.rados.rados.0.smithi059.stdout:1857: finishing write tid 4 to smithi05955964-9 2022-04-28T19:50:16.821 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 646 (ObjNum 607 snap 170 seq_num 607) dirty exists 2022-04-28T19:50:16.822 INFO:tasks.rados.rados.0.smithi059.stdout:1857: left oid 9 (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:16.822 INFO:tasks.rados.rados.0.smithi059.stdout:1857: done (1 left) 2022-04-28T19:50:16.822 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 170 2022-04-28T19:50:16.822 INFO:tasks.rados.rados.0.smithi059.stdout:1861: seq_num 608 ranges {6545408=589824} 2022-04-28T19:50:16.862 INFO:tasks.rados.rados.0.smithi059.stdout:1861: writing smithi05955964-6 from 6545408 to 7135232 tid 1 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1862: read oid 46 snap -1 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1862: expect (ObjNum 569 snap 162 seq_num 569) 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1863: read oid 40 snap -1 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1863: expect deleted 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1864: copy_from oid 36 from oid 38 current snap is 170 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1860: finishing copy_from to smithi05955964-15 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 823 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:50:16.863 INFO:tasks.rados.rados.0.smithi059.stdout:1860: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:50:16.864 INFO:tasks.rados.rados.0.smithi059.stdout:1860: done (4 left) 2022-04-28T19:50:16.864 INFO:tasks.rados.rados.0.smithi059.stdout:1865: setattr oid 27 current snap is 170 2022-04-28T19:50:16.865 INFO:tasks.rados.rados.0.smithi059.stdout:1863: done (4 left) 2022-04-28T19:50:16.865 INFO:tasks.rados.rados.0.smithi059.stdout:1866: read oid 7 snap -1 2022-04-28T19:50:16.865 INFO:tasks.rados.rados.0.smithi059.stdout:1866: expect deleted 2022-04-28T19:50:16.865 INFO:tasks.rados.rados.0.smithi059.stdout:1867: read oid 12 snap -1 2022-04-28T19:50:16.865 INFO:tasks.rados.rados.0.smithi059.stdout:1867: expect deleted 2022-04-28T19:50:16.865 INFO:tasks.rados.rados.0.smithi059.stdout:1868: snap_create 2022-04-28T19:50:16.868 INFO:tasks.rados.rados.0.smithi059.stdout:1861: finishing write tid 1 to smithi05955964-6 2022-04-28T19:50:16.870 INFO:tasks.rados.rados.0.smithi059.stdout:1861: finishing write tid 3 to smithi05955964-6 2022-04-28T19:50:16.870 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 870 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:50:16.873 INFO:tasks.rados.rados.0.smithi059.stdout:1861: finishing write tid 4 to smithi05955964-6 2022-04-28T19:50:16.873 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 757 (ObjNum 608 snap 170 seq_num 608) dirty exists 2022-04-28T19:50:16.874 INFO:tasks.rados.rados.0.smithi059.stdout:1861: left oid 6 (ObjNum 608 snap 170 seq_num 608) 2022-04-28T19:50:16.875 INFO:tasks.rados.rados.0.smithi059.stdout:1864: finishing copy_from to smithi05955964-36 2022-04-28T19:50:16.875 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 712 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:50:16.876 INFO:tasks.rados.rados.0.smithi059.stdout:1864: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:50:16.919 INFO:tasks.rados.rados.0.smithi059.stdout:1862: expect (ObjNum 569 snap 162 seq_num 569) 2022-04-28T19:50:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[61152]: cluster 2022-04-28T19:50:15.904528+0000 mon.a (mon.0) 193 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2022-04-28T19:50:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[61152]: cluster 2022-04-28T19:50:16.818566+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[61152]: audit 2022-04-28T19:50:16.880697+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[61152]: audit 2022-04-28T19:50:16.880987+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[61152]: audit 2022-04-28T19:50:16.881238+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[59745]: cluster 2022-04-28T19:50:15.904528+0000 mon.a (mon.0) 193 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[59745]: cluster 2022-04-28T19:50:16.818566+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[59745]: audit 2022-04-28T19:50:16.880697+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:50:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[59745]: audit 2022-04-28T19:50:16.880987+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T19:50:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:16 smithi059 bash[59745]: audit 2022-04-28T19:50:16.881238+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T19:50:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:16 smithi139 bash[43301]: cluster 2022-04-28T19:50:15.904528+0000 mon.a (mon.0) 193 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2022-04-28T19:50:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:16 smithi139 bash[43301]: cluster 2022-04-28T19:50:16.818566+0000 mon.a (mon.0) 194 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2022-04-28T19:50:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:16 smithi139 bash[43301]: audit 2022-04-28T19:50:16.880697+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:50:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:16 smithi139 bash[43301]: audit 2022-04-28T19:50:16.880987+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T19:50:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:16 smithi139 bash[43301]: audit 2022-04-28T19:50:16.881238+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T19:50:17.833 INFO:tasks.rados.rados.0.smithi059.stdout:1861: done (6 left) 2022-04-28T19:50:17.881 INFO:tasks.rados.rados.0.smithi059.stdout:1862: done (5 left) 2022-04-28T19:50:17.885 INFO:tasks.rados.rados.0.smithi059.stdout:1864: done (4 left) 2022-04-28T19:50:17.885 INFO:tasks.rados.rados.0.smithi059.stdout:1865: done (3 left) 2022-04-28T19:50:17.885 INFO:tasks.rados.rados.0.smithi059.stdout:1866: done (2 left) 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1867: done (1 left) 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1868: done (0 left) 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1869: copy_from oid 43 from oid 29 current snap is 171 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1870: delete oid 37 current snap is 171 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1870: done (1 left) 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1871: read oid 5 snap -1 2022-04-28T19:50:17.886 INFO:tasks.rados.rados.0.smithi059.stdout:1871: expect (ObjNum 574 snap 164 seq_num 574) 2022-04-28T19:50:17.887 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 171 2022-04-28T19:50:17.887 INFO:tasks.rados.rados.0.smithi059.stdout:1872: seq_num 610 ranges {6266880=622592} 2022-04-28T19:50:17.887 INFO:tasks.rados.rados.0.smithi059.stdout:1872: writing smithi05955964-41 from 6266880 to 6889472 tid 1 2022-04-28T19:50:17.888 INFO:tasks.rados.rados.0.smithi059.stdout:1873: rollback oid 44 current snap is 171 2022-04-28T19:50:17.888 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 44 to 159 2022-04-28T19:50:17.888 INFO:tasks.rados.rados.0.smithi059.stdout:1871: expect (ObjNum 574 snap 164 seq_num 574) 2022-04-28T19:50:17.888 INFO:tasks.rados.rados.0.smithi059.stdout:1869: finishing copy_from to smithi05955964-43 2022-04-28T19:50:17.888 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 806 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:50:17.888 INFO:tasks.rados.rados.0.smithi059.stdout:1869: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:50:17.889 INFO:tasks.rados.rados.0.smithi059.stdout:1869: done (3 left) 2022-04-28T19:50:17.889 INFO:tasks.rados.rados.0.smithi059.stdout:1871: done (2 left) 2022-04-28T19:50:17.889 INFO:tasks.rados.rados.0.smithi059.stdout:1874: read oid 10 snap -1 2022-04-28T19:50:17.889 INFO:tasks.rados.rados.0.smithi059.stdout:1874: expect (ObjNum 583 snap 166 seq_num 583) 2022-04-28T19:50:17.889 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 171 2022-04-28T19:50:17.889 INFO:tasks.rados.rados.0.smithi059.stdout:1875: seq_num 611 ranges {0=638976} 2022-04-28T19:50:17.890 INFO:tasks.rados.rados.0.smithi059.stdout:1875: writing smithi05955964-8 from 0 to 638976 tid 1 2022-04-28T19:50:17.890 INFO:tasks.rados.rados.0.smithi059.stdout:1876: copy_from oid 21 from oid 13 current snap is 171 2022-04-28T19:50:17.890 INFO:tasks.rados.rados.0.smithi059.stdout:1873: finishing rollback tid 1 to smithi05955964-44 2022-04-28T19:50:17.890 INFO:tasks.rados.rados.0.smithi059.stdout:1873: finishing rollback tid 2 to smithi05955964-44 2022-04-28T19:50:17.891 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 648 (ObjNum 507 snap 144 seq_num 507) dirty exists 2022-04-28T19:50:17.891 INFO:tasks.rados.rados.0.smithi059.stdout:1872: finishing write tid 1 to smithi05955964-41 2022-04-28T19:50:17.891 INFO:tasks.rados.rados.0.smithi059.stdout:1873: done (4 left) 2022-04-28T19:50:17.891 INFO:tasks.rados.rados.0.smithi059.stdout:1877: setattr oid 6 current snap is 171 2022-04-28T19:50:17.891 INFO:tasks.rados.rados.0.smithi059.stdout:1872: finishing write tid 3 to smithi05955964-41 2022-04-28T19:50:17.891 INFO:tasks.rados.rados.0.smithi059.stdout:1874: expect (ObjNum 583 snap 166 seq_num 583) 2022-04-28T19:50:18.055 INFO:tasks.rados.rados.0.smithi059.stdout:1872: finishing write tid 4 to smithi05955964-41 2022-04-28T19:50:18.055 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 624 (ObjNum 610 snap 171 seq_num 610) dirty exists 2022-04-28T19:50:18.055 INFO:tasks.rados.rados.0.smithi059.stdout:1872: left oid 41 (ObjNum 610 snap 171 seq_num 610) 2022-04-28T19:50:18.055 INFO:tasks.rados.rados.0.smithi059.stdout:1875: finishing write tid 1 to smithi05955964-8 2022-04-28T19:50:18.055 INFO:tasks.rados.rados.0.smithi059.stdout:1875: finishing write tid 3 to smithi05955964-8 2022-04-28T19:50:18.055 INFO:tasks.rados.rados.0.smithi059.stdout:1875: finishing write tid 4 to smithi05955964-8 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 833 (ObjNum 611 snap 171 seq_num 611) dirty exists 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1875: left oid 8 (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1876: finishing copy_from to smithi05955964-21 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 410 (ObjNum 585 snap 166 seq_num 585) dirty exists 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1876: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1872: done (4 left) 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1874: done (3 left) 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1875: done (2 left) 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1876: done (1 left) 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 171 2022-04-28T19:50:18.056 INFO:tasks.rados.rados.0.smithi059.stdout:1878: seq_num 612 ranges {8994816=655360} 2022-04-28T19:50:18.110 INFO:tasks.rados.rados.0.smithi059.stdout:1878: writing smithi05955964-35 from 8994816 to 9650176 tid 1 2022-04-28T19:50:18.110 INFO:tasks.rados.rados.0.smithi059.stdout:1879: snap_create 2022-04-28T19:50:18.110 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 759 (ObjNum 608 snap 170 seq_num 608) dirty exists 2022-04-28T19:50:18.119 INFO:tasks.rados.rados.0.smithi059.stdout:1878: finishing write tid 1 to smithi05955964-35 2022-04-28T19:50:18.119 INFO:tasks.rados.rados.0.smithi059.stdout:1878: finishing write tid 3 to smithi05955964-35 2022-04-28T19:50:18.121 INFO:tasks.rados.rados.0.smithi059.stdout:1878: finishing write tid 4 to smithi05955964-35 2022-04-28T19:50:18.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 729 (ObjNum 612 snap 171 seq_num 612) dirty exists 2022-04-28T19:50:18.121 INFO:tasks.rados.rados.0.smithi059.stdout:1878: left oid 35 (ObjNum 612 snap 171 seq_num 612) 2022-04-28T19:50:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: cluster 2022-04-28T19:50:16.868467+0000 mgr.y (mgr.24961) 68 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: audit 2022-04-28T19:50:16.967238+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: cephadm 2022-04-28T19:50:16.969005+0000 mgr.y (mgr.24961) 69 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2022-04-28T19:50:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: cephadm 2022-04-28T19:50:16.970732+0000 mgr.y (mgr.24961) 70 : cephadm [INF] Reconfiguring daemon osd.6 on smithi139 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: audit 2022-04-28T19:50:16.971649+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "container_image"}]: dispatch 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: audit 2022-04-28T19:50:17.110746+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: audit 2022-04-28T19:50:17.124999+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[61152]: cluster 2022-04-28T19:50:17.832360+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: cluster 2022-04-28T19:50:16.868467+0000 mgr.y (mgr.24961) 68 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: audit 2022-04-28T19:50:16.967238+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: cephadm 2022-04-28T19:50:16.969005+0000 mgr.y (mgr.24961) 69 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2022-04-28T19:50:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: cephadm 2022-04-28T19:50:16.970732+0000 mgr.y (mgr.24961) 70 : cephadm [INF] Reconfiguring daemon osd.6 on smithi139 2022-04-28T19:50:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: audit 2022-04-28T19:50:16.971649+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "container_image"}]: dispatch 2022-04-28T19:50:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: audit 2022-04-28T19:50:17.110746+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:50:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: audit 2022-04-28T19:50:17.124999+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:50:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:17 smithi059 bash[59745]: cluster 2022-04-28T19:50:17.832360+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2022-04-28T19:50:18.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: cluster 2022-04-28T19:50:16.868467+0000 mgr.y (mgr.24961) 68 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 246 MiB data, 405 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:18.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: audit 2022-04-28T19:50:16.967238+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:18.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: cephadm 2022-04-28T19:50:16.969005+0000 mgr.y (mgr.24961) 69 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2022-04-28T19:50:18.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: cephadm 2022-04-28T19:50:16.970732+0000 mgr.y (mgr.24961) 70 : cephadm [INF] Reconfiguring daemon osd.6 on smithi139 2022-04-28T19:50:18.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: audit 2022-04-28T19:50:16.971649+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "container_image"}]: dispatch 2022-04-28T19:50:18.351 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: audit 2022-04-28T19:50:17.110746+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:50:18.352 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: audit 2022-04-28T19:50:17.124999+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:50:18.352 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:17 smithi139 bash[43301]: cluster 2022-04-28T19:50:17.832360+0000 mon.a (mon.0) 202 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2022-04-28T19:50:18.985 INFO:tasks.rados.rados.0.smithi059.stdout:1877: done (2 left) 2022-04-28T19:50:18.985 INFO:tasks.rados.rados.0.smithi059.stdout:1878: done (1 left) 2022-04-28T19:50:18.986 INFO:tasks.rados.rados.0.smithi059.stdout:1879: done (0 left) 2022-04-28T19:50:18.986 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 172 2022-04-28T19:50:18.986 INFO:tasks.rados.rados.0.smithi059.stdout:1880: seq_num 613 ranges {5054464=671744} 2022-04-28T19:50:19.018 INFO:tasks.rados.rados.0.smithi059.stdout:1880: writing smithi05955964-42 from 5054464 to 5726208 tid 1 2022-04-28T19:50:19.018 INFO:tasks.rados.rados.0.smithi059.stdout:1881: snap_create 2022-04-28T19:50:19.029 INFO:tasks.rados.rados.0.smithi059.stdout:1880: finishing write tid 1 to smithi05955964-42 2022-04-28T19:50:19.030 INFO:tasks.rados.rados.0.smithi059.stdout:1880: finishing write tid 3 to smithi05955964-42 2022-04-28T19:50:19.032 INFO:tasks.rados.rados.0.smithi059.stdout:1880: finishing write tid 4 to smithi05955964-42 2022-04-28T19:50:19.032 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 850 (ObjNum 613 snap 172 seq_num 613) dirty exists 2022-04-28T19:50:19.032 INFO:tasks.rados.rados.0.smithi059.stdout:1880: left oid 42 (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:50:20.028 INFO:tasks.rados.rados.0.smithi059.stdout:1880: done (1 left) 2022-04-28T19:50:20.028 INFO:tasks.rados.rados.0.smithi059.stdout:1881: done (0 left) 2022-04-28T19:50:20.028 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 173 2022-04-28T19:50:20.028 INFO:tasks.rados.rados.0.smithi059.stdout:1882: seq_num 614 ranges {2392064=688128} 2022-04-28T19:50:20.050 INFO:tasks.rados.rados.0.smithi059.stdout:1882: writing smithi05955964-18 from 2392064 to 3080192 tid 1 2022-04-28T19:50:20.050 INFO:tasks.rados.rados.0.smithi059.stdout:1883: snap_create 2022-04-28T19:50:20.061 INFO:tasks.rados.rados.0.smithi059.stdout:1882: finishing write tid 1 to smithi05955964-18 2022-04-28T19:50:20.061 INFO:tasks.rados.rados.0.smithi059.stdout:1882: finishing write tid 3 to smithi05955964-18 2022-04-28T19:50:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:1882: finishing write tid 4 to smithi05955964-18 2022-04-28T19:50:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 853 (ObjNum 614 snap 173 seq_num 614) dirty exists 2022-04-28T19:50:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:1882: left oid 18 (ObjNum 614 snap 173 seq_num 614) 2022-04-28T19:50:20.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:19 smithi139 bash[43301]: cluster 2022-04-28T19:50:18.869326+0000 mgr.y (mgr.24961) 71 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 231 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2022-04-28T19:50:20.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:19 smithi139 bash[43301]: cluster 2022-04-28T19:50:18.984399+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2022-04-28T19:50:20.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:19 smithi139 bash[43301]: audit 2022-04-28T19:50:19.019737+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:20.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:19 smithi139 bash[43301]: cephadm 2022-04-28T19:50:19.021135+0000 mgr.y (mgr.24961) 72 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2022-04-28T19:50:20.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:19 smithi139 bash[43301]: cephadm 2022-04-28T19:50:19.022665+0000 mgr.y (mgr.24961) 73 : cephadm [INF] Reconfiguring daemon osd.7 on smithi139 2022-04-28T19:50:20.318 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:19 smithi139 bash[43301]: audit 2022-04-28T19:50:19.023618+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "container_image"}]: dispatch 2022-04-28T19:50:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[61152]: cluster 2022-04-28T19:50:18.869326+0000 mgr.y (mgr.24961) 71 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 231 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2022-04-28T19:50:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[61152]: cluster 2022-04-28T19:50:18.984399+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2022-04-28T19:50:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[61152]: audit 2022-04-28T19:50:19.019737+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[61152]: cephadm 2022-04-28T19:50:19.021135+0000 mgr.y (mgr.24961) 72 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[61152]: cephadm 2022-04-28T19:50:19.022665+0000 mgr.y (mgr.24961) 73 : cephadm [INF] Reconfiguring daemon osd.7 on smithi139 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[61152]: audit 2022-04-28T19:50:19.023618+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "container_image"}]: dispatch 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[59745]: cluster 2022-04-28T19:50:18.869326+0000 mgr.y (mgr.24961) 71 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 231 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[59745]: cluster 2022-04-28T19:50:18.984399+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[59745]: audit 2022-04-28T19:50:19.019737+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[59745]: cephadm 2022-04-28T19:50:19.021135+0000 mgr.y (mgr.24961) 72 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2022-04-28T19:50:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[59745]: cephadm 2022-04-28T19:50:19.022665+0000 mgr.y (mgr.24961) 73 : cephadm [INF] Reconfiguring daemon osd.7 on smithi139 2022-04-28T19:50:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:19 smithi059 bash[59745]: audit 2022-04-28T19:50:19.023618+0000 mon.a (mon.0) 205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "container_image"}]: dispatch 2022-04-28T19:50:20.894 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:50:21.032 INFO:tasks.rados.rados.0.smithi059.stdout:1882: done (1 left) 2022-04-28T19:50:21.032 INFO:tasks.rados.rados.0.smithi059.stdout:1883: done (0 left) 2022-04-28T19:50:21.032 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 174 2022-04-28T19:50:21.032 INFO:tasks.rados.rados.0.smithi059.stdout:1884: seq_num 615 ranges {6807552=704512} 2022-04-28T19:50:21.074 INFO:tasks.rados.rados.0.smithi059.stdout:1884: writing smithi05955964-17 from 6807552 to 7512064 tid 1 2022-04-28T19:50:21.075 INFO:tasks.rados.rados.0.smithi059.stdout:1885: rollback oid 22 current snap is 174 2022-04-28T19:50:21.075 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 171 2022-04-28T19:50:21.075 INFO:tasks.rados.rados.0.smithi059.stdout:1886: delete oid 39 current snap is 174 2022-04-28T19:50:21.081 INFO:tasks.rados.rados.0.smithi059.stdout:1885: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:50:21.081 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 0 (ObjNum 288 snap 0 seq_num 161) dirty dne 2022-04-28T19:50:21.083 INFO:tasks.rados.rados.0.smithi059.stdout:1885: done (2 left) 2022-04-28T19:50:21.083 INFO:tasks.rados.rados.0.smithi059.stdout:1886: done (1 left) 2022-04-28T19:50:21.083 INFO:tasks.rados.rados.0.smithi059.stdout:1887: snap_remove snap 167 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (9m) 45s ago 9m 14.4M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (9m) 28s ago 9m 25.8M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (7m) 45s ago 7m 14.4M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (7m) 28s ago 7m 17.6M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (92s) 28s ago 13m 375M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (2m) 45s ago 15m 408M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (80s) 45s ago 15m 42.1M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:50:21.238 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (40s) 28s ago 13m 19.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (52s) 45s ago 14m 26.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (9m) 45s ago 9m 10.8M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (9m) 28s ago 9m 10.7M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (13m) 45s ago 13m 219M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 547b5580e073 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (12m) 45s ago 12m 159M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (12m) 45s ago 12m 230M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (11m) 45s ago 12m 176M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (11m) 28s ago 11m 196M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (11m) 28s ago 11m 214M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:50:21.239 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (10m) 28s ago 10m 162M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:50:21.240 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (10m) 28s ago 10m 138M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:50:21.240 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (2m) 28s ago 9m 36.6M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:50:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[61152]: cluster 2022-04-28T19:50:20.023523+0000 mon.a (mon.0) 206 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2022-04-28T19:50:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[61152]: audit 2022-04-28T19:50:20.948692+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[61152]: audit 2022-04-28T19:50:20.952232+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:50:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[61152]: audit 2022-04-28T19:50:20.953944+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[61152]: audit 2022-04-28T19:50:20.956328+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "container_image"}]: dispatch 2022-04-28T19:50:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[59745]: cluster 2022-04-28T19:50:20.023523+0000 mon.a (mon.0) 206 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2022-04-28T19:50:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[59745]: audit 2022-04-28T19:50:20.948692+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[59745]: audit 2022-04-28T19:50:20.952232+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:50:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[59745]: audit 2022-04-28T19:50:20.953944+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[59745]: audit 2022-04-28T19:50:20.956328+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "container_image"}]: dispatch 2022-04-28T19:50:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:21 smithi139 bash[43301]: cluster 2022-04-28T19:50:20.023523+0000 mon.a (mon.0) 206 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2022-04-28T19:50:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:21 smithi139 bash[43301]: audit 2022-04-28T19:50:20.948692+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:21 smithi139 bash[43301]: audit 2022-04-28T19:50:20.952232+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:50:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:21 smithi139 bash[43301]: audit 2022-04-28T19:50:20.953944+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:21 smithi139 bash[43301]: audit 2022-04-28T19:50:20.956328+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "container_image"}]: dispatch 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:50:21.609 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 10, 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 5 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:50:21.610 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:50:21.820 INFO:tasks.rados.rados.0.smithi059.stdout:1887: done (1 left) 2022-04-28T19:50:21.820 INFO:tasks.rados.rados.0.smithi059.stdout:1888: snap_create 2022-04-28T19:50:21.820 INFO:tasks.rados.rados.0.smithi059.stdout:1884: finishing write tid 1 to smithi05955964-17 2022-04-28T19:50:21.821 INFO:tasks.rados.rados.0.smithi059.stdout:1884: finishing write tid 3 to smithi05955964-17 2022-04-28T19:50:21.821 INFO:tasks.rados.rados.0.smithi059.stdout:1884: finishing write tid 4 to smithi05955964-17 2022-04-28T19:50:21.821 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 439 (ObjNum 615 snap 174 seq_num 615) dirty exists 2022-04-28T19:50:21.821 INFO:tasks.rados.rados.0.smithi059.stdout:1884: left oid 17 (ObjNum 615 snap 174 seq_num 615) 2022-04-28T19:50:22.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:21.627Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:22.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:21.627Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:22.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:21.627Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: cluster 2022-04-28T19:50:20.869759+0000 mgr.y (mgr.24961) 74 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 231 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2022-04-28T19:50:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:20.880683+0000 mgr.y (mgr.24961) 75 : audit [DBG] from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: cephadm 2022-04-28T19:50:20.951684+0000 mgr.y (mgr.24961) 76 : cephadm [INF] Reconfiguring mds.foo.smithi139.oefiza (monmap changed)... 2022-04-28T19:50:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: cephadm 2022-04-28T19:50:20.955411+0000 mgr.y (mgr.24961) 77 : cephadm [INF] Reconfiguring daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:50:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: cluster 2022-04-28T19:50:21.030235+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2022-04-28T19:50:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.206809+0000 mgr.y (mgr.24961) 78 : audit [DBG] from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.211652+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.212518+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.213233+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.213964+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.214674+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.215409+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.220127+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.220866+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.221672+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.222351+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.222984+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: audit 2022-04-28T19:50:21.607846+0000 mon.a (mon.0) 223 : audit [DBG] from='client.? 172.21.15.59:0/2271279420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[61152]: cluster 2022-04-28T19:50:21.817703+0000 mon.a (mon.0) 224 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: cluster 2022-04-28T19:50:20.869759+0000 mgr.y (mgr.24961) 74 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 231 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:20.880683+0000 mgr.y (mgr.24961) 75 : audit [DBG] from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: cephadm 2022-04-28T19:50:20.951684+0000 mgr.y (mgr.24961) 76 : cephadm [INF] Reconfiguring mds.foo.smithi139.oefiza (monmap changed)... 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: cephadm 2022-04-28T19:50:20.955411+0000 mgr.y (mgr.24961) 77 : cephadm [INF] Reconfiguring daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: cluster 2022-04-28T19:50:21.030235+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.206809+0000 mgr.y (mgr.24961) 78 : audit [DBG] from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.211652+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.212518+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.213233+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.213964+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.214674+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.215409+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.220127+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.220866+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.221672+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.222351+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.222984+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: audit 2022-04-28T19:50:21.607846+0000 mon.a (mon.0) 223 : audit [DBG] from='client.? 172.21.15.59:0/2271279420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:22.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:22 smithi059 bash[59745]: cluster 2022-04-28T19:50:21.817703+0000 mon.a (mon.0) 224 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2022-04-28T19:50:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: cluster 2022-04-28T19:50:20.869759+0000 mgr.y (mgr.24961) 74 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 231 MiB data, 383 MiB used, 707 GiB / 715 GiB avail; 13 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2022-04-28T19:50:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:20.880683+0000 mgr.y (mgr.24961) 75 : audit [DBG] from='client.44172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: cephadm 2022-04-28T19:50:20.951684+0000 mgr.y (mgr.24961) 76 : cephadm [INF] Reconfiguring mds.foo.smithi139.oefiza (monmap changed)... 2022-04-28T19:50:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: cephadm 2022-04-28T19:50:20.955411+0000 mgr.y (mgr.24961) 77 : cephadm [INF] Reconfiguring daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: cluster 2022-04-28T19:50:21.030235+0000 mon.a (mon.0) 211 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.206809+0000 mgr.y (mgr.24961) 78 : audit [DBG] from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.211652+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.212518+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.213233+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.213964+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.214674+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.215409+0000 mon.a (mon.0) 217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.220127+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.220866+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.221672+0000 mon.a (mon.0) 220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.222351+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.222984+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: audit 2022-04-28T19:50:21.607846+0000 mon.a (mon.0) 223 : audit [DBG] from='client.? 172.21.15.59:0/2271279420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:22.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:22 smithi139 bash[43301]: cluster 2022-04-28T19:50:21.817703+0000 mon.a (mon.0) 224 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2022-04-28T19:50:22.830 INFO:tasks.rados.rados.0.smithi059.stdout:1884: done (1 left) 2022-04-28T19:50:22.830 INFO:tasks.rados.rados.0.smithi059.stdout:1888: done (0 left) 2022-04-28T19:50:22.830 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 175 2022-04-28T19:50:22.831 INFO:tasks.rados.rados.0.smithi059.stdout:1889: seq_num 617 ranges {2498560=737280} 2022-04-28T19:50:22.854 INFO:tasks.rados.rados.0.smithi059.stdout:1889: writing smithi05955964-10 from 2498560 to 3235840 tid 1 2022-04-28T19:50:22.855 INFO:tasks.rados.rados.0.smithi059.stdout:1890: snap_create 2022-04-28T19:50:22.862 INFO:tasks.rados.rados.0.smithi059.stdout:1889: finishing write tid 1 to smithi05955964-10 2022-04-28T19:50:22.862 INFO:tasks.rados.rados.0.smithi059.stdout:1889: finishing write tid 3 to smithi05955964-10 2022-04-28T19:50:22.864 INFO:tasks.rados.rados.0.smithi059.stdout:1889: finishing write tid 4 to smithi05955964-10 2022-04-28T19:50:22.864 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 877 (ObjNum 617 snap 175 seq_num 617) dirty exists 2022-04-28T19:50:22.864 INFO:tasks.rados.rados.0.smithi059.stdout:1889: left oid 10 (ObjNum 617 snap 175 seq_num 617) 2022-04-28T19:50:23.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: cluster 2022-04-28T19:50:22.829062+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2022-04-28T19:50:23.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: cluster 2022-04-28T19:50:22.870152+0000 mgr.y (mgr.24961) 79 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2022-04-28T19:50:23.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: cluster 2022-04-28T19:50:22.829062+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2022-04-28T19:50:23.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: cluster 2022-04-28T19:50:22.870152+0000 mgr.y (mgr.24961) 79 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2022-04-28T19:50:23.837 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:23.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:23.837 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:23.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:23.838 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:23.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:23.838 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:23.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:23.838 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:23.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:23.838 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:23.525Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:23.842 INFO:tasks.rados.rados.0.smithi059.stdout:1889: done (1 left) 2022-04-28T19:50:23.842 INFO:tasks.rados.rados.0.smithi059.stdout:1890: done (0 left) 2022-04-28T19:50:23.842 INFO:tasks.rados.rados.0.smithi059.stdout:1891: rollback oid 27 current snap is 176 2022-04-28T19:50:23.842 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 174 2022-04-28T19:50:23.842 INFO:tasks.rados.rados.0.smithi059.stdout:1892: delete oid 39 current snap is 176 2022-04-28T19:50:23.850 INFO:tasks.rados.rados.0.smithi059.stdout:1892: done (1 left) 2022-04-28T19:50:23.850 INFO:tasks.rados.rados.0.smithi059.stdout:1893: rollback oid 5 current snap is 176 2022-04-28T19:50:23.851 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 170 2022-04-28T19:50:23.851 INFO:tasks.rados.rados.0.smithi059.stdout:1894: copy_from oid 6 from oid 24 current snap is 176 2022-04-28T19:50:23.851 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 176 2022-04-28T19:50:23.852 INFO:tasks.rados.rados.0.smithi059.stdout:1895: seq_num 619 ranges {7512064=770048} 2022-04-28T19:50:23.898 INFO:tasks.rados.rados.0.smithi059.stdout:1895: writing smithi05955964-17 from 7512064 to 8282112 tid 1 2022-04-28T19:50:23.898 INFO:tasks.rados.rados.0.smithi059.stdout:1896: snap_create 2022-04-28T19:50:23.898 INFO:tasks.rados.rados.0.smithi059.stdout:1891: finishing rollback tid 0 to smithi05955964-27 2022-04-28T19:50:23.898 INFO:tasks.rados.rados.0.smithi059.stdout:1891: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:50:23.898 INFO:tasks.rados.rados.0.smithi059.stdout:1891: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:50:23.898 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 881 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:1893: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:1893: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:1893: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 841 (ObjNum 574 snap 164 seq_num 574) dirty exists 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:1894: finishing copy_from to smithi05955964-6 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:1894: got expected ENOENT (src dne) 2022-04-28T19:50:23.899 INFO:tasks.rados.rados.0.smithi059.stdout:1894: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:50:23.903 INFO:tasks.rados.rados.0.smithi059.stdout:1895: finishing write tid 1 to smithi05955964-17 2022-04-28T19:50:23.903 INFO:tasks.rados.rados.0.smithi059.stdout:1895: finishing write tid 3 to smithi05955964-17 2022-04-28T19:50:23.904 INFO:tasks.rados.rados.0.smithi059.stdout:1895: finishing write tid 4 to smithi05955964-17 2022-04-28T19:50:23.905 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 444 (ObjNum 619 snap 176 seq_num 619) dirty exists 2022-04-28T19:50:23.905 INFO:tasks.rados.rados.0.smithi059.stdout:1895: left oid 17 (ObjNum 619 snap 176 seq_num 619) 2022-04-28T19:50:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.056216+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.059649+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.063770+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.066201+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: cephadm 2022-04-28T19:50:23.067701+0000 mgr.y (mgr.24961) 80 : cephadm [INF] Upgrade: Setting container_image for all mon 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.073763+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.075270+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.083270+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.084824+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.092812+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2022-04-28T19:50:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.094200+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.099870+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.101364+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: cephadm 2022-04-28T19:50:23.102684+0000 mgr.y (mgr.24961) 81 : cephadm [INF] Upgrade: Setting container_image for all crash 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.108770+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.111126+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: audit 2022-04-28T19:50:23.111400+0000 mgr.y (mgr.24961) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[59745]: cephadm 2022-04-28T19:50:23.113050+0000 mgr.y (mgr.24961) 83 : cephadm [INF] Upgrade: osd.0 is safe to restart 2022-04-28T19:50:24.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.056216+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.059649+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.063770+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.066201+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: cephadm 2022-04-28T19:50:23.067701+0000 mgr.y (mgr.24961) 80 : cephadm [INF] Upgrade: Setting container_image for all mon 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.073763+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.075270+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.083270+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.084824+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.092812+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2022-04-28T19:50:24.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.094200+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.099870+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.101364+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: cephadm 2022-04-28T19:50:23.102684+0000 mgr.y (mgr.24961) 81 : cephadm [INF] Upgrade: Setting container_image for all crash 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.108770+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.111126+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: audit 2022-04-28T19:50:23.111400+0000 mgr.y (mgr.24961) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-04-28T19:50:24.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:23 smithi059 bash[61152]: cephadm 2022-04-28T19:50:23.113050+0000 mgr.y (mgr.24961) 83 : cephadm [INF] Upgrade: osd.0 is safe to restart 2022-04-28T19:50:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: cluster 2022-04-28T19:50:22.829062+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2022-04-28T19:50:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: cluster 2022-04-28T19:50:22.870152+0000 mgr.y (mgr.24961) 79 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2022-04-28T19:50:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.056216+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.059649+0000 mon.a (mon.0) 227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.063770+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.066201+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: cephadm 2022-04-28T19:50:23.067701+0000 mgr.y (mgr.24961) 80 : cephadm [INF] Upgrade: Setting container_image for all mon 2022-04-28T19:50:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.073763+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.075270+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2022-04-28T19:50:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.083270+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2022-04-28T19:50:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.084824+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2022-04-28T19:50:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.092812+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2022-04-28T19:50:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.094200+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2022-04-28T19:50:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.099870+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2022-04-28T19:50:24.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.101364+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:24.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: cephadm 2022-04-28T19:50:23.102684+0000 mgr.y (mgr.24961) 81 : cephadm [INF] Upgrade: Setting container_image for all crash 2022-04-28T19:50:24.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.108770+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:24.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.111126+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-04-28T19:50:24.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: audit 2022-04-28T19:50:23.111400+0000 mgr.y (mgr.24961) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2022-04-28T19:50:24.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:23 smithi139 bash[43301]: cephadm 2022-04-28T19:50:23.113050+0000 mgr.y (mgr.24961) 83 : cephadm [INF] Upgrade: osd.0 is safe to restart 2022-04-28T19:50:24.843 INFO:tasks.rados.rados.0.smithi059.stdout:1891: done (4 left) 2022-04-28T19:50:24.843 INFO:tasks.rados.rados.0.smithi059.stdout:1893: done (3 left) 2022-04-28T19:50:24.843 INFO:tasks.rados.rados.0.smithi059.stdout:1894: done (2 left) 2022-04-28T19:50:24.843 INFO:tasks.rados.rados.0.smithi059.stdout:1895: done (1 left) 2022-04-28T19:50:24.843 INFO:tasks.rados.rados.0.smithi059.stdout:1896: done (0 left) 2022-04-28T19:50:24.843 INFO:tasks.rados.rados.0.smithi059.stdout:1897: snap_remove snap 174 2022-04-28T19:50:25.051 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:24 smithi059 bash[61152]: cluster 2022-04-28T19:50:23.841157+0000 mon.a (mon.0) 240 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2022-04-28T19:50:25.052 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:24 smithi059 bash[59745]: cluster 2022-04-28T19:50:23.841157+0000 mon.a (mon.0) 240 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2022-04-28T19:50:25.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:24 smithi139 bash[43301]: cluster 2022-04-28T19:50:23.841157+0000 mon.a (mon.0) 240 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2022-04-28T19:50:25.866 INFO:tasks.rados.rados.0.smithi059.stdout:1897: done (0 left) 2022-04-28T19:50:25.866 INFO:tasks.rados.rados.0.smithi059.stdout:1898: read oid 42 snap 175 2022-04-28T19:50:25.866 INFO:tasks.rados.rados.0.smithi059.stdout:1898: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:50:25.866 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 177 2022-04-28T19:50:25.866 INFO:tasks.rados.rados.0.smithi059.stdout:1899: seq_num 620 ranges {5619712=794624} 2022-04-28T19:50:25.909 INFO:tasks.rados.rados.0.smithi059.stdout:1899: writing smithi05955964-33 from 5619712 to 6414336 tid 1 2022-04-28T19:50:25.910 INFO:tasks.rados.rados.0.smithi059.stdout:1900: snap_create 2022-04-28T19:50:25.910 INFO:tasks.rados.rados.0.smithi059.stdout:1898: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:50:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:25 smithi139 bash[43301]: cluster 2022-04-28T19:50:24.840555+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2022-04-28T19:50:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:25 smithi139 bash[43301]: cluster 2022-04-28T19:50:24.870788+0000 mgr.y (mgr.24961) 84 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2022-04-28T19:50:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:25 smithi139 bash[43301]: audit 2022-04-28T19:50:25.662475+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:25 smithi139 bash[43301]: audit 2022-04-28T19:50:25.665775+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "container_image"}]: dispatch 2022-04-28T19:50:26.203 INFO:tasks.rados.rados.0.smithi059.stdout:1899: finishing write tid 1 to smithi05955964-33 2022-04-28T19:50:26.203 INFO:tasks.rados.rados.0.smithi059.stdout:1899: finishing write tid 3 to smithi05955964-33 2022-04-28T19:50:26.203 INFO:tasks.rados.rados.0.smithi059.stdout:1899: finishing write tid 4 to smithi05955964-33 2022-04-28T19:50:26.203 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 873 (ObjNum 620 snap 177 seq_num 620) dirty exists 2022-04-28T19:50:26.203 INFO:tasks.rados.rados.0.smithi059.stdout:1899: left oid 33 (ObjNum 620 snap 177 seq_num 620) 2022-04-28T19:50:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[59745]: cluster 2022-04-28T19:50:24.840555+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2022-04-28T19:50:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[59745]: cluster 2022-04-28T19:50:24.870788+0000 mgr.y (mgr.24961) 84 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2022-04-28T19:50:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[59745]: audit 2022-04-28T19:50:25.662475+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[59745]: audit 2022-04-28T19:50:25.665775+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "container_image"}]: dispatch 2022-04-28T19:50:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[61152]: cluster 2022-04-28T19:50:24.840555+0000 mon.a (mon.0) 241 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2022-04-28T19:50:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[61152]: cluster 2022-04-28T19:50:24.870788+0000 mgr.y (mgr.24961) 84 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.2 MiB/s wr, 14 op/s 2022-04-28T19:50:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[61152]: audit 2022-04-28T19:50:25.662475+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:25 smithi059 bash[61152]: audit 2022-04-28T19:50:25.665775+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "container_image"}]: dispatch 2022-04-28T19:50:26.823 INFO:tasks.rados.rados.0.smithi059.stdout:1898: done (2 left) 2022-04-28T19:50:26.823 INFO:tasks.rados.rados.0.smithi059.stdout:1899: done (1 left) 2022-04-28T19:50:26.824 INFO:tasks.rados.rados.0.smithi059.stdout:1900: done (0 left) 2022-04-28T19:50:26.824 INFO:tasks.rados.rados.0.smithi059.stdout:1901: setattr oid 44 current snap is 178 2022-04-28T19:50:26.825 INFO:tasks.rados.rados.0.smithi059.stdout:1902: rollback oid 20 current snap is 178 2022-04-28T19:50:26.825 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 177 2022-04-28T19:50:26.825 INFO:tasks.rados.rados.0.smithi059.stdout:1903: snap_remove snap 172 2022-04-28T19:50:27.002 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[59745]: cephadm 2022-04-28T19:50:25.655719+0000 mgr.y (mgr.24961) 85 : cephadm [INF] Upgrade: Updating osd.0 2022-04-28T19:50:27.002 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[59745]: cephadm 2022-04-28T19:50:25.664641+0000 mgr.y (mgr.24961) 86 : cephadm [INF] Deploying daemon osd.0 on smithi059 2022-04-28T19:50:27.002 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[59745]: cluster 2022-04-28T19:50:25.865090+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2022-04-28T19:50:27.002 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[59745]: cluster 2022-04-28T19:50:26.820820+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2022-04-28T19:50:27.002 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[61152]: cephadm 2022-04-28T19:50:25.655719+0000 mgr.y (mgr.24961) 85 : cephadm [INF] Upgrade: Updating osd.0 2022-04-28T19:50:27.002 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[61152]: cephadm 2022-04-28T19:50:25.664641+0000 mgr.y (mgr.24961) 86 : cephadm [INF] Deploying daemon osd.0 on smithi059 2022-04-28T19:50:27.003 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[61152]: cluster 2022-04-28T19:50:25.865090+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2022-04-28T19:50:27.003 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:26 smithi059 bash[61152]: cluster 2022-04-28T19:50:26.820820+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2022-04-28T19:50:27.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:26 smithi139 bash[43301]: cephadm 2022-04-28T19:50:25.655719+0000 mgr.y (mgr.24961) 85 : cephadm [INF] Upgrade: Updating osd.0 2022-04-28T19:50:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:26 smithi139 bash[43301]: cephadm 2022-04-28T19:50:25.664641+0000 mgr.y (mgr.24961) 86 : cephadm [INF] Deploying daemon osd.0 on smithi059 2022-04-28T19:50:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:26 smithi139 bash[43301]: cluster 2022-04-28T19:50:25.865090+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2022-04-28T19:50:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:26 smithi139 bash[43301]: cluster 2022-04-28T19:50:26.820820+0000 mon.a (mon.0) 245 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2022-04-28T19:50:27.835 INFO:tasks.rados.rados.0.smithi059.stdout:1903: done (2 left) 2022-04-28T19:50:27.836 INFO:tasks.rados.rados.0.smithi059.stdout:1904: copy_from oid 39 from oid 18 current snap is 178 2022-04-28T19:50:27.836 INFO:tasks.rados.rados.0.smithi059.stdout:1902: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:50:27.836 INFO:tasks.rados.rados.0.smithi059.stdout:1902: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:50:27.836 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 649 (ObjNum 507 snap 144 seq_num 507) dirty exists 2022-04-28T19:50:27.836 INFO:tasks.rados.rados.0.smithi059.stdout:1902: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:50:27.837 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 767 (ObjNum 461 snap 136 seq_num 461) dirty exists 2022-04-28T19:50:27.837 INFO:tasks.rados.rados.0.smithi059.stdout:1901: done (2 left) 2022-04-28T19:50:27.837 INFO:tasks.rados.rados.0.smithi059.stdout:1902: done (1 left) 2022-04-28T19:50:27.837 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 178 2022-04-28T19:50:27.837 INFO:tasks.rados.rados.0.smithi059.stdout:1905: seq_num 621 ranges {335872=409600,745472=401408} 2022-04-28T19:50:27.840 INFO:tasks.rados.rados.0.smithi059.stdout:1905: writing smithi05955964-23 from 335872 to 745472 tid 1 2022-04-28T19:50:27.842 INFO:tasks.rados.rados.0.smithi059.stdout:1905: writing smithi05955964-23 from 745472 to 1146880 tid 2 2022-04-28T19:50:27.843 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 178 2022-04-28T19:50:27.843 INFO:tasks.rados.rados.0.smithi059.stdout:1906: seq_num 622 ranges {0=425984,425984=401408} 2022-04-28T19:50:27.846 INFO:tasks.rados.rados.0.smithi059.stdout:1906: writing smithi05955964-30 from 0 to 425984 tid 1 2022-04-28T19:50:27.850 INFO:tasks.rados.rados.0.smithi059.stdout:1906: writing smithi05955964-30 from 425984 to 827392 tid 2 2022-04-28T19:50:27.850 INFO:tasks.rados.rados.0.smithi059.stdout:1907: read oid 19 snap -1 2022-04-28T19:50:27.851 INFO:tasks.rados.rados.0.smithi059.stdout:1907: expect (ObjNum 601 snap 169 seq_num 601) 2022-04-28T19:50:27.851 INFO:tasks.rados.rados.0.smithi059.stdout:1908: read oid 1 snap -1 2022-04-28T19:50:27.851 INFO:tasks.rados.rados.0.smithi059.stdout:1908: expect (ObjNum 602 snap 169 seq_num 602) 2022-04-28T19:50:27.851 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 178 2022-04-28T19:50:27.851 INFO:tasks.rados.rados.0.smithi059.stdout:1909: seq_num 623 ranges {3014656=442368,3457024=401408} 2022-04-28T19:50:27.870 INFO:tasks.rados.rados.0.smithi059.stdout:1909: writing smithi05955964-32 from 3014656 to 3457024 tid 1 2022-04-28T19:50:27.873 INFO:tasks.rados.rados.0.smithi059.stdout:1909: writing smithi05955964-32 from 3457024 to 3858432 tid 2 2022-04-28T19:50:27.873 INFO:tasks.rados.rados.0.smithi059.stdout:1910: setattr oid 9 current snap is 178 2022-04-28T19:50:27.873 INFO:tasks.rados.rados.0.smithi059.stdout:1905: finishing write tid 1 to smithi05955964-23 2022-04-28T19:50:27.873 INFO:tasks.rados.rados.0.smithi059.stdout:1906: finishing write tid 1 to smithi05955964-30 2022-04-28T19:50:27.873 INFO:tasks.rados.rados.0.smithi059.stdout:1906: finishing write tid 2 to smithi05955964-30 2022-04-28T19:50:27.873 INFO:tasks.rados.rados.0.smithi059.stdout:1907: expect (ObjNum 601 snap 169 seq_num 601) 2022-04-28T19:50:27.885 INFO:tasks.rados.rados.0.smithi059.stdout:1906: finishing write tid 4 to smithi05955964-30 2022-04-28T19:50:27.886 INFO:tasks.rados.rados.0.smithi059.stdout:1906: finishing write tid 5 to smithi05955964-30 2022-04-28T19:50:27.886 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 868 (ObjNum 622 snap 178 seq_num 622) dirty exists 2022-04-28T19:50:27.886 INFO:tasks.rados.rados.0.smithi059.stdout:1906: left oid 30 (ObjNum 622 snap 178 seq_num 622) 2022-04-28T19:50:27.886 INFO:tasks.rados.rados.0.smithi059.stdout:1905: finishing write tid 2 to smithi05955964-23 2022-04-28T19:50:27.886 INFO:tasks.rados.rados.0.smithi059.stdout:1905: finishing write tid 4 to smithi05955964-23 2022-04-28T19:50:27.887 INFO:tasks.rados.rados.0.smithi059.stdout:1904: finishing copy_from to smithi05955964-39 2022-04-28T19:50:27.887 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 853 (ObjNum 614 snap 173 seq_num 614) dirty exists 2022-04-28T19:50:27.887 INFO:tasks.rados.rados.0.smithi059.stdout:1905: finishing write tid 5 to smithi05955964-23 2022-04-28T19:50:27.887 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 893 (ObjNum 621 snap 178 seq_num 621) dirty exists 2022-04-28T19:50:27.888 INFO:tasks.rados.rados.0.smithi059.stdout:1905: left oid 23 (ObjNum 621 snap 178 seq_num 621) 2022-04-28T19:50:27.888 INFO:tasks.rados.rados.0.smithi059.stdout:1904: finishing copy_from racing read to smithi05955964-39 2022-04-28T19:50:27.888 INFO:tasks.rados.rados.0.smithi059.stdout:1909: finishing write tid 1 to smithi05955964-32 2022-04-28T19:50:27.888 INFO:tasks.rados.rados.0.smithi059.stdout:1909: finishing write tid 2 to smithi05955964-32 2022-04-28T19:50:27.888 INFO:tasks.rados.rados.0.smithi059.stdout:1909: finishing write tid 4 to smithi05955964-32 2022-04-28T19:50:27.888 INFO:tasks.rados.rados.0.smithi059.stdout:1909: finishing write tid 5 to smithi05955964-32 2022-04-28T19:50:27.889 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 882 (ObjNum 623 snap 178 seq_num 623) dirty exists 2022-04-28T19:50:27.889 INFO:tasks.rados.rados.0.smithi059.stdout:1909: left oid 32 (ObjNum 623 snap 178 seq_num 623) 2022-04-28T19:50:27.889 INFO:tasks.rados.rados.0.smithi059.stdout:1908: expect (ObjNum 602 snap 169 seq_num 602) 2022-04-28T19:50:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:27 smithi139 bash[43301]: cluster 2022-04-28T19:50:26.871393+0000 mgr.y (mgr.24961) 87 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:27 smithi139 bash[43301]: cluster 2022-04-28T19:50:27.832307+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2022-04-28T19:50:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:27 smithi059 bash[59745]: cluster 2022-04-28T19:50:26.871393+0000 mgr.y (mgr.24961) 87 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:27 smithi059 bash[59745]: cluster 2022-04-28T19:50:27.832307+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2022-04-28T19:50:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:27 smithi059 bash[61152]: cluster 2022-04-28T19:50:26.871393+0000 mgr.y (mgr.24961) 87 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 244 MiB data, 396 MiB used, 707 GiB / 715 GiB avail 2022-04-28T19:50:28.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:27 smithi059 bash[61152]: cluster 2022-04-28T19:50:27.832307+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2022-04-28T19:50:28.481 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 650 (ObjNum 607 snap 170 seq_num 607) dirty exists 2022-04-28T19:50:28.481 INFO:tasks.rados.rados.0.smithi059.stdout:1904: done (6 left) 2022-04-28T19:50:28.481 INFO:tasks.rados.rados.0.smithi059.stdout:1905: done (5 left) 2022-04-28T19:50:28.481 INFO:tasks.rados.rados.0.smithi059.stdout:1906: done (4 left) 2022-04-28T19:50:28.481 INFO:tasks.rados.rados.0.smithi059.stdout:1907: done (3 left) 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:1908: done (2 left) 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:1909: done (1 left) 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:1910: done (0 left) 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:1911: rollback oid 5 current snap is 178 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 175 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:1912: read oid 11 snap -1 2022-04-28T19:50:28.482 INFO:tasks.rados.rados.0.smithi059.stdout:1912: expect (ObjNum 554 snap 157 seq_num 554) 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1913: read oid 43 snap -1 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1913: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1914: read oid 35 snap -1 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1914: expect (ObjNum 612 snap 171 seq_num 612) 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1915: rmattr oid 32 current snap is 178 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1915: done (4 left) 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 178 2022-04-28T19:50:28.483 INFO:tasks.rados.rados.0.smithi059.stdout:1916: seq_num 624 ranges {0=458752,458752=401408} 2022-04-28T19:50:28.490 INFO:tasks.rados.rados.0.smithi059.stdout:1916: writing smithi05955964-7 from 0 to 458752 tid 1 2022-04-28T19:50:28.496 INFO:tasks.rados.rados.0.smithi059.stdout:1916: writing smithi05955964-7 from 458752 to 860160 tid 2 2022-04-28T19:50:28.496 INFO:tasks.rados.rados.0.smithi059.stdout:1917: rmattr oid 16 current snap is 178 2022-04-28T19:50:28.497 INFO:tasks.rados.rados.0.smithi059.stdout:1911: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:50:28.497 INFO:tasks.rados.rados.0.smithi059.stdout:1911: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:50:28.497 INFO:tasks.rados.rados.0.smithi059.stdout:1911: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:50:28.497 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 849 (ObjNum 574 snap 164 seq_num 574) dirty exists 2022-04-28T19:50:28.497 INFO:tasks.rados.rados.0.smithi059.stdout:1911: done (5 left) 2022-04-28T19:50:28.497 INFO:tasks.rados.rados.0.smithi059.stdout:1918: snap_remove snap 176 2022-04-28T19:50:28.876 INFO:tasks.rados.rados.0.smithi059.stdout:1918: done (5 left) 2022-04-28T19:50:28.877 INFO:tasks.rados.rados.0.smithi059.stdout:1919: snap_create 2022-04-28T19:50:28.877 INFO:tasks.rados.rados.0.smithi059.stdout:1916: finishing write tid 1 to smithi05955964-7 2022-04-28T19:50:28.877 INFO:tasks.rados.rados.0.smithi059.stdout:1913: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:1916: finishing write tid 2 to smithi05955964-7 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:1916: finishing write tid 4 to smithi05955964-7 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:1916: finishing write tid 5 to smithi05955964-7 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 826 (ObjNum 624 snap 178 seq_num 624) dirty exists 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:1916: left oid 7 (ObjNum 624 snap 178 seq_num 624) 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 701 (ObjNum 597 snap 169 seq_num 597) dirty exists 2022-04-28T19:50:28.971 INFO:tasks.rados.rados.0.smithi059.stdout:1912: expect (ObjNum 554 snap 157 seq_num 554) 2022-04-28T19:50:29.279 INFO:tasks.rados.rados.0.smithi059.stdout:1914: expect (ObjNum 612 snap 171 seq_num 612) 2022-04-28T19:50:29.794 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:29 smithi059 systemd[1]: Stopping Ceph osd.0 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:50:29.795 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[31905]: debug 2022-04-28T19:50:29.688+0000 7ff833fda700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:50:29.795 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[31905]: debug 2022-04-28T19:50:29.688+0000 7ff833fda700 -1 osd.0 411 *** Got signal Terminated *** 2022-04-28T19:50:29.795 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[31905]: debug 2022-04-28T19:50:29.688+0000 7ff833fda700 -1 osd.0 411 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:50:29.924 INFO:tasks.rados.rados.0.smithi059.stdout:1912: done (5 left) 2022-04-28T19:50:29.924 INFO:tasks.rados.rados.0.smithi059.stdout:1913: done (4 left) 2022-04-28T19:50:29.924 INFO:tasks.rados.rados.0.smithi059.stdout:1914: done (3 left) 2022-04-28T19:50:29.925 INFO:tasks.rados.rados.0.smithi059.stdout:1916: done (2 left) 2022-04-28T19:50:29.925 INFO:tasks.rados.rados.0.smithi059.stdout:1917: done (1 left) 2022-04-28T19:50:29.925 INFO:tasks.rados.rados.0.smithi059.stdout:1919: done (0 left) 2022-04-28T19:50:29.925 INFO:tasks.rados.rados.0.smithi059.stdout:1920: delete oid 34 current snap is 179 2022-04-28T19:50:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[59745]: cluster 2022-04-28T19:50:28.872077+0000 mgr.y (mgr.24961) 88 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 246 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 4.2 MiB/s rd, 562 KiB/s wr, 9 op/s 2022-04-28T19:50:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[59745]: cluster 2022-04-28T19:50:28.874895+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2022-04-28T19:50:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[61152]: cluster 2022-04-28T19:50:28.872077+0000 mgr.y (mgr.24961) 88 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 246 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 4.2 MiB/s rd, 562 KiB/s wr, 9 op/s 2022-04-28T19:50:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:29 smithi059 bash[61152]: cluster 2022-04-28T19:50:28.874895+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2022-04-28T19:50:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:29 smithi139 bash[43301]: cluster 2022-04-28T19:50:28.872077+0000 mgr.y (mgr.24961) 88 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 246 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 4.2 MiB/s rd, 562 KiB/s wr, 9 op/s 2022-04-28T19:50:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:29 smithi139 bash[43301]: cluster 2022-04-28T19:50:28.874895+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2022-04-28T19:50:30.843 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[63317]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.0 2022-04-28T19:50:30.844 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[63317]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-0 2022-04-28T19:50:30.844 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[63317]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.0 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.878455+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.927073+0000 mon.a (mon.0) 249 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.927198+0000 mon.a (mon.0) 250 : cluster [INF] osd.0 failed (root=default,host=smithi059) (connection refused reported by osd.5) 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.988111+0000 mon.a (mon.0) 251 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.988317+0000 mon.a (mon.0) 252 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.988542+0000 mon.a (mon.0) 253 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.988719+0000 mon.a (mon.0) 254 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.988884+0000 mon.a (mon.0) 255 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.989094+0000 mon.a (mon.0) 256 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.990913+0000 mon.a (mon.0) 257 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.991141+0000 mon.a (mon.0) 258 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.991310+0000 mon.a (mon.0) 259 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.991479+0000 mon.a (mon.0) 260 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.991820+0000 mon.a (mon.0) 261 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.992088+0000 mon.a (mon.0) 262 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.992392+0000 mon.a (mon.0) 263 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:29.992626+0000 mon.a (mon.0) 264 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.127663+0000 mon.a (mon.0) 265 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.392207+0000 mon.a (mon.0) 266 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.392473+0000 mon.a (mon.0) 267 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.392680+0000 mon.a (mon.0) 268 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.392890+0000 mon.a (mon.0) 269 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.393011+0000 mon.a (mon.0) 270 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.393292+0000 mon.a (mon.0) 271 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.395200+0000 mon.a (mon.0) 272 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.395729+0000 mon.a (mon.0) 273 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.396053+0000 mon.a (mon.0) 274 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.396407+0000 mon.a (mon.0) 275 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.396706+0000 mon.a (mon.0) 276 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.397067+0000 mon.a (mon.0) 277 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.397389+0000 mon.a (mon.0) 278 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.397858+0000 mon.a (mon.0) 279 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.535662+0000 mon.a (mon.0) 280 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.878455+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.927073+0000 mon.a (mon.0) 249 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.927198+0000 mon.a (mon.0) 250 : cluster [INF] osd.0 failed (root=default,host=smithi059) (connection refused reported by osd.5) 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.988111+0000 mon.a (mon.0) 251 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.988317+0000 mon.a (mon.0) 252 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.988542+0000 mon.a (mon.0) 253 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.988719+0000 mon.a (mon.0) 254 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.988884+0000 mon.a (mon.0) 255 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.989094+0000 mon.a (mon.0) 256 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.990913+0000 mon.a (mon.0) 257 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.991141+0000 mon.a (mon.0) 258 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.991310+0000 mon.a (mon.0) 259 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.991479+0000 mon.a (mon.0) 260 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.991820+0000 mon.a (mon.0) 261 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.992088+0000 mon.a (mon.0) 262 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.992392+0000 mon.a (mon.0) 263 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:29.992626+0000 mon.a (mon.0) 264 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.127663+0000 mon.a (mon.0) 265 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.392207+0000 mon.a (mon.0) 266 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.392473+0000 mon.a (mon.0) 267 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.392680+0000 mon.a (mon.0) 268 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.392890+0000 mon.a (mon.0) 269 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.393011+0000 mon.a (mon.0) 270 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.393292+0000 mon.a (mon.0) 271 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.395200+0000 mon.a (mon.0) 272 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.395729+0000 mon.a (mon.0) 273 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.396053+0000 mon.a (mon.0) 274 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.396407+0000 mon.a (mon.0) 275 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.396706+0000 mon.a (mon.0) 276 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.397067+0000 mon.a (mon.0) 277 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.397389+0000 mon.a (mon.0) 278 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.397858+0000 mon.a (mon.0) 279 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:30 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.535662+0000 mon.a (mon.0) 280 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.878455+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.927073+0000 mon.a (mon.0) 249 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.927198+0000 mon.a (mon.0) 250 : cluster [INF] osd.0 failed (root=default,host=smithi059) (connection refused reported by osd.5) 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.988111+0000 mon.a (mon.0) 251 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.988317+0000 mon.a (mon.0) 252 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.988542+0000 mon.a (mon.0) 253 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.988719+0000 mon.a (mon.0) 254 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.988884+0000 mon.a (mon.0) 255 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.989094+0000 mon.a (mon.0) 256 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.990913+0000 mon.a (mon.0) 257 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.991141+0000 mon.a (mon.0) 258 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.991310+0000 mon.a (mon.0) 259 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.991479+0000 mon.a (mon.0) 260 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.991820+0000 mon.a (mon.0) 261 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.992088+0000 mon.a (mon.0) 262 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.992392+0000 mon.a (mon.0) 263 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:29.992626+0000 mon.a (mon.0) 264 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.127663+0000 mon.a (mon.0) 265 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.392207+0000 mon.a (mon.0) 266 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.392473+0000 mon.a (mon.0) 267 : cluster [DBG] osd.0 reported immediately failed by osd.2 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.392680+0000 mon.a (mon.0) 268 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.392890+0000 mon.a (mon.0) 269 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.393011+0000 mon.a (mon.0) 270 : cluster [DBG] osd.0 reported immediately failed by osd.1 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.393292+0000 mon.a (mon.0) 271 : cluster [DBG] osd.0 reported immediately failed by osd.3 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.395200+0000 mon.a (mon.0) 272 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.395729+0000 mon.a (mon.0) 273 : cluster [DBG] osd.0 reported immediately failed by osd.6 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.396053+0000 mon.a (mon.0) 274 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.396407+0000 mon.a (mon.0) 275 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.396706+0000 mon.a (mon.0) 276 : cluster [DBG] osd.0 reported immediately failed by osd.7 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.397067+0000 mon.a (mon.0) 277 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.397389+0000 mon.a (mon.0) 278 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.397858+0000 mon.a (mon.0) 279 : cluster [DBG] osd.0 reported immediately failed by osd.4 2022-04-28T19:50:31.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:30 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.535662+0000 mon.a (mon.0) 280 : cluster [DBG] osd.0 reported immediately failed by osd.5 2022-04-28T19:50:31.834 INFO:tasks.rados.rados.0.smithi059.stdout:1920: done (0 left) 2022-04-28T19:50:31.834 INFO:tasks.rados.rados.0.smithi059.stdout:1921: snap_remove snap 175 2022-04-28T19:50:31.885 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:31.627Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:31.886 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:31.627Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:31.886 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:31.627Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:50:32.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.872822+0000 mgr.y (mgr.24961) 89 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 246 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 554 KiB/s wr, 9 op/s 2022-04-28T19:50:32.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.881952+0000 mon.a (mon.0) 281 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:50:32.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[59745]: cluster 2022-04-28T19:50:30.901510+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e413: 8 total, 7 up, 8 in 2022-04-28T19:50:32.178 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[59745]: cluster 2022-04-28T19:50:31.821397+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e414: 8 total, 7 up, 8 in 2022-04-28T19:50:32.178 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.872822+0000 mgr.y (mgr.24961) 89 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 246 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 554 KiB/s wr, 9 op/s 2022-04-28T19:50:32.178 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.881952+0000 mon.a (mon.0) 281 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:50:32.178 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[61152]: cluster 2022-04-28T19:50:30.901510+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e413: 8 total, 7 up, 8 in 2022-04-28T19:50:32.178 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:31 smithi059 bash[61152]: cluster 2022-04-28T19:50:31.821397+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e414: 8 total, 7 up, 8 in 2022-04-28T19:50:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:31 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.872822+0000 mgr.y (mgr.24961) 89 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 246 MiB data, 400 MiB used, 707 GiB / 715 GiB avail; 4.1 MiB/s rd, 554 KiB/s wr, 9 op/s 2022-04-28T19:50:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:31 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.881952+0000 mon.a (mon.0) 281 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:50:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:31 smithi139 bash[43301]: cluster 2022-04-28T19:50:30.901510+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e413: 8 total, 7 up, 8 in 2022-04-28T19:50:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:31 smithi139 bash[43301]: cluster 2022-04-28T19:50:31.821397+0000 mon.a (mon.0) 283 : cluster [DBG] osdmap e414: 8 total, 7 up, 8 in 2022-04-28T19:50:32.836 INFO:tasks.rados.rados.0.smithi059.stdout:1921: done (0 left) 2022-04-28T19:50:32.836 INFO:tasks.rados.rados.0.smithi059.stdout:1922: copy_from oid 3 from oid 28 current snap is 179 2022-04-28T19:50:32.836 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 179 2022-04-28T19:50:32.837 INFO:tasks.rados.rados.0.smithi059.stdout:1923: seq_num 626 ranges {2342912=491520,2834432=401408} 2022-04-28T19:50:32.856 INFO:tasks.rados.rados.0.smithi059.stdout:1923: writing smithi05955964-45 from 2342912 to 2834432 tid 1 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1923: writing smithi05955964-45 from 2834432 to 3235840 tid 2 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1924: read oid 14 snap -1 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1924: expect (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1925: snap_create 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1922: finishing copy_from to smithi05955964-3 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1922: got expected ENOENT (src dne) 2022-04-28T19:50:32.859 INFO:tasks.rados.rados.0.smithi059.stdout:1922: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:50:32.862 INFO:tasks.rados.rados.0.smithi059.stdout:1923: finishing write tid 1 to smithi05955964-45 2022-04-28T19:50:32.864 INFO:tasks.rados.rados.0.smithi059.stdout:1923: finishing write tid 2 to smithi05955964-45 2022-04-28T19:50:32.864 INFO:tasks.rados.rados.0.smithi059.stdout:1923: finishing write tid 4 to smithi05955964-45 2022-04-28T19:50:32.866 INFO:tasks.rados.rados.0.smithi059.stdout:1924: expect (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:50:32.956 INFO:tasks.rados.rados.0.smithi059.stdout:1923: finishing write tid 5 to smithi05955964-45 2022-04-28T19:50:32.956 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 775 (ObjNum 626 snap 179 seq_num 626) dirty exists 2022-04-28T19:50:32.956 INFO:tasks.rados.rados.0.smithi059.stdout:1923: left oid 45 (ObjNum 626 snap 179 seq_num 626) 2022-04-28T19:50:33.091 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:32 smithi059 systemd[1]: Stopped Ceph osd.0 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:50:33.091 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:32 smithi059 systemd[1]: Started Ceph osd.0 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:50:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:33.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:33.877 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:33.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:33.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:33.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:33.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:33.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:33.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:33.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:33.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:33.525Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1922: done (3 left) 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1923: done (2 left) 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1924: done (1 left) 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1925: done (0 left) 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1926: read oid 31 snap -1 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1926: expect (ObjNum 603 snap 169 seq_num 603) 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 180 2022-04-28T19:50:33.879 INFO:tasks.rados.rados.0.smithi059.stdout:1927: seq_num 627 ranges {6889472=507904,7397376=401408} 2022-04-28T19:50:33.893 INFO:tasks.rados.rados.0.smithi059.stdout:1927: writing smithi05955964-41 from 6889472 to 7397376 tid 1 2022-04-28T19:50:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1927: writing smithi05955964-41 from 7397376 to 7798784 tid 2 2022-04-28T19:50:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1928: rollback oid 20 current snap is 180 2022-04-28T19:50:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 173 2022-04-28T19:50:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1929: read oid 28 snap 179 2022-04-28T19:50:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1929: expect deleted 2022-04-28T19:50:33.896 INFO:tasks.rados.rados.0.smithi059.stdout:1930: setattr oid 11 current snap is 180 2022-04-28T19:50:33.899 INFO:tasks.rados.rados.0.smithi059.stdout:1931: read oid 50 snap -1 2022-04-28T19:50:33.899 INFO:tasks.rados.rados.0.smithi059.stdout:1931: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:50:33.899 INFO:tasks.rados.rados.0.smithi059.stdout:1932: setattr oid 22 current snap is 180 2022-04-28T19:50:33.907 INFO:tasks.rados.rados.0.smithi059.stdout:1926: expect (ObjNum 603 snap 169 seq_num 603) 2022-04-28T19:50:34.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: cluster 2022-04-28T19:50:32.835360+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e415: 8 total, 7 up, 8 in 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: cluster 2022-04-28T19:50:32.873583+0000 mgr.y (mgr.24961) 90 : cluster [DBG] pgmap v107: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 61/429 objects degraded (14.219%) 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: audit 2022-04-28T19:50:33.004049+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: audit 2022-04-28T19:50:33.005367+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: audit 2022-04-28T19:50:33.007627+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: audit 2022-04-28T19:50:33.008746+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: cephadm 2022-04-28T19:50:33.009796+0000 mgr.y (mgr.24961) 91 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:50:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:33 smithi139 bash[43301]: audit 2022-04-28T19:50:33.010758+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:50:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: cluster 2022-04-28T19:50:32.835360+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e415: 8 total, 7 up, 8 in 2022-04-28T19:50:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: cluster 2022-04-28T19:50:32.873583+0000 mgr.y (mgr.24961) 90 : cluster [DBG] pgmap v107: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 61/429 objects degraded (14.219%) 2022-04-28T19:50:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: audit 2022-04-28T19:50:33.004049+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: audit 2022-04-28T19:50:33.005367+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: audit 2022-04-28T19:50:33.007627+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: audit 2022-04-28T19:50:33.008746+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:50:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: cephadm 2022-04-28T19:50:33.009796+0000 mgr.y (mgr.24961) 91 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:50:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[59745]: audit 2022-04-28T19:50:33.010758+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:50:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: cluster 2022-04-28T19:50:32.835360+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e415: 8 total, 7 up, 8 in 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: cluster 2022-04-28T19:50:32.873583+0000 mgr.y (mgr.24961) 90 : cluster [DBG] pgmap v107: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 61/429 objects degraded (14.219%) 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: audit 2022-04-28T19:50:33.004049+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: audit 2022-04-28T19:50:33.005367+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: audit 2022-04-28T19:50:33.007627+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: audit 2022-04-28T19:50:33.008746+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: cephadm 2022-04-28T19:50:33.009796+0000 mgr.y (mgr.24961) 91 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:50:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:33 smithi059 bash[61152]: audit 2022-04-28T19:50:33.010758+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:50:34.491 INFO:tasks.rados.rados.0.smithi059.stdout:1928: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:50:34.491 INFO:tasks.rados.rados.0.smithi059.stdout:1927: finishing write tid 1 to smithi05955964-41 2022-04-28T19:50:34.491 INFO:tasks.rados.rados.0.smithi059.stdout:1927: finishing write tid 2 to smithi05955964-41 2022-04-28T19:50:34.491 INFO:tasks.rados.rados.0.smithi059.stdout:1928: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:1928: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 779 (ObjNum 461 snap 136 seq_num 461) dirty exists 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:1927: finishing write tid 4 to smithi05955964-41 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 781 (ObjNum 554 snap 157 seq_num 554) dirty exists 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:1927: finishing write tid 5 to smithi05955964-41 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 627 (ObjNum 627 snap 180 seq_num 627) dirty exists 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:1927: left oid 41 (ObjNum 627 snap 180 seq_num 627) 2022-04-28T19:50:34.492 INFO:tasks.rados.rados.0.smithi059.stdout:1931: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:50:34.841 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2022-04-28T19:50:34.841 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2022-04-28T19:50:34.841 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2022-04-28T19:50:34.841 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2022-04-28T19:50:34.842 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2022-04-28T19:50:34.842 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2022-04-28T19:50:34.842 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[63562]: --> ceph-volume lvm activate successful for osd ID: 0 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 863 (ObjNum 1465930839 snap 1936282722 seq_num 1532583780) dirty exists 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1926: done (6 left) 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1927: done (5 left) 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1928: done (4 left) 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1929: done (3 left) 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1930: done (2 left) 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1931: done (1 left) 2022-04-28T19:50:34.863 INFO:tasks.rados.rados.0.smithi059.stdout:1932: done (0 left) 2022-04-28T19:50:34.864 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 180 2022-04-28T19:50:34.864 INFO:tasks.rados.rados.0.smithi059.stdout:1933: seq_num 628 ranges {270336=524288,794624=401408} 2022-04-28T19:50:34.873 INFO:tasks.rados.rados.0.smithi059.stdout:1933: writing smithi05955964-4 from 270336 to 794624 tid 1 2022-04-28T19:50:34.876 INFO:tasks.rados.rados.0.smithi059.stdout:1933: writing smithi05955964-4 from 794624 to 1196032 tid 2 2022-04-28T19:50:34.876 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 180 2022-04-28T19:50:34.876 INFO:tasks.rados.rados.0.smithi059.stdout:1934: seq_num 629 ranges {3080192=540672,3620864=401408} 2022-04-28T19:50:34.897 INFO:tasks.rados.rados.0.smithi059.stdout:1934: writing smithi05955964-3 from 3080192 to 3620864 tid 1 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1934: writing smithi05955964-3 from 3620864 to 4022272 tid 2 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1935: read oid 9 snap -1 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1935: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1936: copy_from oid 22 from oid 45 current snap is 180 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1933: finishing write tid 1 to smithi05955964-4 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1933: finishing write tid 2 to smithi05955964-4 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1933: finishing write tid 4 to smithi05955964-4 2022-04-28T19:50:34.900 INFO:tasks.rados.rados.0.smithi059.stdout:1933: finishing write tid 5 to smithi05955964-4 2022-04-28T19:50:34.901 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 785 (ObjNum 628 snap 180 seq_num 628) dirty exists 2022-04-28T19:50:34.901 INFO:tasks.rados.rados.0.smithi059.stdout:1933: left oid 4 (ObjNum 628 snap 180 seq_num 628) 2022-04-28T19:50:34.901 INFO:tasks.rados.rados.0.smithi059.stdout:1933: done (3 left) 2022-04-28T19:50:34.901 INFO:tasks.rados.rados.0.smithi059.stdout:1937: snap_remove snap 159 2022-04-28T19:50:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:34 smithi139 bash[43301]: cluster 2022-04-28T19:50:33.831183+0000 mon.a (mon.0) 290 : cluster [WRN] Health check failed: Degraded data redundancy: 61/429 objects degraded (14.219%), 14 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:34 smithi139 bash[43301]: cluster 2022-04-28T19:50:33.847976+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e416: 8 total, 7 up, 8 in 2022-04-28T19:50:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[61152]: cluster 2022-04-28T19:50:33.831183+0000 mon.a (mon.0) 290 : cluster [WRN] Health check failed: Degraded data redundancy: 61/429 objects degraded (14.219%), 14 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[61152]: cluster 2022-04-28T19:50:33.847976+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e416: 8 total, 7 up, 8 in 2022-04-28T19:50:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[59745]: cluster 2022-04-28T19:50:33.831183+0000 mon.a (mon.0) 290 : cluster [WRN] Health check failed: Degraded data redundancy: 61/429 objects degraded (14.219%), 14 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:34 smithi059 bash[59745]: cluster 2022-04-28T19:50:33.847976+0000 mon.a (mon.0) 291 : cluster [DBG] osdmap e416: 8 total, 7 up, 8 in 2022-04-28T19:50:35.859 INFO:tasks.rados.rados.0.smithi059.stdout:1937: done (3 left) 2022-04-28T19:50:35.859 INFO:tasks.rados.rados.0.smithi059.stdout:1938: setattr oid 17 current snap is 180 2022-04-28T19:50:35.859 INFO:tasks.rados.rados.0.smithi059.stdout:1934: finishing write tid 1 to smithi05955964-3 2022-04-28T19:50:35.859 INFO:tasks.rados.rados.0.smithi059.stdout:1934: finishing write tid 2 to smithi05955964-3 2022-04-28T19:50:35.860 INFO:tasks.rados.rados.0.smithi059.stdout:1934: finishing write tid 4 to smithi05955964-3 2022-04-28T19:50:35.860 INFO:tasks.rados.rados.0.smithi059.stdout:1935: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:35.984 INFO:tasks.rados.rados.0.smithi059.stdout:1934: finishing write tid 5 to smithi05955964-3 2022-04-28T19:50:36.031 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 783 (ObjNum 629 snap 180 seq_num 629) dirty exists 2022-04-28T19:50:36.032 INFO:tasks.rados.rados.0.smithi059.stdout:1934: left oid 3 (ObjNum 629 snap 180 seq_num 629) 2022-04-28T19:50:36.032 INFO:tasks.rados.rados.0.smithi059.stdout:1936: finishing copy_from to smithi05955964-22 2022-04-28T19:50:36.032 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 775 (ObjNum 626 snap 179 seq_num 626) dirty exists 2022-04-28T19:50:36.032 INFO:tasks.rados.rados.0.smithi059.stdout:1936: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:50:36.033 INFO:tasks.rados.rados.0.smithi059.stdout:1934: done (3 left) 2022-04-28T19:50:36.033 INFO:tasks.rados.rados.0.smithi059.stdout:1935: done (2 left) 2022-04-28T19:50:36.033 INFO:tasks.rados.rados.0.smithi059.stdout:1936: done (1 left) 2022-04-28T19:50:36.033 INFO:tasks.rados.rados.0.smithi059.stdout:1939: copy_from oid 7 from oid 35 current snap is 180 2022-04-28T19:50:36.033 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 180 2022-04-28T19:50:36.034 INFO:tasks.rados.rados.0.smithi059.stdout:1940: seq_num 630 ranges {2605056=557056,3162112=401408} 2022-04-28T19:50:36.039 INFO:tasks.rados.rados.0.smithi059.stdout:1940: writing smithi05955964-21 from 2605056 to 3162112 tid 1 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:1940: writing smithi05955964-21 from 3162112 to 3563520 tid 2 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:1941: rollback oid 48 current snap is 180 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 170 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 450 (ObjNum 619 snap 176 seq_num 619) dirty exists 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:1938: done (3 left) 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:1942: delete oid 18 current snap is 180 2022-04-28T19:50:36.040 INFO:tasks.rados.rados.0.smithi059.stdout:1942: done (3 left) 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 180 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1943: seq_num 632 ranges {1409024=589824,1998848=401408} 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1943: writing smithi05955964-38 from 1409024 to 1998848 tid 1 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1943: writing smithi05955964-38 from 1998848 to 2400256 tid 2 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1944: rollback oid 29 current snap is 180 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 170 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1941: finishing rollback tid 0 to smithi05955964-48 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1941: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:1941: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:50:36.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 874 (ObjNum 584 snap 166 seq_num 584) dirty exists 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:1940: finishing write tid 1 to smithi05955964-21 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:1940: finishing write tid 2 to smithi05955964-21 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:1940: finishing write tid 4 to smithi05955964-21 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:1940: finishing write tid 5 to smithi05955964-21 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 698 (ObjNum 630 snap 180 seq_num 630) dirty exists 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:1940: left oid 21 (ObjNum 630 snap 180 seq_num 630) 2022-04-28T19:50:36.042 INFO:tasks.rados.rados.0.smithi059.stdout:1940: done (4 left) 2022-04-28T19:50:36.043 INFO:tasks.rados.rados.0.smithi059.stdout:1941: done (3 left) 2022-04-28T19:50:36.043 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 180 2022-04-28T19:50:36.043 INFO:tasks.rados.rados.0.smithi059.stdout:1945: seq_num 633 ranges {5988352=606208,6594560=401408} 2022-04-28T19:50:36.068 INFO:tasks.rados.rados.0.smithi059.stdout:1945: writing smithi05955964-50 from 5988352 to 6594560 tid 1 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1945: writing smithi05955964-50 from 6594560 to 6995968 tid 2 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1946: rollback oid 28 current snap is 180 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 28 to 162 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1939: finishing copy_from to smithi05955964-7 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 729 (ObjNum 612 snap 171 seq_num 612) dirty exists 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1947: snap_create 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1939: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1943: finishing write tid 1 to smithi05955964-38 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1944: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:50:36.071 INFO:tasks.rados.rados.0.smithi059.stdout:1944: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:1943: finishing write tid 2 to smithi05955964-38 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:1944: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 863 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:1943: finishing write tid 4 to smithi05955964-38 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:1943: finishing write tid 5 to smithi05955964-38 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 789 (ObjNum 632 snap 180 seq_num 632) dirty exists 2022-04-28T19:50:36.072 INFO:tasks.rados.rados.0.smithi059.stdout:1943: left oid 38 (ObjNum 632 snap 180 seq_num 632) 2022-04-28T19:50:36.075 INFO:tasks.rados.rados.0.smithi059.stdout:1946: finishing rollback tid 1 to smithi05955964-28 2022-04-28T19:50:36.075 INFO:tasks.rados.rados.0.smithi059.stdout:1946: finishing rollback tid 2 to smithi05955964-28 2022-04-28T19:50:36.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 703 (ObjNum 404 snap 124 seq_num 404) dirty exists 2022-04-28T19:50:36.076 INFO:tasks.rados.rados.0.smithi059.stdout:1945: finishing write tid 1 to smithi05955964-50 2022-04-28T19:50:36.077 INFO:tasks.rados.rados.0.smithi059.stdout:1945: finishing write tid 2 to smithi05955964-50 2022-04-28T19:50:36.077 INFO:tasks.rados.rados.0.smithi059.stdout:1945: finishing write tid 4 to smithi05955964-50 2022-04-28T19:50:36.078 INFO:tasks.rados.rados.0.smithi059.stdout:1945: finishing write tid 5 to smithi05955964-50 2022-04-28T19:50:36.078 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 900 (ObjNum 633 snap 180 seq_num 633) dirty exists 2022-04-28T19:50:36.078 INFO:tasks.rados.rados.0.smithi059.stdout:1945: left oid 50 (ObjNum 633 snap 180 seq_num 633) 2022-04-28T19:50:36.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:35 smithi139 bash[43301]: cluster 2022-04-28T19:50:34.874531+0000 mgr.y (mgr.24961) 92 : cluster [DBG] pgmap v109: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 61/429 objects degraded (14.219%) 2022-04-28T19:50:36.160 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:35 smithi059 bash[61152]: cluster 2022-04-28T19:50:34.874531+0000 mgr.y (mgr.24961) 92 : cluster [DBG] pgmap v109: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 61/429 objects degraded (14.219%) 2022-04-28T19:50:36.161 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:35 smithi059 bash[59745]: cluster 2022-04-28T19:50:34.874531+0000 mgr.y (mgr.24961) 92 : cluster [DBG] pgmap v109: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 15 op/s; 61/429 objects degraded (14.219%) 2022-04-28T19:50:36.591 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:36 smithi059 bash[63562]: debug 2022-04-28T19:50:36.288+0000 7efd83755700 1 -- 172.21.15.59:0/60396786 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564ef989a4e0 con 0x564ef9913400 2022-04-28T19:50:36.825 INFO:tasks.rados.rados.0.smithi059.stdout:1939: done (5 left) 2022-04-28T19:50:36.825 INFO:tasks.rados.rados.0.smithi059.stdout:1943: done (4 left) 2022-04-28T19:50:36.826 INFO:tasks.rados.rados.0.smithi059.stdout:1944: done (3 left) 2022-04-28T19:50:36.826 INFO:tasks.rados.rados.0.smithi059.stdout:1945: done (2 left) 2022-04-28T19:50:36.827 INFO:tasks.rados.rados.0.smithi059.stdout:1946: done (1 left) 2022-04-28T19:50:36.827 INFO:tasks.rados.rados.0.smithi059.stdout:1947: done (0 left) 2022-04-28T19:50:36.827 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 181 2022-04-28T19:50:36.827 INFO:tasks.rados.rados.0.smithi059.stdout:1948: seq_num 634 ranges {6995968=622592,7618560=401408} 2022-04-28T19:50:36.870 INFO:tasks.rados.rados.0.smithi059.stdout:1948: writing smithi05955964-50 from 6995968 to 7618560 tid 1 2022-04-28T19:50:36.872 INFO:tasks.rados.rados.0.smithi059.stdout:1948: writing smithi05955964-50 from 7618560 to 8019968 tid 2 2022-04-28T19:50:36.872 INFO:tasks.rados.rados.0.smithi059.stdout:1949: read oid 28 snap 178 2022-04-28T19:50:36.873 INFO:tasks.rados.rados.0.smithi059.stdout:1949: expect deleted 2022-04-28T19:50:36.873 INFO:tasks.rados.rados.0.smithi059.stdout:1950: snap_remove snap 166 2022-04-28T19:50:37.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:36 smithi139 bash[43301]: cluster 2022-04-28T19:50:35.858311+0000 mon.a (mon.0) 292 : cluster [DBG] osdmap e417: 8 total, 7 up, 8 in 2022-04-28T19:50:37.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:36 smithi139 bash[43301]: cluster 2022-04-28T19:50:36.823650+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e418: 8 total, 7 up, 8 in 2022-04-28T19:50:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:36 smithi059 bash[59745]: cluster 2022-04-28T19:50:35.858311+0000 mon.a (mon.0) 292 : cluster [DBG] osdmap e417: 8 total, 7 up, 8 in 2022-04-28T19:50:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:36 smithi059 bash[59745]: cluster 2022-04-28T19:50:36.823650+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e418: 8 total, 7 up, 8 in 2022-04-28T19:50:37.341 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:36 smithi059 bash[63562]: debug 2022-04-28T19:50:36.836+0000 7efd8afb0200 -1 Falling back to public interface 2022-04-28T19:50:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:36 smithi059 bash[61152]: cluster 2022-04-28T19:50:35.858311+0000 mon.a (mon.0) 292 : cluster [DBG] osdmap e417: 8 total, 7 up, 8 in 2022-04-28T19:50:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:36 smithi059 bash[61152]: cluster 2022-04-28T19:50:36.823650+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e418: 8 total, 7 up, 8 in 2022-04-28T19:50:37.877 INFO:tasks.rados.rados.0.smithi059.stdout:1950: done (2 left) 2022-04-28T19:50:37.877 INFO:tasks.rados.rados.0.smithi059.stdout:1951: read oid 36 snap -1 2022-04-28T19:50:37.877 INFO:tasks.rados.rados.0.smithi059.stdout:1951: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:50:37.878 INFO:tasks.rados.rados.0.smithi059.stdout:1952: snap_remove snap 162 2022-04-28T19:50:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:37 smithi059 bash[59745]: cluster 2022-04-28T19:50:36.875148+0000 mgr.y (mgr.24961) 93 : cluster [DBG] pgmap v112: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 61/429 objects degraded (14.219%) 2022-04-28T19:50:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:37 smithi059 bash[59745]: audit 2022-04-28T19:50:37.694899+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:38.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[57771]: debug 2022-04-28T19:50:38.004+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:50:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:37 smithi059 bash[61152]: cluster 2022-04-28T19:50:36.875148+0000 mgr.y (mgr.24961) 93 : cluster [DBG] pgmap v112: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 61/429 objects degraded (14.219%) 2022-04-28T19:50:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:37 smithi059 bash[61152]: audit 2022-04-28T19:50:37.694899+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:37 smithi139 bash[43301]: cluster 2022-04-28T19:50:36.875148+0000 mgr.y (mgr.24961) 93 : cluster [DBG] pgmap v112: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 252 MiB data, 409 MiB used, 707 GiB / 715 GiB avail; 61/429 objects degraded (14.219%) 2022-04-28T19:50:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:37 smithi139 bash[43301]: audit 2022-04-28T19:50:37.694899+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:38.887 INFO:tasks.rados.rados.0.smithi059.stdout:1952: done (3 left) 2022-04-28T19:50:38.887 INFO:tasks.rados.rados.0.smithi059.stdout:1953: copy_from oid 49 from oid 25 current snap is 181 2022-04-28T19:50:38.887 INFO:tasks.rados.rados.0.smithi059.stdout:1948: finishing write tid 1 to smithi05955964-50 2022-04-28T19:50:38.888 INFO:tasks.rados.rados.0.smithi059.stdout:1948: finishing write tid 2 to smithi05955964-50 2022-04-28T19:50:38.888 INFO:tasks.rados.rados.0.smithi059.stdout:1948: finishing write tid 4 to smithi05955964-50 2022-04-28T19:50:38.888 INFO:tasks.rados.rados.0.smithi059.stdout:1948: finishing write tid 5 to smithi05955964-50 2022-04-28T19:50:38.888 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 904 (ObjNum 634 snap 181 seq_num 634) dirty exists 2022-04-28T19:50:38.888 INFO:tasks.rados.rados.0.smithi059.stdout:1948: left oid 50 (ObjNum 634 snap 181 seq_num 634) 2022-04-28T19:50:38.888 INFO:tasks.rados.rados.0.smithi059.stdout:1951: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:50:38.932 INFO:tasks.rados.rados.0.smithi059.stdout:1953: finishing copy_from to smithi05955964-49 2022-04-28T19:50:38.932 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 821 (ObjNum 605 snap 169 seq_num 605) dirty exists 2022-04-28T19:50:38.932 INFO:tasks.rados.rados.0.smithi059.stdout:1953: finishing copy_from racing read to smithi05955964-49 2022-04-28T19:50:38.932 INFO:tasks.rados.rados.0.smithi059.stdout:1948: done (3 left) 2022-04-28T19:50:38.933 INFO:tasks.rados.rados.0.smithi059.stdout:1949: done (2 left) 2022-04-28T19:50:38.933 INFO:tasks.rados.rados.0.smithi059.stdout:1951: done (1 left) 2022-04-28T19:50:38.933 INFO:tasks.rados.rados.0.smithi059.stdout:1953: done (0 left) 2022-04-28T19:50:38.933 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 181 2022-04-28T19:50:38.933 INFO:tasks.rados.rados.0.smithi059.stdout:1954: seq_num 635 ranges {9650176=638976,10289152=401408} 2022-04-28T19:50:38.993 INFO:tasks.rados.rados.0.smithi059.stdout:1954: writing smithi05955964-35 from 9650176 to 10289152 tid 1 2022-04-28T19:50:38.995 INFO:tasks.rados.rados.0.smithi059.stdout:1954: writing smithi05955964-35 from 10289152 to 10690560 tid 2 2022-04-28T19:50:38.995 INFO:tasks.rados.rados.0.smithi059.stdout:1955: rollback oid 1 current snap is 181 2022-04-28T19:50:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 170 2022-04-28T19:50:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:1956: copy_from oid 13 from oid 21 current snap is 181 2022-04-28T19:50:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:1957: snap_create 2022-04-28T19:50:39.003 INFO:tasks.rados.rados.0.smithi059.stdout:1955: finishing rollback tid 0 to smithi05955964-1 2022-04-28T19:50:39.004 INFO:tasks.rados.rados.0.smithi059.stdout:1954: finishing write tid 1 to smithi05955964-35 2022-04-28T19:50:39.004 INFO:tasks.rados.rados.0.smithi059.stdout:1954: finishing write tid 2 to smithi05955964-35 2022-04-28T19:50:39.005 INFO:tasks.rados.rados.0.smithi059.stdout:1954: finishing write tid 4 to smithi05955964-35 2022-04-28T19:50:39.006 INFO:tasks.rados.rados.0.smithi059.stdout:1955: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:50:39.007 INFO:tasks.rados.rados.0.smithi059.stdout:1954: finishing write tid 5 to smithi05955964-35 2022-04-28T19:50:39.007 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 732 (ObjNum 635 snap 181 seq_num 635) dirty exists 2022-04-28T19:50:39.007 INFO:tasks.rados.rados.0.smithi059.stdout:1954: left oid 35 (ObjNum 635 snap 181 seq_num 635) 2022-04-28T19:50:39.007 INFO:tasks.rados.rados.0.smithi059.stdout:1955: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:50:39.007 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 824 (ObjNum 602 snap 169 seq_num 602) dirty exists 2022-04-28T19:50:39.019 INFO:tasks.rados.rados.0.smithi059.stdout:1956: finishing copy_from to smithi05955964-13 2022-04-28T19:50:39.100 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 698 (ObjNum 630 snap 180 seq_num 630) dirty exists 2022-04-28T19:50:39.101 INFO:tasks.rados.rados.0.smithi059.stdout:1956: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:50:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: cluster 2022-04-28T19:50:37.875963+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e419: 8 total, 7 up, 8 in 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:37.992525+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:38.000838+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:38.003973+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:38.005689+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:38.006887+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:38.008811+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: audit 2022-04-28T19:50:38.009065+0000 mgr.y (mgr.24961) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[59745]: cephadm 2022-04-28T19:50:38.010444+0000 mgr.y (mgr.24961) 95 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: cluster 2022-04-28T19:50:37.875963+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e419: 8 total, 7 up, 8 in 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:37.992525+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:38.000838+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:38.003973+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:38.005689+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:38.006887+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:38.008811+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: audit 2022-04-28T19:50:38.009065+0000 mgr.y (mgr.24961) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:38 smithi059 bash[61152]: cephadm 2022-04-28T19:50:38.010444+0000 mgr.y (mgr.24961) 95 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:50:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: cluster 2022-04-28T19:50:37.875963+0000 mon.a (mon.0) 295 : cluster [DBG] osdmap e419: 8 total, 7 up, 8 in 2022-04-28T19:50:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:37.992525+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:38.000838+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:50:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:38.003973+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:38.005689+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:38.006887+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:38.008811+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: audit 2022-04-28T19:50:38.009065+0000 mgr.y (mgr.24961) 94 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:38 smithi139 bash[43301]: cephadm 2022-04-28T19:50:38.010444+0000 mgr.y (mgr.24961) 95 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:50:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:1954: done (3 left) 2022-04-28T19:50:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:1955: done (2 left) 2022-04-28T19:50:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:1956: done (1 left) 2022-04-28T19:50:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:1957: done (0 left) 2022-04-28T19:50:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:1958: copy_from oid 35 from oid 2 current snap is 182 2022-04-28T19:50:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:1959: copy_from oid 24 from oid 12 current snap is 182 2022-04-28T19:50:39.898 INFO:tasks.rados.rados.0.smithi059.stdout:1960: copy_from oid 44 from oid 36 current snap is 182 2022-04-28T19:50:39.898 INFO:tasks.rados.rados.0.smithi059.stdout:1961: snap_create 2022-04-28T19:50:39.898 INFO:tasks.rados.rados.0.smithi059.stdout:1958: finishing copy_from to smithi05955964-35 2022-04-28T19:50:39.898 INFO:tasks.rados.rados.0.smithi059.stdout:1958: got expected ENOENT (src dne) 2022-04-28T19:50:39.899 INFO:tasks.rados.rados.0.smithi059.stdout:1958: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:50:39.905 INFO:tasks.rados.rados.0.smithi059.stdout:1959: finishing copy_from to smithi05955964-24 2022-04-28T19:50:39.905 INFO:tasks.rados.rados.0.smithi059.stdout:1959: got expected ENOENT (src dne) 2022-04-28T19:50:39.905 INFO:tasks.rados.rados.0.smithi059.stdout:1959: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:50:39.910 INFO:tasks.rados.rados.0.smithi059.stdout:1960: finishing copy_from to smithi05955964-44 2022-04-28T19:50:39.910 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 712 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:50:39.910 INFO:tasks.rados.rados.0.smithi059.stdout:1960: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:50:40.259 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:39 smithi059 bash[61152]: cluster 2022-04-28T19:50:38.875658+0000 mgr.y (mgr.24961) 96 : cluster [DBG] pgmap v114: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 259 MiB data, 415 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 3.9 MiB/s wr, 24 op/s; 70/450 objects degraded (15.556%) 2022-04-28T19:50:40.259 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:39 smithi059 bash[61152]: cluster 2022-04-28T19:50:38.886316+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e420: 8 total, 7 up, 8 in 2022-04-28T19:50:40.260 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:39 smithi059 bash[61152]: cluster 2022-04-28T19:50:39.871615+0000 mon.a (mon.0) 303 : cluster [WRN] Health check update: Degraded data redundancy: 70/450 objects degraded (15.556%), 14 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:40.260 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:39 smithi059 bash[59745]: cluster 2022-04-28T19:50:38.875658+0000 mgr.y (mgr.24961) 96 : cluster [DBG] pgmap v114: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 259 MiB data, 415 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 3.9 MiB/s wr, 24 op/s; 70/450 objects degraded (15.556%) 2022-04-28T19:50:40.260 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:39 smithi059 bash[59745]: cluster 2022-04-28T19:50:38.886316+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e420: 8 total, 7 up, 8 in 2022-04-28T19:50:40.260 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:39 smithi059 bash[59745]: cluster 2022-04-28T19:50:39.871615+0000 mon.a (mon.0) 303 : cluster [WRN] Health check update: Degraded data redundancy: 70/450 objects degraded (15.556%), 14 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:39 smithi139 bash[43301]: cluster 2022-04-28T19:50:38.875658+0000 mgr.y (mgr.24961) 96 : cluster [DBG] pgmap v114: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 259 MiB data, 415 MiB used, 707 GiB / 715 GiB avail; 11 MiB/s rd, 3.9 MiB/s wr, 24 op/s; 70/450 objects degraded (15.556%) 2022-04-28T19:50:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:39 smithi139 bash[43301]: cluster 2022-04-28T19:50:38.886316+0000 mon.a (mon.0) 302 : cluster [DBG] osdmap e420: 8 total, 7 up, 8 in 2022-04-28T19:50:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:39 smithi139 bash[43301]: cluster 2022-04-28T19:50:39.871615+0000 mon.a (mon.0) 303 : cluster [WRN] Health check update: Degraded data redundancy: 70/450 objects degraded (15.556%), 14 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:40.591 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[63562]: debug 2022-04-28T19:50:40.252+0000 7efd8afb0200 -1 osd.0 411 log_to_monitors {default=true} 2022-04-28T19:50:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:1958: done (3 left) 2022-04-28T19:50:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:1959: done (2 left) 2022-04-28T19:50:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:1960: done (1 left) 2022-04-28T19:50:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:1961: done (0 left) 2022-04-28T19:50:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 183 2022-04-28T19:50:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:1962: seq_num 636 ranges {3235840=655360,3891200=401408} 2022-04-28T19:50:40.934 INFO:tasks.rados.rados.0.smithi059.stdout:1962: writing smithi05955964-45 from 3235840 to 3891200 tid 1 2022-04-28T19:50:40.937 INFO:tasks.rados.rados.0.smithi059.stdout:1962: writing smithi05955964-45 from 3891200 to 4292608 tid 2 2022-04-28T19:50:40.937 INFO:tasks.rados.rados.0.smithi059.stdout:1963: rollback oid 15 current snap is 183 2022-04-28T19:50:40.937 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 182 2022-04-28T19:50:40.938 INFO:tasks.rados.rados.0.smithi059.stdout:1964: snap_remove snap 178 2022-04-28T19:50:41.341 INFO:journalctl@ceph.osd.0.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[63562]: debug 2022-04-28T19:50:40.928+0000 7efd7bec9700 -1 osd.0 411 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:50:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[59745]: cluster 2022-04-28T19:50:39.895443+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e421: 8 total, 7 up, 8 in 2022-04-28T19:50:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[59745]: audit 2022-04-28T19:50:40.262273+0000 mon.c (mon.1) 4 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:50:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[59745]: audit 2022-04-28T19:50:40.262600+0000 mon.a (mon.0) 305 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:50:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[61152]: cluster 2022-04-28T19:50:39.895443+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e421: 8 total, 7 up, 8 in 2022-04-28T19:50:41.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[61152]: audit 2022-04-28T19:50:40.262273+0000 mon.c (mon.1) 4 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:50:41.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:40 smithi059 bash[61152]: audit 2022-04-28T19:50:40.262600+0000 mon.a (mon.0) 305 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:50:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:40 smithi139 bash[43301]: cluster 2022-04-28T19:50:39.895443+0000 mon.a (mon.0) 304 : cluster [DBG] osdmap e421: 8 total, 7 up, 8 in 2022-04-28T19:50:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:40 smithi139 bash[43301]: audit 2022-04-28T19:50:40.262273+0000 mon.c (mon.1) 4 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:50:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:40 smithi139 bash[43301]: audit 2022-04-28T19:50:40.262600+0000 mon.a (mon.0) 305 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2022-04-28T19:50:41.921 INFO:tasks.rados.rados.0.smithi059.stdout:1964: done (2 left) 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:1965: read oid 8 snap 180 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:1965: expect (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:1966: rollback oid 47 current snap is 183 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 180 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:1962: finishing write tid 1 to smithi05955964-45 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:1962: finishing write tid 2 to smithi05955964-45 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 183 2022-04-28T19:50:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:1967: seq_num 637 ranges {7143424=671744,7815168=401408} 2022-04-28T19:50:41.927 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:41.628Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:41.927 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:41.628Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:41.927 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:41.628Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:41.927 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[59745]: cluster 2022-04-28T19:50:40.876634+0000 mgr.y (mgr.24961) 97 : cluster [DBG] pgmap v117: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 259 MiB data, 415 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 4.8 MiB/s wr, 30 op/s; 70/450 objects degraded (15.556%) 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[59745]: audit 2022-04-28T19:50:40.902494+0000 mon.a (mon.0) 306 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[59745]: cluster 2022-04-28T19:50:40.902653+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e422: 8 total, 7 up, 8 in 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[59745]: audit 2022-04-28T19:50:40.904911+0000 mon.c (mon.1) 5 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[59745]: audit 2022-04-28T19:50:40.912482+0000 mon.a (mon.0) 308 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[61152]: cluster 2022-04-28T19:50:40.876634+0000 mgr.y (mgr.24961) 97 : cluster [DBG] pgmap v117: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 259 MiB data, 415 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 4.8 MiB/s wr, 30 op/s; 70/450 objects degraded (15.556%) 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[61152]: audit 2022-04-28T19:50:40.902494+0000 mon.a (mon.0) 306 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[61152]: cluster 2022-04-28T19:50:40.902653+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e422: 8 total, 7 up, 8 in 2022-04-28T19:50:41.928 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[61152]: audit 2022-04-28T19:50:40.904911+0000 mon.c (mon.1) 5 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:50:41.929 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:41 smithi059 bash[61152]: audit 2022-04-28T19:50:40.912482+0000 mon.a (mon.0) 308 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:50:41.971 INFO:tasks.rados.rados.0.smithi059.stdout:1967: writing smithi05955964-11 from 7143424 to 7815168 tid 1 2022-04-28T19:50:41.975 INFO:tasks.rados.rados.0.smithi059.stdout:1967: writing smithi05955964-11 from 7815168 to 8216576 tid 2 2022-04-28T19:50:41.975 INFO:tasks.rados.rados.0.smithi059.stdout:1968: copy_from oid 23 from oid 42 current snap is 183 2022-04-28T19:50:41.975 INFO:tasks.rados.rados.0.smithi059.stdout:1962: finishing write tid 4 to smithi05955964-45 2022-04-28T19:50:41.975 INFO:tasks.rados.rados.0.smithi059.stdout:1962: finishing write tid 5 to smithi05955964-45 2022-04-28T19:50:41.975 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 787 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:50:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:1962: left oid 45 (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:50:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:1963: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:50:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:1962: done (5 left) 2022-04-28T19:50:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 183 2022-04-28T19:50:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:1969: seq_num 638 ranges {3235840=688128,3923968=401408} 2022-04-28T19:50:41.996 INFO:tasks.rados.rados.0.smithi059.stdout:1969: writing smithi05955964-10 from 3235840 to 3923968 tid 1 2022-04-28T19:50:41.999 INFO:tasks.rados.rados.0.smithi059.stdout:1969: writing smithi05955964-10 from 3923968 to 4325376 tid 2 2022-04-28T19:50:41.999 INFO:tasks.rados.rados.0.smithi059.stdout:1970: snap_remove snap 170 2022-04-28T19:50:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:41 smithi139 bash[43301]: cluster 2022-04-28T19:50:40.876634+0000 mgr.y (mgr.24961) 97 : cluster [DBG] pgmap v117: 89 pgs: 14 active+undersized+degraded, 20 active+undersized, 55 active+clean; 259 MiB data, 415 MiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 4.8 MiB/s wr, 30 op/s; 70/450 objects degraded (15.556%) 2022-04-28T19:50:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:41 smithi139 bash[43301]: audit 2022-04-28T19:50:40.902494+0000 mon.a (mon.0) 306 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2022-04-28T19:50:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:41 smithi139 bash[43301]: cluster 2022-04-28T19:50:40.902653+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e422: 8 total, 7 up, 8 in 2022-04-28T19:50:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:41 smithi139 bash[43301]: audit 2022-04-28T19:50:40.904911+0000 mon.c (mon.1) 5 : audit [INF] from='osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:50:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:41 smithi139 bash[43301]: audit 2022-04-28T19:50:40.912482+0000 mon.a (mon.0) 308 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:50:42.922 INFO:tasks.rados.rados.0.smithi059.stdout:1970: done (6 left) 2022-04-28T19:50:42.922 INFO:tasks.rados.rados.0.smithi059.stdout:1971: snap_remove snap 179 2022-04-28T19:50:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[61152]: cluster 2022-04-28T19:50:41.906258+0000 mon.a (mon.0) 309 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:50:43.385 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[61152]: cluster 2022-04-28T19:50:41.917571+0000 mon.a (mon.0) 310 : cluster [INF] osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435] boot 2022-04-28T19:50:43.385 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[61152]: cluster 2022-04-28T19:50:41.917783+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2022-04-28T19:50:43.385 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[61152]: audit 2022-04-28T19:50:41.919287+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:50:43.385 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[61152]: cluster 2022-04-28T19:50:42.920780+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2022-04-28T19:50:43.386 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[59745]: cluster 2022-04-28T19:50:41.906258+0000 mon.a (mon.0) 309 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:50:43.386 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[59745]: cluster 2022-04-28T19:50:41.917571+0000 mon.a (mon.0) 310 : cluster [INF] osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435] boot 2022-04-28T19:50:43.386 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[59745]: cluster 2022-04-28T19:50:41.917783+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2022-04-28T19:50:43.386 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[59745]: audit 2022-04-28T19:50:41.919287+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:50:43.387 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:42 smithi059 bash[59745]: cluster 2022-04-28T19:50:42.920780+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2022-04-28T19:50:43.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:42 smithi139 bash[43301]: cluster 2022-04-28T19:50:41.906258+0000 mon.a (mon.0) 309 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:50:43.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:42 smithi139 bash[43301]: cluster 2022-04-28T19:50:41.917571+0000 mon.a (mon.0) 310 : cluster [INF] osd.0 [v2:172.21.15.59:6802/787267435,v1:172.21.15.59:6803/787267435] boot 2022-04-28T19:50:43.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:42 smithi139 bash[43301]: cluster 2022-04-28T19:50:41.917783+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2022-04-28T19:50:43.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:42 smithi139 bash[43301]: audit 2022-04-28T19:50:41.919287+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2022-04-28T19:50:43.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:42 smithi139 bash[43301]: cluster 2022-04-28T19:50:42.920780+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2022-04-28T19:50:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:43.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:43.521Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:43.521Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:50:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:43.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:43.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:43.526Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:43.924 INFO:tasks.rados.rados.0.smithi059.stdout:1971: done (6 left) 2022-04-28T19:50:43.924 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 183 2022-04-28T19:50:43.924 INFO:tasks.rados.rados.0.smithi059.stdout:1972: seq_num 639 ranges {1974272=704512,2678784=401408} 2022-04-28T19:50:43.940 INFO:tasks.rados.rados.0.smithi059.stdout:1972: writing smithi05955964-28 from 1974272 to 2678784 tid 1 2022-04-28T19:50:43.943 INFO:tasks.rados.rados.0.smithi059.stdout:1972: writing smithi05955964-28 from 2678784 to 3080192 tid 2 2022-04-28T19:50:43.943 INFO:tasks.rados.rados.0.smithi059.stdout:1973: setattr oid 35 current snap is 183 2022-04-28T19:50:43.943 INFO:tasks.rados.rados.0.smithi059.stdout:1963: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:50:43.943 INFO:tasks.rados.rados.0.smithi059.stdout:1963: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:50:43.943 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 829 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:50:43.944 INFO:tasks.rados.rados.0.smithi059.stdout:1965: expect (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:50:43.963 INFO:tasks.rados.rados.0.smithi059.stdout:1966: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1966: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1966: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 923 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1969: finishing write tid 1 to smithi05955964-10 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1969: finishing write tid 2 to smithi05955964-10 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1969: finishing write tid 4 to smithi05955964-10 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1969: finishing write tid 5 to smithi05955964-10 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 927 (ObjNum 638 snap 183 seq_num 638) dirty exists 2022-04-28T19:50:43.964 INFO:tasks.rados.rados.0.smithi059.stdout:1969: left oid 10 (ObjNum 638 snap 183 seq_num 638) 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1968: finishing copy_from to smithi05955964-23 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 850 (ObjNum 613 snap 172 seq_num 613) dirty exists 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1968: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1967: finishing write tid 1 to smithi05955964-11 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1967: finishing write tid 2 to smithi05955964-11 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1967: finishing write tid 4 to smithi05955964-11 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1967: finishing write tid 5 to smithi05955964-11 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 793 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1967: left oid 11 (ObjNum 637 snap 183 seq_num 637) 2022-04-28T19:50:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:1972: finishing write tid 1 to smithi05955964-28 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1972: finishing write tid 2 to smithi05955964-28 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1972: finishing write tid 4 to smithi05955964-28 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1972: finishing write tid 5 to smithi05955964-28 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 706 (ObjNum 639 snap 183 seq_num 639) dirty exists 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1972: left oid 28 (ObjNum 639 snap 183 seq_num 639) 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1963: done (7 left) 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1965: done (6 left) 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1966: done (5 left) 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1967: done (4 left) 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1968: done (3 left) 2022-04-28T19:50:43.966 INFO:tasks.rados.rados.0.smithi059.stdout:1969: done (2 left) 2022-04-28T19:50:43.967 INFO:tasks.rados.rados.0.smithi059.stdout:1972: done (1 left) 2022-04-28T19:50:43.967 INFO:tasks.rados.rados.0.smithi059.stdout:1974: rollback oid 14 current snap is 183 2022-04-28T19:50:43.967 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 182 2022-04-28T19:50:43.967 INFO:tasks.rados.rados.0.smithi059.stdout:1975: read oid 37 snap -1 2022-04-28T19:50:43.967 INFO:tasks.rados.rados.0.smithi059.stdout:1975: expect deleted 2022-04-28T19:50:43.967 INFO:tasks.rados.rados.0.smithi059.stdout:1976: setattr oid 11 current snap is 183 2022-04-28T19:50:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:1975: done (3 left) 2022-04-28T19:50:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:1977: read oid 50 snap 173 2022-04-28T19:50:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:1977: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:1978: rollback oid 36 current snap is 183 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 180 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:1979: read oid 9 snap -1 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:1979: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:1980: rollback oid 34 current snap is 183 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 34 to 182 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:1981: setattr oid 18 current snap is 183 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:1980: finishing rollback tid 1 to smithi05955964-34 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 0 (ObjNum 1886022488 snap 1869440866 seq_num 1583567216) dirty dne 2022-04-28T19:50:43.970 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 733 (ObjNum 635 snap 181 seq_num 635) dirty exists 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1974: finishing rollback tid 0 to smithi05955964-14 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1973: done (7 left) 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1980: done (6 left) 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1982: read oid 23 snap 180 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1982: expect (ObjNum 621 snap 178 seq_num 621) 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1983: rollback oid 31 current snap is 183 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 177 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1984: setattr oid 19 current snap is 183 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1974: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:1974: finishing rollback tid 2 to smithi05955964-14 2022-04-28T19:50:43.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 894 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:43.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 794 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:50:43.972 INFO:tasks.rados.rados.0.smithi059.stdout:1974: done (8 left) 2022-04-28T19:50:43.972 INFO:tasks.rados.rados.0.smithi059.stdout:1976: done (7 left) 2022-04-28T19:50:43.972 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 183 2022-04-28T19:50:43.973 INFO:tasks.rados.rados.0.smithi059.stdout:1985: seq_num 640 ranges {8282112=729088,9011200=401408} 2022-04-28T19:50:44.022 INFO:tasks.rados.rados.0.smithi059.stdout:1985: writing smithi05955964-17 from 8282112 to 9011200 tid 1 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:1985: writing smithi05955964-17 from 9011200 to 9412608 tid 2 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:1986: snap_create 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:1983: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 890 (ObjNum 288 snap 0 seq_num 289) dirty exists 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:1983: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 896 (ObjNum 601 snap 169 seq_num 601) dirty exists 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:1983: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 840 (ObjNum 603 snap 169 seq_num 603) dirty exists 2022-04-28T19:50:44.025 INFO:tasks.rados.rados.0.smithi059.stdout:1978: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:50:44.026 INFO:tasks.rados.rados.0.smithi059.stdout:1982: expect (ObjNum 621 snap 178 seq_num 621) 2022-04-28T19:50:44.065 INFO:tasks.rados.rados.0.smithi059.stdout:1978: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:50:44.066 INFO:tasks.rados.rados.0.smithi059.stdout:1978: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:50:44.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 827 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:50:44.066 INFO:tasks.rados.rados.0.smithi059.stdout:1979: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:44.177 INFO:tasks.rados.rados.0.smithi059.stdout:1977: expect (ObjNum 516 snap 145 seq_num 516) 2022-04-28T19:50:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[61152]: cluster 2022-04-28T19:50:42.877969+0000 mgr.y (mgr.24961) 98 : cluster [DBG] pgmap v120: 89 pgs: 21 peering, 4 active+undersized+degraded, 9 active+undersized, 55 active+clean; 223 MiB data, 380 MiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.8 MiB/s wr, 7 op/s; 11/408 objects degraded (2.696%) 2022-04-28T19:50:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[61152]: cluster 2022-04-28T19:50:43.922325+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2022-04-28T19:50:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[59745]: cluster 2022-04-28T19:50:42.877969+0000 mgr.y (mgr.24961) 98 : cluster [DBG] pgmap v120: 89 pgs: 21 peering, 4 active+undersized+degraded, 9 active+undersized, 55 active+clean; 223 MiB data, 380 MiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.8 MiB/s wr, 7 op/s; 11/408 objects degraded (2.696%) 2022-04-28T19:50:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:43 smithi059 bash[59745]: cluster 2022-04-28T19:50:43.922325+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2022-04-28T19:50:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:43 smithi139 bash[43301]: cluster 2022-04-28T19:50:42.877969+0000 mgr.y (mgr.24961) 98 : cluster [DBG] pgmap v120: 89 pgs: 21 peering, 4 active+undersized+degraded, 9 active+undersized, 55 active+clean; 223 MiB data, 380 MiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.8 MiB/s wr, 7 op/s; 11/408 objects degraded (2.696%) 2022-04-28T19:50:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:43 smithi139 bash[43301]: cluster 2022-04-28T19:50:43.922325+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2022-04-28T19:50:44.547 INFO:tasks.rados.rados.0.smithi059.stdout:1985: finishing write tid 1 to smithi05955964-17 2022-04-28T19:50:44.547 INFO:tasks.rados.rados.0.smithi059.stdout:1985: finishing write tid 2 to smithi05955964-17 2022-04-28T19:50:44.547 INFO:tasks.rados.rados.0.smithi059.stdout:1985: finishing write tid 4 to smithi05955964-17 2022-04-28T19:50:44.547 INFO:tasks.rados.rados.0.smithi059.stdout:1985: finishing write tid 5 to smithi05955964-17 2022-04-28T19:50:44.547 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 701 (ObjNum 640 snap 183 seq_num 640) dirty exists 2022-04-28T19:50:44.547 INFO:tasks.rados.rados.0.smithi059.stdout:1985: left oid 17 (ObjNum 640 snap 183 seq_num 640) 2022-04-28T19:50:44.962 INFO:tasks.rados.rados.0.smithi059.stdout:1977: done (8 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1978: done (7 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1979: done (6 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1981: done (5 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1982: done (4 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1983: done (3 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1984: done (2 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1985: done (1 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1986: done (0 left) 2022-04-28T19:50:44.963 INFO:tasks.rados.rados.0.smithi059.stdout:1987: delete oid 9 current snap is 184 2022-04-28T19:50:44.969 INFO:tasks.rados.rados.0.smithi059.stdout:1987: done (0 left) 2022-04-28T19:50:44.969 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 184 2022-04-28T19:50:44.969 INFO:tasks.rados.rados.0.smithi059.stdout:1988: seq_num 642 ranges {10690560=761856,11452416=401408} 2022-04-28T19:50:45.032 INFO:tasks.rados.rados.0.smithi059.stdout:1988: writing smithi05955964-35 from 10690560 to 11452416 tid 1 2022-04-28T19:50:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:1988: writing smithi05955964-35 from 11452416 to 11853824 tid 2 2022-04-28T19:50:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 184 2022-04-28T19:50:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:1989: seq_num 643 ranges {7135232=778240,7913472=401408} 2022-04-28T19:50:45.078 INFO:tasks.rados.rados.0.smithi059.stdout:1989: writing smithi05955964-6 from 7135232 to 7913472 tid 1 2022-04-28T19:50:45.080 INFO:tasks.rados.rados.0.smithi059.stdout:1989: writing smithi05955964-6 from 7913472 to 8314880 tid 2 2022-04-28T19:50:45.080 INFO:tasks.rados.rados.0.smithi059.stdout:1990: delete oid 37 current snap is 184 2022-04-28T19:50:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:1988: finishing write tid 1 to smithi05955964-35 2022-04-28T19:50:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:1988: finishing write tid 2 to smithi05955964-35 2022-04-28T19:50:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:1988: finishing write tid 4 to smithi05955964-35 2022-04-28T19:50:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:1988: finishing write tid 5 to smithi05955964-35 2022-04-28T19:50:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 736 (ObjNum 642 snap 184 seq_num 642) dirty exists 2022-04-28T19:50:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:1988: left oid 35 (ObjNum 642 snap 184 seq_num 642) 2022-04-28T19:50:45.084 INFO:tasks.rados.rados.0.smithi059.stdout:1988: done (2 left) 2022-04-28T19:50:45.085 INFO:tasks.rados.rados.0.smithi059.stdout:1990: done (1 left) 2022-04-28T19:50:45.085 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 184 2022-04-28T19:50:45.085 INFO:tasks.rados.rados.0.smithi059.stdout:1991: seq_num 645 ranges {0=409600,409600=778240,1187840=24576} 2022-04-28T19:50:45.087 INFO:tasks.rados.rados.0.smithi059.stdout:1991: writing smithi05955964-40 from 0 to 409600 tid 1 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1991: writing smithi05955964-40 from 409600 to 1187840 tid 2 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1991: writing smithi05955964-40 from 1187840 to 1212416 tid 3 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1992: copy_from oid 31 from oid 21 current snap is 184 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1989: finishing write tid 1 to smithi05955964-6 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1989: finishing write tid 2 to smithi05955964-6 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1989: finishing write tid 4 to smithi05955964-6 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:1989: finishing write tid 5 to smithi05955964-6 2022-04-28T19:50:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 817 (ObjNum 643 snap 184 seq_num 643) dirty exists 2022-04-28T19:50:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:1989: left oid 6 (ObjNum 643 snap 184 seq_num 643) 2022-04-28T19:50:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:1991: finishing write tid 1 to smithi05955964-40 2022-04-28T19:50:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:1989: done (2 left) 2022-04-28T19:50:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 184 2022-04-28T19:50:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:1993: seq_num 646 ranges {4325376=425984,4751360=655360,5406720=147456} 2022-04-28T19:50:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:1993: writing smithi05955964-10 from 4325376 to 4751360 tid 1 2022-04-28T19:50:45.129 INFO:tasks.rados.rados.0.smithi059.stdout:1993: writing smithi05955964-10 from 4751360 to 5406720 tid 2 2022-04-28T19:50:45.129 INFO:tasks.rados.rados.0.smithi059.stdout:1993: writing smithi05955964-10 from 5406720 to 5554176 tid 3 2022-04-28T19:50:45.130 INFO:tasks.rados.rados.0.smithi059.stdout:1994: copy_from oid 19 from oid 47 current snap is 184 2022-04-28T19:50:45.130 INFO:tasks.rados.rados.0.smithi059.stdout:1991: finishing write tid 2 to smithi05955964-40 2022-04-28T19:50:45.130 INFO:tasks.rados.rados.0.smithi059.stdout:1991: finishing write tid 3 to smithi05955964-40 2022-04-28T19:50:45.130 INFO:tasks.rados.rados.0.smithi059.stdout:1991: finishing write tid 5 to smithi05955964-40 2022-04-28T19:50:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:1991: finishing write tid 6 to smithi05955964-40 2022-04-28T19:50:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 831 (ObjNum 645 snap 184 seq_num 645) dirty exists 2022-04-28T19:50:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:1991: left oid 40 (ObjNum 645 snap 184 seq_num 645) 2022-04-28T19:50:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:1992: finishing copy_from to smithi05955964-31 2022-04-28T19:50:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 698 (ObjNum 630 snap 180 seq_num 630) dirty exists 2022-04-28T19:50:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:1992: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:50:45.132 INFO:tasks.rados.rados.0.smithi059.stdout:1993: finishing write tid 1 to smithi05955964-10 2022-04-28T19:50:45.132 INFO:tasks.rados.rados.0.smithi059.stdout:1991: done (3 left) 2022-04-28T19:50:45.132 INFO:tasks.rados.rados.0.smithi059.stdout:1992: done (2 left) 2022-04-28T19:50:45.132 INFO:tasks.rados.rados.0.smithi059.stdout:1995: delete oid 31 current snap is 184 2022-04-28T19:50:45.132 INFO:tasks.rados.rados.0.smithi059.stdout:1995: done (2 left) 2022-04-28T19:50:45.133 INFO:tasks.rados.rados.0.smithi059.stdout:1996: snap_remove snap 171 2022-04-28T19:50:45.971 INFO:tasks.rados.rados.0.smithi059.stdout:1996: done (2 left) 2022-04-28T19:50:45.971 INFO:tasks.rados.rados.0.smithi059.stdout:1997: snap_remove snap 183 2022-04-28T19:50:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:45 smithi059 bash[59745]: cluster 2022-04-28T19:50:44.878748+0000 mgr.y (mgr.24961) 99 : cluster [DBG] pgmap v123: 89 pgs: 21 peering, 4 active+undersized+degraded, 9 active+undersized, 55 active+clean; 223 MiB data, 380 MiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.8 MiB/s wr, 7 op/s; 11/408 objects degraded (2.696%) 2022-04-28T19:50:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:45 smithi059 bash[59745]: cluster 2022-04-28T19:50:44.960375+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2022-04-28T19:50:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:45 smithi059 bash[61152]: cluster 2022-04-28T19:50:44.878748+0000 mgr.y (mgr.24961) 99 : cluster [DBG] pgmap v123: 89 pgs: 21 peering, 4 active+undersized+degraded, 9 active+undersized, 55 active+clean; 223 MiB data, 380 MiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.8 MiB/s wr, 7 op/s; 11/408 objects degraded (2.696%) 2022-04-28T19:50:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:45 smithi059 bash[61152]: cluster 2022-04-28T19:50:44.960375+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2022-04-28T19:50:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:45 smithi139 bash[43301]: cluster 2022-04-28T19:50:44.878748+0000 mgr.y (mgr.24961) 99 : cluster [DBG] pgmap v123: 89 pgs: 21 peering, 4 active+undersized+degraded, 9 active+undersized, 55 active+clean; 223 MiB data, 380 MiB used, 708 GiB / 715 GiB avail; 1.5 MiB/s rd, 2.8 MiB/s wr, 7 op/s; 11/408 objects degraded (2.696%) 2022-04-28T19:50:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:45 smithi139 bash[43301]: cluster 2022-04-28T19:50:44.960375+0000 mon.a (mon.0) 315 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:1997: done (2 left) 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:1998: copy_from oid 30 from oid 31 current snap is 184 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:1993: finishing write tid 2 to smithi05955964-10 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:1993: finishing write tid 3 to smithi05955964-10 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:1993: finishing write tid 5 to smithi05955964-10 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:1993: finishing write tid 6 to smithi05955964-10 2022-04-28T19:50:46.978 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 948 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1993: left oid 10 (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1994: finishing copy_from to smithi05955964-19 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 923 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1994: finishing copy_from racing read to smithi05955964-19 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1993: done (2 left) 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1994: done (1 left) 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1999: read oid 21 snap -1 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:1999: expect (ObjNum 630 snap 180 seq_num 630) 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 184 2022-04-28T19:50:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:2000: seq_num 648 ranges {4022272=458752,4481024=475136,4956160=327680} 2022-04-28T19:50:47.007 INFO:tasks.rados.rados.0.smithi059.stdout:2000: writing smithi05955964-3 from 4022272 to 4481024 tid 1 2022-04-28T19:50:47.009 INFO:tasks.rados.rados.0.smithi059.stdout:2000: writing smithi05955964-3 from 4481024 to 4956160 tid 2 2022-04-28T19:50:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:2000: writing smithi05955964-3 from 4956160 to 5283840 tid 3 2022-04-28T19:50:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:2001: delete oid 36 current snap is 184 2022-04-28T19:50:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:1998: finishing copy_from to smithi05955964-30 2022-04-28T19:50:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:1998: got expected ENOENT (src dne) 2022-04-28T19:50:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:1998: finishing copy_from racing read to smithi05955964-30 2022-04-28T19:50:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:1999: expect (ObjNum 630 snap 180 seq_num 630) 2022-04-28T19:50:47.196 INFO:tasks.rados.rados.0.smithi059.stdout:2000: finishing write tid 1 to smithi05955964-3 2022-04-28T19:50:47.196 INFO:tasks.rados.rados.0.smithi059.stdout:2000: finishing write tid 2 to smithi05955964-3 2022-04-28T19:50:47.196 INFO:tasks.rados.rados.0.smithi059.stdout:2000: finishing write tid 3 to smithi05955964-3 2022-04-28T19:50:47.196 INFO:tasks.rados.rados.0.smithi059.stdout:2000: finishing write tid 5 to smithi05955964-3 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:2000: finishing write tid 6 to smithi05955964-3 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 832 (ObjNum 648 snap 184 seq_num 648) dirty exists 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:2000: left oid 3 (ObjNum 648 snap 184 seq_num 648) 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:1998: done (3 left) 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:1999: done (2 left) 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:2000: done (1 left) 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:2001: done (0 left) 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:2002: copy_from oid 17 from oid 37 current snap is 184 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 184 2022-04-28T19:50:47.197 INFO:tasks.rados.rados.0.smithi059.stdout:2003: seq_num 650 ranges {0=491520,491520=647168,1138688=155648} 2022-04-28T19:50:47.198 INFO:tasks.rados.rados.0.smithi059.stdout:2003: writing smithi05955964-24 from 0 to 491520 tid 1 2022-04-28T19:50:47.202 INFO:tasks.rados.rados.0.smithi059.stdout:2003: writing smithi05955964-24 from 491520 to 1138688 tid 2 2022-04-28T19:50:47.204 INFO:tasks.rados.rados.0.smithi059.stdout:2003: writing smithi05955964-24 from 1138688 to 1294336 tid 3 2022-04-28T19:50:47.204 INFO:tasks.rados.rados.0.smithi059.stdout:2004: delete oid 1 current snap is 184 2022-04-28T19:50:47.204 INFO:tasks.rados.rados.0.smithi059.stdout:2002: finishing copy_from to smithi05955964-17 2022-04-28T19:50:47.204 INFO:tasks.rados.rados.0.smithi059.stdout:2002: got expected ENOENT (src dne) 2022-04-28T19:50:47.204 INFO:tasks.rados.rados.0.smithi059.stdout:2002: finishing copy_from racing read to smithi05955964-17 2022-04-28T19:50:47.208 INFO:tasks.rados.rados.0.smithi059.stdout:2003: finishing write tid 1 to smithi05955964-24 2022-04-28T19:50:47.209 INFO:tasks.rados.rados.0.smithi059.stdout:2003: finishing write tid 2 to smithi05955964-24 2022-04-28T19:50:47.211 INFO:tasks.rados.rados.0.smithi059.stdout:2003: finishing write tid 3 to smithi05955964-24 2022-04-28T19:50:47.211 INFO:tasks.rados.rados.0.smithi059.stdout:2003: finishing write tid 5 to smithi05955964-24 2022-04-28T19:50:47.212 INFO:tasks.rados.rados.0.smithi059.stdout:2002: done (2 left) 2022-04-28T19:50:47.212 INFO:tasks.rados.rados.0.smithi059.stdout:2004: done (1 left) 2022-04-28T19:50:47.212 INFO:tasks.rados.rados.0.smithi059.stdout:2005: rmattr oid 27 current snap is 184 2022-04-28T19:50:47.212 INFO:tasks.rados.rados.0.smithi059.stdout:2006: snap_create 2022-04-28T19:50:47.213 INFO:tasks.rados.rados.0.smithi059.stdout:2003: finishing write tid 6 to smithi05955964-24 2022-04-28T19:50:47.213 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 798 (ObjNum 650 snap 184 seq_num 650) dirty exists 2022-04-28T19:50:47.213 INFO:tasks.rados.rados.0.smithi059.stdout:2003: left oid 24 (ObjNum 650 snap 184 seq_num 650) 2022-04-28T19:50:47.215 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 960 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:50:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:46 smithi059 bash[61152]: cluster 2022-04-28T19:50:45.968032+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2022-04-28T19:50:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:46 smithi059 bash[61152]: cluster 2022-04-28T19:50:46.820308+0000 mon.a (mon.0) 317 : cluster [WRN] Health check update: Degraded data redundancy: 11/408 objects degraded (2.696%), 4 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:46 smithi059 bash[59745]: cluster 2022-04-28T19:50:45.968032+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2022-04-28T19:50:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:46 smithi059 bash[59745]: cluster 2022-04-28T19:50:46.820308+0000 mon.a (mon.0) 317 : cluster [WRN] Health check update: Degraded data redundancy: 11/408 objects degraded (2.696%), 4 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:47.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:46 smithi139 bash[43301]: cluster 2022-04-28T19:50:45.968032+0000 mon.a (mon.0) 316 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2022-04-28T19:50:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:46 smithi139 bash[43301]: cluster 2022-04-28T19:50:46.820308+0000 mon.a (mon.0) 317 : cluster [WRN] Health check update: Degraded data redundancy: 11/408 objects degraded (2.696%), 4 pgs degraded (PG_DEGRADED) 2022-04-28T19:50:47.997 INFO:tasks.rados.rados.0.smithi059.stdout:2003: done (2 left) 2022-04-28T19:50:47.997 INFO:tasks.rados.rados.0.smithi059.stdout:2005: done (1 left) 2022-04-28T19:50:47.997 INFO:tasks.rados.rados.0.smithi059.stdout:2006: done (0 left) 2022-04-28T19:50:47.998 INFO:tasks.rados.rados.0.smithi059.stdout:2007: rollback oid 36 current snap is 185 2022-04-28T19:50:47.998 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 173 2022-04-28T19:50:47.998 INFO:tasks.rados.rados.0.smithi059.stdout:2008: delete oid 18 current snap is 185 2022-04-28T19:50:48.003 INFO:tasks.rados.rados.0.smithi059.stdout:2007: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:2007: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 837 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:2007: done (1 left) 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:2008: done (0 left) 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:2009: read oid 37 snap -1 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:2009: expect deleted 2022-04-28T19:50:48.005 INFO:tasks.rados.rados.0.smithi059.stdout:2010: copy_from oid 28 from oid 46 current snap is 185 2022-04-28T19:50:48.006 INFO:tasks.rados.rados.0.smithi059.stdout:2011: rollback oid 8 current snap is 185 2022-04-28T19:50:48.006 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 8 to 177 2022-04-28T19:50:48.006 INFO:tasks.rados.rados.0.smithi059.stdout:2012: rollback oid 9 current snap is 185 2022-04-28T19:50:48.006 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 173 2022-04-28T19:50:48.006 INFO:tasks.rados.rados.0.smithi059.stdout:2013: snap_create 2022-04-28T19:50:48.009 INFO:tasks.rados.rados.0.smithi059.stdout:2012: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:50:48.009 INFO:tasks.rados.rados.0.smithi059.stdout:2012: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:50:48.009 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 715 (ObjNum 607 snap 170 seq_num 607) dirty exists 2022-04-28T19:50:48.011 INFO:tasks.rados.rados.0.smithi059.stdout:2011: finishing rollback tid 0 to smithi05955964-8 2022-04-28T19:50:48.011 INFO:tasks.rados.rados.0.smithi059.stdout:2011: finishing rollback tid 1 to smithi05955964-8 2022-04-28T19:50:48.012 INFO:tasks.rados.rados.0.smithi059.stdout:2011: finishing rollback tid 2 to smithi05955964-8 2022-04-28T19:50:48.012 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 891 (ObjNum 611 snap 171 seq_num 611) dirty exists 2022-04-28T19:50:48.050 INFO:tasks.rados.rados.0.smithi059.stdout:2010: finishing copy_from to smithi05955964-28 2022-04-28T19:50:48.051 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 777 (ObjNum 569 snap 162 seq_num 569) dirty exists 2022-04-28T19:50:48.051 INFO:tasks.rados.rados.0.smithi059.stdout:2010: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:50:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[61152]: cluster 2022-04-28T19:50:46.879471+0000 mgr.y (mgr.24961) 100 : cluster [DBG] pgmap v126: 89 pgs: 21 peering, 68 active+clean; 224 MiB data, 424 MiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 262 KiB/s wr, 5 op/s; 4.7 MiB/s, 1 objects/s recovering 2022-04-28T19:50:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[61152]: cluster 2022-04-28T19:50:46.972664+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2022-04-28T19:50:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[61152]: cluster 2022-04-28T19:50:47.823373+0000 mon.a (mon.0) 319 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/408 objects degraded (2.696%), 4 pgs degraded) 2022-04-28T19:50:48.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[61152]: cluster 2022-04-28T19:50:47.823443+0000 mon.a (mon.0) 320 : cluster [INF] Cluster is now healthy 2022-04-28T19:50:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[59745]: cluster 2022-04-28T19:50:46.879471+0000 mgr.y (mgr.24961) 100 : cluster [DBG] pgmap v126: 89 pgs: 21 peering, 68 active+clean; 224 MiB data, 424 MiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 262 KiB/s wr, 5 op/s; 4.7 MiB/s, 1 objects/s recovering 2022-04-28T19:50:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[59745]: cluster 2022-04-28T19:50:46.972664+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2022-04-28T19:50:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[59745]: cluster 2022-04-28T19:50:47.823373+0000 mon.a (mon.0) 319 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/408 objects degraded (2.696%), 4 pgs degraded) 2022-04-28T19:50:48.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:47 smithi059 bash[59745]: cluster 2022-04-28T19:50:47.823443+0000 mon.a (mon.0) 320 : cluster [INF] Cluster is now healthy 2022-04-28T19:50:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:47 smithi139 bash[43301]: cluster 2022-04-28T19:50:46.879471+0000 mgr.y (mgr.24961) 100 : cluster [DBG] pgmap v126: 89 pgs: 21 peering, 68 active+clean; 224 MiB data, 424 MiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 262 KiB/s wr, 5 op/s; 4.7 MiB/s, 1 objects/s recovering 2022-04-28T19:50:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:47 smithi139 bash[43301]: cluster 2022-04-28T19:50:46.972664+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2022-04-28T19:50:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:47 smithi139 bash[43301]: cluster 2022-04-28T19:50:47.823373+0000 mon.a (mon.0) 319 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/408 objects degraded (2.696%), 4 pgs degraded) 2022-04-28T19:50:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:47 smithi139 bash[43301]: cluster 2022-04-28T19:50:47.823443+0000 mon.a (mon.0) 320 : cluster [INF] Cluster is now healthy 2022-04-28T19:50:49.017 INFO:tasks.rados.rados.0.smithi059.stdout:2009: done (4 left) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2010: done (3 left) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2011: done (2 left) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2012: done (1 left) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2013: done (0 left) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2014: read oid 42 snap 184 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2014: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2015: read oid 8 snap -1 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2015: expect (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:50:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:2016: snap_remove snap 180 2022-04-28T19:50:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:49 smithi059 bash[61152]: cluster 2022-04-28T19:50:47.994523+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2022-04-28T19:50:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:49 smithi059 bash[59745]: cluster 2022-04-28T19:50:47.994523+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2022-04-28T19:50:49.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:49 smithi139 bash[43301]: cluster 2022-04-28T19:50:47.994523+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2022-04-28T19:50:50.011 INFO:tasks.rados.rados.0.smithi059.stdout:2016: done (2 left) 2022-04-28T19:50:50.012 INFO:tasks.rados.rados.0.smithi059.stdout:2017: copy_from oid 13 from oid 30 current snap is 186 2022-04-28T19:50:50.012 INFO:tasks.rados.rados.0.smithi059.stdout:2015: expect (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:50:50.038 INFO:tasks.rados.rados.0.smithi059.stdout:2014: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:50:50.324 INFO:tasks.rados.rados.0.smithi059.stdout:2017: finishing copy_from to smithi05955964-13 2022-04-28T19:50:50.324 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 868 (ObjNum 622 snap 178 seq_num 622) dirty exists 2022-04-28T19:50:50.324 INFO:tasks.rados.rados.0.smithi059.stdout:2017: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:50:50.324 INFO:tasks.rados.rados.0.smithi059.stdout:2014: done (2 left) 2022-04-28T19:50:50.325 INFO:tasks.rados.rados.0.smithi059.stdout:2015: done (1 left) 2022-04-28T19:50:50.325 INFO:tasks.rados.rados.0.smithi059.stdout:2017: done (0 left) 2022-04-28T19:50:50.325 INFO:tasks.rados.rados.0.smithi059.stdout:2018: snap_create 2022-04-28T19:50:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:50 smithi059 bash[59745]: cluster 2022-04-28T19:50:48.880243+0000 mgr.y (mgr.24961) 101 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 209 MiB data, 417 MiB used, 708 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.9 MiB/s wr, 40 op/s; 25 MiB/s, 6 objects/s recovering 2022-04-28T19:50:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:50 smithi059 bash[59745]: cluster 2022-04-28T19:50:49.015307+0000 mon.a (mon.0) 322 : cluster [DBG] mgrmap e31: y(active, since 99s), standbys: x 2022-04-28T19:50:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:50 smithi059 bash[59745]: cluster 2022-04-28T19:50:49.015392+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2022-04-28T19:50:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:50 smithi059 bash[61152]: cluster 2022-04-28T19:50:48.880243+0000 mgr.y (mgr.24961) 101 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 209 MiB data, 417 MiB used, 708 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.9 MiB/s wr, 40 op/s; 25 MiB/s, 6 objects/s recovering 2022-04-28T19:50:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:50 smithi059 bash[61152]: cluster 2022-04-28T19:50:49.015307+0000 mon.a (mon.0) 322 : cluster [DBG] mgrmap e31: y(active, since 99s), standbys: x 2022-04-28T19:50:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:50 smithi059 bash[61152]: cluster 2022-04-28T19:50:49.015392+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2022-04-28T19:50:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:50 smithi139 bash[43301]: cluster 2022-04-28T19:50:48.880243+0000 mgr.y (mgr.24961) 101 : cluster [DBG] pgmap v129: 89 pgs: 89 active+clean; 209 MiB data, 417 MiB used, 708 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.9 MiB/s wr, 40 op/s; 25 MiB/s, 6 objects/s recovering 2022-04-28T19:50:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:50 smithi139 bash[43301]: cluster 2022-04-28T19:50:49.015307+0000 mon.a (mon.0) 322 : cluster [DBG] mgrmap e31: y(active, since 99s), standbys: x 2022-04-28T19:50:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:50 smithi139 bash[43301]: cluster 2022-04-28T19:50:49.015392+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2022-04-28T19:50:51.032 INFO:tasks.rados.rados.0.smithi059.stdout:2018: done (0 left) 2022-04-28T19:50:51.032 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 187 2022-04-28T19:50:51.032 INFO:tasks.rados.rados.0.smithi059.stdout:2019: seq_num 653 ranges {3563520=540672,4104192=696320,4800512=106496} 2022-04-28T19:50:51.062 INFO:tasks.rados.rados.0.smithi059.stdout:2019: writing smithi05955964-21 from 3563520 to 4104192 tid 1 2022-04-28T19:50:51.065 INFO:tasks.rados.rados.0.smithi059.stdout:2019: writing smithi05955964-21 from 4104192 to 4800512 tid 2 2022-04-28T19:50:51.066 INFO:tasks.rados.rados.0.smithi059.stdout:2019: writing smithi05955964-21 from 4800512 to 4907008 tid 3 2022-04-28T19:50:51.066 INFO:tasks.rados.rados.0.smithi059.stdout:2020: snap_remove snap 184 2022-04-28T19:50:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:51 smithi059 bash[59745]: cluster 2022-04-28T19:50:50.009700+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2022-04-28T19:50:51.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:51 smithi059 bash[61152]: cluster 2022-04-28T19:50:50.009700+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2022-04-28T19:50:51.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:51 smithi139 bash[43301]: cluster 2022-04-28T19:50:50.009700+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2020: done (1 left) 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2021: read oid 46 snap -1 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2021: expect (ObjNum 569 snap 162 seq_num 569) 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2019: finishing write tid 1 to smithi05955964-21 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2019: finishing write tid 2 to smithi05955964-21 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2019: finishing write tid 3 to smithi05955964-21 2022-04-28T19:50:51.832 INFO:tasks.rados.rados.0.smithi059.stdout:2019: finishing write tid 5 to smithi05955964-21 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:2019: finishing write tid 6 to smithi05955964-21 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 702 (ObjNum 653 snap 187 seq_num 653) dirty exists 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:2019: left oid 21 (ObjNum 653 snap 187 seq_num 653) 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:2019: done (1 left) 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:2022: copy_from oid 16 from oid 48 current snap is 187 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 187 2022-04-28T19:50:51.833 INFO:tasks.rados.rados.0.smithi059.stdout:2023: seq_num 654 ranges {2400256=557056,2957312=630784,3588096=172032} 2022-04-28T19:50:51.853 INFO:tasks.rados.rados.0.smithi059.stdout:2023: writing smithi05955964-38 from 2400256 to 2957312 tid 1 2022-04-28T19:50:51.857 INFO:tasks.rados.rados.0.smithi059.stdout:2023: writing smithi05955964-38 from 2957312 to 3588096 tid 2 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2023: writing smithi05955964-38 from 3588096 to 3760128 tid 3 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2024: read oid 35 snap -1 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2024: expect (ObjNum 642 snap 184 seq_num 642) 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2025: delete oid 32 current snap is 187 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2022: finishing copy_from to smithi05955964-16 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 874 (ObjNum 584 snap 166 seq_num 584) dirty exists 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2022: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:50:51.859 INFO:tasks.rados.rados.0.smithi059.stdout:2023: finishing write tid 1 to smithi05955964-38 2022-04-28T19:50:51.862 INFO:tasks.rados.rados.0.smithi059.stdout:2023: finishing write tid 2 to smithi05955964-38 2022-04-28T19:50:51.862 INFO:tasks.rados.rados.0.smithi059.stdout:2023: finishing write tid 3 to smithi05955964-38 2022-04-28T19:50:51.862 INFO:tasks.rados.rados.0.smithi059.stdout:2023: finishing write tid 5 to smithi05955964-38 2022-04-28T19:50:51.869 INFO:tasks.rados.rados.0.smithi059.stdout:2023: finishing write tid 6 to smithi05955964-38 2022-04-28T19:50:51.869 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 803 (ObjNum 654 snap 187 seq_num 654) dirty exists 2022-04-28T19:50:51.869 INFO:tasks.rados.rados.0.smithi059.stdout:2023: left oid 38 (ObjNum 654 snap 187 seq_num 654) 2022-04-28T19:50:51.883 INFO:tasks.rados.rados.0.smithi059.stdout:2022: done (4 left) 2022-04-28T19:50:51.883 INFO:tasks.rados.rados.0.smithi059.stdout:2023: done (3 left) 2022-04-28T19:50:51.883 INFO:tasks.rados.rados.0.smithi059.stdout:2025: done (2 left) 2022-04-28T19:50:51.884 INFO:tasks.rados.rados.0.smithi059.stdout:2026: copy_from oid 34 from oid 7 current snap is 187 2022-04-28T19:50:51.884 INFO:tasks.rados.rados.0.smithi059.stdout:2027: read oid 27 snap -1 2022-04-28T19:50:51.884 INFO:tasks.rados.rados.0.smithi059.stdout:2027: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:50:51.884 INFO:tasks.rados.rados.0.smithi059.stdout:2028: snap_create 2022-04-28T19:50:51.901 INFO:tasks.rados.rados.0.smithi059.stdout:2027: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:50:51.983 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:50:52.001 INFO:tasks.rados.rados.0.smithi059.stdout:2021: expect (ObjNum 569 snap 162 seq_num 569) 2022-04-28T19:50:52.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:51.628Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:52.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:51.628Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:52.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:51.628Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:52.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:52 smithi059 bash[59745]: cluster 2022-04-28T19:50:50.880971+0000 mgr.y (mgr.24961) 102 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 209 MiB data, 417 MiB used, 708 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.7 MiB/s wr, 34 op/s; 20 MiB/s, 4 objects/s recovering 2022-04-28T19:50:52.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:52 smithi059 bash[59745]: cluster 2022-04-28T19:50:51.030693+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2022-04-28T19:50:52.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:52 smithi059 bash[59745]: cluster 2022-04-28T19:50:51.830684+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2022-04-28T19:50:52.304 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:50:52.304 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (9m) 14s ago 10m 14.5M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:50:52.304 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (9m) 59s ago 9m 25.8M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:50:52.304 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (8m) 14s ago 8m 14.6M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (8m) 59s ago 8m 17.6M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (2m) 59s ago 14m 375M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (2m) 14s ago 16m 413M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (111s) 14s ago 16m 53.1M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (71s) 59s ago 14m 19.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (83s) 14s ago 14m 47.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (10m) 14s ago 10m 10.8M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (10m) 59s ago 10m 10.7M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (17s) 14s ago 13m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:50:52.305 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (13m) 14s ago 13m 185M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 b45bb9f9a44a 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (12m) 14s ago 12m 270M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (12m) 14s ago 12m 204M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (12m) 59s ago 12m 196M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (11m) 59s ago 11m 214M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (11m) 59s ago 11m 162M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (10m) 59s ago 10m 138M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:50:52.306 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (2m) 59s ago 10m 36.6M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:50:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:52 smithi059 bash[61152]: cluster 2022-04-28T19:50:50.880971+0000 mgr.y (mgr.24961) 102 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 209 MiB data, 417 MiB used, 708 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.7 MiB/s wr, 34 op/s; 20 MiB/s, 4 objects/s recovering 2022-04-28T19:50:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:52 smithi059 bash[61152]: cluster 2022-04-28T19:50:51.030693+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2022-04-28T19:50:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:52 smithi059 bash[61152]: cluster 2022-04-28T19:50:51.830684+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2022-04-28T19:50:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:52 smithi139 bash[43301]: cluster 2022-04-28T19:50:50.880971+0000 mgr.y (mgr.24961) 102 : cluster [DBG] pgmap v132: 89 pgs: 89 active+clean; 209 MiB data, 417 MiB used, 708 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.7 MiB/s wr, 34 op/s; 20 MiB/s, 4 objects/s recovering 2022-04-28T19:50:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:52 smithi139 bash[43301]: cluster 2022-04-28T19:50:51.030693+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2022-04-28T19:50:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:52 smithi139 bash[43301]: cluster 2022-04-28T19:50:51.830684+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:50:52.667 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 7, 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:50:52.668 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:50:52.669 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 9, 2022-04-28T19:50:52.669 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 6 2022-04-28T19:50:52.669 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:50:52.669 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:50:52.758 INFO:tasks.rados.rados.0.smithi059.stdout:2026: finishing copy_from to smithi05955964-34 2022-04-28T19:50:52.759 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 729 (ObjNum 612 snap 171 seq_num 612) dirty exists 2022-04-28T19:50:52.759 INFO:tasks.rados.rados.0.smithi059.stdout:2026: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:50:52.759 INFO:tasks.rados.rados.0.smithi059.stdout:2024: expect (ObjNum 642 snap 184 seq_num 642) 2022-04-28T19:50:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:51.970678+0000 mgr.y (mgr.24961) 103 : audit [DBG] from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.281458+0000 mgr.y (mgr.24961) 104 : audit [DBG] from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.283870+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.284636+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.285616+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.286440+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.287327+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.288125+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.290334+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.290727+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.291207+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.291698+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.292131+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:52.665827+0000 mon.c (mon.1) 6 : audit [DBG] from='client.? 172.21.15.59:0/3143445688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: cluster 2022-04-28T19:50:52.843709+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2022-04-28T19:50:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[59745]: audit 2022-04-28T19:50:53.024124+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:53.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:51.970678+0000 mgr.y (mgr.24961) 103 : audit [DBG] from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:53.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.281458+0000 mgr.y (mgr.24961) 104 : audit [DBG] from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:53.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.283870+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.284636+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.285616+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.286440+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.287327+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.288125+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.290334+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.290727+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.291207+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.291698+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.292131+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:52.665827+0000 mon.c (mon.1) 6 : audit [DBG] from='client.? 172.21.15.59:0/3143445688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:53.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: cluster 2022-04-28T19:50:52.843709+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2022-04-28T19:50:53.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[61152]: audit 2022-04-28T19:50:53.024124+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:53.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:51.970678+0000 mgr.y (mgr.24961) 103 : audit [DBG] from='client.44196 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:53.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.281458+0000 mgr.y (mgr.24961) 104 : audit [DBG] from='client.44202 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:50:53.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.283870+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.284636+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.285616+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.286440+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.287327+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.288125+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.290334+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:50:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.290727+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.291207+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.291698+0000 mon.a (mon.0) 336 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.292131+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:50:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:52.665827+0000 mon.c (mon.1) 6 : audit [DBG] from='client.? 172.21.15.59:0/3143445688' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:50:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: cluster 2022-04-28T19:50:52.843709+0000 mon.a (mon.0) 338 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2022-04-28T19:50:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:53 smithi139 bash[43301]: audit 2022-04-28T19:50:53.024124+0000 mon.a (mon.0) 339 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:53.780 INFO:tasks.rados.rados.0.smithi059.stdout:2021: done (4 left) 2022-04-28T19:50:53.782 INFO:tasks.rados.rados.0.smithi059.stdout:2024: done (3 left) 2022-04-28T19:50:53.783 INFO:tasks.rados.rados.0.smithi059.stdout:2026: done (2 left) 2022-04-28T19:50:53.783 INFO:tasks.rados.rados.0.smithi059.stdout:2027: done (1 left) 2022-04-28T19:50:53.783 INFO:tasks.rados.rados.0.smithi059.stdout:2028: done (0 left) 2022-04-28T19:50:53.783 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 188 2022-04-28T19:50:53.784 INFO:tasks.rados.rados.0.smithi059.stdout:2029: seq_num 656 ranges {1212416=589824,1802240=802816} 2022-04-28T19:50:53.798 INFO:tasks.rados.rados.0.smithi059.stdout:2029: writing smithi05955964-40 from 1212416 to 1802240 tid 1 2022-04-28T19:50:53.803 INFO:tasks.rados.rados.0.smithi059.stdout:2029: writing smithi05955964-40 from 1802240 to 2605056 tid 2 2022-04-28T19:50:53.803 INFO:tasks.rados.rados.0.smithi059.stdout:2030: copy_from oid 26 from oid 4 current snap is 188 2022-04-28T19:50:53.803 INFO:tasks.rados.rados.0.smithi059.stdout:2031: snap_create 2022-04-28T19:50:53.804 INFO:tasks.rados.rados.0.smithi059.stdout:2029: finishing write tid 1 to smithi05955964-40 2022-04-28T19:50:53.809 INFO:tasks.rados.rados.0.smithi059.stdout:2029: finishing write tid 2 to smithi05955964-40 2022-04-28T19:50:53.809 INFO:tasks.rados.rados.0.smithi059.stdout:2029: finishing write tid 4 to smithi05955964-40 2022-04-28T19:50:53.811 INFO:tasks.rados.rados.0.smithi059.stdout:2029: finishing write tid 5 to smithi05955964-40 2022-04-28T19:50:53.811 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 847 (ObjNum 656 snap 188 seq_num 656) dirty exists 2022-04-28T19:50:53.811 INFO:tasks.rados.rados.0.smithi059.stdout:2029: left oid 40 (ObjNum 656 snap 188 seq_num 656) 2022-04-28T19:50:53.811 INFO:tasks.rados.rados.0.smithi059.stdout:2030: finishing copy_from to smithi05955964-26 2022-04-28T19:50:53.812 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 785 (ObjNum 628 snap 180 seq_num 628) dirty exists 2022-04-28T19:50:53.812 INFO:tasks.rados.rados.0.smithi059.stdout:2030: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:50:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:53.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:50:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:53.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:50:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:50:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:50:53.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:50:54.051 INFO:tasks.rados.rados.0.smithi059.stdout:2029: done (2 left) 2022-04-28T19:50:54.051 INFO:tasks.rados.rados.0.smithi059.stdout:2030: done (1 left) 2022-04-28T19:50:54.051 INFO:tasks.rados.rados.0.smithi059.stdout:2031: done (0 left) 2022-04-28T19:50:54.051 INFO:tasks.rados.rados.0.smithi059.stdout:2032: snap_remove snap 187 2022-04-28T19:50:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:54 smithi059 bash[59745]: cluster 2022-04-28T19:50:52.881464+0000 mgr.y (mgr.24961) 105 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 236 MiB data, 450 MiB used, 708 GiB / 715 GiB avail; 24 MiB/s rd, 6.1 MiB/s wr, 11 op/s 2022-04-28T19:50:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:54 smithi059 bash[59745]: audit 2022-04-28T19:50:53.024440+0000 mgr.y (mgr.24961) 106 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:54 smithi059 bash[59745]: cephadm 2022-04-28T19:50:53.026378+0000 mgr.y (mgr.24961) 107 : cephadm [INF] Upgrade: osd.1 is safe to restart 2022-04-28T19:50:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:54 smithi059 bash[61152]: cluster 2022-04-28T19:50:52.881464+0000 mgr.y (mgr.24961) 105 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 236 MiB data, 450 MiB used, 708 GiB / 715 GiB avail; 24 MiB/s rd, 6.1 MiB/s wr, 11 op/s 2022-04-28T19:50:54.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:54 smithi059 bash[61152]: audit 2022-04-28T19:50:53.024440+0000 mgr.y (mgr.24961) 106 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:54.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:54 smithi059 bash[61152]: cephadm 2022-04-28T19:50:53.026378+0000 mgr.y (mgr.24961) 107 : cephadm [INF] Upgrade: osd.1 is safe to restart 2022-04-28T19:50:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:54 smithi139 bash[43301]: cluster 2022-04-28T19:50:52.881464+0000 mgr.y (mgr.24961) 105 : cluster [DBG] pgmap v136: 89 pgs: 89 active+clean; 236 MiB data, 450 MiB used, 708 GiB / 715 GiB avail; 24 MiB/s rd, 6.1 MiB/s wr, 11 op/s 2022-04-28T19:50:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:54 smithi139 bash[43301]: audit 2022-04-28T19:50:53.024440+0000 mgr.y (mgr.24961) 106 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2022-04-28T19:50:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:54 smithi139 bash[43301]: cephadm 2022-04-28T19:50:53.026378+0000 mgr.y (mgr.24961) 107 : cephadm [INF] Upgrade: osd.1 is safe to restart 2022-04-28T19:50:55.056 INFO:tasks.rados.rados.0.smithi059.stdout:2032: done (0 left) 2022-04-28T19:50:55.056 INFO:tasks.rados.rados.0.smithi059.stdout:2033: delete oid 21 current snap is 189 2022-04-28T19:50:55.063 INFO:tasks.rados.rados.0.smithi059.stdout:2033: done (0 left) 2022-04-28T19:50:55.063 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 189 2022-04-28T19:50:55.063 INFO:tasks.rados.rados.0.smithi059.stdout:2034: seq_num 658 ranges {5726208=622592,6348800=565248,6914048=237568} 2022-04-28T19:50:55.097 INFO:tasks.rados.rados.0.smithi059.stdout:2034: writing smithi05955964-23 from 5726208 to 6348800 tid 1 2022-04-28T19:50:55.101 INFO:tasks.rados.rados.0.smithi059.stdout:2034: writing smithi05955964-23 from 6348800 to 6914048 tid 2 2022-04-28T19:50:55.102 INFO:tasks.rados.rados.0.smithi059.stdout:2034: writing smithi05955964-23 from 6914048 to 7151616 tid 3 2022-04-28T19:50:55.103 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 189 2022-04-28T19:50:55.103 INFO:tasks.rados.rados.0.smithi059.stdout:2035: seq_num 659 ranges {2473984=647168,3121152=450560,3571712=352256} 2022-04-28T19:50:55.120 INFO:tasks.rados.rados.0.smithi059.stdout:2035: writing smithi05955964-43 from 2473984 to 3121152 tid 1 2022-04-28T19:50:55.123 INFO:tasks.rados.rados.0.smithi059.stdout:2035: writing smithi05955964-43 from 3121152 to 3571712 tid 2 2022-04-28T19:50:55.125 INFO:tasks.rados.rados.0.smithi059.stdout:2035: writing smithi05955964-43 from 3571712 to 3923968 tid 3 2022-04-28T19:50:55.125 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 189 2022-04-28T19:50:55.125 INFO:tasks.rados.rados.0.smithi059.stdout:2036: seq_num 660 ranges {188416=663552,851968=737280,1589248=65536} 2022-04-28T19:50:55.130 INFO:tasks.rados.rados.0.smithi059.stdout:2036: writing smithi05955964-48 from 188416 to 851968 tid 1 2022-04-28T19:50:55.134 INFO:tasks.rados.rados.0.smithi059.stdout:2036: writing smithi05955964-48 from 851968 to 1589248 tid 2 2022-04-28T19:50:55.139 INFO:tasks.rados.rados.0.smithi059.stdout:2036: writing smithi05955964-48 from 1589248 to 1654784 tid 3 2022-04-28T19:50:55.139 INFO:tasks.rados.rados.0.smithi059.stdout:2037: read oid 10 snap -1 2022-04-28T19:50:55.139 INFO:tasks.rados.rados.0.smithi059.stdout:2037: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:50:55.139 INFO:tasks.rados.rados.0.smithi059.stdout:2038: read oid 49 snap -1 2022-04-28T19:50:55.139 INFO:tasks.rados.rados.0.smithi059.stdout:2038: expect (ObjNum 605 snap 169 seq_num 605) 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2039: read oid 50 snap -1 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2039: expect (ObjNum 634 snap 181 seq_num 634) 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2040: read oid 9 snap 186 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2040: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2041: rollback oid 36 current snap is 189 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 185 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2034: finishing write tid 1 to smithi05955964-23 2022-04-28T19:50:55.140 INFO:tasks.rados.rados.0.smithi059.stdout:2034: finishing write tid 2 to smithi05955964-23 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2034: finishing write tid 3 to smithi05955964-23 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2034: finishing write tid 5 to smithi05955964-23 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2034: finishing write tid 6 to smithi05955964-23 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 973 (ObjNum 658 snap 189 seq_num 658) dirty exists 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2034: left oid 23 (ObjNum 658 snap 189 seq_num 658) 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2035: finishing write tid 1 to smithi05955964-43 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2034: done (7 left) 2022-04-28T19:50:55.141 INFO:tasks.rados.rados.0.smithi059.stdout:2042: setattr oid 14 current snap is 189 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2035: finishing write tid 2 to smithi05955964-43 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2035: finishing write tid 3 to smithi05955964-43 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2035: finishing write tid 5 to smithi05955964-43 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2035: finishing write tid 6 to smithi05955964-43 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 955 (ObjNum 659 snap 189 seq_num 659) dirty exists 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2035: left oid 43 (ObjNum 659 snap 189 seq_num 659) 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2036: finishing write tid 1 to smithi05955964-48 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2035: done (7 left) 2022-04-28T19:50:55.142 INFO:tasks.rados.rados.0.smithi059.stdout:2043: read oid 29 snap -1 2022-04-28T19:50:55.143 INFO:tasks.rados.rados.0.smithi059.stdout:2043: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:50:55.143 INFO:tasks.rados.rados.0.smithi059.stdout:2044: snap_remove snap 182 2022-04-28T19:50:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:55 smithi059 bash[59745]: cluster 2022-04-28T19:50:54.049034+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2022-04-28T19:50:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:55 smithi059 bash[61152]: cluster 2022-04-28T19:50:54.049034+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2022-04-28T19:50:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:55 smithi139 bash[43301]: cluster 2022-04-28T19:50:54.049034+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2022-04-28T19:50:56.069 INFO:tasks.rados.rados.0.smithi059.stdout:2044: done (8 left) 2022-04-28T19:50:56.069 INFO:tasks.rados.rados.0.smithi059.stdout:2045: copy_from oid 28 from oid 45 current snap is 189 2022-04-28T19:50:56.070 INFO:tasks.rados.rados.0.smithi059.stdout:2046: delete oid 23 current snap is 189 2022-04-28T19:50:56.070 INFO:tasks.rados.rados.0.smithi059.stdout:2036: finishing write tid 2 to smithi05955964-48 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:2036: finishing write tid 3 to smithi05955964-48 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:2036: finishing write tid 5 to smithi05955964-48 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 928 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:2036: finishing write tid 6 to smithi05955964-48 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 927 (ObjNum 660 snap 189 seq_num 660) dirty exists 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:2036: left oid 48 (ObjNum 660 snap 189 seq_num 660) 2022-04-28T19:50:56.071 INFO:tasks.rados.rados.0.smithi059.stdout:2043: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:50:56.170 INFO:tasks.rados.rados.0.smithi059.stdout:2038: expect (ObjNum 605 snap 169 seq_num 605) 2022-04-28T19:50:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[59745]: cluster 2022-04-28T19:50:54.882021+0000 mgr.y (mgr.24961) 108 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 236 MiB data, 450 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 5.9 MiB/s wr, 11 op/s 2022-04-28T19:50:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[59745]: cluster 2022-04-28T19:50:55.055382+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2022-04-28T19:50:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[59745]: audit 2022-04-28T19:50:55.600449+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[59745]: audit 2022-04-28T19:50:55.603681+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "container_image"}]: dispatch 2022-04-28T19:50:56.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[61152]: cluster 2022-04-28T19:50:54.882021+0000 mgr.y (mgr.24961) 108 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 236 MiB data, 450 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 5.9 MiB/s wr, 11 op/s 2022-04-28T19:50:56.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[61152]: cluster 2022-04-28T19:50:55.055382+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2022-04-28T19:50:56.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[61152]: audit 2022-04-28T19:50:55.600449+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:56.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:56 smithi059 bash[61152]: audit 2022-04-28T19:50:55.603681+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "container_image"}]: dispatch 2022-04-28T19:50:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:56 smithi139 bash[43301]: cluster 2022-04-28T19:50:54.882021+0000 mgr.y (mgr.24961) 108 : cluster [DBG] pgmap v138: 89 pgs: 89 active+clean; 236 MiB data, 450 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 5.9 MiB/s wr, 11 op/s 2022-04-28T19:50:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:56 smithi139 bash[43301]: cluster 2022-04-28T19:50:55.055382+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2022-04-28T19:50:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:56 smithi139 bash[43301]: audit 2022-04-28T19:50:55.600449+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:56 smithi139 bash[43301]: audit 2022-04-28T19:50:55.603681+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "container_image"}]: dispatch 2022-04-28T19:50:56.374 INFO:tasks.rados.rados.0.smithi059.stdout:2041: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:50:56.374 INFO:tasks.rados.rados.0.smithi059.stdout:2041: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:50:56.374 INFO:tasks.rados.rados.0.smithi059.stdout:2041: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:50:56.374 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 853 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:50:56.374 INFO:tasks.rados.rados.0.smithi059.stdout:2040: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:50:56.479 INFO:tasks.rados.rados.0.smithi059.stdout:2037: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:50:56.872 INFO:tasks.rados.rados.0.smithi059.stdout:2039: expect (ObjNum 634 snap 181 seq_num 634) 2022-04-28T19:50:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[61152]: cephadm 2022-04-28T19:50:55.589503+0000 mgr.y (mgr.24961) 109 : cephadm [INF] Upgrade: Updating osd.1 2022-04-28T19:50:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[61152]: cephadm 2022-04-28T19:50:55.602839+0000 mgr.y (mgr.24961) 110 : cephadm [INF] Deploying daemon osd.1 on smithi059 2022-04-28T19:50:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[61152]: cluster 2022-04-28T19:50:56.061376+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2022-04-28T19:50:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[61152]: cluster 2022-04-28T19:50:56.831806+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2022-04-28T19:50:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[59745]: cephadm 2022-04-28T19:50:55.589503+0000 mgr.y (mgr.24961) 109 : cephadm [INF] Upgrade: Updating osd.1 2022-04-28T19:50:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[59745]: cephadm 2022-04-28T19:50:55.602839+0000 mgr.y (mgr.24961) 110 : cephadm [INF] Deploying daemon osd.1 on smithi059 2022-04-28T19:50:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[59745]: cluster 2022-04-28T19:50:56.061376+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2022-04-28T19:50:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:57 smithi059 bash[59745]: cluster 2022-04-28T19:50:56.831806+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2022-04-28T19:50:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:57 smithi139 bash[43301]: cephadm 2022-04-28T19:50:55.589503+0000 mgr.y (mgr.24961) 109 : cephadm [INF] Upgrade: Updating osd.1 2022-04-28T19:50:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:57 smithi139 bash[43301]: cephadm 2022-04-28T19:50:55.602839+0000 mgr.y (mgr.24961) 110 : cephadm [INF] Deploying daemon osd.1 on smithi059 2022-04-28T19:50:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:57 smithi139 bash[43301]: cluster 2022-04-28T19:50:56.061376+0000 mon.a (mon.0) 344 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2022-04-28T19:50:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:57 smithi139 bash[43301]: cluster 2022-04-28T19:50:56.831806+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2022-04-28T19:50:57.501 INFO:tasks.rados.rados.0.smithi059.stdout:2045: finishing copy_from to smithi05955964-28 2022-04-28T19:50:57.501 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 787 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:50:57.501 INFO:tasks.rados.rados.0.smithi059.stdout:2045: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2036: done (9 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2037: done (8 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2038: done (7 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2039: done (6 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2040: done (5 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2041: done (4 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2042: done (3 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2043: done (2 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2045: done (1 left) 2022-04-28T19:50:57.502 INFO:tasks.rados.rados.0.smithi059.stdout:2046: done (0 left) 2022-04-28T19:50:57.503 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 189 2022-04-28T19:50:57.503 INFO:tasks.rados.rados.0.smithi059.stdout:2047: seq_num 662 ranges {0=696320,696320=557056,1253376=245760} 2022-04-28T19:50:57.512 INFO:tasks.rados.rados.0.smithi059.stdout:2047: writing smithi05955964-23 from 0 to 696320 tid 1 2022-04-28T19:50:57.517 INFO:tasks.rados.rados.0.smithi059.stdout:2047: writing smithi05955964-23 from 696320 to 1253376 tid 2 2022-04-28T19:50:57.519 INFO:tasks.rados.rados.0.smithi059.stdout:2047: writing smithi05955964-23 from 1253376 to 1499136 tid 3 2022-04-28T19:50:57.519 INFO:tasks.rados.rados.0.smithi059.stdout:2048: snap_remove snap 185 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2048: done (1 left) 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2049: delete oid 30 current snap is 189 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2047: finishing write tid 1 to smithi05955964-23 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2047: finishing write tid 2 to smithi05955964-23 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2047: finishing write tid 3 to smithi05955964-23 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2047: finishing write tid 5 to smithi05955964-23 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:2047: finishing write tid 6 to smithi05955964-23 2022-04-28T19:50:58.178 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 986 (ObjNum 662 snap 189 seq_num 662) dirty exists 2022-04-28T19:50:58.179 INFO:tasks.rados.rados.0.smithi059.stdout:2047: left oid 23 (ObjNum 662 snap 189 seq_num 662) 2022-04-28T19:50:58.183 INFO:tasks.rados.rados.0.smithi059.stdout:2047: done (1 left) 2022-04-28T19:50:58.184 INFO:tasks.rados.rados.0.smithi059.stdout:2049: done (0 left) 2022-04-28T19:50:58.184 INFO:tasks.rados.rados.0.smithi059.stdout:2050: setattr oid 22 current snap is 189 2022-04-28T19:50:58.189 INFO:tasks.rados.rados.0.smithi059.stdout:2051: snap_create 2022-04-28T19:50:58.192 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 937 (ObjNum 626 snap 179 seq_num 626) dirty exists 2022-04-28T19:50:58.437 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:58 smithi059 bash[61152]: cluster 2022-04-28T19:50:56.882657+0000 mgr.y (mgr.24961) 111 : cluster [DBG] pgmap v142: 89 pgs: 89 active+clean; 238 MiB data, 457 MiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 332 KiB/s wr, 1 op/s 2022-04-28T19:50:58.437 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:58 smithi059 bash[61152]: audit 2022-04-28T19:50:57.164898+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:58.437 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:58 smithi059 bash[59745]: cluster 2022-04-28T19:50:56.882657+0000 mgr.y (mgr.24961) 111 : cluster [DBG] pgmap v142: 89 pgs: 89 active+clean; 238 MiB data, 457 MiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 332 KiB/s wr, 1 op/s 2022-04-28T19:50:58.437 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:58 smithi059 bash[59745]: audit 2022-04-28T19:50:57.164898+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:58 smithi139 bash[43301]: cluster 2022-04-28T19:50:56.882657+0000 mgr.y (mgr.24961) 111 : cluster [DBG] pgmap v142: 89 pgs: 89 active+clean; 238 MiB data, 457 MiB used, 708 GiB / 715 GiB avail; 255 B/s rd, 332 KiB/s wr, 1 op/s 2022-04-28T19:50:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:58 smithi139 bash[43301]: audit 2022-04-28T19:50:57.164898+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:50:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:2050: done (1 left) 2022-04-28T19:50:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:2051: done (0 left) 2022-04-28T19:50:59.192 INFO:tasks.rados.rados.0.smithi059.stdout:2052: snap_remove snap 181 2022-04-28T19:50:59.533 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:50:59 smithi059 bash[59745]: cluster 2022-04-28T19:50:58.175667+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2022-04-28T19:50:59.534 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:50:59 smithi059 bash[61152]: cluster 2022-04-28T19:50:58.175667+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2022-04-28T19:50:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:50:59 smithi139 bash[43301]: cluster 2022-04-28T19:50:58.175667+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2022-04-28T19:50:59.793 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:50:59 smithi059 systemd[1]: Stopping Ceph osd.1 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:50:59.793 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:50:59 smithi059 bash[33921]: debug 2022-04-28T19:50:59.648+0000 7fa4733bb700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:50:59.793 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:50:59 smithi059 bash[33921]: debug 2022-04-28T19:50:59.648+0000 7fa4733bb700 -1 osd.1 440 *** Got signal Terminated *** 2022-04-28T19:50:59.794 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:50:59 smithi059 bash[33921]: debug 2022-04-28T19:50:59.648+0000 7fa4733bb700 -1 osd.1 440 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:51:00.197 INFO:tasks.rados.rados.0.smithi059.stdout:2052: done (0 left) 2022-04-28T19:51:00.197 INFO:tasks.rados.rados.0.smithi059.stdout:2053: read oid 8 snap -1 2022-04-28T19:51:00.197 INFO:tasks.rados.rados.0.smithi059.stdout:2053: expect (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:51:00.198 INFO:tasks.rados.rados.0.smithi059.stdout:2054: read oid 35 snap -1 2022-04-28T19:51:00.198 INFO:tasks.rados.rados.0.smithi059.stdout:2054: expect (ObjNum 642 snap 184 seq_num 642) 2022-04-28T19:51:00.198 INFO:tasks.rados.rados.0.smithi059.stdout:2055: rollback oid 5 current snap is 190 2022-04-28T19:51:00.198 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 177 2022-04-28T19:51:00.198 INFO:tasks.rados.rados.0.smithi059.stdout:2056: read oid 44 snap -1 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2056: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2057: read oid 15 snap -1 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2057: expect (ObjNum 416 snap 125 seq_num 416) 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2058: copy_from oid 18 from oid 19 current snap is 190 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2059: read oid 27 snap -1 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2059: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 190 2022-04-28T19:51:00.199 INFO:tasks.rados.rados.0.smithi059.stdout:2060: seq_num 664 ranges {1499136=729088,2228224=729088,2957312=73728} 2022-04-28T19:51:00.211 INFO:tasks.rados.rados.0.smithi059.stdout:2060: writing smithi05955964-23 from 1499136 to 2228224 tid 1 2022-04-28T19:51:00.216 INFO:tasks.rados.rados.0.smithi059.stdout:2060: writing smithi05955964-23 from 2228224 to 2957312 tid 2 2022-04-28T19:51:00.217 INFO:tasks.rados.rados.0.smithi059.stdout:2060: writing smithi05955964-23 from 2957312 to 3031040 tid 3 2022-04-28T19:51:00.217 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 190 2022-04-28T19:51:00.217 INFO:tasks.rados.rados.0.smithi059.stdout:2061: seq_num 665 ranges {188416=745472,933888=606208,1540096=196608} 2022-04-28T19:51:00.222 INFO:tasks.rados.rados.0.smithi059.stdout:2061: writing smithi05955964-16 from 188416 to 933888 tid 1 2022-04-28T19:51:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:2061: writing smithi05955964-16 from 933888 to 1540096 tid 2 2022-04-28T19:51:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:2061: writing smithi05955964-16 from 1540096 to 1736704 tid 3 2022-04-28T19:51:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:2062: snap_create 2022-04-28T19:51:00.228 INFO:tasks.rados.rados.0.smithi059.stdout:2059: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2060: finishing write tid 1 to smithi05955964-23 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2061: finishing write tid 1 to smithi05955964-16 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2060: finishing write tid 2 to smithi05955964-23 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2061: finishing write tid 2 to smithi05955964-16 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2061: finishing write tid 3 to smithi05955964-16 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2061: finishing write tid 5 to smithi05955964-16 2022-04-28T19:51:00.337 INFO:tasks.rados.rados.0.smithi059.stdout:2061: finishing write tid 6 to smithi05955964-16 2022-04-28T19:51:00.338 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 878 (ObjNum 665 snap 190 seq_num 665) dirty exists 2022-04-28T19:51:00.338 INFO:tasks.rados.rados.0.smithi059.stdout:2061: left oid 16 (ObjNum 665 snap 190 seq_num 665) 2022-04-28T19:51:00.338 INFO:tasks.rados.rados.0.smithi059.stdout:2060: finishing write tid 3 to smithi05955964-23 2022-04-28T19:51:00.338 INFO:tasks.rados.rados.0.smithi059.stdout:2060: finishing write tid 5 to smithi05955964-23 2022-04-28T19:51:00.338 INFO:tasks.rados.rados.0.smithi059.stdout:2060: finishing write tid 6 to smithi05955964-23 2022-04-28T19:51:00.339 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 997 (ObjNum 664 snap 190 seq_num 664) dirty exists 2022-04-28T19:51:00.339 INFO:tasks.rados.rados.0.smithi059.stdout:2060: left oid 23 (ObjNum 664 snap 190 seq_num 664) 2022-04-28T19:51:00.339 INFO:tasks.rados.rados.0.smithi059.stdout:2057: expect (ObjNum 416 snap 125 seq_num 416) 2022-04-28T19:51:00.469 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[64958]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.1 2022-04-28T19:51:00.469 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[64958]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-1 2022-04-28T19:51:00.469 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:58.883398+0000 mgr.y (mgr.24961) 112 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 248 MiB data, 469 MiB used, 708 GiB / 715 GiB avail; 16 MiB/s rd, 3.4 MiB/s wr, 19 op/s 2022-04-28T19:51:00.469 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.187750+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2022-04-28T19:51:00.469 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.971670+0000 mon.a (mon.0) 349 : cluster [DBG] osd.1 reported immediately failed by osd.6 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.971955+0000 mon.a (mon.0) 350 : cluster [INF] osd.1 failed (root=default,host=smithi059) (connection refused reported by osd.6) 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.972545+0000 mon.a (mon.0) 351 : cluster [DBG] osd.1 reported immediately failed by osd.7 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.972754+0000 mon.a (mon.0) 352 : cluster [DBG] osd.1 reported immediately failed by osd.2 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.973029+0000 mon.a (mon.0) 353 : cluster [DBG] osd.1 reported immediately failed by osd.5 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.973240+0000 mon.a (mon.0) 354 : cluster [DBG] osd.1 reported immediately failed by osd.3 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.973438+0000 mon.a (mon.0) 355 : cluster [DBG] osd.1 reported immediately failed by osd.6 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.973654+0000 mon.a (mon.0) 356 : cluster [DBG] osd.1 reported immediately failed by osd.7 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.973765+0000 mon.a (mon.0) 357 : cluster [DBG] osd.1 reported immediately failed by osd.2 2022-04-28T19:51:00.470 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.974121+0000 mon.a (mon.0) 358 : cluster [DBG] osd.1 reported immediately failed by osd.4 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.974256+0000 mon.a (mon.0) 359 : cluster [DBG] osd.1 reported immediately failed by osd.3 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.974693+0000 mon.a (mon.0) 360 : cluster [DBG] osd.1 reported immediately failed by osd.5 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.974887+0000 mon.a (mon.0) 361 : cluster [DBG] osd.1 reported immediately failed by osd.4 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.975271+0000 mon.a (mon.0) 362 : cluster [DBG] osd.1 reported immediately failed by osd.0 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[59745]: cluster 2022-04-28T19:50:59.975462+0000 mon.a (mon.0) 363 : cluster [DBG] osd.1 reported immediately failed by osd.0 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:58.883398+0000 mgr.y (mgr.24961) 112 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 248 MiB data, 469 MiB used, 708 GiB / 715 GiB avail; 16 MiB/s rd, 3.4 MiB/s wr, 19 op/s 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.187750+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2022-04-28T19:51:00.471 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.971670+0000 mon.a (mon.0) 349 : cluster [DBG] osd.1 reported immediately failed by osd.6 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.971955+0000 mon.a (mon.0) 350 : cluster [INF] osd.1 failed (root=default,host=smithi059) (connection refused reported by osd.6) 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.972545+0000 mon.a (mon.0) 351 : cluster [DBG] osd.1 reported immediately failed by osd.7 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.972754+0000 mon.a (mon.0) 352 : cluster [DBG] osd.1 reported immediately failed by osd.2 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.973029+0000 mon.a (mon.0) 353 : cluster [DBG] osd.1 reported immediately failed by osd.5 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.973240+0000 mon.a (mon.0) 354 : cluster [DBG] osd.1 reported immediately failed by osd.3 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.973438+0000 mon.a (mon.0) 355 : cluster [DBG] osd.1 reported immediately failed by osd.6 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.973654+0000 mon.a (mon.0) 356 : cluster [DBG] osd.1 reported immediately failed by osd.7 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.973765+0000 mon.a (mon.0) 357 : cluster [DBG] osd.1 reported immediately failed by osd.2 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.974121+0000 mon.a (mon.0) 358 : cluster [DBG] osd.1 reported immediately failed by osd.4 2022-04-28T19:51:00.472 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.974256+0000 mon.a (mon.0) 359 : cluster [DBG] osd.1 reported immediately failed by osd.3 2022-04-28T19:51:00.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.974693+0000 mon.a (mon.0) 360 : cluster [DBG] osd.1 reported immediately failed by osd.5 2022-04-28T19:51:00.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.974887+0000 mon.a (mon.0) 361 : cluster [DBG] osd.1 reported immediately failed by osd.4 2022-04-28T19:51:00.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.975271+0000 mon.a (mon.0) 362 : cluster [DBG] osd.1 reported immediately failed by osd.0 2022-04-28T19:51:00.473 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[61152]: cluster 2022-04-28T19:50:59.975462+0000 mon.a (mon.0) 363 : cluster [DBG] osd.1 reported immediately failed by osd.0 2022-04-28T19:51:00.526 INFO:tasks.rados.rados.0.smithi059.stdout:2054: expect (ObjNum 642 snap 184 seq_num 642) 2022-04-28T19:51:00.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:58.883398+0000 mgr.y (mgr.24961) 112 : cluster [DBG] pgmap v144: 89 pgs: 89 active+clean; 248 MiB data, 469 MiB used, 708 GiB / 715 GiB avail; 16 MiB/s rd, 3.4 MiB/s wr, 19 op/s 2022-04-28T19:51:00.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.187750+0000 mon.a (mon.0) 348 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.971670+0000 mon.a (mon.0) 349 : cluster [DBG] osd.1 reported immediately failed by osd.6 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.971955+0000 mon.a (mon.0) 350 : cluster [INF] osd.1 failed (root=default,host=smithi059) (connection refused reported by osd.6) 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.972545+0000 mon.a (mon.0) 351 : cluster [DBG] osd.1 reported immediately failed by osd.7 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.972754+0000 mon.a (mon.0) 352 : cluster [DBG] osd.1 reported immediately failed by osd.2 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.973029+0000 mon.a (mon.0) 353 : cluster [DBG] osd.1 reported immediately failed by osd.5 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.973240+0000 mon.a (mon.0) 354 : cluster [DBG] osd.1 reported immediately failed by osd.3 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.973438+0000 mon.a (mon.0) 355 : cluster [DBG] osd.1 reported immediately failed by osd.6 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.973654+0000 mon.a (mon.0) 356 : cluster [DBG] osd.1 reported immediately failed by osd.7 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.973765+0000 mon.a (mon.0) 357 : cluster [DBG] osd.1 reported immediately failed by osd.2 2022-04-28T19:51:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.974121+0000 mon.a (mon.0) 358 : cluster [DBG] osd.1 reported immediately failed by osd.4 2022-04-28T19:51:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.974256+0000 mon.a (mon.0) 359 : cluster [DBG] osd.1 reported immediately failed by osd.3 2022-04-28T19:51:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.974693+0000 mon.a (mon.0) 360 : cluster [DBG] osd.1 reported immediately failed by osd.5 2022-04-28T19:51:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.974887+0000 mon.a (mon.0) 361 : cluster [DBG] osd.1 reported immediately failed by osd.4 2022-04-28T19:51:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.975271+0000 mon.a (mon.0) 362 : cluster [DBG] osd.1 reported immediately failed by osd.0 2022-04-28T19:51:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:00 smithi139 bash[43301]: cluster 2022-04-28T19:50:59.975462+0000 mon.a (mon.0) 363 : cluster [DBG] osd.1 reported immediately failed by osd.0 2022-04-28T19:51:00.841 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:00 smithi059 bash[64958]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.1 2022-04-28T19:51:01.553 INFO:tasks.rados.rados.0.smithi059.stdout:2053: expect (ObjNum 611 snap 171 seq_num 611) 2022-04-28T19:51:01.568 INFO:tasks.rados.rados.0.smithi059.stdout:2055: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:51:01.568 INFO:tasks.rados.rados.0.smithi059.stdout:2056: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:51:01.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[61152]: cluster 2022-04-28T19:51:00.182696+0000 mon.a (mon.0) 364 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:51:01.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[61152]: cluster 2022-04-28T19:51:00.194897+0000 mon.a (mon.0) 365 : cluster [DBG] osdmap e441: 8 total, 7 up, 8 in 2022-04-28T19:51:01.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[61152]: cluster 2022-04-28T19:51:01.192646+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e442: 8 total, 7 up, 8 in 2022-04-28T19:51:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[59745]: cluster 2022-04-28T19:51:00.182696+0000 mon.a (mon.0) 364 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:51:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[59745]: cluster 2022-04-28T19:51:00.194897+0000 mon.a (mon.0) 365 : cluster [DBG] osdmap e441: 8 total, 7 up, 8 in 2022-04-28T19:51:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[59745]: cluster 2022-04-28T19:51:01.192646+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e442: 8 total, 7 up, 8 in 2022-04-28T19:51:01.606 INFO:tasks.rados.rados.0.smithi059.stdout:2055: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:51:01.606 INFO:tasks.rados.rados.0.smithi059.stdout:2055: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:51:01.607 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 901 (ObjNum 574 snap 164 seq_num 574) dirty exists 2022-04-28T19:51:01.607 INFO:tasks.rados.rados.0.smithi059.stdout:2058: finishing copy_from to smithi05955964-18 2022-04-28T19:51:01.607 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 923 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:51:01.607 INFO:tasks.rados.rados.0.smithi059.stdout:2058: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:51:01.607 INFO:tasks.rados.rados.0.smithi059.stdout:2053: done (9 left) 2022-04-28T19:51:01.607 INFO:tasks.rados.rados.0.smithi059.stdout:2054: done (8 left) 2022-04-28T19:51:01.608 INFO:tasks.rados.rados.0.smithi059.stdout:2055: done (7 left) 2022-04-28T19:51:01.608 INFO:tasks.rados.rados.0.smithi059.stdout:2056: done (6 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2057: done (5 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2058: done (4 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2059: done (3 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2060: done (2 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2061: done (1 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2062: done (0 left) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2063: rmattr oid 11 current snap is 191 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2064: read oid 20 snap -1 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2064: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:51:01.609 INFO:tasks.rados.rados.0.smithi059.stdout:2065: snap_create 2022-04-28T19:51:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:01 smithi139 bash[43301]: cluster 2022-04-28T19:51:00.182696+0000 mon.a (mon.0) 364 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:51:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:01 smithi139 bash[43301]: cluster 2022-04-28T19:51:00.194897+0000 mon.a (mon.0) 365 : cluster [DBG] osdmap e441: 8 total, 7 up, 8 in 2022-04-28T19:51:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:01 smithi139 bash[43301]: cluster 2022-04-28T19:51:01.192646+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e442: 8 total, 7 up, 8 in 2022-04-28T19:51:01.620 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 805 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:51:01.633 INFO:tasks.rados.rados.0.smithi059.stdout:2064: expect (ObjNum 461 snap 136 seq_num 461) 2022-04-28T19:51:01.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:01.628Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:01.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:01.628Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:01.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:01.628Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:51:02.140 INFO:tasks.rados.rados.0.smithi059.stdout:2063: done (2 left) 2022-04-28T19:51:02.140 INFO:tasks.rados.rados.0.smithi059.stdout:2064: done (1 left) 2022-04-28T19:51:02.140 INFO:tasks.rados.rados.0.smithi059.stdout:2065: done (0 left) 2022-04-28T19:51:02.141 INFO:tasks.rados.rados.0.smithi059.stdout:2066: snap_remove snap 186 2022-04-28T19:51:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:02 smithi059 bash[61152]: cluster 2022-04-28T19:51:00.884127+0000 mgr.y (mgr.24961) 113 : cluster [DBG] pgmap v147: 89 pgs: 11 stale+active+clean, 78 active+clean; 248 MiB data, 469 MiB used, 708 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2022-04-28T19:51:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:02 smithi059 bash[61152]: cluster 2022-04-28T19:51:01.834481+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e443: 8 total, 7 up, 8 in 2022-04-28T19:51:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:02 smithi059 bash[59745]: cluster 2022-04-28T19:51:00.884127+0000 mgr.y (mgr.24961) 113 : cluster [DBG] pgmap v147: 89 pgs: 11 stale+active+clean, 78 active+clean; 248 MiB data, 469 MiB used, 708 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2022-04-28T19:51:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:02 smithi059 bash[59745]: cluster 2022-04-28T19:51:01.834481+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e443: 8 total, 7 up, 8 in 2022-04-28T19:51:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:02 smithi139 bash[43301]: cluster 2022-04-28T19:51:00.884127+0000 mgr.y (mgr.24961) 113 : cluster [DBG] pgmap v147: 89 pgs: 11 stale+active+clean, 78 active+clean; 248 MiB data, 469 MiB used, 708 GiB / 715 GiB avail; 16 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2022-04-28T19:51:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:02 smithi139 bash[43301]: cluster 2022-04-28T19:51:01.834481+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e443: 8 total, 7 up, 8 in 2022-04-28T19:51:02.851 INFO:tasks.rados.rados.0.smithi059.stdout:2066: done (0 left) 2022-04-28T19:51:02.851 INFO:tasks.rados.rados.0.smithi059.stdout:2067: rollback oid 11 current snap is 192 2022-04-28T19:51:02.851 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 189 2022-04-28T19:51:02.852 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 192 2022-04-28T19:51:02.852 INFO:tasks.rados.rados.0.smithi059.stdout:2068: seq_num 666 ranges {8314880=761856,9076736=491520,9568256=311296} 2022-04-28T19:51:02.899 INFO:tasks.rados.rados.0.smithi059.stdout:2068: writing smithi05955964-6 from 8314880 to 9076736 tid 1 2022-04-28T19:51:02.916 INFO:tasks.rados.rados.0.smithi059.stdout:2068: writing smithi05955964-6 from 9076736 to 9568256 tid 2 2022-04-28T19:51:02.916 INFO:tasks.rados.rados.0.smithi059.stdout:2068: writing smithi05955964-6 from 9568256 to 9879552 tid 3 2022-04-28T19:51:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2067: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:51:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2067: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:51:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2067: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:51:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 809 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:51:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2067: done (1 left) 2022-04-28T19:51:02.918 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 192 2022-04-28T19:51:02.918 INFO:tasks.rados.rados.0.smithi059.stdout:2069: seq_num 667 ranges {4292608=778240,5070848=778240,5849088=24576} 2022-04-28T19:51:02.934 INFO:tasks.rados.rados.0.smithi059.stdout:2069: writing smithi05955964-28 from 4292608 to 5070848 tid 1 2022-04-28T19:51:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2069: writing smithi05955964-28 from 5070848 to 5849088 tid 2 2022-04-28T19:51:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2069: writing smithi05955964-28 from 5849088 to 5873664 tid 3 2022-04-28T19:51:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2070: snap_create 2022-04-28T19:51:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2068: finishing write tid 1 to smithi05955964-6 2022-04-28T19:51:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2068: finishing write tid 2 to smithi05955964-6 2022-04-28T19:51:02.940 INFO:tasks.rados.rados.0.smithi059.stdout:2068: finishing write tid 3 to smithi05955964-6 2022-04-28T19:51:02.940 INFO:tasks.rados.rados.0.smithi059.stdout:2068: finishing write tid 5 to smithi05955964-6 2022-04-28T19:51:02.940 INFO:tasks.rados.rados.0.smithi059.stdout:2068: finishing write tid 6 to smithi05955964-6 2022-04-28T19:51:02.940 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 849 (ObjNum 666 snap 192 seq_num 666) dirty exists 2022-04-28T19:51:02.940 INFO:tasks.rados.rados.0.smithi059.stdout:2068: left oid 6 (ObjNum 666 snap 192 seq_num 666) 2022-04-28T19:51:02.941 INFO:tasks.rados.rados.0.smithi059.stdout:2069: finishing write tid 1 to smithi05955964-28 2022-04-28T19:51:02.941 INFO:tasks.rados.rados.0.smithi059.stdout:2069: finishing write tid 2 to smithi05955964-28 2022-04-28T19:51:02.942 INFO:tasks.rados.rados.0.smithi059.stdout:2069: finishing write tid 3 to smithi05955964-28 2022-04-28T19:51:02.943 INFO:tasks.rados.rados.0.smithi059.stdout:2069: finishing write tid 5 to smithi05955964-28 2022-04-28T19:51:02.944 INFO:tasks.rados.rados.0.smithi059.stdout:2069: finishing write tid 6 to smithi05955964-28 2022-04-28T19:51:02.944 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 882 (ObjNum 667 snap 192 seq_num 667) dirty exists 2022-04-28T19:51:02.944 INFO:tasks.rados.rados.0.smithi059.stdout:2069: left oid 28 (ObjNum 667 snap 192 seq_num 667) 2022-04-28T19:51:03.091 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:02 smithi059 systemd[1]: Stopped Ceph osd.1 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:51:03.091 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:02 smithi059 systemd[1]: Started Ceph osd.1 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:51:03.819 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:03.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:03.819 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:03.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:03.819 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:03.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:51:03.854 INFO:tasks.rados.rados.0.smithi059.stdout:2068: done (2 left) 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:2069: done (1 left) 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:2070: done (0 left) 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:2071: rollback oid 16 current snap is 193 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 189 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:2072: read oid 42 snap -1 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:2072: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:51:03.855 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 193 2022-04-28T19:51:03.856 INFO:tasks.rados.rados.0.smithi059.stdout:2073: seq_num 668 ranges {11485184=794624,12279808=663552,12943360=139264} 2022-04-28T19:51:03.929 INFO:tasks.rados.rados.0.smithi059.stdout:2073: writing smithi05955964-46 from 11485184 to 12279808 tid 1 2022-04-28T19:51:03.933 INFO:tasks.rados.rados.0.smithi059.stdout:2073: writing smithi05955964-46 from 12279808 to 12943360 tid 2 2022-04-28T19:51:03.935 INFO:tasks.rados.rados.0.smithi059.stdout:2073: writing smithi05955964-46 from 12943360 to 13082624 tid 3 2022-04-28T19:51:03.935 INFO:tasks.rados.rados.0.smithi059.stdout:2074: read oid 48 snap -1 2022-04-28T19:51:03.935 INFO:tasks.rados.rados.0.smithi059.stdout:2074: expect (ObjNum 660 snap 189 seq_num 660) 2022-04-28T19:51:03.935 INFO:tasks.rados.rados.0.smithi059.stdout:2075: setattr oid 14 current snap is 193 2022-04-28T19:51:03.935 INFO:tasks.rados.rados.0.smithi059.stdout:2071: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:51:03.936 INFO:tasks.rados.rados.0.smithi059.stdout:2071: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:51:03.936 INFO:tasks.rados.rados.0.smithi059.stdout:2071: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:51:03.936 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 885 (ObjNum 584 snap 166 seq_num 584) dirty exists 2022-04-28T19:51:03.936 INFO:tasks.rados.rados.0.smithi059.stdout:2072: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:51:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: audit 2022-04-28T19:51:02.825117+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: audit 2022-04-28T19:51:02.826242+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: audit 2022-04-28T19:51:02.827891+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: audit 2022-04-28T19:51:02.828967+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: cephadm 2022-04-28T19:51:02.829879+0000 mgr.y (mgr.24961) 114 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: audit 2022-04-28T19:51:02.830731+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: cluster 2022-04-28T19:51:02.842790+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e444: 8 total, 7 up, 8 in 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[61152]: cluster 2022-04-28T19:51:02.884732+0000 mgr.y (mgr.24961) 115 : cluster [DBG] pgmap v151: 89 pgs: 8 active+undersized+degraded, 5 peering, 29 active+undersized, 47 active+clean; 247 MiB data, 467 MiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 16 op/s; 50/399 objects degraded (12.531%) 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: audit 2022-04-28T19:51:02.825117+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: audit 2022-04-28T19:51:02.826242+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: audit 2022-04-28T19:51:02.827891+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:51:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: audit 2022-04-28T19:51:02.828967+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:51:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: cephadm 2022-04-28T19:51:02.829879+0000 mgr.y (mgr.24961) 114 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:51:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: audit 2022-04-28T19:51:02.830731+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:51:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: cluster 2022-04-28T19:51:02.842790+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e444: 8 total, 7 up, 8 in 2022-04-28T19:51:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:03 smithi059 bash[59745]: cluster 2022-04-28T19:51:02.884732+0000 mgr.y (mgr.24961) 115 : cluster [DBG] pgmap v151: 89 pgs: 8 active+undersized+degraded, 5 peering, 29 active+undersized, 47 active+clean; 247 MiB data, 467 MiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 16 op/s; 50/399 objects degraded (12.531%) 2022-04-28T19:51:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: audit 2022-04-28T19:51:02.825117+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: audit 2022-04-28T19:51:02.826242+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: audit 2022-04-28T19:51:02.827891+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:51:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: audit 2022-04-28T19:51:02.828967+0000 mon.a (mon.0) 371 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:51:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: cephadm 2022-04-28T19:51:02.829879+0000 mgr.y (mgr.24961) 114 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:51:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: audit 2022-04-28T19:51:02.830731+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:51:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: cluster 2022-04-28T19:51:02.842790+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e444: 8 total, 7 up, 8 in 2022-04-28T19:51:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:03 smithi139 bash[43301]: cluster 2022-04-28T19:51:02.884732+0000 mgr.y (mgr.24961) 115 : cluster [DBG] pgmap v151: 89 pgs: 8 active+undersized+degraded, 5 peering, 29 active+undersized, 47 active+clean; 247 MiB data, 467 MiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 16 op/s; 50/399 objects degraded (12.531%) 2022-04-28T19:51:04.243 INFO:tasks.rados.rados.0.smithi059.stdout:2073: finishing write tid 1 to smithi05955964-46 2022-04-28T19:51:04.243 INFO:tasks.rados.rados.0.smithi059.stdout:2073: finishing write tid 2 to smithi05955964-46 2022-04-28T19:51:04.243 INFO:tasks.rados.rados.0.smithi059.stdout:2073: finishing write tid 3 to smithi05955964-46 2022-04-28T19:51:04.243 INFO:tasks.rados.rados.0.smithi059.stdout:2073: finishing write tid 5 to smithi05955964-46 2022-04-28T19:51:04.243 INFO:tasks.rados.rados.0.smithi059.stdout:2073: finishing write tid 6 to smithi05955964-46 2022-04-28T19:51:04.243 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 906 (ObjNum 668 snap 193 seq_num 668) dirty exists 2022-04-28T19:51:04.244 INFO:tasks.rados.rados.0.smithi059.stdout:2073: left oid 46 (ObjNum 668 snap 193 seq_num 668) 2022-04-28T19:51:04.244 INFO:tasks.rados.rados.0.smithi059.stdout:2074: expect (ObjNum 660 snap 189 seq_num 660) 2022-04-28T19:51:04.286 INFO:tasks.rados.rados.0.smithi059.stdout:2071: done (4 left) 2022-04-28T19:51:04.287 INFO:tasks.rados.rados.0.smithi059.stdout:2072: done (3 left) 2022-04-28T19:51:04.287 INFO:tasks.rados.rados.0.smithi059.stdout:2073: done (2 left) 2022-04-28T19:51:04.287 INFO:tasks.rados.rados.0.smithi059.stdout:2074: done (1 left) 2022-04-28T19:51:04.287 INFO:tasks.rados.rados.0.smithi059.stdout:2076: rollback oid 31 current snap is 193 2022-04-28T19:51:04.287 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 189 2022-04-28T19:51:04.288 INFO:tasks.rados.rados.0.smithi059.stdout:2077: snap_create 2022-04-28T19:51:04.290 INFO:tasks.rados.rados.0.smithi059.stdout:2076: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:51:04.290 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 843 (ObjNum -132777024 snap 21893 seq_num 4026544032) dirty dne 2022-04-28T19:51:04.291 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 939 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:51:04.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2022-04-28T19:51:04.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2022-04-28T19:51:04.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2022-04-28T19:51:04.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2022-04-28T19:51:04.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2022-04-28T19:51:04.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2022-04-28T19:51:04.592 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[65202]: --> ceph-volume lvm activate successful for osd ID: 1 2022-04-28T19:51:04.859 INFO:tasks.rados.rados.0.smithi059.stdout:2075: done (2 left) 2022-04-28T19:51:04.859 INFO:tasks.rados.rados.0.smithi059.stdout:2076: done (1 left) 2022-04-28T19:51:04.859 INFO:tasks.rados.rados.0.smithi059.stdout:2077: done (0 left) 2022-04-28T19:51:04.859 INFO:tasks.rados.rados.0.smithi059.stdout:2078: read oid 7 snap -1 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:2078: expect (ObjNum 612 snap 171 seq_num 612) 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:2079: copy_from oid 16 from oid 46 current snap is 194 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:2080: read oid 49 snap -1 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:2080: expect (ObjNum 605 snap 169 seq_num 605) 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:2081: rollback oid 32 current snap is 194 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 191 2022-04-28T19:51:04.860 INFO:tasks.rados.rados.0.smithi059.stdout:2082: snap_remove snap 188 2022-04-28T19:51:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[61152]: cluster 2022-04-28T19:51:03.839184+0000 mon.a (mon.0) 374 : cluster [WRN] Health check failed: Degraded data redundancy: 50/399 objects degraded (12.531%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[61152]: cluster 2022-04-28T19:51:03.851477+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e445: 8 total, 7 up, 8 in 2022-04-28T19:51:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[59745]: cluster 2022-04-28T19:51:03.839184+0000 mon.a (mon.0) 374 : cluster [WRN] Health check failed: Degraded data redundancy: 50/399 objects degraded (12.531%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:04 smithi059 bash[59745]: cluster 2022-04-28T19:51:03.851477+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e445: 8 total, 7 up, 8 in 2022-04-28T19:51:05.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:04 smithi139 bash[43301]: cluster 2022-04-28T19:51:03.839184+0000 mon.a (mon.0) 374 : cluster [WRN] Health check failed: Degraded data redundancy: 50/399 objects degraded (12.531%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:05.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:04 smithi139 bash[43301]: cluster 2022-04-28T19:51:03.851477+0000 mon.a (mon.0) 375 : cluster [DBG] osdmap e445: 8 total, 7 up, 8 in 2022-04-28T19:51:05.869 INFO:tasks.rados.rados.0.smithi059.stdout:2082: done (4 left) 2022-04-28T19:51:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:2083: rmattr oid 11 current snap is 194 2022-04-28T19:51:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:2081: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:51:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 950 (ObjNum 288 snap 0 seq_num 33) dirty dne 2022-04-28T19:51:05.870 INFO:tasks.rados.rados.0.smithi059.stdout:2080: expect (ObjNum 605 snap 169 seq_num 605) 2022-04-28T19:51:06.087 INFO:tasks.rados.rados.0.smithi059.stdout:2078: expect (ObjNum 612 snap 171 seq_num 612) 2022-04-28T19:51:06.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:05 smithi139 bash[43301]: cluster 2022-04-28T19:51:04.856718+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e446: 8 total, 7 up, 8 in 2022-04-28T19:51:06.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:05 smithi139 bash[43301]: cluster 2022-04-28T19:51:04.885278+0000 mgr.y (mgr.24961) 116 : cluster [DBG] pgmap v154: 89 pgs: 8 active+undersized+degraded, 5 peering, 29 active+undersized, 47 active+clean; 247 MiB data, 467 MiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 16 op/s; 50/399 objects degraded (12.531%) 2022-04-28T19:51:06.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:05 smithi059 bash[61152]: cluster 2022-04-28T19:51:04.856718+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e446: 8 total, 7 up, 8 in 2022-04-28T19:51:06.123 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:05 smithi059 bash[61152]: cluster 2022-04-28T19:51:04.885278+0000 mgr.y (mgr.24961) 116 : cluster [DBG] pgmap v154: 89 pgs: 8 active+undersized+degraded, 5 peering, 29 active+undersized, 47 active+clean; 247 MiB data, 467 MiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 16 op/s; 50/399 objects degraded (12.531%) 2022-04-28T19:51:06.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:05 smithi059 bash[59745]: cluster 2022-04-28T19:51:04.856718+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e446: 8 total, 7 up, 8 in 2022-04-28T19:51:06.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:05 smithi059 bash[59745]: cluster 2022-04-28T19:51:04.885278+0000 mgr.y (mgr.24961) 116 : cluster [DBG] pgmap v154: 89 pgs: 8 active+undersized+degraded, 5 peering, 29 active+undersized, 47 active+clean; 247 MiB data, 467 MiB used, 708 GiB / 715 GiB avail; 22 MiB/s rd, 1.3 MiB/s wr, 16 op/s; 50/399 objects degraded (12.531%) 2022-04-28T19:51:06.591 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:06 smithi059 bash[65202]: debug 2022-04-28T19:51:06.152+0000 7f6ec14f6700 1 -- 172.21.15.59:0/410568170 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563b2f5dc4e0 con 0x563b2f655000 2022-04-28T19:51:06.759 INFO:tasks.rados.rados.0.smithi059.stdout:2079: finishing copy_from to smithi05955964-16 2022-04-28T19:51:06.759 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 906 (ObjNum 668 snap 193 seq_num 668) dirty exists 2022-04-28T19:51:06.760 INFO:tasks.rados.rados.0.smithi059.stdout:2079: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:51:06.760 INFO:tasks.rados.rados.0.smithi059.stdout:2078: done (4 left) 2022-04-28T19:51:06.760 INFO:tasks.rados.rados.0.smithi059.stdout:2079: done (3 left) 2022-04-28T19:51:06.760 INFO:tasks.rados.rados.0.smithi059.stdout:2080: done (2 left) 2022-04-28T19:51:06.760 INFO:tasks.rados.rados.0.smithi059.stdout:2081: done (1 left) 2022-04-28T19:51:06.760 INFO:tasks.rados.rados.0.smithi059.stdout:2084: read oid 3 snap -1 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:2084: expect (ObjNum 648 snap 184 seq_num 648) 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:2085: rollback oid 5 current snap is 194 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 177 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 811 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:2083: done (2 left) 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:2086: read oid 45 snap -1 2022-04-28T19:51:06.761 INFO:tasks.rados.rados.0.smithi059.stdout:2086: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:51:06.762 INFO:tasks.rados.rados.0.smithi059.stdout:2087: copy_from oid 22 from oid 15 current snap is 194 2022-04-28T19:51:06.762 INFO:tasks.rados.rados.0.smithi059.stdout:2088: read oid 19 snap -1 2022-04-28T19:51:06.762 INFO:tasks.rados.rados.0.smithi059.stdout:2088: expect (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:51:06.762 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 194 2022-04-28T19:51:06.762 INFO:tasks.rados.rados.0.smithi059.stdout:2089: seq_num 669 ranges {0=409600,409600=548864,958464=655360} 2022-04-28T19:51:06.763 INFO:tasks.rados.rados.0.smithi059.stdout:2089: writing smithi05955964-30 from 0 to 409600 tid 1 2022-04-28T19:51:06.768 INFO:tasks.rados.rados.0.smithi059.stdout:2089: writing smithi05955964-30 from 409600 to 958464 tid 2 2022-04-28T19:51:06.773 INFO:tasks.rados.rados.0.smithi059.stdout:2089: writing smithi05955964-30 from 958464 to 1613824 tid 3 2022-04-28T19:51:06.773 INFO:tasks.rados.rados.0.smithi059.stdout:2090: setattr oid 33 current snap is 194 2022-04-28T19:51:06.774 INFO:tasks.rados.rados.0.smithi059.stdout:2091: copy_from oid 40 from oid 48 current snap is 194 2022-04-28T19:51:06.775 INFO:tasks.rados.rados.0.smithi059.stdout:2092: snap_remove snap 192 2022-04-28T19:51:06.837 INFO:tasks.rados.rados.0.smithi059.stdout:2092: done (8 left) 2022-04-28T19:51:06.837 INFO:tasks.rados.rados.0.smithi059.stdout:2093: rmattr oid 16 current snap is 194 2022-04-28T19:51:06.837 INFO:tasks.rados.rados.0.smithi059.stdout:2094: snap_create 2022-04-28T19:51:06.837 INFO:tasks.rados.rados.0.smithi059.stdout:2085: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:51:06.837 INFO:tasks.rados.rados.0.smithi059.stdout:2089: finishing write tid 1 to smithi05955964-30 2022-04-28T19:51:06.837 INFO:tasks.rados.rados.0.smithi059.stdout:2085: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:51:06.838 INFO:tasks.rados.rados.0.smithi059.stdout:2085: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:51:06.838 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 910 (ObjNum 574 snap 164 seq_num 574) dirty exists 2022-04-28T19:51:06.838 INFO:tasks.rados.rados.0.smithi059.stdout:2088: expect (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:51:06.924 INFO:tasks.rados.rados.0.smithi059.stdout:2089: finishing write tid 2 to smithi05955964-30 2022-04-28T19:51:06.924 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 976 (ObjNum 620 snap 177 seq_num 620) dirty exists 2022-04-28T19:51:06.925 INFO:tasks.rados.rados.0.smithi059.stdout:2089: finishing write tid 3 to smithi05955964-30 2022-04-28T19:51:06.925 INFO:tasks.rados.rados.0.smithi059.stdout:2089: finishing write tid 5 to smithi05955964-30 2022-04-28T19:51:06.925 INFO:tasks.rados.rados.0.smithi059.stdout:2089: finishing write tid 6 to smithi05955964-30 2022-04-28T19:51:06.925 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 927 (ObjNum 669 snap 194 seq_num 669) dirty exists 2022-04-28T19:51:06.925 INFO:tasks.rados.rados.0.smithi059.stdout:2089: left oid 30 (ObjNum 669 snap 194 seq_num 669) 2022-04-28T19:51:06.925 INFO:tasks.rados.rados.0.smithi059.stdout:2084: expect (ObjNum 648 snap 184 seq_num 648) 2022-04-28T19:51:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:06 smithi059 bash[61152]: cluster 2022-04-28T19:51:05.868642+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e447: 8 total, 7 up, 8 in 2022-04-28T19:51:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:06 smithi059 bash[61152]: cluster 2022-04-28T19:51:06.835561+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e448: 8 total, 7 up, 8 in 2022-04-28T19:51:07.092 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:06 smithi059 bash[65202]: debug 2022-04-28T19:51:06.716+0000 7f6ec8d51200 -1 Falling back to public interface 2022-04-28T19:51:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:06 smithi059 bash[59745]: cluster 2022-04-28T19:51:05.868642+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e447: 8 total, 7 up, 8 in 2022-04-28T19:51:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:06 smithi059 bash[59745]: cluster 2022-04-28T19:51:06.835561+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e448: 8 total, 7 up, 8 in 2022-04-28T19:51:07.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:06 smithi139 bash[43301]: cluster 2022-04-28T19:51:05.868642+0000 mon.a (mon.0) 377 : cluster [DBG] osdmap e447: 8 total, 7 up, 8 in 2022-04-28T19:51:07.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:06 smithi139 bash[43301]: cluster 2022-04-28T19:51:06.835561+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e448: 8 total, 7 up, 8 in 2022-04-28T19:51:07.253 INFO:tasks.rados.rados.0.smithi059.stdout:2086: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:51:07.522 INFO:tasks.rados.rados.0.smithi059.stdout:2087: finishing copy_from to smithi05955964-22 2022-04-28T19:51:07.523 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 829 (ObjNum 416 snap 125 seq_num 416) dirty exists 2022-04-28T19:51:07.523 INFO:tasks.rados.rados.0.smithi059.stdout:2087: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:51:07.523 INFO:tasks.rados.rados.0.smithi059.stdout:2091: finishing copy_from to smithi05955964-40 2022-04-28T19:51:07.523 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 927 (ObjNum 660 snap 189 seq_num 660) dirty exists 2022-04-28T19:51:07.523 INFO:tasks.rados.rados.0.smithi059.stdout:2091: finishing copy_from racing read to smithi05955964-40 2022-04-28T19:51:07.524 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 907 (ObjNum 668 snap 193 seq_num 668) dirty exists 2022-04-28T19:51:07.841 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[57771]: debug 2022-04-28T19:51:07.348+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2084: done (9 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2085: done (8 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2086: done (7 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2087: done (6 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2088: done (5 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2089: done (4 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2090: done (3 left) 2022-04-28T19:51:07.866 INFO:tasks.rados.rados.0.smithi059.stdout:2091: done (2 left) 2022-04-28T19:51:07.867 INFO:tasks.rados.rados.0.smithi059.stdout:2093: done (1 left) 2022-04-28T19:51:07.867 INFO:tasks.rados.rados.0.smithi059.stdout:2094: done (0 left) 2022-04-28T19:51:07.867 INFO:tasks.rados.rados.0.smithi059.stdout:2095: delete oid 15 current snap is 195 2022-04-28T19:51:07.867 INFO:tasks.rados.rados.0.smithi059.stdout:2095: done (0 left) 2022-04-28T19:51:07.867 INFO:tasks.rados.rados.0.smithi059.stdout:2096: snap_remove snap 173 2022-04-28T19:51:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: cluster 2022-04-28T19:51:06.885865+0000 mgr.y (mgr.24961) 117 : cluster [DBG] pgmap v157: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 247 MiB data, 479 MiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 53/408 objects degraded (12.990%) 2022-04-28T19:51:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.331133+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.337858+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.347269+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.350103+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.351621+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.352759+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: audit 2022-04-28T19:51:07.354406+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[59745]: cluster 2022-04-28T19:51:07.845783+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e449: 8 total, 7 up, 8 in 2022-04-28T19:51:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: cluster 2022-04-28T19:51:06.885865+0000 mgr.y (mgr.24961) 117 : cluster [DBG] pgmap v157: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 247 MiB data, 479 MiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 53/408 objects degraded (12.990%) 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.331133+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.337858+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.347269+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.350103+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.351621+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.352759+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: audit 2022-04-28T19:51:07.354406+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:07 smithi059 bash[61152]: cluster 2022-04-28T19:51:07.845783+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e449: 8 total, 7 up, 8 in 2022-04-28T19:51:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: cluster 2022-04-28T19:51:06.885865+0000 mgr.y (mgr.24961) 117 : cluster [DBG] pgmap v157: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 247 MiB data, 479 MiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 22 op/s; 53/408 objects degraded (12.990%) 2022-04-28T19:51:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.331133+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.337858+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.347269+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.350103+0000 mon.a (mon.0) 382 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.351621+0000 mon.a (mon.0) 383 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.352759+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: audit 2022-04-28T19:51:07.354406+0000 mon.a (mon.0) 385 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:07 smithi139 bash[43301]: cluster 2022-04-28T19:51:07.845783+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e449: 8 total, 7 up, 8 in 2022-04-28T19:51:08.849 INFO:tasks.rados.rados.0.smithi059.stdout:2096: done (0 left) 2022-04-28T19:51:08.850 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 195 2022-04-28T19:51:08.850 INFO:tasks.rados.rados.0.smithi059.stdout:2097: seq_num 671 ranges {0=442368,442368=761856,1204224=442368} 2022-04-28T19:51:08.860 INFO:tasks.rados.rados.0.smithi059.stdout:2097: writing smithi05955964-21 from 0 to 442368 tid 1 2022-04-28T19:51:08.869 INFO:tasks.rados.rados.0.smithi059.stdout:2097: writing smithi05955964-21 from 442368 to 1204224 tid 2 2022-04-28T19:51:08.870 INFO:tasks.rados.rados.0.smithi059.stdout:2097: writing smithi05955964-21 from 1204224 to 1646592 tid 3 2022-04-28T19:51:08.870 INFO:tasks.rados.rados.0.smithi059.stdout:2098: rmattr oid 18 current snap is 195 2022-04-28T19:51:08.870 INFO:tasks.rados.rados.0.smithi059.stdout:2099: copy_from oid 46 from oid 10 current snap is 195 2022-04-28T19:51:08.871 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 195 2022-04-28T19:51:08.871 INFO:tasks.rados.rados.0.smithi059.stdout:2100: seq_num 672 ranges {9650176=458752,10108928=647168,10756096=466944,11223040=90112} 2022-04-28T19:51:08.919 INFO:tasks.rados.rados.0.smithi059.stdout:2100: writing smithi05955964-7 from 9650176 to 10108928 tid 1 2022-04-28T19:51:08.990 INFO:tasks.rados.rados.0.smithi059.stdout:2100: writing smithi05955964-7 from 10108928 to 10756096 tid 2 2022-04-28T19:51:08.991 INFO:tasks.rados.rados.0.smithi059.stdout:2100: writing smithi05955964-7 from 10756096 to 11223040 tid 3 2022-04-28T19:51:08.991 INFO:tasks.rados.rados.0.smithi059.stdout:2100: writing smithi05955964-7 from 11223040 to 11313152 tid 4 2022-04-28T19:51:08.991 INFO:tasks.rados.rados.0.smithi059.stdout:2101: snap_remove snap 177 2022-04-28T19:51:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:08 smithi059 bash[61152]: audit 2022-04-28T19:51:07.354588+0000 mgr.y (mgr.24961) 118 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:08 smithi059 bash[61152]: cephadm 2022-04-28T19:51:07.355713+0000 mgr.y (mgr.24961) 119 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:51:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:08 smithi059 bash[61152]: cluster 2022-04-28T19:51:08.847632+0000 mon.a (mon.0) 387 : cluster [DBG] osdmap e450: 8 total, 7 up, 8 in 2022-04-28T19:51:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:08 smithi059 bash[59745]: audit 2022-04-28T19:51:07.354588+0000 mgr.y (mgr.24961) 118 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:08 smithi059 bash[59745]: cephadm 2022-04-28T19:51:07.355713+0000 mgr.y (mgr.24961) 119 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:51:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:08 smithi059 bash[59745]: cluster 2022-04-28T19:51:08.847632+0000 mon.a (mon.0) 387 : cluster [DBG] osdmap e450: 8 total, 7 up, 8 in 2022-04-28T19:51:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:08 smithi139 bash[43301]: audit 2022-04-28T19:51:07.354588+0000 mgr.y (mgr.24961) 118 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:08 smithi139 bash[43301]: cephadm 2022-04-28T19:51:07.355713+0000 mgr.y (mgr.24961) 119 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:51:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:08 smithi139 bash[43301]: cluster 2022-04-28T19:51:08.847632+0000 mon.a (mon.0) 387 : cluster [DBG] osdmap e450: 8 total, 7 up, 8 in 2022-04-28T19:51:09.889 INFO:tasks.rados.rados.0.smithi059.stdout:2101: done (4 left) 2022-04-28T19:51:09.889 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 195 2022-04-28T19:51:09.889 INFO:tasks.rados.rados.0.smithi059.stdout:2102: seq_num 673 ranges {1654784=475136,2129920=532480,2662400=671744} 2022-04-28T19:51:09.902 INFO:tasks.rados.rados.0.smithi059.stdout:2102: writing smithi05955964-48 from 1654784 to 2129920 tid 1 2022-04-28T19:51:09.906 INFO:tasks.rados.rados.0.smithi059.stdout:2102: writing smithi05955964-48 from 2129920 to 2662400 tid 2 2022-04-28T19:51:09.911 INFO:tasks.rados.rados.0.smithi059.stdout:2102: writing smithi05955964-48 from 2662400 to 3334144 tid 3 2022-04-28T19:51:09.911 INFO:tasks.rados.rados.0.smithi059.stdout:2103: rollback oid 9 current snap is 195 2022-04-28T19:51:09.911 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 191 2022-04-28T19:51:09.912 INFO:tasks.rados.rados.0.smithi059.stdout:2097: finishing write tid 1 to smithi05955964-21 2022-04-28T19:51:09.912 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 195 2022-04-28T19:51:09.912 INFO:tasks.rados.rados.0.smithi059.stdout:2104: seq_num 674 ranges {2113536=491520,2605056=417792,3022848=483328,3506176=303104} 2022-04-28T19:51:09.927 INFO:tasks.rados.rados.0.smithi059.stdout:2104: writing smithi05955964-19 from 2113536 to 2605056 tid 1 2022-04-28T19:51:09.930 INFO:tasks.rados.rados.0.smithi059.stdout:2104: writing smithi05955964-19 from 2605056 to 3022848 tid 2 2022-04-28T19:51:09.934 INFO:tasks.rados.rados.0.smithi059.stdout:2104: writing smithi05955964-19 from 3022848 to 3506176 tid 3 2022-04-28T19:51:09.936 INFO:tasks.rados.rados.0.smithi059.stdout:2104: writing smithi05955964-19 from 3506176 to 3809280 tid 4 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2105: snap_create 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 928 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2097: finishing write tid 2 to smithi05955964-21 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2097: finishing write tid 3 to smithi05955964-21 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2097: finishing write tid 5 to smithi05955964-21 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2097: finishing write tid 6 to smithi05955964-21 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 707 (ObjNum 671 snap 195 seq_num 671) dirty exists 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2097: left oid 21 (ObjNum 671 snap 195 seq_num 671) 2022-04-28T19:51:09.937 INFO:tasks.rados.rados.0.smithi059.stdout:2099: finishing copy_from to smithi05955964-46 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 948 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2099: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: finishing write tid 1 to smithi05955964-7 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: finishing write tid 2 to smithi05955964-7 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: finishing write tid 3 to smithi05955964-7 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: finishing write tid 4 to smithi05955964-7 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: finishing write tid 6 to smithi05955964-7 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: finishing write tid 7 to smithi05955964-7 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 874 (ObjNum 672 snap 195 seq_num 672) dirty exists 2022-04-28T19:51:09.938 INFO:tasks.rados.rados.0.smithi059.stdout:2100: left oid 7 (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2102: finishing write tid 1 to smithi05955964-48 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2102: finishing write tid 2 to smithi05955964-48 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2103: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2103: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2103: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 718 (ObjNum 607 snap 170 seq_num 607) dirty exists 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2102: finishing write tid 3 to smithi05955964-48 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2102: finishing write tid 5 to smithi05955964-48 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2102: finishing write tid 6 to smithi05955964-48 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 946 (ObjNum 673 snap 195 seq_num 673) dirty exists 2022-04-28T19:51:09.939 INFO:tasks.rados.rados.0.smithi059.stdout:2102: left oid 48 (ObjNum 673 snap 195 seq_num 673) 2022-04-28T19:51:09.940 INFO:tasks.rados.rados.0.smithi059.stdout:2104: finishing write tid 1 to smithi05955964-19 2022-04-28T19:51:09.940 INFO:tasks.rados.rados.0.smithi059.stdout:2104: finishing write tid 2 to smithi05955964-19 2022-04-28T19:51:09.940 INFO:tasks.rados.rados.0.smithi059.stdout:2104: finishing write tid 3 to smithi05955964-19 2022-04-28T19:51:09.940 INFO:tasks.rados.rados.0.smithi059.stdout:2104: finishing write tid 4 to smithi05955964-19 2022-04-28T19:51:09.940 INFO:tasks.rados.rados.0.smithi059.stdout:2104: finishing write tid 6 to smithi05955964-19 2022-04-28T19:51:09.941 INFO:tasks.rados.rados.0.smithi059.stdout:2104: finishing write tid 7 to smithi05955964-19 2022-04-28T19:51:09.941 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 952 (ObjNum 674 snap 195 seq_num 674) dirty exists 2022-04-28T19:51:09.941 INFO:tasks.rados.rados.0.smithi059.stdout:2104: left oid 19 (ObjNum 674 snap 195 seq_num 674) 2022-04-28T19:51:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:09 smithi059 bash[61152]: cluster 2022-04-28T19:51:08.886477+0000 mgr.y (mgr.24961) 120 : cluster [DBG] pgmap v160: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 255 MiB data, 483 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 6.2 MiB/s wr, 48 op/s; 58/420 objects degraded (13.810%) 2022-04-28T19:51:10.342 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[65202]: debug 2022-04-28T19:51:10.040+0000 7f6ec8d51200 -1 osd.1 440 log_to_monitors {default=true} 2022-04-28T19:51:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:09 smithi059 bash[59745]: cluster 2022-04-28T19:51:08.886477+0000 mgr.y (mgr.24961) 120 : cluster [DBG] pgmap v160: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 255 MiB data, 483 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 6.2 MiB/s wr, 48 op/s; 58/420 objects degraded (13.810%) 2022-04-28T19:51:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:09 smithi139 bash[43301]: cluster 2022-04-28T19:51:08.886477+0000 mgr.y (mgr.24961) 120 : cluster [DBG] pgmap v160: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 255 MiB data, 483 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 6.2 MiB/s wr, 48 op/s; 58/420 objects degraded (13.810%) 2022-04-28T19:51:10.901 INFO:tasks.rados.rados.0.smithi059.stdout:2097: done (7 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2098: done (6 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2099: done (5 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2100: done (4 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2102: done (3 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2103: done (2 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2104: done (1 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2105: done (0 left) 2022-04-28T19:51:10.902 INFO:tasks.rados.rados.0.smithi059.stdout:2106: snap_remove snap 195 2022-04-28T19:51:11.341 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[65202]: debug 2022-04-28T19:51:10.912+0000 7f6ebac6c700 -1 osd.1 440 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[61152]: cluster 2022-04-28T19:51:09.873346+0000 mon.a (mon.0) 388 : cluster [WRN] Health check update: Degraded data redundancy: 58/420 objects degraded (13.810%), 10 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[61152]: cluster 2022-04-28T19:51:09.888256+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e451: 8 total, 7 up, 8 in 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[61152]: audit 2022-04-28T19:51:10.047496+0000 mon.c (mon.1) 7 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[61152]: audit 2022-04-28T19:51:10.047788+0000 mon.a (mon.0) 390 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[59745]: cluster 2022-04-28T19:51:09.873346+0000 mon.a (mon.0) 388 : cluster [WRN] Health check update: Degraded data redundancy: 58/420 objects degraded (13.810%), 10 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[59745]: cluster 2022-04-28T19:51:09.888256+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e451: 8 total, 7 up, 8 in 2022-04-28T19:51:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[59745]: audit 2022-04-28T19:51:10.047496+0000 mon.c (mon.1) 7 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:51:11.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:10 smithi059 bash[59745]: audit 2022-04-28T19:51:10.047788+0000 mon.a (mon.0) 390 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:51:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:10 smithi139 bash[43301]: cluster 2022-04-28T19:51:09.873346+0000 mon.a (mon.0) 388 : cluster [WRN] Health check update: Degraded data redundancy: 58/420 objects degraded (13.810%), 10 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:10 smithi139 bash[43301]: cluster 2022-04-28T19:51:09.888256+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e451: 8 total, 7 up, 8 in 2022-04-28T19:51:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:10 smithi139 bash[43301]: audit 2022-04-28T19:51:10.047496+0000 mon.c (mon.1) 7 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:51:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:10 smithi139 bash[43301]: audit 2022-04-28T19:51:10.047788+0000 mon.a (mon.0) 390 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2022-04-28T19:51:11.899 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:11.629Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:11.899 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:11.629Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:11.899 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:11.629Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:51:11.899 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[59745]: cluster 2022-04-28T19:51:10.887300+0000 mgr.y (mgr.24961) 121 : cluster [DBG] pgmap v162: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 255 MiB data, 483 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 6.1 MiB/s wr, 25 op/s; 58/420 objects degraded (13.810%) 2022-04-28T19:51:11.900 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[59745]: audit 2022-04-28T19:51:10.900940+0000 mon.a (mon.0) 391 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-04-28T19:51:11.900 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[59745]: cluster 2022-04-28T19:51:10.901003+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e452: 8 total, 7 up, 8 in 2022-04-28T19:51:11.900 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[59745]: audit 2022-04-28T19:51:10.902109+0000 mon.c (mon.1) 8 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:11.900 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[59745]: audit 2022-04-28T19:51:10.902703+0000 mon.a (mon.0) 393 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:11.910 INFO:tasks.rados.rados.0.smithi059.stdout:2106: done (0 left) 2022-04-28T19:51:11.911 INFO:tasks.rados.rados.0.smithi059.stdout:2107: read oid 16 snap -1 2022-04-28T19:51:11.911 INFO:tasks.rados.rados.0.smithi059.stdout:2107: expect (ObjNum 668 snap 193 seq_num 668) 2022-04-28T19:51:11.911 INFO:tasks.rados.rados.0.smithi059.stdout:2108: delete oid 35 current snap is 196 2022-04-28T19:51:11.931 INFO:tasks.rados.rados.0.smithi059.stdout:2108: done (1 left) 2022-04-28T19:51:11.931 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 196 2022-04-28T19:51:11.931 INFO:tasks.rados.rados.0.smithi059.stdout:2109: seq_num 676 ranges {3497984=524288,4022272=589824,4612096=450560,5062656=163840} 2022-04-28T19:51:11.954 INFO:tasks.rados.rados.0.smithi059.stdout:2109: writing smithi05955964-49 from 3497984 to 4022272 tid 1 2022-04-28T19:51:11.957 INFO:tasks.rados.rados.0.smithi059.stdout:2109: writing smithi05955964-49 from 4022272 to 4612096 tid 2 2022-04-28T19:51:11.960 INFO:tasks.rados.rados.0.smithi059.stdout:2109: writing smithi05955964-49 from 4612096 to 5062656 tid 3 2022-04-28T19:51:11.961 INFO:tasks.rados.rados.0.smithi059.stdout:2109: writing smithi05955964-49 from 5062656 to 5226496 tid 4 2022-04-28T19:51:11.962 INFO:tasks.rados.rados.0.smithi059.stdout:2110: snap_create 2022-04-28T19:51:11.962 INFO:tasks.rados.rados.0.smithi059.stdout:2109: finishing write tid 1 to smithi05955964-49 2022-04-28T19:51:11.963 INFO:tasks.rados.rados.0.smithi059.stdout:2109: finishing write tid 2 to smithi05955964-49 2022-04-28T19:51:11.965 INFO:tasks.rados.rados.0.smithi059.stdout:2109: finishing write tid 3 to smithi05955964-49 2022-04-28T19:51:11.966 INFO:tasks.rados.rados.0.smithi059.stdout:2109: finishing write tid 4 to smithi05955964-49 2022-04-28T19:51:11.966 INFO:tasks.rados.rados.0.smithi059.stdout:2109: finishing write tid 6 to smithi05955964-49 2022-04-28T19:51:11.968 INFO:tasks.rados.rados.0.smithi059.stdout:2109: finishing write tid 7 to smithi05955964-49 2022-04-28T19:51:11.968 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 932 (ObjNum 676 snap 196 seq_num 676) dirty exists 2022-04-28T19:51:11.968 INFO:tasks.rados.rados.0.smithi059.stdout:2109: left oid 49 (ObjNum 676 snap 196 seq_num 676) 2022-04-28T19:51:11.985 INFO:tasks.rados.rados.0.smithi059.stdout:2107: expect (ObjNum 668 snap 193 seq_num 668) 2022-04-28T19:51:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[61152]: cluster 2022-04-28T19:51:10.887300+0000 mgr.y (mgr.24961) 121 : cluster [DBG] pgmap v162: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 255 MiB data, 483 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 6.1 MiB/s wr, 25 op/s; 58/420 objects degraded (13.810%) 2022-04-28T19:51:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[61152]: audit 2022-04-28T19:51:10.900940+0000 mon.a (mon.0) 391 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-04-28T19:51:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[61152]: cluster 2022-04-28T19:51:10.901003+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e452: 8 total, 7 up, 8 in 2022-04-28T19:51:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[61152]: audit 2022-04-28T19:51:10.902109+0000 mon.c (mon.1) 8 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:11 smithi059 bash[61152]: audit 2022-04-28T19:51:10.902703+0000 mon.a (mon.0) 393 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:11 smithi139 bash[43301]: cluster 2022-04-28T19:51:10.887300+0000 mgr.y (mgr.24961) 121 : cluster [DBG] pgmap v162: 89 pgs: 10 active+undersized+degraded, 32 active+undersized, 47 active+clean; 255 MiB data, 483 MiB used, 708 GiB / 715 GiB avail; 23 MiB/s rd, 6.1 MiB/s wr, 25 op/s; 58/420 objects degraded (13.810%) 2022-04-28T19:51:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:11 smithi139 bash[43301]: audit 2022-04-28T19:51:10.900940+0000 mon.a (mon.0) 391 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2022-04-28T19:51:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:11 smithi139 bash[43301]: cluster 2022-04-28T19:51:10.901003+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e452: 8 total, 7 up, 8 in 2022-04-28T19:51:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:11 smithi139 bash[43301]: audit 2022-04-28T19:51:10.902109+0000 mon.c (mon.1) 8 : audit [INF] from='osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:11 smithi139 bash[43301]: audit 2022-04-28T19:51:10.902703+0000 mon.a (mon.0) 393 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:12.973 INFO:tasks.rados.rados.0.smithi059.stdout:2107: done (2 left) 2022-04-28T19:51:12.974 INFO:tasks.rados.rados.0.smithi059.stdout:2109: done (1 left) 2022-04-28T19:51:12.974 INFO:tasks.rados.rados.0.smithi059.stdout:2110: done (0 left) 2022-04-28T19:51:12.974 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 197 2022-04-28T19:51:12.974 INFO:tasks.rados.rados.0.smithi059.stdout:2111: seq_num 677 ranges {0=540672,540672=516096,1056768=655360,1712128=32768} 2022-04-28T19:51:12.977 INFO:tasks.rados.rados.0.smithi059.stdout:2111: writing smithi05955964-32 from 0 to 540672 tid 1 2022-04-28T19:51:12.980 INFO:tasks.rados.rados.0.smithi059.stdout:2111: writing smithi05955964-32 from 540672 to 1056768 tid 2 2022-04-28T19:51:12.984 INFO:tasks.rados.rados.0.smithi059.stdout:2111: writing smithi05955964-32 from 1056768 to 1712128 tid 3 2022-04-28T19:51:12.985 INFO:tasks.rados.rados.0.smithi059.stdout:2111: writing smithi05955964-32 from 1712128 to 1744896 tid 4 2022-04-28T19:51:12.985 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 197 2022-04-28T19:51:12.985 INFO:tasks.rados.rados.0.smithi059.stdout:2112: seq_num 678 ranges {0=565248,565248=802816,1368064=401408} 2022-04-28T19:51:12.988 INFO:tasks.rados.rados.0.smithi059.stdout:2112: writing smithi05955964-12 from 0 to 565248 tid 1 2022-04-28T19:51:12.992 INFO:tasks.rados.rados.0.smithi059.stdout:2112: writing smithi05955964-12 from 565248 to 1368064 tid 2 2022-04-28T19:51:12.995 INFO:tasks.rados.rados.0.smithi059.stdout:2112: writing smithi05955964-12 from 1368064 to 1769472 tid 3 2022-04-28T19:51:12.995 INFO:tasks.rados.rados.0.smithi059.stdout:2113: rmattr oid 2 current snap is 197 2022-04-28T19:51:12.995 INFO:tasks.rados.rados.0.smithi059.stdout:2113: done (2 left) 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2114: rollback oid 46 current snap is 197 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 196 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2111: finishing write tid 1 to smithi05955964-32 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2111: finishing write tid 2 to smithi05955964-32 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2111: finishing write tid 3 to smithi05955964-32 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2111: finishing write tid 4 to smithi05955964-32 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2111: finishing write tid 6 to smithi05955964-32 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2112: finishing write tid 1 to smithi05955964-12 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:2111: finishing write tid 7 to smithi05955964-32 2022-04-28T19:51:12.996 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1001 (ObjNum 677 snap 197 seq_num 677) dirty exists 2022-04-28T19:51:12.997 INFO:tasks.rados.rados.0.smithi059.stdout:2111: left oid 32 (ObjNum 677 snap 197 seq_num 677) 2022-04-28T19:51:12.997 INFO:tasks.rados.rados.0.smithi059.stdout:2111: done (2 left) 2022-04-28T19:51:12.997 INFO:tasks.rados.rados.0.smithi059.stdout:2115: delete oid 14 current snap is 197 2022-04-28T19:51:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:2112: finishing write tid 2 to smithi05955964-12 2022-04-28T19:51:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:2112: finishing write tid 3 to smithi05955964-12 2022-04-28T19:51:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:2112: finishing write tid 5 to smithi05955964-12 2022-04-28T19:51:13.002 INFO:tasks.rados.rados.0.smithi059.stdout:2112: finishing write tid 6 to smithi05955964-12 2022-04-28T19:51:13.002 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 882 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:51:13.002 INFO:tasks.rados.rados.0.smithi059.stdout:2112: left oid 12 (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:51:13.012 INFO:tasks.rados.rados.0.smithi059.stdout:2112: done (2 left) 2022-04-28T19:51:13.012 INFO:tasks.rados.rados.0.smithi059.stdout:2115: done (1 left) 2022-04-28T19:51:13.012 INFO:tasks.rados.rados.0.smithi059.stdout:2116: setattr oid 47 current snap is 197 2022-04-28T19:51:13.014 INFO:tasks.rados.rados.0.smithi059.stdout:2117: rmattr oid 48 current snap is 197 2022-04-28T19:51:13.014 INFO:tasks.rados.rados.0.smithi059.stdout:2117: done (2 left) 2022-04-28T19:51:13.015 INFO:tasks.rados.rados.0.smithi059.stdout:2118: read oid 13 snap -1 2022-04-28T19:51:13.015 INFO:tasks.rados.rados.0.smithi059.stdout:2118: expect (ObjNum 622 snap 178 seq_num 622) 2022-04-28T19:51:13.015 INFO:tasks.rados.rados.0.smithi059.stdout:2119: copy_from oid 20 from oid 36 current snap is 197 2022-04-28T19:51:13.015 INFO:tasks.rados.rados.0.smithi059.stdout:2120: setattr oid 8 current snap is 197 2022-04-28T19:51:13.017 INFO:tasks.rados.rados.0.smithi059.stdout:2121: rollback oid 16 current snap is 197 2022-04-28T19:51:13.017 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 194 2022-04-28T19:51:13.017 INFO:tasks.rados.rados.0.smithi059.stdout:2122: rollback oid 37 current snap is 197 2022-04-28T19:51:13.017 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 37 to 196 2022-04-28T19:51:13.018 INFO:tasks.rados.rados.0.smithi059.stdout:2123: copy_from oid 29 from oid 15 current snap is 197 2022-04-28T19:51:13.018 INFO:tasks.rados.rados.0.smithi059.stdout:2124: read oid 33 snap 190 2022-04-28T19:51:13.018 INFO:tasks.rados.rados.0.smithi059.stdout:2124: expect (ObjNum 620 snap 177 seq_num 620) 2022-04-28T19:51:13.018 INFO:tasks.rados.rados.0.smithi059.stdout:2125: delete oid 39 current snap is 197 2022-04-28T19:51:13.019 INFO:tasks.rados.rados.0.smithi059.stdout:2122: finishing rollback tid 1 to smithi05955964-37 2022-04-28T19:51:13.020 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 0 (ObjNum 0 snap 0 seq_num 188341440) dirty dne 2022-04-28T19:51:13.023 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1023 (ObjNum 581 snap 166 seq_num 581) dirty exists 2022-04-28T19:51:13.024 INFO:tasks.rados.rados.0.smithi059.stdout:2121: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:51:13.024 INFO:tasks.rados.rados.0.smithi059.stdout:2118: expect (ObjNum 622 snap 178 seq_num 622) 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:2119: finishing copy_from to smithi05955964-20 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 853 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:2121: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:2121: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 910 (ObjNum 668 snap 193 seq_num 668) dirty exists 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:2119: finishing copy_from racing read to smithi05955964-20 2022-04-28T19:51:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:2123: finishing copy_from to smithi05955964-29 2022-04-28T19:51:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:2123: got expected ENOENT (src dne) 2022-04-28T19:51:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 949 (ObjNum 611 snap 171 seq_num 611) dirty exists 2022-04-28T19:51:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:2123: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:51:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:2114: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:51:13.051 INFO:tasks.rados.rados.0.smithi059.stdout:2114: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:51:13.051 INFO:tasks.rados.rados.0.smithi059.stdout:2114: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:51:13.051 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 952 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:51:13.057 INFO:tasks.rados.rados.0.smithi059.stdout:2124: expect (ObjNum 620 snap 177 seq_num 620) 2022-04-28T19:51:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[61152]: cluster 2022-04-28T19:51:11.895192+0000 mon.a (mon.0) 394 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:51:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[61152]: cluster 2022-04-28T19:51:11.909778+0000 mon.a (mon.0) 395 : cluster [INF] osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580] boot 2022-04-28T19:51:13.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[61152]: cluster 2022-04-28T19:51:11.909927+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2022-04-28T19:51:13.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[61152]: audit 2022-04-28T19:51:11.910733+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:51:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[59745]: cluster 2022-04-28T19:51:11.895192+0000 mon.a (mon.0) 394 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:51:13.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[59745]: cluster 2022-04-28T19:51:11.909778+0000 mon.a (mon.0) 395 : cluster [INF] osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580] boot 2022-04-28T19:51:13.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[59745]: cluster 2022-04-28T19:51:11.909927+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2022-04-28T19:51:13.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:12 smithi059 bash[59745]: audit 2022-04-28T19:51:11.910733+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:51:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:12 smithi139 bash[43301]: cluster 2022-04-28T19:51:11.895192+0000 mon.a (mon.0) 394 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:51:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:12 smithi139 bash[43301]: cluster 2022-04-28T19:51:11.909778+0000 mon.a (mon.0) 395 : cluster [INF] osd.1 [v2:172.21.15.59:6810/518678580,v1:172.21.15.59:6811/518678580] boot 2022-04-28T19:51:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:12 smithi139 bash[43301]: cluster 2022-04-28T19:51:11.909927+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2022-04-28T19:51:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:12 smithi139 bash[43301]: audit 2022-04-28T19:51:11.910733+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2022-04-28T19:51:13.540 INFO:tasks.rados.rados.0.smithi059.stdout:2114: done (9 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2116: done (8 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2118: done (7 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2119: done (6 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2120: done (5 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2121: done (4 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2122: done (3 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2123: done (2 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2124: done (1 left) 2022-04-28T19:51:13.541 INFO:tasks.rados.rados.0.smithi059.stdout:2125: done (0 left) 2022-04-28T19:51:13.542 INFO:tasks.rados.rados.0.smithi059.stdout:2126: delete oid 47 current snap is 197 2022-04-28T19:51:13.544 INFO:tasks.rados.rados.0.smithi059.stdout:2126: done (0 left) 2022-04-28T19:51:13.544 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 197 2022-04-28T19:51:13.544 INFO:tasks.rados.rados.0.smithi059.stdout:2127: seq_num 682 ranges {638976=630784,1269760=745472,2015232=442368,2457600=16384} 2022-04-28T19:51:13.554 INFO:tasks.rados.rados.0.smithi059.stdout:2127: writing smithi05955964-8 from 638976 to 1269760 tid 1 2022-04-28T19:51:13.559 INFO:tasks.rados.rados.0.smithi059.stdout:2127: writing smithi05955964-8 from 1269760 to 2015232 tid 2 2022-04-28T19:51:13.562 INFO:tasks.rados.rados.0.smithi059.stdout:2127: writing smithi05955964-8 from 2015232 to 2457600 tid 3 2022-04-28T19:51:13.562 INFO:tasks.rados.rados.0.smithi059.stdout:2127: writing smithi05955964-8 from 2457600 to 2473984 tid 4 2022-04-28T19:51:13.562 INFO:tasks.rados.rados.0.smithi059.stdout:2128: setattr oid 49 current snap is 197 2022-04-28T19:51:13.563 INFO:tasks.rados.rados.0.smithi059.stdout:2129: snap_remove snap 194 2022-04-28T19:51:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:13.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:13.855 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:13.522Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:13.855 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:13.522Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:51:13.930 INFO:tasks.rados.rados.0.smithi059.stdout:2129: done (2 left) 2022-04-28T19:51:13.930 INFO:tasks.rados.rados.0.smithi059.stdout:2130: delete oid 41 current snap is 197 2022-04-28T19:51:13.930 INFO:tasks.rados.rados.0.smithi059.stdout:2127: finishing write tid 1 to smithi05955964-8 2022-04-28T19:51:13.930 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 933 (ObjNum 676 snap 196 seq_num 676) dirty exists 2022-04-28T19:51:13.930 INFO:tasks.rados.rados.0.smithi059.stdout:2127: finishing write tid 2 to smithi05955964-8 2022-04-28T19:51:13.931 INFO:tasks.rados.rados.0.smithi059.stdout:2127: finishing write tid 3 to smithi05955964-8 2022-04-28T19:51:13.931 INFO:tasks.rados.rados.0.smithi059.stdout:2127: finishing write tid 4 to smithi05955964-8 2022-04-28T19:51:13.931 INFO:tasks.rados.rados.0.smithi059.stdout:2127: finishing write tid 6 to smithi05955964-8 2022-04-28T19:51:13.931 INFO:tasks.rados.rados.0.smithi059.stdout:2127: finishing write tid 7 to smithi05955964-8 2022-04-28T19:51:13.931 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 957 (ObjNum 682 snap 197 seq_num 682) dirty exists 2022-04-28T19:51:13.931 INFO:tasks.rados.rados.0.smithi059.stdout:2127: left oid 8 (ObjNum 682 snap 197 seq_num 682) 2022-04-28T19:51:13.943 INFO:tasks.rados.rados.0.smithi059.stdout:2127: done (2 left) 2022-04-28T19:51:13.943 INFO:tasks.rados.rados.0.smithi059.stdout:2128: done (1 left) 2022-04-28T19:51:13.943 INFO:tasks.rados.rados.0.smithi059.stdout:2130: done (0 left) 2022-04-28T19:51:13.943 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 197 2022-04-28T19:51:13.943 INFO:tasks.rados.rados.0.smithi059.stdout:2131: seq_num 684 ranges {1646592=663552,2310144=507904,2818048=409600,3227648=286720} 2022-04-28T19:51:13.957 INFO:tasks.rados.rados.0.smithi059.stdout:2131: writing smithi05955964-21 from 1646592 to 2310144 tid 1 2022-04-28T19:51:13.959 INFO:tasks.rados.rados.0.smithi059.stdout:2131: writing smithi05955964-21 from 2310144 to 2818048 tid 2 2022-04-28T19:51:13.962 INFO:tasks.rados.rados.0.smithi059.stdout:2131: writing smithi05955964-21 from 2818048 to 3227648 tid 3 2022-04-28T19:51:13.964 INFO:tasks.rados.rados.0.smithi059.stdout:2131: writing smithi05955964-21 from 3227648 to 3514368 tid 4 2022-04-28T19:51:13.964 INFO:tasks.rados.rados.0.smithi059.stdout:2132: rollback oid 24 current snap is 197 2022-04-28T19:51:13.964 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 193 2022-04-28T19:51:13.964 INFO:tasks.rados.rados.0.smithi059.stdout:2133: copy_from oid 44 from oid 15 current snap is 197 2022-04-28T19:51:13.964 INFO:tasks.rados.rados.0.smithi059.stdout:2134: rmattr oid 11 current snap is 197 2022-04-28T19:51:13.965 INFO:tasks.rados.rados.0.smithi059.stdout:2135: rmattr oid 16 current snap is 197 2022-04-28T19:51:13.965 INFO:tasks.rados.rados.0.smithi059.stdout:2135: done (4 left) 2022-04-28T19:51:13.965 INFO:tasks.rados.rados.0.smithi059.stdout:2136: delete oid 47 current snap is 197 2022-04-28T19:51:13.968 INFO:tasks.rados.rados.0.smithi059.stdout:2133: finishing copy_from to smithi05955964-44 2022-04-28T19:51:13.968 INFO:tasks.rados.rados.0.smithi059.stdout:2133: got expected ENOENT (src dne) 2022-04-28T19:51:13.970 INFO:tasks.rados.rados.0.smithi059.stdout:2136: done (4 left) 2022-04-28T19:51:13.971 INFO:tasks.rados.rados.0.smithi059.stdout:2137: read oid 32 snap -1 2022-04-28T19:51:13.971 INFO:tasks.rados.rados.0.smithi059.stdout:2137: expect (ObjNum 677 snap 197 seq_num 677) 2022-04-28T19:51:13.971 INFO:tasks.rados.rados.0.smithi059.stdout:2138: read oid 18 snap -1 2022-04-28T19:51:13.971 INFO:tasks.rados.rados.0.smithi059.stdout:2138: expect (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:51:13.971 INFO:tasks.rados.rados.0.smithi059.stdout:2139: snap_create 2022-04-28T19:51:13.973 INFO:tasks.rados.rados.0.smithi059.stdout:2133: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:51:13.973 INFO:tasks.rados.rados.0.smithi059.stdout:2131: finishing write tid 1 to smithi05955964-21 2022-04-28T19:51:13.973 INFO:tasks.rados.rados.0.smithi059.stdout:2131: finishing write tid 2 to smithi05955964-21 2022-04-28T19:51:13.973 INFO:tasks.rados.rados.0.smithi059.stdout:2131: finishing write tid 3 to smithi05955964-21 2022-04-28T19:51:13.973 INFO:tasks.rados.rados.0.smithi059.stdout:2131: finishing write tid 4 to smithi05955964-21 2022-04-28T19:51:13.974 INFO:tasks.rados.rados.0.smithi059.stdout:2131: finishing write tid 6 to smithi05955964-21 2022-04-28T19:51:13.974 INFO:tasks.rados.rados.0.smithi059.stdout:2132: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:51:13.974 INFO:tasks.rados.rados.0.smithi059.stdout:2132: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:51:13.975 INFO:tasks.rados.rados.0.smithi059.stdout:2132: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:51:13.975 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 815 (ObjNum 650 snap 184 seq_num 650) dirty exists 2022-04-28T19:51:13.976 INFO:tasks.rados.rados.0.smithi059.stdout:2131: finishing write tid 7 to smithi05955964-21 2022-04-28T19:51:13.976 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 712 (ObjNum 684 snap 197 seq_num 684) dirty exists 2022-04-28T19:51:13.976 INFO:tasks.rados.rados.0.smithi059.stdout:2131: left oid 21 (ObjNum 684 snap 197 seq_num 684) 2022-04-28T19:51:13.976 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 817 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:51:13.986 INFO:tasks.rados.rados.0.smithi059.stdout:2137: expect (ObjNum 677 snap 197 seq_num 677) 2022-04-28T19:51:14.041 INFO:tasks.rados.rados.0.smithi059.stdout:2138: expect (ObjNum 581 snap 166 seq_num 581) 2022-04-28T19:51:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[59745]: cluster 2022-04-28T19:51:12.888420+0000 mgr.y (mgr.24961) 122 : cluster [DBG] pgmap v165: 89 pgs: 27 peering, 3 active+undersized+degraded, 12 active+undersized, 47 active+clean; 236 MiB data, 458 MiB used, 709 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.9 MiB/s wr, 9 op/s; 7/372 objects degraded (1.882%) 2022-04-28T19:51:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[59745]: cluster 2022-04-28T19:51:12.914592+0000 mon.a (mon.0) 398 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2022-04-28T19:51:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[61152]: cluster 2022-04-28T19:51:12.888420+0000 mgr.y (mgr.24961) 122 : cluster [DBG] pgmap v165: 89 pgs: 27 peering, 3 active+undersized+degraded, 12 active+undersized, 47 active+clean; 236 MiB data, 458 MiB used, 709 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.9 MiB/s wr, 9 op/s; 7/372 objects degraded (1.882%) 2022-04-28T19:51:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:13 smithi059 bash[61152]: cluster 2022-04-28T19:51:12.914592+0000 mon.a (mon.0) 398 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2022-04-28T19:51:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:13 smithi139 bash[43301]: cluster 2022-04-28T19:51:12.888420+0000 mgr.y (mgr.24961) 122 : cluster [DBG] pgmap v165: 89 pgs: 27 peering, 3 active+undersized+degraded, 12 active+undersized, 47 active+clean; 236 MiB data, 458 MiB used, 709 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.9 MiB/s wr, 9 op/s; 7/372 objects degraded (1.882%) 2022-04-28T19:51:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:13 smithi139 bash[43301]: cluster 2022-04-28T19:51:12.914592+0000 mon.a (mon.0) 398 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2022-04-28T19:51:14.947 INFO:tasks.rados.rados.0.smithi059.stdout:2131: done (6 left) 2022-04-28T19:51:14.947 INFO:tasks.rados.rados.0.smithi059.stdout:2132: done (5 left) 2022-04-28T19:51:14.947 INFO:tasks.rados.rados.0.smithi059.stdout:2133: done (4 left) 2022-04-28T19:51:14.947 INFO:tasks.rados.rados.0.smithi059.stdout:2134: done (3 left) 2022-04-28T19:51:14.948 INFO:tasks.rados.rados.0.smithi059.stdout:2137: done (2 left) 2022-04-28T19:51:14.948 INFO:tasks.rados.rados.0.smithi059.stdout:2138: done (1 left) 2022-04-28T19:51:14.948 INFO:tasks.rados.rados.0.smithi059.stdout:2139: done (0 left) 2022-04-28T19:51:14.948 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 198 2022-04-28T19:51:14.948 INFO:tasks.rados.rados.0.smithi059.stdout:2140: seq_num 686 ranges {16384=696320,712704=729088,1441792=425984,1867776=49152} 2022-04-28T19:51:14.955 INFO:tasks.rados.rados.0.smithi059.stdout:2140: writing smithi05955964-5 from 16384 to 712704 tid 1 2022-04-28T19:51:14.969 INFO:tasks.rados.rados.0.smithi059.stdout:2140: writing smithi05955964-5 from 712704 to 1441792 tid 2 2022-04-28T19:51:14.969 INFO:tasks.rados.rados.0.smithi059.stdout:2140: writing smithi05955964-5 from 1441792 to 1867776 tid 3 2022-04-28T19:51:14.970 INFO:tasks.rados.rados.0.smithi059.stdout:2140: writing smithi05955964-5 from 1867776 to 1916928 tid 4 2022-04-28T19:51:14.970 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 198 2022-04-28T19:51:14.970 INFO:tasks.rados.rados.0.smithi059.stdout:2141: seq_num 687 ranges {0=712704,712704=614400,1327104=589824} 2022-04-28T19:51:14.970 INFO:tasks.rados.rados.0.smithi059.stdout:2141: writing smithi05955964-31 from 0 to 712704 tid 1 2022-04-28T19:51:14.972 INFO:tasks.rados.rados.0.smithi059.stdout:2141: writing smithi05955964-31 from 712704 to 1327104 tid 2 2022-04-28T19:51:14.976 INFO:tasks.rados.rados.0.smithi059.stdout:2141: writing smithi05955964-31 from 1327104 to 1916928 tid 3 2022-04-28T19:51:14.976 INFO:tasks.rados.rados.0.smithi059.stdout:2142: rmattr oid 17 current snap is 198 2022-04-28T19:51:14.976 INFO:tasks.rados.rados.0.smithi059.stdout:2140: finishing write tid 1 to smithi05955964-5 2022-04-28T19:51:14.976 INFO:tasks.rados.rados.0.smithi059.stdout:2140: finishing write tid 2 to smithi05955964-5 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2140: finishing write tid 3 to smithi05955964-5 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2140: finishing write tid 4 to smithi05955964-5 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2140: finishing write tid 6 to smithi05955964-5 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2140: finishing write tid 7 to smithi05955964-5 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 962 (ObjNum 686 snap 198 seq_num 686) dirty exists 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2140: left oid 5 (ObjNum 686 snap 198 seq_num 686) 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2141: finishing write tid 1 to smithi05955964-31 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2140: done (2 left) 2022-04-28T19:51:14.977 INFO:tasks.rados.rados.0.smithi059.stdout:2143: snap_remove snap 189 2022-04-28T19:51:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:14 smithi059 bash[59745]: cluster 2022-04-28T19:51:13.925136+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2022-04-28T19:51:15.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:14 smithi059 bash[61152]: cluster 2022-04-28T19:51:13.925136+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2022-04-28T19:51:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:14 smithi139 bash[43301]: cluster 2022-04-28T19:51:13.925136+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2022-04-28T19:51:15.959 INFO:tasks.rados.rados.0.smithi059.stdout:2143: done (2 left) 2022-04-28T19:51:15.959 INFO:tasks.rados.rados.0.smithi059.stdout:2144: snap_create 2022-04-28T19:51:15.959 INFO:tasks.rados.rados.0.smithi059.stdout:2141: finishing write tid 2 to smithi05955964-31 2022-04-28T19:51:15.960 INFO:tasks.rados.rados.0.smithi059.stdout:2141: finishing write tid 3 to smithi05955964-31 2022-04-28T19:51:15.960 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 869 (ObjNum 640 snap 183 seq_num 640) dirty exists 2022-04-28T19:51:15.960 INFO:tasks.rados.rados.0.smithi059.stdout:2141: finishing write tid 5 to smithi05955964-31 2022-04-28T19:51:15.960 INFO:tasks.rados.rados.0.smithi059.stdout:2141: finishing write tid 6 to smithi05955964-31 2022-04-28T19:51:15.960 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 890 (ObjNum 687 snap 198 seq_num 687) dirty exists 2022-04-28T19:51:15.961 INFO:tasks.rados.rados.0.smithi059.stdout:2141: left oid 31 (ObjNum 687 snap 198 seq_num 687) 2022-04-28T19:51:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:15 smithi059 bash[61152]: cluster 2022-04-28T19:51:14.889144+0000 mgr.y (mgr.24961) 123 : cluster [DBG] pgmap v168: 89 pgs: 27 peering, 3 active+undersized+degraded, 12 active+undersized, 47 active+clean; 236 MiB data, 458 MiB used, 709 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.9 MiB/s wr, 9 op/s; 7/372 objects degraded (1.882%) 2022-04-28T19:51:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:15 smithi059 bash[61152]: cluster 2022-04-28T19:51:14.945221+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2022-04-28T19:51:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:15 smithi059 bash[59745]: cluster 2022-04-28T19:51:14.889144+0000 mgr.y (mgr.24961) 123 : cluster [DBG] pgmap v168: 89 pgs: 27 peering, 3 active+undersized+degraded, 12 active+undersized, 47 active+clean; 236 MiB data, 458 MiB used, 709 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.9 MiB/s wr, 9 op/s; 7/372 objects degraded (1.882%) 2022-04-28T19:51:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:15 smithi059 bash[59745]: cluster 2022-04-28T19:51:14.945221+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2022-04-28T19:51:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:15 smithi139 bash[43301]: cluster 2022-04-28T19:51:14.889144+0000 mgr.y (mgr.24961) 123 : cluster [DBG] pgmap v168: 89 pgs: 27 peering, 3 active+undersized+degraded, 12 active+undersized, 47 active+clean; 236 MiB data, 458 MiB used, 709 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.9 MiB/s wr, 9 op/s; 7/372 objects degraded (1.882%) 2022-04-28T19:51:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:15 smithi139 bash[43301]: cluster 2022-04-28T19:51:14.945221+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2022-04-28T19:51:16.960 INFO:tasks.rados.rados.0.smithi059.stdout:2141: done (2 left) 2022-04-28T19:51:16.960 INFO:tasks.rados.rados.0.smithi059.stdout:2142: done (1 left) 2022-04-28T19:51:16.960 INFO:tasks.rados.rados.0.smithi059.stdout:2144: done (0 left) 2022-04-28T19:51:16.960 INFO:tasks.rados.rados.0.smithi059.stdout:2145: rollback oid 30 current snap is 199 2022-04-28T19:51:16.961 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 30 to 196 2022-04-28T19:51:16.961 INFO:tasks.rados.rados.0.smithi059.stdout:2146: rollback oid 21 current snap is 199 2022-04-28T19:51:16.961 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 196 2022-04-28T19:51:16.961 INFO:tasks.rados.rados.0.smithi059.stdout:2147: delete oid 10 current snap is 199 2022-04-28T19:51:16.967 INFO:tasks.rados.rados.0.smithi059.stdout:2146: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:51:16.967 INFO:tasks.rados.rados.0.smithi059.stdout:2146: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:51:16.968 INFO:tasks.rados.rados.0.smithi059.stdout:2146: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:51:16.968 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 715 (ObjNum 671 snap 195 seq_num 671) dirty exists 2022-04-28T19:51:16.969 INFO:tasks.rados.rados.0.smithi059.stdout:2146: done (2 left) 2022-04-28T19:51:16.969 INFO:tasks.rados.rados.0.smithi059.stdout:2147: done (1 left) 2022-04-28T19:51:16.969 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 199 2022-04-28T19:51:16.969 INFO:tasks.rados.rados.0.smithi059.stdout:2148: seq_num 689 ranges {9879552=745472,10625024=786432,11411456=417792} 2022-04-28T19:51:17.029 INFO:tasks.rados.rados.0.smithi059.stdout:2148: writing smithi05955964-6 from 9879552 to 10625024 tid 1 2022-04-28T19:51:17.034 INFO:tasks.rados.rados.0.smithi059.stdout:2148: writing smithi05955964-6 from 10625024 to 11411456 tid 2 2022-04-28T19:51:17.037 INFO:tasks.rados.rados.0.smithi059.stdout:2148: writing smithi05955964-6 from 11411456 to 11829248 tid 3 2022-04-28T19:51:17.037 INFO:tasks.rados.rados.0.smithi059.stdout:2149: snap_create 2022-04-28T19:51:17.037 INFO:tasks.rados.rados.0.smithi059.stdout:2145: finishing rollback tid 0 to smithi05955964-30 2022-04-28T19:51:17.037 INFO:tasks.rados.rados.0.smithi059.stdout:2145: finishing rollback tid 1 to smithi05955964-30 2022-04-28T19:51:17.037 INFO:tasks.rados.rados.0.smithi059.stdout:2145: finishing rollback tid 2 to smithi05955964-30 2022-04-28T19:51:17.037 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 931 (ObjNum 669 snap 194 seq_num 669) dirty exists 2022-04-28T19:51:17.038 INFO:tasks.rados.rados.0.smithi059.stdout:2148: finishing write tid 1 to smithi05955964-6 2022-04-28T19:51:17.042 INFO:tasks.rados.rados.0.smithi059.stdout:2148: finishing write tid 2 to smithi05955964-6 2022-04-28T19:51:17.042 INFO:tasks.rados.rados.0.smithi059.stdout:2148: finishing write tid 3 to smithi05955964-6 2022-04-28T19:51:17.042 INFO:tasks.rados.rados.0.smithi059.stdout:2148: finishing write tid 5 to smithi05955964-6 2022-04-28T19:51:17.043 INFO:tasks.rados.rados.0.smithi059.stdout:2148: finishing write tid 6 to smithi05955964-6 2022-04-28T19:51:17.043 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 876 (ObjNum 689 snap 199 seq_num 689) dirty exists 2022-04-28T19:51:17.044 INFO:tasks.rados.rados.0.smithi059.stdout:2148: left oid 6 (ObjNum 689 snap 199 seq_num 689) 2022-04-28T19:51:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:16 smithi059 bash[61152]: cluster 2022-04-28T19:51:15.953365+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2022-04-28T19:51:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:16 smithi059 bash[61152]: cluster 2022-04-28T19:51:16.831422+0000 mon.a (mon.0) 402 : cluster [WRN] Health check update: Degraded data redundancy: 7/372 objects degraded (1.882%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:16 smithi059 bash[59745]: cluster 2022-04-28T19:51:15.953365+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2022-04-28T19:51:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:16 smithi059 bash[59745]: cluster 2022-04-28T19:51:16.831422+0000 mon.a (mon.0) 402 : cluster [WRN] Health check update: Degraded data redundancy: 7/372 objects degraded (1.882%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:16 smithi139 bash[43301]: cluster 2022-04-28T19:51:15.953365+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2022-04-28T19:51:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:16 smithi139 bash[43301]: cluster 2022-04-28T19:51:16.831422+0000 mon.a (mon.0) 402 : cluster [WRN] Health check update: Degraded data redundancy: 7/372 objects degraded (1.882%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:17.982 INFO:tasks.rados.rados.0.smithi059.stdout:2145: done (2 left) 2022-04-28T19:51:17.982 INFO:tasks.rados.rados.0.smithi059.stdout:2148: done (1 left) 2022-04-28T19:51:17.983 INFO:tasks.rados.rados.0.smithi059.stdout:2149: done (0 left) 2022-04-28T19:51:17.983 INFO:tasks.rados.rados.0.smithi059.stdout:2150: rmattr oid 36 current snap is 200 2022-04-28T19:51:17.983 INFO:tasks.rados.rados.0.smithi059.stdout:2150: done (0 left) 2022-04-28T19:51:17.983 INFO:tasks.rados.rados.0.smithi059.stdout:2151: delete oid 24 current snap is 200 2022-04-28T19:51:17.986 INFO:tasks.rados.rados.0.smithi059.stdout:2151: done (0 left) 2022-04-28T19:51:17.986 INFO:tasks.rados.rados.0.smithi059.stdout:2152: read oid 46 snap -1 2022-04-28T19:51:17.986 INFO:tasks.rados.rados.0.smithi059.stdout:2152: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:51:17.987 INFO:tasks.rados.rados.0.smithi059.stdout:2153: copy_from oid 18 from oid 20 current snap is 200 2022-04-28T19:51:17.987 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 200 2022-04-28T19:51:17.987 INFO:tasks.rados.rados.0.smithi059.stdout:2154: seq_num 691 ranges {1916928=778240,2695168=548864,3244032=614400,3858432=40960} 2022-04-28T19:51:18.003 INFO:tasks.rados.rados.0.smithi059.stdout:2154: writing smithi05955964-31 from 1916928 to 2695168 tid 1 2022-04-28T19:51:18.007 INFO:tasks.rados.rados.0.smithi059.stdout:2154: writing smithi05955964-31 from 2695168 to 3244032 tid 2 2022-04-28T19:51:18.012 INFO:tasks.rados.rados.0.smithi059.stdout:2154: writing smithi05955964-31 from 3244032 to 3858432 tid 3 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2154: writing smithi05955964-31 from 3858432 to 3899392 tid 4 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2155: read oid 11 snap 199 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2155: expect (ObjNum 637 snap 183 seq_num 637) 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2156: snap_create 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2153: finishing copy_from to smithi05955964-18 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 853 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2153: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:51:18.013 INFO:tasks.rados.rados.0.smithi059.stdout:2154: finishing write tid 1 to smithi05955964-31 2022-04-28T19:51:18.014 INFO:tasks.rados.rados.0.smithi059.stdout:2154: finishing write tid 2 to smithi05955964-31 2022-04-28T19:51:18.017 INFO:tasks.rados.rados.0.smithi059.stdout:2154: finishing write tid 3 to smithi05955964-31 2022-04-28T19:51:18.018 INFO:tasks.rados.rados.0.smithi059.stdout:2154: finishing write tid 4 to smithi05955964-31 2022-04-28T19:51:18.018 INFO:tasks.rados.rados.0.smithi059.stdout:2152: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:51:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[59745]: cluster 2022-04-28T19:51:16.889854+0000 mgr.y (mgr.24961) 124 : cluster [DBG] pgmap v171: 89 pgs: 27 peering, 62 active+clean; 239 MiB data, 490 MiB used, 709 GiB / 715 GiB avail; 255 B/s rd, 858 KiB/s wr, 9 op/s; 402 KiB/s, 0 objects/s recovering 2022-04-28T19:51:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[59745]: cluster 2022-04-28T19:51:16.957096+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2022-04-28T19:51:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[59745]: audit 2022-04-28T19:51:17.115840+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:51:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[59745]: audit 2022-04-28T19:51:17.130929+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:51:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[59745]: cluster 2022-04-28T19:51:17.835096+0000 mon.a (mon.0) 406 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/372 objects degraded (1.882%), 3 pgs degraded) 2022-04-28T19:51:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[59745]: cluster 2022-04-28T19:51:17.835169+0000 mon.a (mon.0) 407 : cluster [INF] Cluster is now healthy 2022-04-28T19:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[61152]: cluster 2022-04-28T19:51:16.889854+0000 mgr.y (mgr.24961) 124 : cluster [DBG] pgmap v171: 89 pgs: 27 peering, 62 active+clean; 239 MiB data, 490 MiB used, 709 GiB / 715 GiB avail; 255 B/s rd, 858 KiB/s wr, 9 op/s; 402 KiB/s, 0 objects/s recovering 2022-04-28T19:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[61152]: cluster 2022-04-28T19:51:16.957096+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2022-04-28T19:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[61152]: audit 2022-04-28T19:51:17.115840+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[61152]: audit 2022-04-28T19:51:17.130929+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[61152]: cluster 2022-04-28T19:51:17.835096+0000 mon.a (mon.0) 406 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/372 objects degraded (1.882%), 3 pgs degraded) 2022-04-28T19:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:17 smithi059 bash[61152]: cluster 2022-04-28T19:51:17.835169+0000 mon.a (mon.0) 407 : cluster [INF] Cluster is now healthy 2022-04-28T19:51:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:17 smithi139 bash[43301]: cluster 2022-04-28T19:51:16.889854+0000 mgr.y (mgr.24961) 124 : cluster [DBG] pgmap v171: 89 pgs: 27 peering, 62 active+clean; 239 MiB data, 490 MiB used, 709 GiB / 715 GiB avail; 255 B/s rd, 858 KiB/s wr, 9 op/s; 402 KiB/s, 0 objects/s recovering 2022-04-28T19:51:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:17 smithi139 bash[43301]: cluster 2022-04-28T19:51:16.957096+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2022-04-28T19:51:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:17 smithi139 bash[43301]: audit 2022-04-28T19:51:17.115840+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:51:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:17 smithi139 bash[43301]: audit 2022-04-28T19:51:17.130929+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:51:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:17 smithi139 bash[43301]: cluster 2022-04-28T19:51:17.835096+0000 mon.a (mon.0) 406 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/372 objects degraded (1.882%), 3 pgs degraded) 2022-04-28T19:51:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:17 smithi139 bash[43301]: cluster 2022-04-28T19:51:17.835169+0000 mon.a (mon.0) 407 : cluster [INF] Cluster is now healthy 2022-04-28T19:51:18.422 INFO:tasks.rados.rados.0.smithi059.stdout:2154: finishing write tid 6 to smithi05955964-31 2022-04-28T19:51:18.422 INFO:tasks.rados.rados.0.smithi059.stdout:2154: finishing write tid 7 to smithi05955964-31 2022-04-28T19:51:18.423 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 900 (ObjNum 691 snap 200 seq_num 691) dirty exists 2022-04-28T19:51:18.423 INFO:tasks.rados.rados.0.smithi059.stdout:2154: left oid 31 (ObjNum 691 snap 200 seq_num 691) 2022-04-28T19:51:18.423 INFO:tasks.rados.rados.0.smithi059.stdout:2155: expect (ObjNum 637 snap 183 seq_num 637) 2022-04-28T19:51:18.992 INFO:tasks.rados.rados.0.smithi059.stdout:2152: done (4 left) 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2153: done (3 left) 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2154: done (2 left) 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2155: done (1 left) 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2156: done (0 left) 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2157: copy_from oid 28 from oid 37 current snap is 201 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2158: copy_from oid 24 from oid 14 current snap is 201 2022-04-28T19:51:18.995 INFO:tasks.rados.rados.0.smithi059.stdout:2159: delete oid 36 current snap is 201 2022-04-28T19:51:18.996 INFO:tasks.rados.rados.0.smithi059.stdout:2158: finishing copy_from to smithi05955964-24 2022-04-28T19:51:18.997 INFO:tasks.rados.rados.0.smithi059.stdout:2158: got expected ENOENT (src dne) 2022-04-28T19:51:18.997 INFO:tasks.rados.rados.0.smithi059.stdout:2158: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:51:18.997 INFO:tasks.rados.rados.0.smithi059.stdout:2158: done (2 left) 2022-04-28T19:51:18.997 INFO:tasks.rados.rados.0.smithi059.stdout:2159: done (1 left) 2022-04-28T19:51:18.997 INFO:tasks.rados.rados.0.smithi059.stdout:2160: read oid 9 snap -1 2022-04-28T19:51:18.998 INFO:tasks.rados.rados.0.smithi059.stdout:2160: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:51:18.998 INFO:tasks.rados.rados.0.smithi059.stdout:2161: snap_remove snap 197 2022-04-28T19:51:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:18 smithi059 bash[61152]: cluster 2022-04-28T19:51:17.981858+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2022-04-28T19:51:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:18 smithi059 bash[59745]: cluster 2022-04-28T19:51:17.981858+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2022-04-28T19:51:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:18 smithi139 bash[43301]: cluster 2022-04-28T19:51:17.981858+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2022-04-28T19:51:19.993 INFO:tasks.rados.rados.0.smithi059.stdout:2161: done (2 left) 2022-04-28T19:51:19.994 INFO:tasks.rados.rados.0.smithi059.stdout:2162: copy_from oid 16 from oid 48 current snap is 201 2022-04-28T19:51:19.994 INFO:tasks.rados.rados.0.smithi059.stdout:2163: delete oid 27 current snap is 201 2022-04-28T19:51:19.994 INFO:tasks.rados.rados.0.smithi059.stdout:2157: finishing copy_from to smithi05955964-28 2022-04-28T19:51:19.994 INFO:tasks.rados.rados.0.smithi059.stdout:2157: got expected ENOENT (src dne) 2022-04-28T19:51:19.994 INFO:tasks.rados.rados.0.smithi059.stdout:2157: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:51:19.994 INFO:tasks.rados.rados.0.smithi059.stdout:2160: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:51:20.110 INFO:tasks.rados.rados.0.smithi059.stdout:2162: finishing copy_from to smithi05955964-16 2022-04-28T19:51:20.110 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 946 (ObjNum 673 snap 195 seq_num 673) dirty exists 2022-04-28T19:51:20.110 INFO:tasks.rados.rados.0.smithi059.stdout:2162: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:51:20.110 INFO:tasks.rados.rados.0.smithi059.stdout:2157: done (3 left) 2022-04-28T19:51:20.111 INFO:tasks.rados.rados.0.smithi059.stdout:2160: done (2 left) 2022-04-28T19:51:20.111 INFO:tasks.rados.rados.0.smithi059.stdout:2162: done (1 left) 2022-04-28T19:51:20.111 INFO:tasks.rados.rados.0.smithi059.stdout:2163: done (0 left) 2022-04-28T19:51:20.111 INFO:tasks.rados.rados.0.smithi059.stdout:2164: read oid 42 snap -1 2022-04-28T19:51:20.111 INFO:tasks.rados.rados.0.smithi059.stdout:2164: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:51:20.111 INFO:tasks.rados.rados.0.smithi059.stdout:2165: delete oid 5 current snap is 201 2022-04-28T19:51:20.114 INFO:tasks.rados.rados.0.smithi059.stdout:2165: done (1 left) 2022-04-28T19:51:20.114 INFO:tasks.rados.rados.0.smithi059.stdout:2166: snap_create 2022-04-28T19:51:20.143 INFO:tasks.rados.rados.0.smithi059.stdout:2164: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:51:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:19 smithi059 bash[61152]: cluster 2022-04-28T19:51:18.890575+0000 mgr.y (mgr.24961) 125 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 238 MiB data, 495 MiB used, 709 GiB / 715 GiB avail; 8.0 MiB/s rd, 3.8 MiB/s wr, 44 op/s; 20 MiB/s, 5 objects/s recovering 2022-04-28T19:51:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:19 smithi059 bash[61152]: cluster 2022-04-28T19:51:18.990278+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2022-04-28T19:51:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:19 smithi059 bash[59745]: cluster 2022-04-28T19:51:18.890575+0000 mgr.y (mgr.24961) 125 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 238 MiB data, 495 MiB used, 709 GiB / 715 GiB avail; 8.0 MiB/s rd, 3.8 MiB/s wr, 44 op/s; 20 MiB/s, 5 objects/s recovering 2022-04-28T19:51:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:19 smithi059 bash[59745]: cluster 2022-04-28T19:51:18.990278+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2022-04-28T19:51:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:19 smithi139 bash[43301]: cluster 2022-04-28T19:51:18.890575+0000 mgr.y (mgr.24961) 125 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 238 MiB data, 495 MiB used, 709 GiB / 715 GiB avail; 8.0 MiB/s rd, 3.8 MiB/s wr, 44 op/s; 20 MiB/s, 5 objects/s recovering 2022-04-28T19:51:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:19 smithi139 bash[43301]: cluster 2022-04-28T19:51:18.990278+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2022-04-28T19:51:21.012 INFO:tasks.rados.rados.0.smithi059.stdout:2164: done (1 left) 2022-04-28T19:51:21.012 INFO:tasks.rados.rados.0.smithi059.stdout:2166: done (0 left) 2022-04-28T19:51:21.012 INFO:tasks.rados.rados.0.smithi059.stdout:2167: snap_create 2022-04-28T19:51:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:21 smithi059 bash[61152]: cluster 2022-04-28T19:51:19.992577+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2022-04-28T19:51:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:21 smithi059 bash[59745]: cluster 2022-04-28T19:51:19.992577+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2022-04-28T19:51:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:21 smithi139 bash[43301]: cluster 2022-04-28T19:51:19.992577+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2022-04-28T19:51:21.845 INFO:tasks.rados.rados.0.smithi059.stdout:2167: done (0 left) 2022-04-28T19:51:21.845 INFO:tasks.rados.rados.0.smithi059.stdout:2168: delete oid 49 current snap is 203 2022-04-28T19:51:21.852 INFO:tasks.rados.rados.0.smithi059.stdout:2168: done (0 left) 2022-04-28T19:51:21.852 INFO:tasks.rados.rados.0.smithi059.stdout:2169: delete oid 41 current snap is 203 2022-04-28T19:51:21.854 INFO:tasks.rados.rados.0.smithi059.stdout:2169: done (0 left) 2022-04-28T19:51:21.854 INFO:tasks.rados.rados.0.smithi059.stdout:2170: copy_from oid 45 from oid 18 current snap is 203 2022-04-28T19:51:21.855 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 203 2022-04-28T19:51:21.855 INFO:tasks.rados.rados.0.smithi059.stdout:2171: seq_num 697 ranges {11829248=475136,12304384=704512,13008896=557056,13565952=344064} 2022-04-28T19:51:21.924 INFO:tasks.rados.rados.0.smithi059.stdout:2171: writing smithi05955964-6 from 11829248 to 12304384 tid 1 2022-04-28T19:51:21.960 INFO:tasks.rados.rados.0.smithi059.stdout:2171: writing smithi05955964-6 from 12304384 to 13008896 tid 2 2022-04-28T19:51:21.961 INFO:tasks.rados.rados.0.smithi059.stdout:2171: writing smithi05955964-6 from 13008896 to 13565952 tid 3 2022-04-28T19:51:21.961 INFO:tasks.rados.rados.0.smithi059.stdout:2171: writing smithi05955964-6 from 13565952 to 13910016 tid 4 2022-04-28T19:51:21.961 INFO:tasks.rados.rados.0.smithi059.stdout:2172: copy_from oid 41 from oid 11 current snap is 203 2022-04-28T19:51:21.961 INFO:tasks.rados.rados.0.smithi059.stdout:2170: finishing copy_from to smithi05955964-45 2022-04-28T19:51:21.961 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 853 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:21.961 INFO:tasks.rados.rados.0.smithi059.stdout:2170: finishing copy_from racing read to smithi05955964-45 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2171: finishing write tid 1 to smithi05955964-6 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2171: finishing write tid 2 to smithi05955964-6 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2170: done (2 left) 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2173: setattr oid 30 current snap is 203 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 203 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2174: seq_num 698 ranges {0=499712,499712=589824,1089536=770048,1859584=245760} 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2174: writing smithi05955964-47 from 0 to 499712 tid 1 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2174: writing smithi05955964-47 from 499712 to 1089536 tid 2 2022-04-28T19:51:21.962 INFO:tasks.rados.rados.0.smithi059.stdout:2174: writing smithi05955964-47 from 1089536 to 1859584 tid 3 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2174: writing smithi05955964-47 from 1859584 to 2105344 tid 4 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2175: rollback oid 32 current snap is 203 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 191 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2171: finishing write tid 3 to smithi05955964-6 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2171: finishing write tid 4 to smithi05955964-6 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 932 (ObjNum 669 snap 194 seq_num 669) dirty exists 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2174: finishing write tid 1 to smithi05955964-47 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2171: finishing write tid 6 to smithi05955964-6 2022-04-28T19:51:21.963 INFO:tasks.rados.rados.0.smithi059.stdout:2174: finishing write tid 2 to smithi05955964-47 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2171: finishing write tid 7 to smithi05955964-6 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 886 (ObjNum 697 snap 203 seq_num 697) dirty exists 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2171: left oid 6 (ObjNum 697 snap 203 seq_num 697) 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2171: done (4 left) 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2173: done (3 left) 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2176: setattr oid 20 current snap is 203 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2177: copy_from oid 15 from oid 49 current snap is 203 2022-04-28T19:51:21.964 INFO:tasks.rados.rados.0.smithi059.stdout:2178: rmattr oid 39 current snap is 203 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2178: done (5 left) 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2179: copy_from oid 50 from oid 9 current snap is 203 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2180: read oid 28 snap -1 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2180: expect (ObjNum 667 snap 192 seq_num 667) 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2181: delete oid 37 current snap is 203 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2177: finishing copy_from to smithi05955964-15 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2177: got expected ENOENT (src dne) 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2177: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 888 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:2176: done (7 left) 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2177: done (6 left) 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2181: done (5 left) 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2182: rmattr oid 43 current snap is 203 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2182: done (5 left) 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2183: delete oid 48 current snap is 203 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2174: finishing write tid 3 to smithi05955964-47 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2174: finishing write tid 4 to smithi05955964-47 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2174: finishing write tid 6 to smithi05955964-47 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2175: finishing rollback tid 0 to smithi05955964-32 2022-04-28T19:51:21.966 INFO:tasks.rados.rados.0.smithi059.stdout:2175: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:51:21.967 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1012 (ObjNum -133457568 snap 21893 seq_num 4026534096) dirty dne 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:2175: done (5 left) 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:2183: done (4 left) 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:2184: rollback oid 4 current snap is 203 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 199 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:2185: rollback oid 31 current snap is 203 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 191 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:2186: read oid 33 snap 190 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:2186: expect (ObjNum 620 snap 177 seq_num 620) 2022-04-28T19:51:21.968 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 203 2022-04-28T19:51:21.969 INFO:tasks.rados.rados.0.smithi059.stdout:2187: seq_num 701 ranges {9650176=548864,10199040=696320,10895360=540672,11436032=368640} 2022-04-28T19:51:22.006 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:21.629Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:22.006 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:21.629Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:22.006 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:21.629Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:51:22.020 INFO:tasks.rados.rados.0.smithi059.stdout:2187: writing smithi05955964-34 from 9650176 to 10199040 tid 1 2022-04-28T19:51:22.024 INFO:tasks.rados.rados.0.smithi059.stdout:2187: writing smithi05955964-34 from 10199040 to 10895360 tid 2 2022-04-28T19:51:22.027 INFO:tasks.rados.rados.0.smithi059.stdout:2187: writing smithi05955964-34 from 10895360 to 11436032 tid 3 2022-04-28T19:51:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:2187: writing smithi05955964-34 from 11436032 to 11804672 tid 4 2022-04-28T19:51:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:2188: snap_remove snap 200 2022-04-28T19:51:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:22 smithi059 bash[59745]: cluster 2022-04-28T19:51:20.891434+0000 mgr.y (mgr.24961) 126 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 238 MiB data, 495 MiB used, 709 GiB / 715 GiB avail; 8.0 MiB/s rd, 3.0 MiB/s wr, 34 op/s; 20 MiB/s, 5 objects/s recovering 2022-04-28T19:51:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:22 smithi059 bash[59745]: cluster 2022-04-28T19:51:21.011026+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2022-04-28T19:51:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:22 smithi059 bash[59745]: cluster 2022-04-28T19:51:21.843386+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2022-04-28T19:51:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:22 smithi059 bash[61152]: cluster 2022-04-28T19:51:20.891434+0000 mgr.y (mgr.24961) 126 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 238 MiB data, 495 MiB used, 709 GiB / 715 GiB avail; 8.0 MiB/s rd, 3.0 MiB/s wr, 34 op/s; 20 MiB/s, 5 objects/s recovering 2022-04-28T19:51:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:22 smithi059 bash[61152]: cluster 2022-04-28T19:51:21.011026+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2022-04-28T19:51:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:22 smithi059 bash[61152]: cluster 2022-04-28T19:51:21.843386+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2022-04-28T19:51:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:22 smithi139 bash[43301]: cluster 2022-04-28T19:51:20.891434+0000 mgr.y (mgr.24961) 126 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 238 MiB data, 495 MiB used, 709 GiB / 715 GiB avail; 8.0 MiB/s rd, 3.0 MiB/s wr, 34 op/s; 20 MiB/s, 5 objects/s recovering 2022-04-28T19:51:22.425 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:22 smithi139 bash[43301]: cluster 2022-04-28T19:51:21.011026+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2022-04-28T19:51:22.425 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:22 smithi139 bash[43301]: cluster 2022-04-28T19:51:21.843386+0000 mon.a (mon.0) 412 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2022-04-28T19:51:23.013 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:51:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:2188: done (8 left) 2022-04-28T19:51:23.022 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 203 2022-04-28T19:51:23.029 INFO:tasks.rados.rados.0.smithi059.stdout:2189: seq_num 702 ranges {5054464=565248,5619712=581632,6201344=745472,6946816=278528} 2022-04-28T19:51:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2189: writing smithi05955964-22 from 5054464 to 5619712 tid 1 2022-04-28T19:51:23.058 INFO:tasks.rados.rados.0.smithi059.stdout:2189: writing smithi05955964-22 from 5619712 to 6201344 tid 2 2022-04-28T19:51:23.063 INFO:tasks.rados.rados.0.smithi059.stdout:2189: writing smithi05955964-22 from 6201344 to 6946816 tid 3 2022-04-28T19:51:23.065 INFO:tasks.rados.rados.0.smithi059.stdout:2189: writing smithi05955964-22 from 6946816 to 7225344 tid 4 2022-04-28T19:51:23.065 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 203 2022-04-28T19:51:23.065 INFO:tasks.rados.rados.0.smithi059.stdout:2190: seq_num 703 ranges {0=581632,581632=458752,1040384=557056,1597440=417792,2015232=172032} 2022-04-28T19:51:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2190: writing smithi05955964-32 from 0 to 581632 tid 1 2022-04-28T19:51:23.071 INFO:tasks.rados.rados.0.smithi059.stdout:2190: writing smithi05955964-32 from 581632 to 1040384 tid 2 2022-04-28T19:51:23.075 INFO:tasks.rados.rados.0.smithi059.stdout:2190: writing smithi05955964-32 from 1040384 to 1597440 tid 3 2022-04-28T19:51:23.077 INFO:tasks.rados.rados.0.smithi059.stdout:2190: writing smithi05955964-32 from 1597440 to 2015232 tid 4 2022-04-28T19:51:23.078 INFO:tasks.rados.rados.0.smithi059.stdout:2190: writing smithi05955964-32 from 2015232 to 2187264 tid 5 2022-04-28T19:51:23.079 INFO:tasks.rados.rados.0.smithi059.stdout:2191: snap_remove snap 193 2022-04-28T19:51:23.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:23 smithi059 bash[61152]: audit 2022-04-28T19:51:22.363866+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:23 smithi059 bash[59745]: audit 2022-04-28T19:51:22.363866+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:23.347 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:51:23.347 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (10m) 16s ago 10m 14.6M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:51:23.347 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (10m) 90s ago 10m 25.8M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (8m) 16s ago 8m 14.7M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (8m) 90s ago 8m 17.6M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (2m) 90s ago 14m 375M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (3m) 16s ago 16m 414M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (2m) 16s ago 16m 57.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (102s) 90s ago 14m 19.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (114s) 16s ago 15m 52.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (10m) 16s ago 10m 11.0M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (10m) 90s ago 10m 10.7M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:51:23.348 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (48s) 16s ago 14m 256M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (18s) 16s ago 13m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (13m) 16s ago 13m 286M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 35cdc0c6604a 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (12m) 16s ago 13m 220M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (12m) 90s ago 12m 196M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (12m) 90s ago 12m 214M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (11m) 90s ago 11m 162M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (11m) 90s ago 11m 138M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:51:23.349 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (3m) 90s ago 11m 36.6M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:51:23.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:23 smithi139 bash[43301]: audit 2022-04-28T19:51:22.363866+0000 mon.a (mon.0) 413 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:23.728 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 6, 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 8, 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 7 2022-04-28T19:51:23.729 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:51:23.730 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:51:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:23.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:23.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:23.523Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:51:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:2174: finishing write tid 7 to smithi05955964-47 2022-04-28T19:51:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1042 (ObjNum 698 snap 203 seq_num 698) dirty exists 2022-04-28T19:51:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:2174: left oid 47 (ObjNum 698 snap 203 seq_num 698) 2022-04-28T19:51:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:2185: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2185: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 903 (ObjNum 0 snap 0 seq_num 0) dirty dne 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2184: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2172: finishing copy_from to smithi05955964-41 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 817 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2184: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2184: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 823 (ObjNum 628 snap 180 seq_num 628) dirty exists 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2172: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:51:24.040 INFO:tasks.rados.rados.0.smithi059.stdout:2180: expect (ObjNum 667 snap 192 seq_num 667) 2022-04-28T19:51:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:22.364274+0000 mgr.y (mgr.24961) 127 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: cephadm 2022-04-28T19:51:22.366400+0000 mgr.y (mgr.24961) 128 : cephadm [INF] Upgrade: osd.2 is safe to restart 2022-04-28T19:51:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: cluster 2022-04-28T19:51:22.892575+0000 mgr.y (mgr.24961) 129 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 262 MiB data, 531 MiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 6.1 MiB/s wr, 28 op/s 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:22.998536+0000 mgr.y (mgr.24961) 130 : audit [DBG] from='client.54169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: cluster 2022-04-28T19:51:23.020977+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.321776+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.322746+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.323715+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.324695+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.325530+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.326348+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.332301+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.332910+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.333511+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.334068+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.334605+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[59745]: audit 2022-04-28T19:51:23.727057+0000 mon.a (mon.0) 426 : audit [DBG] from='client.? 172.21.15.59:0/1438106270' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:22.364274+0000 mgr.y (mgr.24961) 127 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: cephadm 2022-04-28T19:51:22.366400+0000 mgr.y (mgr.24961) 128 : cephadm [INF] Upgrade: osd.2 is safe to restart 2022-04-28T19:51:24.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: cluster 2022-04-28T19:51:22.892575+0000 mgr.y (mgr.24961) 129 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 262 MiB data, 531 MiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 6.1 MiB/s wr, 28 op/s 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:22.998536+0000 mgr.y (mgr.24961) 130 : audit [DBG] from='client.54169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: cluster 2022-04-28T19:51:23.020977+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.321776+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.322746+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.323715+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.324695+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.325530+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.326348+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.332301+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.332910+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.333511+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.334068+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.334605+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:24 smithi059 bash[61152]: audit 2022-04-28T19:51:23.727057+0000 mon.a (mon.0) 426 : audit [DBG] from='client.? 172.21.15.59:0/1438106270' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:22.364274+0000 mgr.y (mgr.24961) 127 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2022-04-28T19:51:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: cephadm 2022-04-28T19:51:22.366400+0000 mgr.y (mgr.24961) 128 : cephadm [INF] Upgrade: osd.2 is safe to restart 2022-04-28T19:51:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: cluster 2022-04-28T19:51:22.892575+0000 mgr.y (mgr.24961) 129 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 262 MiB data, 531 MiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 6.1 MiB/s wr, 28 op/s 2022-04-28T19:51:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:22.998536+0000 mgr.y (mgr.24961) 130 : audit [DBG] from='client.54169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: cluster 2022-04-28T19:51:23.020977+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.321776+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.322746+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.323715+0000 mon.a (mon.0) 417 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.324695+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.325530+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.326348+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.332301+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.332910+0000 mon.a (mon.0) 422 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.333511+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.334068+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.334605+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:24.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:24 smithi139 bash[43301]: audit 2022-04-28T19:51:23.727057+0000 mon.a (mon.0) 426 : audit [DBG] from='client.? 172.21.15.59:0/1438106270' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:2179: finishing copy_from to smithi05955964-50 2022-04-28T19:51:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 718 (ObjNum 607 snap 170 seq_num 607) dirty exists 2022-04-28T19:51:24.417 INFO:tasks.rados.rados.0.smithi059.stdout:2179: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:51:24.418 INFO:tasks.rados.rados.0.smithi059.stdout:2186: expect (ObjNum 620 snap 177 seq_num 620) 2022-04-28T19:51:24.888 INFO:tasks.rados.rados.0.smithi059.stdout:2187: finishing write tid 1 to smithi05955964-34 2022-04-28T19:51:24.888 INFO:tasks.rados.rados.0.smithi059.stdout:2187: finishing write tid 2 to smithi05955964-34 2022-04-28T19:51:24.888 INFO:tasks.rados.rados.0.smithi059.stdout:2187: finishing write tid 3 to smithi05955964-34 2022-04-28T19:51:24.890 INFO:tasks.rados.rados.0.smithi059.stdout:2187: finishing write tid 4 to smithi05955964-34 2022-04-28T19:51:24.890 INFO:tasks.rados.rados.0.smithi059.stdout:2187: finishing write tid 6 to smithi05955964-34 2022-04-28T19:51:24.890 INFO:tasks.rados.rados.0.smithi059.stdout:2187: finishing write tid 7 to smithi05955964-34 2022-04-28T19:51:24.891 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 937 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:51:24.891 INFO:tasks.rados.rados.0.smithi059.stdout:2187: left oid 34 (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:51:24.891 INFO:tasks.rados.rados.0.smithi059.stdout:2189: finishing write tid 1 to smithi05955964-22 2022-04-28T19:51:24.891 INFO:tasks.rados.rados.0.smithi059.stdout:2189: finishing write tid 2 to smithi05955964-22 2022-04-28T19:51:24.891 INFO:tasks.rados.rados.0.smithi059.stdout:2189: finishing write tid 3 to smithi05955964-22 2022-04-28T19:51:24.892 INFO:tasks.rados.rados.0.smithi059.stdout:2189: finishing write tid 4 to smithi05955964-22 2022-04-28T19:51:24.892 INFO:tasks.rados.rados.0.smithi059.stdout:2189: finishing write tid 6 to smithi05955964-22 2022-04-28T19:51:24.892 INFO:tasks.rados.rados.0.smithi059.stdout:2189: finishing write tid 7 to smithi05955964-22 2022-04-28T19:51:24.893 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 996 (ObjNum 702 snap 203 seq_num 702) dirty exists 2022-04-28T19:51:24.893 INFO:tasks.rados.rados.0.smithi059.stdout:2189: left oid 22 (ObjNum 702 snap 203 seq_num 702) 2022-04-28T19:51:24.894 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 1 to smithi05955964-32 2022-04-28T19:51:24.894 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 2 to smithi05955964-32 2022-04-28T19:51:24.894 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 3 to smithi05955964-32 2022-04-28T19:51:24.894 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 4 to smithi05955964-32 2022-04-28T19:51:24.894 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 5 to smithi05955964-32 2022-04-28T19:51:24.895 INFO:tasks.rados.rados.0.smithi059.stdout:2172: done (10 left) 2022-04-28T19:51:24.895 INFO:tasks.rados.rados.0.smithi059.stdout:2174: done (9 left) 2022-04-28T19:51:24.895 INFO:tasks.rados.rados.0.smithi059.stdout:2179: done (8 left) 2022-04-28T19:51:24.895 INFO:tasks.rados.rados.0.smithi059.stdout:2180: done (7 left) 2022-04-28T19:51:24.896 INFO:tasks.rados.rados.0.smithi059.stdout:2184: done (6 left) 2022-04-28T19:51:24.896 INFO:tasks.rados.rados.0.smithi059.stdout:2185: done (5 left) 2022-04-28T19:51:24.896 INFO:tasks.rados.rados.0.smithi059.stdout:2186: done (4 left) 2022-04-28T19:51:24.896 INFO:tasks.rados.rados.0.smithi059.stdout:2187: done (3 left) 2022-04-28T19:51:24.897 INFO:tasks.rados.rados.0.smithi059.stdout:2189: done (2 left) 2022-04-28T19:51:24.897 INFO:tasks.rados.rados.0.smithi059.stdout:2191: done (1 left) 2022-04-28T19:51:24.897 INFO:tasks.rados.rados.0.smithi059.stdout:2192: setattr oid 17 current snap is 203 2022-04-28T19:51:24.897 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 7 to smithi05955964-32 2022-04-28T19:51:24.897 INFO:tasks.rados.rados.0.smithi059.stdout:2190: finishing write tid 8 to smithi05955964-32 2022-04-28T19:51:24.898 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1024 (ObjNum 703 snap 203 seq_num 703) dirty exists 2022-04-28T19:51:24.898 INFO:tasks.rados.rados.0.smithi059.stdout:2190: left oid 32 (ObjNum 703 snap 203 seq_num 703) 2022-04-28T19:51:24.899 INFO:tasks.rados.rados.0.smithi059.stdout:2190: done (1 left) 2022-04-28T19:51:24.899 INFO:tasks.rados.rados.0.smithi059.stdout:2193: rmattr oid 39 current snap is 203 2022-04-28T19:51:24.899 INFO:tasks.rados.rados.0.smithi059.stdout:2193: done (1 left) 2022-04-28T19:51:24.900 INFO:tasks.rados.rados.0.smithi059.stdout:2194: rollback oid 23 current snap is 203 2022-04-28T19:51:24.900 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 196 2022-04-28T19:51:24.900 INFO:tasks.rados.rados.0.smithi059.stdout:2195: snap_remove snap 191 2022-04-28T19:51:25.048 INFO:tasks.rados.rados.0.smithi059.stdout:2195: done (2 left) 2022-04-28T19:51:25.048 INFO:tasks.rados.rados.0.smithi059.stdout:2196: delete oid 19 current snap is 203 2022-04-28T19:51:25.049 INFO:tasks.rados.rados.0.smithi059.stdout:2194: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:51:25.049 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 870 (ObjNum 640 snap 183 seq_num 640) dirty exists 2022-04-28T19:51:25.049 INFO:tasks.rados.rados.0.smithi059.stdout:2194: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:51:25.049 INFO:tasks.rados.rados.0.smithi059.stdout:2194: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:51:25.049 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1054 (ObjNum 664 snap 190 seq_num 664) dirty exists 2022-04-28T19:51:25.056 INFO:tasks.rados.rados.0.smithi059.stdout:2192: done (2 left) 2022-04-28T19:51:25.056 INFO:tasks.rados.rados.0.smithi059.stdout:2194: done (1 left) 2022-04-28T19:51:25.057 INFO:tasks.rados.rados.0.smithi059.stdout:2196: done (0 left) 2022-04-28T19:51:25.057 INFO:tasks.rados.rados.0.smithi059.stdout:2197: read oid 50 snap -1 2022-04-28T19:51:25.057 INFO:tasks.rados.rados.0.smithi059.stdout:2197: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:51:25.057 INFO:tasks.rados.rados.0.smithi059.stdout:2198: rollback oid 21 current snap is 203 2022-04-28T19:51:25.057 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 199 2022-04-28T19:51:25.057 INFO:tasks.rados.rados.0.smithi059.stdout:2199: delete oid 35 current snap is 203 2022-04-28T19:51:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:2199: done (2 left) 2022-04-28T19:51:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:2200: delete oid 33 current snap is 203 2022-04-28T19:51:25.068 INFO:tasks.rados.rados.0.smithi059.stdout:2198: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:51:25.069 INFO:tasks.rados.rados.0.smithi059.stdout:2198: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:51:25.070 INFO:tasks.rados.rados.0.smithi059.stdout:2198: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:51:25.070 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 718 (ObjNum 671 snap 195 seq_num 671) dirty exists 2022-04-28T19:51:25.083 INFO:tasks.rados.rados.0.smithi059.stdout:2197: expect (ObjNum 607 snap 170 seq_num 607) 2022-04-28T19:51:25.209 INFO:tasks.rados.rados.0.smithi059.stdout:2197: done (2 left) 2022-04-28T19:51:25.210 INFO:tasks.rados.rados.0.smithi059.stdout:2198: done (1 left) 2022-04-28T19:51:25.210 INFO:tasks.rados.rados.0.smithi059.stdout:2200: done (0 left) 2022-04-28T19:51:25.210 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 203 2022-04-28T19:51:25.210 INFO:tasks.rados.rados.0.smithi059.stdout:2201: seq_num 707 ranges {0=647168,647168=802816,1449984=532480,1982464=270336} 2022-04-28T19:51:25.216 INFO:tasks.rados.rados.0.smithi059.stdout:2201: writing smithi05955964-35 from 0 to 647168 tid 1 2022-04-28T19:51:25.223 INFO:tasks.rados.rados.0.smithi059.stdout:2201: writing smithi05955964-35 from 647168 to 1449984 tid 2 2022-04-28T19:51:25.225 INFO:tasks.rados.rados.0.smithi059.stdout:2201: writing smithi05955964-35 from 1449984 to 1982464 tid 3 2022-04-28T19:51:25.227 INFO:tasks.rados.rados.0.smithi059.stdout:2201: writing smithi05955964-35 from 1982464 to 2252800 tid 4 2022-04-28T19:51:25.227 INFO:tasks.rados.rados.0.smithi059.stdout:2202: delete oid 17 current snap is 203 2022-04-28T19:51:25.228 INFO:tasks.rados.rados.0.smithi059.stdout:2201: finishing write tid 1 to smithi05955964-35 2022-04-28T19:51:25.230 INFO:tasks.rados.rados.0.smithi059.stdout:2201: finishing write tid 2 to smithi05955964-35 2022-04-28T19:51:25.231 INFO:tasks.rados.rados.0.smithi059.stdout:2202: done (1 left) 2022-04-28T19:51:25.231 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 203 2022-04-28T19:51:25.231 INFO:tasks.rados.rados.0.smithi059.stdout:2203: seq_num 709 ranges {5283840=679936,5963776=614400,6578176=499712,7077888=491520} 2022-04-28T19:51:25.264 INFO:tasks.rados.rados.0.smithi059.stdout:2203: writing smithi05955964-3 from 5283840 to 5963776 tid 1 2022-04-28T19:51:25.268 INFO:tasks.rados.rados.0.smithi059.stdout:2203: writing smithi05955964-3 from 5963776 to 6578176 tid 2 2022-04-28T19:51:25.271 INFO:tasks.rados.rados.0.smithi059.stdout:2203: writing smithi05955964-3 from 6578176 to 7077888 tid 3 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2203: writing smithi05955964-3 from 7077888 to 7569408 tid 4 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2204: snap_create 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2201: finishing write tid 3 to smithi05955964-35 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2201: finishing write tid 4 to smithi05955964-35 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2201: finishing write tid 6 to smithi05955964-35 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2201: finishing write tid 7 to smithi05955964-35 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 742 (ObjNum 707 snap 203 seq_num 707) dirty exists 2022-04-28T19:51:25.275 INFO:tasks.rados.rados.0.smithi059.stdout:2201: left oid 35 (ObjNum 707 snap 203 seq_num 707) 2022-04-28T19:51:25.276 INFO:tasks.rados.rados.0.smithi059.stdout:2203: finishing write tid 1 to smithi05955964-3 2022-04-28T19:51:25.276 INFO:tasks.rados.rados.0.smithi059.stdout:2203: finishing write tid 2 to smithi05955964-3 2022-04-28T19:51:25.276 INFO:tasks.rados.rados.0.smithi059.stdout:2203: finishing write tid 3 to smithi05955964-3 2022-04-28T19:51:25.278 INFO:tasks.rados.rados.0.smithi059.stdout:2203: finishing write tid 4 to smithi05955964-3 2022-04-28T19:51:25.278 INFO:tasks.rados.rados.0.smithi059.stdout:2203: finishing write tid 6 to smithi05955964-3 2022-04-28T19:51:25.280 INFO:tasks.rados.rados.0.smithi059.stdout:2203: finishing write tid 7 to smithi05955964-3 2022-04-28T19:51:25.280 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 912 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:51:25.281 INFO:tasks.rados.rados.0.smithi059.stdout:2203: left oid 3 (ObjNum 709 snap 203 seq_num 709) 2022-04-28T19:51:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:25 smithi059 bash[59745]: audit 2022-04-28T19:51:23.319160+0000 mgr.y (mgr.24961) 131 : audit [DBG] from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:25 smithi059 bash[59745]: cluster 2022-04-28T19:51:24.030569+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2022-04-28T19:51:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:25 smithi059 bash[61152]: audit 2022-04-28T19:51:23.319160+0000 mgr.y (mgr.24961) 131 : audit [DBG] from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:25 smithi059 bash[61152]: cluster 2022-04-28T19:51:24.030569+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2022-04-28T19:51:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:25 smithi139 bash[43301]: audit 2022-04-28T19:51:23.319160+0000 mgr.y (mgr.24961) 131 : audit [DBG] from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:25 smithi139 bash[43301]: cluster 2022-04-28T19:51:24.030569+0000 mon.a (mon.0) 427 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2022-04-28T19:51:26.104 INFO:tasks.rados.rados.0.smithi059.stdout:2201: done (2 left) 2022-04-28T19:51:26.104 INFO:tasks.rados.rados.0.smithi059.stdout:2203: done (1 left) 2022-04-28T19:51:26.104 INFO:tasks.rados.rados.0.smithi059.stdout:2204: done (0 left) 2022-04-28T19:51:26.104 INFO:tasks.rados.rados.0.smithi059.stdout:2205: delete oid 47 current snap is 204 2022-04-28T19:51:26.109 INFO:tasks.rados.rados.0.smithi059.stdout:2205: done (0 left) 2022-04-28T19:51:26.109 INFO:tasks.rados.rados.0.smithi059.stdout:2206: copy_from oid 37 from oid 34 current snap is 204 2022-04-28T19:51:26.109 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 204 2022-04-28T19:51:26.110 INFO:tasks.rados.rados.0.smithi059.stdout:2207: seq_num 711 ranges {2965504=712704,3678208=786432,4464640=516096,4980736=303104} 2022-04-28T19:51:26.130 INFO:tasks.rados.rados.0.smithi059.stdout:2207: writing smithi05955964-9 from 2965504 to 3678208 tid 1 2022-04-28T19:51:26.135 INFO:tasks.rados.rados.0.smithi059.stdout:2207: writing smithi05955964-9 from 3678208 to 4464640 tid 2 2022-04-28T19:51:26.138 INFO:tasks.rados.rados.0.smithi059.stdout:2207: writing smithi05955964-9 from 4464640 to 4980736 tid 3 2022-04-28T19:51:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2207: writing smithi05955964-9 from 4980736 to 5283840 tid 4 2022-04-28T19:51:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2208: rollback oid 27 current snap is 204 2022-04-28T19:51:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 198 2022-04-28T19:51:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2207: finishing write tid 1 to smithi05955964-9 2022-04-28T19:51:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2209: delete oid 28 current snap is 204 2022-04-28T19:51:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2207: finishing write tid 2 to smithi05955964-9 2022-04-28T19:51:26.143 INFO:tasks.rados.rados.0.smithi059.stdout:2207: finishing write tid 3 to smithi05955964-9 2022-04-28T19:51:26.144 INFO:tasks.rados.rados.0.smithi059.stdout:2207: finishing write tid 4 to smithi05955964-9 2022-04-28T19:51:26.144 INFO:tasks.rados.rados.0.smithi059.stdout:2208: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:51:26.144 INFO:tasks.rados.rados.0.smithi059.stdout:2207: finishing write tid 6 to smithi05955964-9 2022-04-28T19:51:26.144 INFO:tasks.rados.rados.0.smithi059.stdout:2209: done (3 left) 2022-04-28T19:51:26.145 INFO:tasks.rados.rados.0.smithi059.stdout:2210: delete oid 35 current snap is 204 2022-04-28T19:51:26.146 INFO:tasks.rados.rados.0.smithi059.stdout:2208: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:51:26.146 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1066 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:51:26.146 INFO:tasks.rados.rados.0.smithi059.stdout:2207: finishing write tid 7 to smithi05955964-9 2022-04-28T19:51:26.146 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 723 (ObjNum 711 snap 204 seq_num 711) dirty exists 2022-04-28T19:51:26.146 INFO:tasks.rados.rados.0.smithi059.stdout:2207: left oid 9 (ObjNum 711 snap 204 seq_num 711) 2022-04-28T19:51:26.147 INFO:tasks.rados.rados.0.smithi059.stdout:2207: done (3 left) 2022-04-28T19:51:26.147 INFO:tasks.rados.rados.0.smithi059.stdout:2208: done (2 left) 2022-04-28T19:51:26.147 INFO:tasks.rados.rados.0.smithi059.stdout:2210: done (1 left) 2022-04-28T19:51:26.147 INFO:tasks.rados.rados.0.smithi059.stdout:2211: snap_create 2022-04-28T19:51:26.180 INFO:tasks.rados.rados.0.smithi059.stdout:2206: finishing copy_from to smithi05955964-37 2022-04-28T19:51:26.181 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 937 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:51:26.182 INFO:tasks.rados.rados.0.smithi059.stdout:2206: finishing copy_from racing read to smithi05955964-37 2022-04-28T19:51:26.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[59745]: cluster 2022-04-28T19:51:24.892970+0000 mgr.y (mgr.24961) 132 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 262 MiB data, 531 MiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 6.1 MiB/s wr, 28 op/s 2022-04-28T19:51:26.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[59745]: cluster 2022-04-28T19:51:25.045649+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2022-04-28T19:51:26.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[59745]: cephadm 2022-04-28T19:51:25.088183+0000 mgr.y (mgr.24961) 133 : cephadm [INF] Upgrade: Updating osd.2 2022-04-28T19:51:26.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[59745]: audit 2022-04-28T19:51:25.092164+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:26.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[59745]: cephadm 2022-04-28T19:51:25.093568+0000 mgr.y (mgr.24961) 134 : cephadm [INF] Deploying daemon osd.2 on smithi059 2022-04-28T19:51:26.292 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[59745]: audit 2022-04-28T19:51:25.094020+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "container_image"}]: dispatch 2022-04-28T19:51:26.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[61152]: cluster 2022-04-28T19:51:24.892970+0000 mgr.y (mgr.24961) 132 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 262 MiB data, 531 MiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 6.1 MiB/s wr, 28 op/s 2022-04-28T19:51:26.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[61152]: cluster 2022-04-28T19:51:25.045649+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2022-04-28T19:51:26.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[61152]: cephadm 2022-04-28T19:51:25.088183+0000 mgr.y (mgr.24961) 133 : cephadm [INF] Upgrade: Updating osd.2 2022-04-28T19:51:26.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[61152]: audit 2022-04-28T19:51:25.092164+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:26.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[61152]: cephadm 2022-04-28T19:51:25.093568+0000 mgr.y (mgr.24961) 134 : cephadm [INF] Deploying daemon osd.2 on smithi059 2022-04-28T19:51:26.293 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:26 smithi059 bash[61152]: audit 2022-04-28T19:51:25.094020+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "container_image"}]: dispatch 2022-04-28T19:51:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:26 smithi139 bash[43301]: cluster 2022-04-28T19:51:24.892970+0000 mgr.y (mgr.24961) 132 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 262 MiB data, 531 MiB used, 709 GiB / 715 GiB avail; 25 MiB/s rd, 6.1 MiB/s wr, 28 op/s 2022-04-28T19:51:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:26 smithi139 bash[43301]: cluster 2022-04-28T19:51:25.045649+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2022-04-28T19:51:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:26 smithi139 bash[43301]: cephadm 2022-04-28T19:51:25.088183+0000 mgr.y (mgr.24961) 133 : cephadm [INF] Upgrade: Updating osd.2 2022-04-28T19:51:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:26 smithi139 bash[43301]: audit 2022-04-28T19:51:25.092164+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:26 smithi139 bash[43301]: cephadm 2022-04-28T19:51:25.093568+0000 mgr.y (mgr.24961) 134 : cephadm [INF] Deploying daemon osd.2 on smithi059 2022-04-28T19:51:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:26 smithi139 bash[43301]: audit 2022-04-28T19:51:25.094020+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "container_image"}]: dispatch 2022-04-28T19:51:26.844 INFO:tasks.rados.rados.0.smithi059.stdout:2206: done (1 left) 2022-04-28T19:51:26.845 INFO:tasks.rados.rados.0.smithi059.stdout:2211: done (0 left) 2022-04-28T19:51:26.845 INFO:tasks.rados.rados.0.smithi059.stdout:2212: snap_remove snap 204 2022-04-28T19:51:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:27 smithi139 bash[43301]: cluster 2022-04-28T19:51:26.102384+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2022-04-28T19:51:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:27 smithi139 bash[43301]: cluster 2022-04-28T19:51:26.843926+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2022-04-28T19:51:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:27 smithi059 bash[59745]: cluster 2022-04-28T19:51:26.102384+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2022-04-28T19:51:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:27 smithi059 bash[59745]: cluster 2022-04-28T19:51:26.843926+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2022-04-28T19:51:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:27 smithi059 bash[61152]: cluster 2022-04-28T19:51:26.102384+0000 mon.a (mon.0) 431 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2022-04-28T19:51:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:27 smithi059 bash[61152]: cluster 2022-04-28T19:51:26.843926+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2022-04-28T19:51:27.861 INFO:tasks.rados.rados.0.smithi059.stdout:2212: done (0 left) 2022-04-28T19:51:27.861 INFO:tasks.rados.rados.0.smithi059.stdout:2213: snap_create 2022-04-28T19:51:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:28 smithi059 bash[59745]: cluster 2022-04-28T19:51:26.893631+0000 mgr.y (mgr.24961) 135 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 262 MiB data, 539 MiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2022-04-28T19:51:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:28 smithi059 bash[59745]: audit 2022-04-28T19:51:27.203805+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:28 smithi059 bash[59745]: cluster 2022-04-28T19:51:27.860385+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2022-04-28T19:51:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:28 smithi059 bash[61152]: cluster 2022-04-28T19:51:26.893631+0000 mgr.y (mgr.24961) 135 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 262 MiB data, 539 MiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2022-04-28T19:51:28.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:28 smithi059 bash[61152]: audit 2022-04-28T19:51:27.203805+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:28.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:28 smithi059 bash[61152]: cluster 2022-04-28T19:51:27.860385+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2022-04-28T19:51:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:28 smithi139 bash[43301]: cluster 2022-04-28T19:51:26.893631+0000 mgr.y (mgr.24961) 135 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 262 MiB data, 539 MiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2022-04-28T19:51:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:28 smithi139 bash[43301]: audit 2022-04-28T19:51:27.203805+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:28 smithi139 bash[43301]: cluster 2022-04-28T19:51:27.860385+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2022-04-28T19:51:28.861 INFO:tasks.rados.rados.0.smithi059.stdout:2213: done (0 left) 2022-04-28T19:51:28.861 INFO:tasks.rados.rados.0.smithi059.stdout:2214: snap_remove snap 203 2022-04-28T19:51:29.252 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:29 smithi059 systemd[1]: Stopping Ceph osd.2 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:51:29.591 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[35970]: debug 2022-04-28T19:51:29.244+0000 7ff3ecd57700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:51:29.591 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[35970]: debug 2022-04-28T19:51:29.244+0000 7ff3ecd57700 -1 osd.2 470 *** Got signal Terminated *** 2022-04-28T19:51:29.591 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[35970]: debug 2022-04-28T19:51:29.244+0000 7ff3ecd57700 -1 osd.2 470 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:51:29.875 INFO:tasks.rados.rados.0.smithi059.stdout:2214: done (0 left) 2022-04-28T19:51:29.876 INFO:tasks.rados.rados.0.smithi059.stdout:2215: setattr oid 43 current snap is 206 2022-04-28T19:51:29.877 INFO:tasks.rados.rados.0.smithi059.stdout:2216: setattr oid 29 current snap is 206 2022-04-28T19:51:29.879 INFO:tasks.rados.rados.0.smithi059.stdout:2217: rmattr oid 12 current snap is 206 2022-04-28T19:51:29.879 INFO:tasks.rados.rados.0.smithi059.stdout:2217: done (2 left) 2022-04-28T19:51:29.879 INFO:tasks.rados.rados.0.smithi059.stdout:2218: read oid 3 snap -1 2022-04-28T19:51:29.879 INFO:tasks.rados.rados.0.smithi059.stdout:2218: expect (ObjNum 709 snap 203 seq_num 709) 2022-04-28T19:51:29.880 INFO:tasks.rados.rados.0.smithi059.stdout:2219: rollback oid 50 current snap is 206 2022-04-28T19:51:29.880 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 202 2022-04-28T19:51:29.880 INFO:tasks.rados.rados.0.smithi059.stdout:2220: copy_from oid 47 from oid 19 current snap is 206 2022-04-28T19:51:29.880 INFO:tasks.rados.rados.0.smithi059.stdout:2221: rollback oid 17 current snap is 206 2022-04-28T19:51:29.880 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 199 2022-04-28T19:51:29.881 INFO:tasks.rados.rados.0.smithi059.stdout:2222: read oid 16 snap 202 2022-04-28T19:51:29.881 INFO:tasks.rados.rados.0.smithi059.stdout:2222: expect (ObjNum 673 snap 195 seq_num 673) 2022-04-28T19:51:29.881 INFO:tasks.rados.rados.0.smithi059.stdout:2223: copy_from oid 23 from oid 48 current snap is 206 2022-04-28T19:51:29.881 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 206 2022-04-28T19:51:29.881 INFO:tasks.rados.rados.0.smithi059.stdout:2224: seq_num 714 ranges {3497984=761856,4259840=442368,4702208=688128,5390336=475136} 2022-04-28T19:51:29.903 INFO:tasks.rados.rados.0.smithi059.stdout:2224: writing smithi05955964-25 from 3497984 to 4259840 tid 1 2022-04-28T19:51:29.906 INFO:tasks.rados.rados.0.smithi059.stdout:2224: writing smithi05955964-25 from 4259840 to 4702208 tid 2 2022-04-28T19:51:29.910 INFO:tasks.rados.rados.0.smithi059.stdout:2224: writing smithi05955964-25 from 4702208 to 5390336 tid 3 2022-04-28T19:51:29.913 INFO:tasks.rados.rados.0.smithi059.stdout:2224: writing smithi05955964-25 from 5390336 to 5865472 tid 4 2022-04-28T19:51:29.913 INFO:tasks.rados.rados.0.smithi059.stdout:2225: snap_remove snap 198 2022-04-28T19:51:30.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[66561]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.2 2022-04-28T19:51:30.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:30 smithi059 bash[66561]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-2 2022-04-28T19:51:30.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:30 smithi059 bash[66561]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.2 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:28.859552+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:28.894359+0000 mgr.y (mgr.24961) 136 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 285 MiB data, 556 MiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.7 MiB/s wr, 18 op/s 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.579946+0000 mon.a (mon.0) 436 : cluster [DBG] osd.2 reported immediately failed by osd.7 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.580076+0000 mon.a (mon.0) 437 : cluster [INF] osd.2 failed (root=default,host=smithi059) (connection refused reported by osd.7) 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.580758+0000 mon.a (mon.0) 438 : cluster [DBG] osd.2 reported immediately failed by osd.6 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.581110+0000 mon.a (mon.0) 439 : cluster [DBG] osd.2 reported immediately failed by osd.7 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.581447+0000 mon.a (mon.0) 440 : cluster [DBG] osd.2 reported immediately failed by osd.5 2022-04-28T19:51:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.581749+0000 mon.a (mon.0) 441 : cluster [DBG] osd.2 reported immediately failed by osd.3 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.581973+0000 mon.a (mon.0) 442 : cluster [DBG] osd.2 reported immediately failed by osd.6 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.582261+0000 mon.a (mon.0) 443 : cluster [DBG] osd.2 reported immediately failed by osd.4 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.582433+0000 mon.a (mon.0) 444 : cluster [DBG] osd.2 reported immediately failed by osd.3 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.582833+0000 mon.a (mon.0) 445 : cluster [DBG] osd.2 reported immediately failed by osd.5 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.583101+0000 mon.a (mon.0) 446 : cluster [DBG] osd.2 reported immediately failed by osd.4 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.583617+0000 mon.a (mon.0) 447 : cluster [DBG] osd.2 reported immediately failed by osd.0 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.583984+0000 mon.a (mon.0) 448 : cluster [DBG] osd.2 reported immediately failed by osd.1 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.584323+0000 mon.a (mon.0) 449 : cluster [DBG] osd.2 reported immediately failed by osd.1 2022-04-28T19:51:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.584686+0000 mon.a (mon.0) 450 : cluster [DBG] osd.2 reported immediately failed by osd.0 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:28.859552+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:28.894359+0000 mgr.y (mgr.24961) 136 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 285 MiB data, 556 MiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.7 MiB/s wr, 18 op/s 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.579946+0000 mon.a (mon.0) 436 : cluster [DBG] osd.2 reported immediately failed by osd.7 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.580076+0000 mon.a (mon.0) 437 : cluster [INF] osd.2 failed (root=default,host=smithi059) (connection refused reported by osd.7) 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.580758+0000 mon.a (mon.0) 438 : cluster [DBG] osd.2 reported immediately failed by osd.6 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.581110+0000 mon.a (mon.0) 439 : cluster [DBG] osd.2 reported immediately failed by osd.7 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.581447+0000 mon.a (mon.0) 440 : cluster [DBG] osd.2 reported immediately failed by osd.5 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.581749+0000 mon.a (mon.0) 441 : cluster [DBG] osd.2 reported immediately failed by osd.3 2022-04-28T19:51:30.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.581973+0000 mon.a (mon.0) 442 : cluster [DBG] osd.2 reported immediately failed by osd.6 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.582261+0000 mon.a (mon.0) 443 : cluster [DBG] osd.2 reported immediately failed by osd.4 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.582433+0000 mon.a (mon.0) 444 : cluster [DBG] osd.2 reported immediately failed by osd.3 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.582833+0000 mon.a (mon.0) 445 : cluster [DBG] osd.2 reported immediately failed by osd.5 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.583101+0000 mon.a (mon.0) 446 : cluster [DBG] osd.2 reported immediately failed by osd.4 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.583617+0000 mon.a (mon.0) 447 : cluster [DBG] osd.2 reported immediately failed by osd.0 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.583984+0000 mon.a (mon.0) 448 : cluster [DBG] osd.2 reported immediately failed by osd.1 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.584323+0000 mon.a (mon.0) 449 : cluster [DBG] osd.2 reported immediately failed by osd.1 2022-04-28T19:51:30.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:29 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.584686+0000 mon.a (mon.0) 450 : cluster [DBG] osd.2 reported immediately failed by osd.0 2022-04-28T19:51:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:28.859552+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2022-04-28T19:51:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:28.894359+0000 mgr.y (mgr.24961) 136 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 285 MiB data, 556 MiB used, 709 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.7 MiB/s wr, 18 op/s 2022-04-28T19:51:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.579946+0000 mon.a (mon.0) 436 : cluster [DBG] osd.2 reported immediately failed by osd.7 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.580076+0000 mon.a (mon.0) 437 : cluster [INF] osd.2 failed (root=default,host=smithi059) (connection refused reported by osd.7) 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.580758+0000 mon.a (mon.0) 438 : cluster [DBG] osd.2 reported immediately failed by osd.6 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.581110+0000 mon.a (mon.0) 439 : cluster [DBG] osd.2 reported immediately failed by osd.7 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.581447+0000 mon.a (mon.0) 440 : cluster [DBG] osd.2 reported immediately failed by osd.5 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.581749+0000 mon.a (mon.0) 441 : cluster [DBG] osd.2 reported immediately failed by osd.3 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.581973+0000 mon.a (mon.0) 442 : cluster [DBG] osd.2 reported immediately failed by osd.6 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.582261+0000 mon.a (mon.0) 443 : cluster [DBG] osd.2 reported immediately failed by osd.4 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.582433+0000 mon.a (mon.0) 444 : cluster [DBG] osd.2 reported immediately failed by osd.3 2022-04-28T19:51:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.582833+0000 mon.a (mon.0) 445 : cluster [DBG] osd.2 reported immediately failed by osd.5 2022-04-28T19:51:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.583101+0000 mon.a (mon.0) 446 : cluster [DBG] osd.2 reported immediately failed by osd.4 2022-04-28T19:51:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.583617+0000 mon.a (mon.0) 447 : cluster [DBG] osd.2 reported immediately failed by osd.0 2022-04-28T19:51:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.583984+0000 mon.a (mon.0) 448 : cluster [DBG] osd.2 reported immediately failed by osd.1 2022-04-28T19:51:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.584323+0000 mon.a (mon.0) 449 : cluster [DBG] osd.2 reported immediately failed by osd.1 2022-04-28T19:51:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:29 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.584686+0000 mon.a (mon.0) 450 : cluster [DBG] osd.2 reported immediately failed by osd.0 2022-04-28T19:51:30.878 INFO:tasks.rados.rados.0.smithi059.stdout:2225: done (9 left) 2022-04-28T19:51:30.956 INFO:tasks.rados.rados.0.smithi059.stdout:2226: copy_from oid 9 from oid 35 current snap is 206 2022-04-28T19:51:30.956 INFO:tasks.rados.rados.0.smithi059.stdout:2227: rmattr oid 26 current snap is 206 2022-04-28T19:51:30.956 INFO:tasks.rados.rados.0.smithi059.stdout:2227: done (10 left) 2022-04-28T19:51:30.956 INFO:tasks.rados.rados.0.smithi059.stdout:2228: snap_create 2022-04-28T19:51:30.957 INFO:tasks.rados.rados.0.smithi059.stdout:2221: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:51:30.957 INFO:tasks.rados.rados.0.smithi059.stdout:2221: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:51:30.957 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 873 (ObjNum 640 snap 183 seq_num 640) dirty exists 2022-04-28T19:51:30.957 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:51:30.957 INFO:tasks.rados.rados.0.smithi059.stdout:2222: expect (ObjNum 673 snap 195 seq_num 673) 2022-04-28T19:51:30.998 INFO:tasks.rados.rados.0.smithi059.stdout:2224: finishing write tid 1 to smithi05955964-25 2022-04-28T19:51:30.999 INFO:tasks.rados.rados.0.smithi059.stdout:2224: finishing write tid 2 to smithi05955964-25 2022-04-28T19:51:30.999 INFO:tasks.rados.rados.0.smithi059.stdout:2224: finishing write tid 3 to smithi05955964-25 2022-04-28T19:51:30.999 INFO:tasks.rados.rados.0.smithi059.stdout:2224: finishing write tid 4 to smithi05955964-25 2022-04-28T19:51:30.999 INFO:tasks.rados.rados.0.smithi059.stdout:2224: finishing write tid 6 to smithi05955964-25 2022-04-28T19:51:30.999 INFO:tasks.rados.rados.0.smithi059.stdout:2224: finishing write tid 7 to smithi05955964-25 2022-04-28T19:51:31.000 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 979 (ObjNum 714 snap 206 seq_num 714) dirty exists 2022-04-28T19:51:31.000 INFO:tasks.rados.rados.0.smithi059.stdout:2224: left oid 25 (ObjNum 714 snap 206 seq_num 714) 2022-04-28T19:51:31.000 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1040 (ObjNum 659 snap 189 seq_num 659) dirty exists 2022-04-28T19:51:31.001 INFO:tasks.rados.rados.0.smithi059.stdout:2226: finishing copy_from to smithi05955964-9 2022-04-28T19:51:31.001 INFO:tasks.rados.rados.0.smithi059.stdout:2226: got expected ENOENT (src dne) 2022-04-28T19:51:31.001 INFO:tasks.rados.rados.0.smithi059.stdout:2226: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:51:31.001 INFO:tasks.rados.rados.0.smithi059.stdout:2219: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:51:31.001 INFO:tasks.rados.rados.0.smithi059.stdout:2219: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:51:31.002 INFO:tasks.rados.rados.0.smithi059.stdout:2219: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:51:31.002 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1044 (ObjNum 634 snap 181 seq_num 634) dirty exists 2022-04-28T19:51:31.002 INFO:tasks.rados.rados.0.smithi059.stdout:2223: finishing copy_from to smithi05955964-23 2022-04-28T19:51:31.002 INFO:tasks.rados.rados.0.smithi059.stdout:2223: got expected ENOENT (src dne) 2022-04-28T19:51:31.002 INFO:tasks.rados.rados.0.smithi059.stdout:2223: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:51:31.003 INFO:tasks.rados.rados.0.smithi059.stdout:2220: finishing copy_from to smithi05955964-47 2022-04-28T19:51:31.003 INFO:tasks.rados.rados.0.smithi059.stdout:2220: got expected ENOENT (src dne) 2022-04-28T19:51:31.003 INFO:tasks.rados.rados.0.smithi059.stdout:2220: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:51:31.003 INFO:tasks.rados.rados.0.smithi059.stdout:2218: expect (ObjNum 709 snap 203 seq_num 709) 2022-04-28T19:51:31.172 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:30 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.858006+0000 mon.a (mon.0) 451 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:51:31.173 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:30 smithi059 bash[59745]: cluster 2022-04-28T19:51:29.875446+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e471: 8 total, 7 up, 8 in 2022-04-28T19:51:31.173 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:30 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.858006+0000 mon.a (mon.0) 451 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:51:31.173 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:30 smithi059 bash[61152]: cluster 2022-04-28T19:51:29.875446+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e471: 8 total, 7 up, 8 in 2022-04-28T19:51:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:30 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.858006+0000 mon.a (mon.0) 451 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:51:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:30 smithi139 bash[43301]: cluster 2022-04-28T19:51:29.875446+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e471: 8 total, 7 up, 8 in 2022-04-28T19:51:31.850 INFO:tasks.rados.rados.0.smithi059.stdout:2215: done (10 left) 2022-04-28T19:51:31.850 INFO:tasks.rados.rados.0.smithi059.stdout:2216: done (9 left) 2022-04-28T19:51:31.850 INFO:tasks.rados.rados.0.smithi059.stdout:2218: done (8 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2219: done (7 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2220: done (6 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2221: done (5 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2222: done (4 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2223: done (3 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2224: done (2 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2226: done (1 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2228: done (0 left) 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:2229: rollback oid 50 current snap is 207 2022-04-28T19:51:31.851 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 199 2022-04-28T19:51:31.852 INFO:tasks.rados.rados.0.smithi059.stdout:2230: read oid 14 snap -1 2022-04-28T19:51:31.852 INFO:tasks.rados.rados.0.smithi059.stdout:2230: expect deleted 2022-04-28T19:51:31.852 INFO:tasks.rados.rados.0.smithi059.stdout:2231: setattr oid 45 current snap is 207 2022-04-28T19:51:31.852 INFO:tasks.rados.rados.0.smithi059.stdout:2232: read oid 1 snap 190 2022-04-28T19:51:31.853 INFO:tasks.rados.rados.0.smithi059.stdout:2232: expect deleted 2022-04-28T19:51:31.853 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 207 2022-04-28T19:51:31.853 INFO:tasks.rados.rados.0.smithi059.stdout:2233: seq_num 715 ranges {1409024=778240,2187264=770048,2957312=499712,3457024=335872} 2022-04-28T19:51:31.871 INFO:tasks.rados.rados.0.smithi059.stdout:2233: writing smithi05955964-18 from 1409024 to 2187264 tid 1 2022-04-28T19:51:31.876 INFO:tasks.rados.rados.0.smithi059.stdout:2233: writing smithi05955964-18 from 2187264 to 2957312 tid 2 2022-04-28T19:51:31.878 INFO:tasks.rados.rados.0.smithi059.stdout:2233: writing smithi05955964-18 from 2957312 to 3457024 tid 3 2022-04-28T19:51:31.881 INFO:tasks.rados.rados.0.smithi059.stdout:2233: writing smithi05955964-18 from 3457024 to 3792896 tid 4 2022-04-28T19:51:31.881 INFO:tasks.rados.rados.0.smithi059.stdout:2234: snap_create 2022-04-28T19:51:31.881 INFO:tasks.rados.rados.0.smithi059.stdout:2229: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:51:31.881 INFO:tasks.rados.rados.0.smithi059.stdout:2229: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:51:31.881 INFO:tasks.rados.rados.0.smithi059.stdout:2229: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:51:31.881 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1048 (ObjNum 634 snap 181 seq_num 634) dirty exists 2022-04-28T19:51:31.882 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 914 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:31.882 INFO:tasks.rados.rados.0.smithi059.stdout:2233: finishing write tid 1 to smithi05955964-18 2022-04-28T19:51:31.882 INFO:tasks.rados.rados.0.smithi059.stdout:2233: finishing write tid 2 to smithi05955964-18 2022-04-28T19:51:31.884 INFO:tasks.rados.rados.0.smithi059.stdout:2233: finishing write tid 3 to smithi05955964-18 2022-04-28T19:51:31.885 INFO:tasks.rados.rados.0.smithi059.stdout:2233: finishing write tid 4 to smithi05955964-18 2022-04-28T19:51:31.885 INFO:tasks.rados.rados.0.smithi059.stdout:2233: finishing write tid 6 to smithi05955964-18 2022-04-28T19:51:31.887 INFO:tasks.rados.rados.0.smithi059.stdout:2233: finishing write tid 7 to smithi05955964-18 2022-04-28T19:51:31.887 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 960 (ObjNum 715 snap 207 seq_num 715) dirty exists 2022-04-28T19:51:31.887 INFO:tasks.rados.rados.0.smithi059.stdout:2233: left oid 18 (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:51:31.888 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:31.629Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:31.888 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:31.629Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:31.888 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:31.629Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:51:32.181 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[59745]: cluster 2022-04-28T19:51:30.875450+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e472: 8 total, 7 up, 8 in 2022-04-28T19:51:32.181 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[59745]: cluster 2022-04-28T19:51:30.895039+0000 mgr.y (mgr.24961) 137 : cluster [DBG] pgmap v193: 89 pgs: 7 stale+active+clean, 82 active+clean; 285 MiB data, 556 MiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.5 MiB/s wr, 17 op/s 2022-04-28T19:51:32.181 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[59745]: cluster 2022-04-28T19:51:31.847196+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e473: 8 total, 7 up, 8 in 2022-04-28T19:51:32.182 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[61152]: cluster 2022-04-28T19:51:30.875450+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e472: 8 total, 7 up, 8 in 2022-04-28T19:51:32.182 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[61152]: cluster 2022-04-28T19:51:30.895039+0000 mgr.y (mgr.24961) 137 : cluster [DBG] pgmap v193: 89 pgs: 7 stale+active+clean, 82 active+clean; 285 MiB data, 556 MiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.5 MiB/s wr, 17 op/s 2022-04-28T19:51:32.182 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:31 smithi059 bash[61152]: cluster 2022-04-28T19:51:31.847196+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e473: 8 total, 7 up, 8 in 2022-04-28T19:51:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:31 smithi139 bash[43301]: cluster 2022-04-28T19:51:30.875450+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e472: 8 total, 7 up, 8 in 2022-04-28T19:51:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:31 smithi139 bash[43301]: cluster 2022-04-28T19:51:30.895039+0000 mgr.y (mgr.24961) 137 : cluster [DBG] pgmap v193: 89 pgs: 7 stale+active+clean, 82 active+clean; 285 MiB data, 556 MiB used, 709 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.5 MiB/s wr, 17 op/s 2022-04-28T19:51:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:31 smithi139 bash[43301]: cluster 2022-04-28T19:51:31.847196+0000 mon.a (mon.0) 454 : cluster [DBG] osdmap e473: 8 total, 7 up, 8 in 2022-04-28T19:51:32.591 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:32 smithi059 systemd[1]: Stopped Ceph osd.2 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:51:32.591 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:32 smithi059 systemd[1]: Started Ceph osd.2 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:51:32.896 INFO:tasks.rados.rados.0.smithi059.stdout:2229: done (5 left) 2022-04-28T19:51:32.896 INFO:tasks.rados.rados.0.smithi059.stdout:2230: done (4 left) 2022-04-28T19:51:32.896 INFO:tasks.rados.rados.0.smithi059.stdout:2231: done (3 left) 2022-04-28T19:51:32.896 INFO:tasks.rados.rados.0.smithi059.stdout:2232: done (2 left) 2022-04-28T19:51:32.897 INFO:tasks.rados.rados.0.smithi059.stdout:2233: done (1 left) 2022-04-28T19:51:32.897 INFO:tasks.rados.rados.0.smithi059.stdout:2234: done (0 left) 2022-04-28T19:51:32.897 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 208 2022-04-28T19:51:32.897 INFO:tasks.rados.rados.0.smithi059.stdout:2235: seq_num 716 ranges {5283840=794624,6078464=655360,6733824=704512,7438336=245760} 2022-04-28T19:51:32.935 INFO:tasks.rados.rados.0.smithi059.stdout:2235: writing smithi05955964-9 from 5283840 to 6078464 tid 1 2022-04-28T19:51:32.938 INFO:tasks.rados.rados.0.smithi059.stdout:2235: writing smithi05955964-9 from 6078464 to 6733824 tid 2 2022-04-28T19:51:32.942 INFO:tasks.rados.rados.0.smithi059.stdout:2235: writing smithi05955964-9 from 6733824 to 7438336 tid 3 2022-04-28T19:51:32.944 INFO:tasks.rados.rados.0.smithi059.stdout:2235: writing smithi05955964-9 from 7438336 to 7684096 tid 4 2022-04-28T19:51:32.944 INFO:tasks.rados.rados.0.smithi059.stdout:2236: read oid 18 snap -1 2022-04-28T19:51:32.944 INFO:tasks.rados.rados.0.smithi059.stdout:2236: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:51:32.944 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 208 2022-04-28T19:51:32.945 INFO:tasks.rados.rados.0.smithi059.stdout:2237: seq_num 717 ranges {1646592=16384} 2022-04-28T19:51:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:2237: writing smithi05955964-21 from 1646592 to 1662976 tid 1 2022-04-28T19:51:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:2238: snap_create 2022-04-28T19:51:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:2235: finishing write tid 1 to smithi05955964-9 2022-04-28T19:51:32.954 INFO:tasks.rados.rados.0.smithi059.stdout:2235: finishing write tid 2 to smithi05955964-9 2022-04-28T19:51:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:2235: finishing write tid 3 to smithi05955964-9 2022-04-28T19:51:32.956 INFO:tasks.rados.rados.0.smithi059.stdout:2237: finishing write tid 1 to smithi05955964-21 2022-04-28T19:51:32.957 INFO:tasks.rados.rados.0.smithi059.stdout:2237: finishing write tid 3 to smithi05955964-21 2022-04-28T19:51:32.959 INFO:tasks.rados.rados.0.smithi059.stdout:2237: finishing write tid 4 to smithi05955964-21 2022-04-28T19:51:32.959 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 720 (ObjNum 717 snap 208 seq_num 717) dirty exists 2022-04-28T19:51:32.959 INFO:tasks.rados.rados.0.smithi059.stdout:2237: left oid 21 (ObjNum 717 snap 208 seq_num 717) 2022-04-28T19:51:32.962 INFO:tasks.rados.rados.0.smithi059.stdout:2235: finishing write tid 4 to smithi05955964-9 2022-04-28T19:51:32.962 INFO:tasks.rados.rados.0.smithi059.stdout:2235: finishing write tid 6 to smithi05955964-9 2022-04-28T19:51:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:2236: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:51:33.085 INFO:tasks.rados.rados.0.smithi059.stdout:2235: finishing write tid 7 to smithi05955964-9 2022-04-28T19:51:33.085 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 728 (ObjNum 716 snap 208 seq_num 716) dirty exists 2022-04-28T19:51:33.086 INFO:tasks.rados.rados.0.smithi059.stdout:2235: left oid 9 (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:51:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:33.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:33.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:33.523Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: audit 2022-04-28T19:51:32.389545+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: audit 2022-04-28T19:51:32.390889+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: audit 2022-04-28T19:51:32.392406+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: audit 2022-04-28T19:51:32.393271+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: cephadm 2022-04-28T19:51:32.394116+0000 mgr.y (mgr.24961) 138 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: audit 2022-04-28T19:51:32.394871+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:51:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: cluster 2022-04-28T19:51:32.893679+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e474: 8 total, 7 up, 8 in 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[61152]: cluster 2022-04-28T19:51:32.895615+0000 mgr.y (mgr.24961) 139 : cluster [DBG] pgmap v196: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 311 MiB data, 573 MiB used, 709 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s; 51/456 objects degraded (11.184%) 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: audit 2022-04-28T19:51:32.389545+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: audit 2022-04-28T19:51:32.390889+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: audit 2022-04-28T19:51:32.392406+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: audit 2022-04-28T19:51:32.393271+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: cephadm 2022-04-28T19:51:32.394116+0000 mgr.y (mgr.24961) 138 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:51:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: audit 2022-04-28T19:51:32.394871+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:51:33.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: cluster 2022-04-28T19:51:32.893679+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e474: 8 total, 7 up, 8 in 2022-04-28T19:51:33.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[59745]: cluster 2022-04-28T19:51:32.895615+0000 mgr.y (mgr.24961) 139 : cluster [DBG] pgmap v196: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 311 MiB data, 573 MiB used, 709 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s; 51/456 objects degraded (11.184%) 2022-04-28T19:51:33.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: audit 2022-04-28T19:51:32.389545+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:33.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: audit 2022-04-28T19:51:32.390889+0000 mon.a (mon.0) 456 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:33.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: audit 2022-04-28T19:51:32.392406+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:51:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: audit 2022-04-28T19:51:32.393271+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:51:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: cephadm 2022-04-28T19:51:32.394116+0000 mgr.y (mgr.24961) 138 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:51:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: audit 2022-04-28T19:51:32.394871+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:51:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: cluster 2022-04-28T19:51:32.893679+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e474: 8 total, 7 up, 8 in 2022-04-28T19:51:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:33 smithi139 bash[43301]: cluster 2022-04-28T19:51:32.895615+0000 mgr.y (mgr.24961) 139 : cluster [DBG] pgmap v196: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 311 MiB data, 573 MiB used, 709 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s; 51/456 objects degraded (11.184%) 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2235: done (3 left) 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2236: done (2 left) 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2237: done (1 left) 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2238: done (0 left) 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2239: rmattr oid 11 current snap is 209 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2240: read oid 40 snap -1 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:2240: expect (ObjNum 660 snap 189 seq_num 660) 2022-04-28T19:51:33.898 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 209 2022-04-28T19:51:33.899 INFO:tasks.rados.rados.0.smithi059.stdout:2241: seq_num 718 ranges {0=32768} 2022-04-28T19:51:33.899 INFO:tasks.rados.rados.0.smithi059.stdout:2241: writing smithi05955964-47 from 0 to 32768 tid 1 2022-04-28T19:51:33.899 INFO:tasks.rados.rados.0.smithi059.stdout:2242: read oid 10 snap -1 2022-04-28T19:51:33.900 INFO:tasks.rados.rados.0.smithi059.stdout:2242: expect deleted 2022-04-28T19:51:33.900 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 209 2022-04-28T19:51:33.900 INFO:tasks.rados.rados.0.smithi059.stdout:2243: seq_num 719 ranges {5865472=49152} 2022-04-28T19:51:33.937 INFO:tasks.rados.rados.0.smithi059.stdout:2243: writing smithi05955964-25 from 5865472 to 5914624 tid 1 2022-04-28T19:51:33.937 INFO:tasks.rados.rados.0.smithi059.stdout:2244: snap_create 2022-04-28T19:51:33.937 INFO:tasks.rados.rados.0.smithi059.stdout:2241: finishing write tid 1 to smithi05955964-47 2022-04-28T19:51:33.937 INFO:tasks.rados.rados.0.smithi059.stdout:2241: finishing write tid 3 to smithi05955964-47 2022-04-28T19:51:33.937 INFO:tasks.rados.rados.0.smithi059.stdout:2240: expect (ObjNum 660 snap 189 seq_num 660) 2022-04-28T19:51:33.994 INFO:tasks.rados.rados.0.smithi059.stdout:2241: finishing write tid 4 to smithi05955964-47 2022-04-28T19:51:33.994 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1068 (ObjNum 718 snap 209 seq_num 718) dirty exists 2022-04-28T19:51:33.994 INFO:tasks.rados.rados.0.smithi059.stdout:2241: left oid 47 (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:51:33.994 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 825 (ObjNum 637 snap 183 seq_num 637) dirty exists 2022-04-28T19:51:33.994 INFO:tasks.rados.rados.0.smithi059.stdout:2243: finishing write tid 1 to smithi05955964-25 2022-04-28T19:51:33.995 INFO:tasks.rados.rados.0.smithi059.stdout:2243: finishing write tid 3 to smithi05955964-25 2022-04-28T19:51:33.995 INFO:tasks.rados.rados.0.smithi059.stdout:2243: finishing write tid 4 to smithi05955964-25 2022-04-28T19:51:33.995 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 988 (ObjNum 719 snap 209 seq_num 719) dirty exists 2022-04-28T19:51:33.995 INFO:tasks.rados.rados.0.smithi059.stdout:2243: left oid 25 (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2022-04-28T19:51:34.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:33 smithi059 bash[66806]: --> ceph-volume lvm activate successful for osd ID: 2 2022-04-28T19:51:34.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:34 smithi059 bash[61152]: cluster 2022-04-28T19:51:33.884914+0000 mon.a (mon.0) 461 : cluster [WRN] Health check failed: Degraded data redundancy: 51/456 objects degraded (11.184%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:34.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:34 smithi059 bash[61152]: cluster 2022-04-28T19:51:33.895652+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e475: 8 total, 7 up, 8 in 2022-04-28T19:51:34.865 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:34 smithi059 bash[59745]: cluster 2022-04-28T19:51:33.884914+0000 mon.a (mon.0) 461 : cluster [WRN] Health check failed: Degraded data redundancy: 51/456 objects degraded (11.184%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:34.865 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:34 smithi059 bash[59745]: cluster 2022-04-28T19:51:33.895652+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e475: 8 total, 7 up, 8 in 2022-04-28T19:51:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:34 smithi139 bash[43301]: cluster 2022-04-28T19:51:33.884914+0000 mon.a (mon.0) 461 : cluster [WRN] Health check failed: Degraded data redundancy: 51/456 objects degraded (11.184%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:34 smithi139 bash[43301]: cluster 2022-04-28T19:51:33.895652+0000 mon.a (mon.0) 462 : cluster [DBG] osdmap e475: 8 total, 7 up, 8 in 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2239: done (5 left) 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2240: done (4 left) 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2241: done (3 left) 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2242: done (2 left) 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2243: done (1 left) 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2244: done (0 left) 2022-04-28T19:51:34.902 INFO:tasks.rados.rados.0.smithi059.stdout:2245: delete oid 37 current snap is 210 2022-04-28T19:51:34.908 INFO:tasks.rados.rados.0.smithi059.stdout:2245: done (0 left) 2022-04-28T19:51:34.908 INFO:tasks.rados.rados.0.smithi059.stdout:2246: delete oid 8 current snap is 210 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2246: done (0 left) 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2247: read oid 49 snap 205 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2247: expect deleted 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2248: read oid 36 snap 206 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2248: expect deleted 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2249: copy_from oid 43 from oid 5 current snap is 210 2022-04-28T19:51:34.915 INFO:tasks.rados.rados.0.smithi059.stdout:2250: snap_remove snap 208 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2250: done (3 left) 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2251: copy_from oid 50 from oid 10 current snap is 210 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2249: finishing copy_from to smithi05955964-43 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2249: got expected ENOENT (src dne) 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2249: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2247: done (3 left) 2022-04-28T19:51:35.915 INFO:tasks.rados.rados.0.smithi059.stdout:2248: done (2 left) 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:2249: done (1 left) 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:2252: read oid 37 snap -1 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:2252: expect deleted 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 210 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:2253: seq_num 722 ranges {0=98304} 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:2253: writing smithi05955964-39 from 0 to 98304 tid 1 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 210 2022-04-28T19:51:35.916 INFO:tasks.rados.rados.0.smithi059.stdout:2254: seq_num 723 ranges {0=114688} 2022-04-28T19:51:35.917 INFO:tasks.rados.rados.0.smithi059.stdout:2254: writing smithi05955964-1 from 0 to 114688 tid 1 2022-04-28T19:51:35.917 INFO:tasks.rados.rados.0.smithi059.stdout:2255: delete oid 43 current snap is 210 2022-04-28T19:51:35.921 INFO:tasks.rados.rados.0.smithi059.stdout:2251: finishing copy_from to smithi05955964-50 2022-04-28T19:51:35.921 INFO:tasks.rados.rados.0.smithi059.stdout:2251: got expected ENOENT (src dne) 2022-04-28T19:51:35.921 INFO:tasks.rados.rados.0.smithi059.stdout:2251: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:51:35.925 INFO:tasks.rados.rados.0.smithi059.stdout:2254: finishing write tid 1 to smithi05955964-1 2022-04-28T19:51:35.925 INFO:tasks.rados.rados.0.smithi059.stdout:2253: finishing write tid 1 to smithi05955964-39 2022-04-28T19:51:35.925 INFO:tasks.rados.rados.0.smithi059.stdout:2254: finishing write tid 3 to smithi05955964-1 2022-04-28T19:51:35.925 INFO:tasks.rados.rados.0.smithi059.stdout:2253: finishing write tid 3 to smithi05955964-39 2022-04-28T19:51:35.926 INFO:tasks.rados.rados.0.smithi059.stdout:2251: done (4 left) 2022-04-28T19:51:35.926 INFO:tasks.rados.rados.0.smithi059.stdout:2252: done (3 left) 2022-04-28T19:51:35.926 INFO:tasks.rados.rados.0.smithi059.stdout:2255: done (2 left) 2022-04-28T19:51:35.926 INFO:tasks.rados.rados.0.smithi059.stdout:2256: setattr oid 7 current snap is 210 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:2253: finishing write tid 4 to smithi05955964-39 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1050 (ObjNum 722 snap 210 seq_num 722) dirty exists 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:2253: left oid 39 (ObjNum 722 snap 210 seq_num 722) 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:2253: done (2 left) 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:2257: read oid 5 snap -1 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:2257: expect deleted 2022-04-28T19:51:35.928 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 210 2022-04-28T19:51:35.929 INFO:tasks.rados.rados.0.smithi059.stdout:2258: seq_num 725 ranges {0=147456} 2022-04-28T19:51:35.929 INFO:tasks.rados.rados.0.smithi059.stdout:2258: writing smithi05955964-8 from 0 to 147456 tid 1 2022-04-28T19:51:35.929 INFO:tasks.rados.rados.0.smithi059.stdout:2259: delete oid 43 current snap is 210 2022-04-28T19:51:35.929 INFO:tasks.rados.rados.0.smithi059.stdout:2254: finishing write tid 4 to smithi05955964-1 2022-04-28T19:51:35.930 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 937 (ObjNum 723 snap 210 seq_num 723) dirty exists 2022-04-28T19:51:35.930 INFO:tasks.rados.rados.0.smithi059.stdout:2254: left oid 1 (ObjNum 723 snap 210 seq_num 723) 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 917 (ObjNum 672 snap 195 seq_num 672) dirty exists 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:2254: done (4 left) 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:2256: done (3 left) 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:2257: done (2 left) 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:2259: done (1 left) 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:2260: read oid 27 snap -1 2022-04-28T19:51:35.932 INFO:tasks.rados.rados.0.smithi059.stdout:2260: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:51:35.933 INFO:tasks.rados.rados.0.smithi059.stdout:2261: rollback oid 41 current snap is 210 2022-04-28T19:51:35.933 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 202 2022-04-28T19:51:35.933 INFO:tasks.rados.rados.0.smithi059.stdout:2262: snap_create 2022-04-28T19:51:35.936 INFO:tasks.rados.rados.0.smithi059.stdout:2258: finishing write tid 1 to smithi05955964-8 2022-04-28T19:51:35.936 INFO:tasks.rados.rados.0.smithi059.stdout:2261: finishing rollback tid 0 to smithi05955964-41 2022-04-28T19:51:35.936 INFO:tasks.rados.rados.0.smithi059.stdout:2258: finishing write tid 3 to smithi05955964-8 2022-04-28T19:51:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:2261: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:51:35.937 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 819 (ObjNum 1718315609 snap 1397711713 seq_num 1952865363) dirty dne 2022-04-28T19:51:35.939 INFO:tasks.rados.rados.0.smithi059.stdout:2258: finishing write tid 4 to smithi05955964-8 2022-04-28T19:51:35.939 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 996 (ObjNum 725 snap 210 seq_num 725) dirty exists 2022-04-28T19:51:35.939 INFO:tasks.rados.rados.0.smithi059.stdout:2258: left oid 8 (ObjNum 725 snap 210 seq_num 725) 2022-04-28T19:51:35.946 INFO:tasks.rados.rados.0.smithi059.stdout:2260: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:51:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:35 smithi059 bash[61152]: cluster 2022-04-28T19:51:34.896612+0000 mgr.y (mgr.24961) 140 : cluster [DBG] pgmap v198: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 311 MiB data, 573 MiB used, 709 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s; 51/456 objects degraded (11.184%) 2022-04-28T19:51:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:35 smithi059 bash[61152]: cluster 2022-04-28T19:51:34.898664+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e476: 8 total, 7 up, 8 in 2022-04-28T19:51:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:35 smithi059 bash[59745]: cluster 2022-04-28T19:51:34.896612+0000 mgr.y (mgr.24961) 140 : cluster [DBG] pgmap v198: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 311 MiB data, 573 MiB used, 709 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s; 51/456 objects degraded (11.184%) 2022-04-28T19:51:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:35 smithi059 bash[59745]: cluster 2022-04-28T19:51:34.898664+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e476: 8 total, 7 up, 8 in 2022-04-28T19:51:36.092 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:35 smithi059 bash[66806]: debug 2022-04-28T19:51:35.872+0000 7fb70dbd5700 1 -- 172.21.15.59:0/3100206218 <== mon.0 v2:172.21.15.59:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f13b0564e0 con 0x55f13b0cf000 2022-04-28T19:51:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:35 smithi139 bash[43301]: cluster 2022-04-28T19:51:34.896612+0000 mgr.y (mgr.24961) 140 : cluster [DBG] pgmap v198: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 311 MiB data, 573 MiB used, 709 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s; 51/456 objects degraded (11.184%) 2022-04-28T19:51:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:35 smithi139 bash[43301]: cluster 2022-04-28T19:51:34.898664+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e476: 8 total, 7 up, 8 in 2022-04-28T19:51:36.841 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:36 smithi059 bash[66806]: debug 2022-04-28T19:51:36.456+0000 7fb715430200 -1 Falling back to public interface 2022-04-28T19:51:36.867 INFO:tasks.rados.rados.0.smithi059.stdout:2258: done (3 left) 2022-04-28T19:51:36.867 INFO:tasks.rados.rados.0.smithi059.stdout:2260: done (2 left) 2022-04-28T19:51:36.867 INFO:tasks.rados.rados.0.smithi059.stdout:2261: done (1 left) 2022-04-28T19:51:36.867 INFO:tasks.rados.rados.0.smithi059.stdout:2262: done (0 left) 2022-04-28T19:51:36.867 INFO:tasks.rados.rados.0.smithi059.stdout:2263: copy_from oid 16 from oid 7 current snap is 211 2022-04-28T19:51:36.868 INFO:tasks.rados.rados.0.smithi059.stdout:2264: read oid 14 snap -1 2022-04-28T19:51:36.868 INFO:tasks.rados.rados.0.smithi059.stdout:2264: expect deleted 2022-04-28T19:51:36.868 INFO:tasks.rados.rados.0.smithi059.stdout:2265: delete oid 40 current snap is 211 2022-04-28T19:51:36.871 INFO:tasks.rados.rados.0.smithi059.stdout:2264: done (2 left) 2022-04-28T19:51:36.871 INFO:tasks.rados.rados.0.smithi059.stdout:2265: done (1 left) 2022-04-28T19:51:36.871 INFO:tasks.rados.rados.0.smithi059.stdout:2266: snap_remove snap 196 2022-04-28T19:51:37.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:51:37 smithi059 bash[57771]: debug 2022-04-28T19:51:37.024+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2022-04-28T19:51:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:36 smithi059 bash[59745]: cluster 2022-04-28T19:51:35.911300+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e477: 8 total, 7 up, 8 in 2022-04-28T19:51:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:36 smithi059 bash[59745]: cluster 2022-04-28T19:51:36.865007+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e478: 8 total, 7 up, 8 in 2022-04-28T19:51:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:36 smithi059 bash[61152]: cluster 2022-04-28T19:51:35.911300+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e477: 8 total, 7 up, 8 in 2022-04-28T19:51:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:36 smithi059 bash[61152]: cluster 2022-04-28T19:51:36.865007+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e478: 8 total, 7 up, 8 in 2022-04-28T19:51:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:36 smithi139 bash[43301]: cluster 2022-04-28T19:51:35.911300+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e477: 8 total, 7 up, 8 in 2022-04-28T19:51:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:36 smithi139 bash[43301]: cluster 2022-04-28T19:51:36.865007+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e478: 8 total, 7 up, 8 in 2022-04-28T19:51:37.887 INFO:tasks.rados.rados.0.smithi059.stdout:2266: done (1 left) 2022-04-28T19:51:37.887 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 211 2022-04-28T19:51:37.887 INFO:tasks.rados.rados.0.smithi059.stdout:2267: seq_num 728 ranges {13910016=196608} 2022-04-28T19:51:37.967 INFO:tasks.rados.rados.0.smithi059.stdout:2267: writing smithi05955964-6 from 13910016 to 14106624 tid 1 2022-04-28T19:51:37.968 INFO:tasks.rados.rados.0.smithi059.stdout:2268: snap_remove snap 201 2022-04-28T19:51:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: cluster 2022-04-28T19:51:36.897244+0000 mgr.y (mgr.24961) 141 : cluster [DBG] pgmap v202: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 312 MiB data, 580 MiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s rd, 32 KiB/s wr, 13 op/s; 53/468 objects degraded (11.325%) 2022-04-28T19:51:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.006453+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.012016+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.021707+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.024198+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.025718+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.026966+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.028844+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: audit 2022-04-28T19:51:37.029099+0000 mgr.y (mgr.24961) 142 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: cephadm 2022-04-28T19:51:37.030413+0000 mgr.y (mgr.24961) 143 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2022-04-28T19:51:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[59745]: cluster 2022-04-28T19:51:37.884956+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e479: 8 total, 7 up, 8 in 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: cluster 2022-04-28T19:51:36.897244+0000 mgr.y (mgr.24961) 141 : cluster [DBG] pgmap v202: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 312 MiB data, 580 MiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s rd, 32 KiB/s wr, 13 op/s; 53/468 objects degraded (11.325%) 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.006453+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.012016+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.021707+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.024198+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.025718+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.026966+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.028844+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: audit 2022-04-28T19:51:37.029099+0000 mgr.y (mgr.24961) 142 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: cephadm 2022-04-28T19:51:37.030413+0000 mgr.y (mgr.24961) 143 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2022-04-28T19:51:38.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:38 smithi059 bash[61152]: cluster 2022-04-28T19:51:37.884956+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e479: 8 total, 7 up, 8 in 2022-04-28T19:51:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: cluster 2022-04-28T19:51:36.897244+0000 mgr.y (mgr.24961) 141 : cluster [DBG] pgmap v202: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 312 MiB data, 580 MiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s rd, 32 KiB/s wr, 13 op/s; 53/468 objects degraded (11.325%) 2022-04-28T19:51:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.006453+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.012016+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.021707+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.024198+0000 mon.a (mon.0) 469 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.025718+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.026966+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.028844+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: audit 2022-04-28T19:51:37.029099+0000 mgr.y (mgr.24961) 142 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: cephadm 2022-04-28T19:51:37.030413+0000 mgr.y (mgr.24961) 143 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2022-04-28T19:51:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:38 smithi139 bash[43301]: cluster 2022-04-28T19:51:37.884956+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e479: 8 total, 7 up, 8 in 2022-04-28T19:51:38.894 INFO:tasks.rados.rados.0.smithi059.stdout:2268: done (2 left) 2022-04-28T19:51:38.894 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 211 2022-04-28T19:51:38.894 INFO:tasks.rados.rados.0.smithi059.stdout:2269: seq_num 729 ranges {1196032=212992} 2022-04-28T19:51:38.903 INFO:tasks.rados.rados.0.smithi059.stdout:2269: writing smithi05955964-26 from 1196032 to 1409024 tid 1 2022-04-28T19:51:38.903 INFO:tasks.rados.rados.0.smithi059.stdout:2270: copy_from oid 35 from oid 30 current snap is 211 2022-04-28T19:51:38.903 INFO:tasks.rados.rados.0.smithi059.stdout:2263: finishing copy_from to smithi05955964-16 2022-04-28T19:51:38.903 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 917 (ObjNum 672 snap 195 seq_num 672) dirty exists 2022-04-28T19:51:38.903 INFO:tasks.rados.rados.0.smithi059.stdout:2263: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:51:38.903 INFO:tasks.rados.rados.0.smithi059.stdout:2267: finishing write tid 1 to smithi05955964-6 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2267: finishing write tid 3 to smithi05955964-6 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2267: finishing write tid 4 to smithi05955964-6 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 917 (ObjNum 728 snap 211 seq_num 728) dirty exists 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2267: left oid 6 (ObjNum 728 snap 211 seq_num 728) 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2263: done (3 left) 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2267: done (2 left) 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2271: rollback oid 48 current snap is 211 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 210 2022-04-28T19:51:38.904 INFO:tasks.rados.rados.0.smithi059.stdout:2272: delete oid 22 current snap is 211 2022-04-28T19:51:38.910 INFO:tasks.rados.rados.0.smithi059.stdout:2269: finishing write tid 1 to smithi05955964-26 2022-04-28T19:51:38.910 INFO:tasks.rados.rados.0.smithi059.stdout:2269: finishing write tid 3 to smithi05955964-26 2022-04-28T19:51:38.911 INFO:tasks.rados.rados.0.smithi059.stdout:2271: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:51:38.911 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 988 (ObjNum 0 snap 21893 seq_num 4161878160) dirty dne 2022-04-28T19:51:38.914 INFO:tasks.rados.rados.0.smithi059.stdout:2271: done (3 left) 2022-04-28T19:51:38.914 INFO:tasks.rados.rados.0.smithi059.stdout:2272: done (2 left) 2022-04-28T19:51:38.914 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 211 2022-04-28T19:51:38.914 INFO:tasks.rados.rados.0.smithi059.stdout:2273: seq_num 731 ranges {9412608=245760} 2022-04-28T19:51:38.971 INFO:tasks.rados.rados.0.smithi059.stdout:2273: writing smithi05955964-17 from 9412608 to 9658368 tid 1 2022-04-28T19:51:38.971 INFO:tasks.rados.rados.0.smithi059.stdout:2274: copy_from oid 43 from oid 10 current snap is 211 2022-04-28T19:51:38.971 INFO:tasks.rados.rados.0.smithi059.stdout:2269: finishing write tid 4 to smithi05955964-26 2022-04-28T19:51:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1037 (ObjNum 729 snap 211 seq_num 729) dirty exists 2022-04-28T19:51:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:2269: left oid 26 (ObjNum 729 snap 211 seq_num 729) 2022-04-28T19:51:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:2270: finishing copy_from to smithi05955964-35 2022-04-28T19:51:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 932 (ObjNum 669 snap 194 seq_num 669) dirty exists 2022-04-28T19:51:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:2270: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:51:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2269: done (3 left) 2022-04-28T19:51:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2270: done (2 left) 2022-04-28T19:51:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2275: snap_remove snap 209 2022-04-28T19:51:39.896 INFO:tasks.rados.rados.0.smithi059.stdout:2275: done (2 left) 2022-04-28T19:51:39.896 INFO:tasks.rados.rados.0.smithi059.stdout:2276: read oid 33 snap -1 2022-04-28T19:51:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:2276: expect deleted 2022-04-28T19:51:39.897 INFO:tasks.rados.rados.0.smithi059.stdout:2277: snap_remove snap 210 2022-04-28T19:51:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[61152]: cluster 2022-04-28T19:51:38.893093+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e480: 8 total, 7 up, 8 in 2022-04-28T19:51:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[61152]: cluster 2022-04-28T19:51:38.899959+0000 mgr.y (mgr.24961) 144 : cluster [DBG] pgmap v205: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 325 MiB data, 588 MiB used, 709 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 27 op/s; 56/492 objects degraded (11.382%) 2022-04-28T19:51:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[61152]: audit 2022-04-28T19:51:39.881720+0000 mon.a (mon.0) 475 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:51:40.341 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[66806]: debug 2022-04-28T19:51:39.872+0000 7fb715430200 -1 osd.2 470 log_to_monitors {default=true} 2022-04-28T19:51:40.342 INFO:journalctl@ceph.osd.2.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[66806]: debug 2022-04-28T19:51:39.900+0000 7fb704345700 -1 osd.2 470 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:51:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[59745]: cluster 2022-04-28T19:51:38.893093+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e480: 8 total, 7 up, 8 in 2022-04-28T19:51:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[59745]: cluster 2022-04-28T19:51:38.899959+0000 mgr.y (mgr.24961) 144 : cluster [DBG] pgmap v205: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 325 MiB data, 588 MiB used, 709 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 27 op/s; 56/492 objects degraded (11.382%) 2022-04-28T19:51:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:39 smithi059 bash[59745]: audit 2022-04-28T19:51:39.881720+0000 mon.a (mon.0) 475 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:51:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:39 smithi139 bash[43301]: cluster 2022-04-28T19:51:38.893093+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e480: 8 total, 7 up, 8 in 2022-04-28T19:51:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:39 smithi139 bash[43301]: cluster 2022-04-28T19:51:38.899959+0000 mgr.y (mgr.24961) 144 : cluster [DBG] pgmap v205: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 325 MiB data, 588 MiB used, 709 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 27 op/s; 56/492 objects degraded (11.382%) 2022-04-28T19:51:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:39 smithi139 bash[43301]: audit 2022-04-28T19:51:39.881720+0000 mon.a (mon.0) 475 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2022-04-28T19:51:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:2274: finishing copy_from to smithi05955964-43 2022-04-28T19:51:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:2274: got expected ENOENT (src dne) 2022-04-28T19:51:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:2274: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:51:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:2273: finishing write tid 1 to smithi05955964-17 2022-04-28T19:51:40.904 INFO:tasks.rados.rados.0.smithi059.stdout:2273: finishing write tid 3 to smithi05955964-17 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2273: finishing write tid 4 to smithi05955964-17 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 875 (ObjNum 731 snap 211 seq_num 731) dirty exists 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2273: left oid 17 (ObjNum 731 snap 211 seq_num 731) 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2273: done (3 left) 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2274: done (2 left) 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2276: done (1 left) 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2277: done (0 left) 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2278: read oid 47 snap -1 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2278: expect (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:51:40.905 INFO:tasks.rados.rados.0.smithi059.stdout:2279: rollback oid 38 current snap is 211 2022-04-28T19:51:40.906 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 206 2022-04-28T19:51:40.906 INFO:tasks.rados.rados.0.smithi059.stdout:2280: copy_from oid 39 from oid 9 current snap is 211 2022-04-28T19:51:40.906 INFO:tasks.rados.rados.0.smithi059.stdout:2281: snap_remove snap 205 2022-04-28T19:51:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[59745]: cluster 2022-04-28T19:51:39.883375+0000 mon.a (mon.0) 476 : cluster [WRN] Health check update: Degraded data redundancy: 56/492 objects degraded (11.382%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[59745]: audit 2022-04-28T19:51:39.893280+0000 mon.a (mon.0) 477 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-04-28T19:51:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[59745]: cluster 2022-04-28T19:51:39.893392+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e481: 8 total, 7 up, 8 in 2022-04-28T19:51:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[59745]: audit 2022-04-28T19:51:39.894414+0000 mon.a (mon.0) 479 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[61152]: cluster 2022-04-28T19:51:39.883375+0000 mon.a (mon.0) 476 : cluster [WRN] Health check update: Degraded data redundancy: 56/492 objects degraded (11.382%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[61152]: audit 2022-04-28T19:51:39.893280+0000 mon.a (mon.0) 477 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-04-28T19:51:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[61152]: cluster 2022-04-28T19:51:39.893392+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e481: 8 total, 7 up, 8 in 2022-04-28T19:51:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:40 smithi059 bash[61152]: audit 2022-04-28T19:51:39.894414+0000 mon.a (mon.0) 479 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:40 smithi139 bash[43301]: cluster 2022-04-28T19:51:39.883375+0000 mon.a (mon.0) 476 : cluster [WRN] Health check update: Degraded data redundancy: 56/492 objects degraded (11.382%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:40 smithi139 bash[43301]: audit 2022-04-28T19:51:39.893280+0000 mon.a (mon.0) 477 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2022-04-28T19:51:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:40 smithi139 bash[43301]: cluster 2022-04-28T19:51:39.893392+0000 mon.a (mon.0) 478 : cluster [DBG] osdmap e481: 8 total, 7 up, 8 in 2022-04-28T19:51:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:40 smithi139 bash[43301]: audit 2022-04-28T19:51:39.894414+0000 mon.a (mon.0) 479 : audit [INF] from='osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:51:41.897 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:41.630Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:41.897 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:41.630Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:41.898 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:41.630Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:51:41.921 INFO:tasks.rados.rados.0.smithi059.stdout:2281: done (3 left) 2022-04-28T19:51:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:2282: read oid 34 snap -1 2022-04-28T19:51:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:2282: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:51:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:2283: rmattr oid 45 current snap is 211 2022-04-28T19:51:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:2279: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:51:41.922 INFO:tasks.rados.rados.0.smithi059.stdout:2279: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:51:41.923 INFO:tasks.rados.rados.0.smithi059.stdout:2279: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:51:41.923 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 829 (ObjNum 654 snap 187 seq_num 654) dirty exists 2022-04-28T19:51:41.923 INFO:tasks.rados.rados.0.smithi059.stdout:2279: done (4 left) 2022-04-28T19:51:41.923 INFO:tasks.rados.rados.0.smithi059.stdout:2284: snap_remove snap 202 2022-04-28T19:51:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[59745]: cluster 2022-04-28T19:51:40.891373+0000 mon.a (mon.0) 480 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:51:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[59745]: cluster 2022-04-28T19:51:40.900426+0000 mgr.y (mgr.24961) 145 : cluster [DBG] pgmap v208: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 325 MiB data, 602 MiB used, 710 GiB / 715 GiB avail; 4.4 MiB/s rd, 3.3 MiB/s wr, 13 op/s; 56/492 objects degraded (11.382%) 2022-04-28T19:51:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[59745]: cluster 2022-04-28T19:51:40.902938+0000 mon.a (mon.0) 481 : cluster [INF] osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974] boot 2022-04-28T19:51:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[59745]: cluster 2022-04-28T19:51:40.903034+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2022-04-28T19:51:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[59745]: audit 2022-04-28T19:51:40.903431+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:51:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[61152]: cluster 2022-04-28T19:51:40.891373+0000 mon.a (mon.0) 480 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:51:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[61152]: cluster 2022-04-28T19:51:40.900426+0000 mgr.y (mgr.24961) 145 : cluster [DBG] pgmap v208: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 325 MiB data, 602 MiB used, 710 GiB / 715 GiB avail; 4.4 MiB/s rd, 3.3 MiB/s wr, 13 op/s; 56/492 objects degraded (11.382%) 2022-04-28T19:51:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[61152]: cluster 2022-04-28T19:51:40.902938+0000 mon.a (mon.0) 481 : cluster [INF] osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974] boot 2022-04-28T19:51:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[61152]: cluster 2022-04-28T19:51:40.903034+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2022-04-28T19:51:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:41 smithi059 bash[61152]: audit 2022-04-28T19:51:40.903431+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:51:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:41 smithi139 bash[43301]: cluster 2022-04-28T19:51:40.891373+0000 mon.a (mon.0) 480 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:51:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:41 smithi139 bash[43301]: cluster 2022-04-28T19:51:40.900426+0000 mgr.y (mgr.24961) 145 : cluster [DBG] pgmap v208: 89 pgs: 8 active+undersized+degraded, 15 active+undersized, 66 active+clean; 325 MiB data, 602 MiB used, 710 GiB / 715 GiB avail; 4.4 MiB/s rd, 3.3 MiB/s wr, 13 op/s; 56/492 objects degraded (11.382%) 2022-04-28T19:51:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:41 smithi139 bash[43301]: cluster 2022-04-28T19:51:40.902938+0000 mon.a (mon.0) 481 : cluster [INF] osd.2 [v2:172.21.15.59:6818/3675900974,v1:172.21.15.59:6819/3675900974] boot 2022-04-28T19:51:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:41 smithi139 bash[43301]: cluster 2022-04-28T19:51:40.903034+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2022-04-28T19:51:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:41 smithi139 bash[43301]: audit 2022-04-28T19:51:40.903431+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2022-04-28T19:51:42.918 INFO:tasks.rados.rados.0.smithi059.stdout:2284: done (4 left) 2022-04-28T19:51:42.918 INFO:tasks.rados.rados.0.smithi059.stdout:2285: copy_from oid 36 from oid 32 current snap is 211 2022-04-28T19:51:42.919 INFO:tasks.rados.rados.0.smithi059.stdout:2278: expect (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:51:42.923 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 919 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:42.923 INFO:tasks.rados.rados.0.smithi059.stdout:2282: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:51:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:42 smithi059 bash[61152]: cluster 2022-04-28T19:51:41.914895+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2022-04-28T19:51:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:42 smithi059 bash[61152]: audit 2022-04-28T19:51:42.232448+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:42 smithi059 bash[59745]: cluster 2022-04-28T19:51:41.914895+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2022-04-28T19:51:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:42 smithi059 bash[59745]: audit 2022-04-28T19:51:42.232448+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:43.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:42 smithi139 bash[43301]: cluster 2022-04-28T19:51:41.914895+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2022-04-28T19:51:43.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:42 smithi139 bash[43301]: audit 2022-04-28T19:51:42.232448+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:43.765 INFO:tasks.rados.rados.0.smithi059.stdout:2280: finishing copy_from to smithi05955964-39 2022-04-28T19:51:43.779 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 728 (ObjNum 716 snap 208 seq_num 716) dirty exists 2022-04-28T19:51:43.779 INFO:tasks.rados.rados.0.smithi059.stdout:2280: finishing copy_from racing read to smithi05955964-39 2022-04-28T19:51:43.779 INFO:tasks.rados.rados.0.smithi059.stdout:2278: done (4 left) 2022-04-28T19:51:43.779 INFO:tasks.rados.rados.0.smithi059.stdout:2280: done (3 left) 2022-04-28T19:51:43.779 INFO:tasks.rados.rados.0.smithi059.stdout:2282: done (2 left) 2022-04-28T19:51:43.780 INFO:tasks.rados.rados.0.smithi059.stdout:2283: done (1 left) 2022-04-28T19:51:43.780 INFO:tasks.rados.rados.0.smithi059.stdout:2286: snap_create 2022-04-28T19:51:43.790 INFO:tasks.rados.rados.0.smithi059.stdout:2285: finishing copy_from to smithi05955964-36 2022-04-28T19:51:43.790 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1024 (ObjNum 703 snap 203 seq_num 703) dirty exists 2022-04-28T19:51:43.790 INFO:tasks.rados.rados.0.smithi059.stdout:2285: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:51:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:43.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:43.523Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:43.523Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:51:44.243 INFO:tasks.rados.rados.0.smithi059.stdout:2285: done (1 left) 2022-04-28T19:51:44.243 INFO:tasks.rados.rados.0.smithi059.stdout:2286: done (0 left) 2022-04-28T19:51:44.243 INFO:tasks.rados.rados.0.smithi059.stdout:2287: snap_create 2022-04-28T19:51:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[59745]: cluster 2022-04-28T19:51:42.901123+0000 mgr.y (mgr.24961) 146 : cluster [DBG] pgmap v210: 89 pgs: 6 peering, 5 active+undersized+degraded, 2 active+undersized, 76 active+clean; 331 MiB data, 627 MiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s; 38/486 objects degraded (7.819%); 6.4 MiB/s, 1 objects/s recovering 2022-04-28T19:51:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[59745]: cluster 2022-04-28T19:51:42.914495+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2022-04-28T19:51:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[61152]: cluster 2022-04-28T19:51:42.901123+0000 mgr.y (mgr.24961) 146 : cluster [DBG] pgmap v210: 89 pgs: 6 peering, 5 active+undersized+degraded, 2 active+undersized, 76 active+clean; 331 MiB data, 627 MiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s; 38/486 objects degraded (7.819%); 6.4 MiB/s, 1 objects/s recovering 2022-04-28T19:51:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:43 smithi059 bash[61152]: cluster 2022-04-28T19:51:42.914495+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2022-04-28T19:51:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:43 smithi139 bash[43301]: cluster 2022-04-28T19:51:42.901123+0000 mgr.y (mgr.24961) 146 : cluster [DBG] pgmap v210: 89 pgs: 6 peering, 5 active+undersized+degraded, 2 active+undersized, 76 active+clean; 331 MiB data, 627 MiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s; 38/486 objects degraded (7.819%); 6.4 MiB/s, 1 objects/s recovering 2022-04-28T19:51:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:43 smithi139 bash[43301]: cluster 2022-04-28T19:51:42.914495+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2022-04-28T19:51:45.252 INFO:tasks.rados.rados.0.smithi059.stdout:2287: done (0 left) 2022-04-28T19:51:45.295 INFO:tasks.rados.rados.0.smithi059.stdout:2288: snap_remove snap 206 2022-04-28T19:51:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:45 smithi059 bash[59745]: cluster 2022-04-28T19:51:44.240811+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2022-04-28T19:51:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:45 smithi059 bash[61152]: cluster 2022-04-28T19:51:44.240811+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2022-04-28T19:51:45.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:45 smithi139 bash[43301]: cluster 2022-04-28T19:51:44.240811+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e485: 8 total, 8 up, 8 in 2022-04-28T19:51:46.259 INFO:tasks.rados.rados.0.smithi059.stdout:2288: done (0 left) 2022-04-28T19:51:46.260 INFO:tasks.rados.rados.0.smithi059.stdout:2289: rmattr oid 25 current snap is 213 2022-04-28T19:51:46.260 INFO:tasks.rados.rados.0.smithi059.stdout:2290: read oid 26 snap -1 2022-04-28T19:51:46.260 INFO:tasks.rados.rados.0.smithi059.stdout:2290: expect (ObjNum 729 snap 211 seq_num 729) 2022-04-28T19:51:46.260 INFO:tasks.rados.rados.0.smithi059.stdout:2291: delete oid 30 current snap is 213 2022-04-28T19:51:46.272 INFO:tasks.rados.rados.0.smithi059.stdout:2291: done (2 left) 2022-04-28T19:51:46.272 INFO:tasks.rados.rados.0.smithi059.stdout:2292: rollback oid 50 current snap is 213 2022-04-28T19:51:46.272 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 211 2022-04-28T19:51:46.272 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 0 (ObjNum 719 snap 209 seq_num 719) dirty exists 2022-04-28T19:51:46.272 INFO:tasks.rados.rados.0.smithi059.stdout:2289: done (2 left) 2022-04-28T19:51:46.272 INFO:tasks.rados.rados.0.smithi059.stdout:2293: snap_remove snap 207 2022-04-28T19:51:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:46 smithi059 bash[61152]: cluster 2022-04-28T19:51:44.902018+0000 mgr.y (mgr.24961) 147 : cluster [DBG] pgmap v213: 89 pgs: 6 peering, 5 active+undersized+degraded, 2 active+undersized, 76 active+clean; 331 MiB data, 627 MiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s; 38/486 objects degraded (7.819%); 6.4 MiB/s, 1 objects/s recovering 2022-04-28T19:51:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:46 smithi059 bash[61152]: cluster 2022-04-28T19:51:45.251844+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2022-04-28T19:51:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:46 smithi059 bash[59745]: cluster 2022-04-28T19:51:44.902018+0000 mgr.y (mgr.24961) 147 : cluster [DBG] pgmap v213: 89 pgs: 6 peering, 5 active+undersized+degraded, 2 active+undersized, 76 active+clean; 331 MiB data, 627 MiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s; 38/486 objects degraded (7.819%); 6.4 MiB/s, 1 objects/s recovering 2022-04-28T19:51:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:46 smithi059 bash[59745]: cluster 2022-04-28T19:51:45.251844+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2022-04-28T19:51:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:46 smithi139 bash[43301]: cluster 2022-04-28T19:51:44.902018+0000 mgr.y (mgr.24961) 147 : cluster [DBG] pgmap v213: 89 pgs: 6 peering, 5 active+undersized+degraded, 2 active+undersized, 76 active+clean; 331 MiB data, 627 MiB used, 710 GiB / 715 GiB avail; 8.4 MiB/s rd, 2.3 MiB/s wr, 4 op/s; 38/486 objects degraded (7.819%); 6.4 MiB/s, 1 objects/s recovering 2022-04-28T19:51:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:46 smithi139 bash[43301]: cluster 2022-04-28T19:51:45.251844+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e486: 8 total, 8 up, 8 in 2022-04-28T19:51:47.276 INFO:tasks.rados.rados.0.smithi059.stdout:2293: done (2 left) 2022-04-28T19:51:47.276 INFO:tasks.rados.rados.0.smithi059.stdout:2294: copy_from oid 36 from oid 44 current snap is 213 2022-04-28T19:51:47.276 INFO:tasks.rados.rados.0.smithi059.stdout:2290: expect (ObjNum 729 snap 211 seq_num 729) 2022-04-28T19:51:47.338 INFO:tasks.rados.rados.0.smithi059.stdout:2292: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:51:47.338 INFO:tasks.rados.rados.0.smithi059.stdout:2292: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:51:47.338 INFO:tasks.rados.rados.0.smithi059.stdout:2292: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:51:47.338 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1084 (ObjNum 634 snap 181 seq_num 634) dirty exists 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2294: finishing copy_from to smithi05955964-36 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 712 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2294: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2290: done (2 left) 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2292: done (1 left) 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2294: done (0 left) 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2295: read oid 14 snap -1 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2295: expect deleted 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 213 2022-04-28T19:51:47.339 INFO:tasks.rados.rados.0.smithi059.stdout:2296: seq_num 733 ranges {0=278528} 2022-04-28T19:51:47.342 INFO:tasks.rados.rados.0.smithi059.stdout:2296: writing smithi05955964-10 from 0 to 278528 tid 1 2022-04-28T19:51:47.342 INFO:tasks.rados.rados.0.smithi059.stdout:2297: snap_remove snap 212 2022-04-28T19:51:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:47 smithi059 bash[61152]: cluster 2022-04-28T19:51:46.255693+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2022-04-28T19:51:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:47 smithi059 bash[61152]: cluster 2022-04-28T19:51:46.860677+0000 mon.a (mon.0) 490 : cluster [WRN] Health check update: Degraded data redundancy: 38/486 objects degraded (7.819%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:47 smithi059 bash[59745]: cluster 2022-04-28T19:51:46.255693+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2022-04-28T19:51:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:47 smithi059 bash[59745]: cluster 2022-04-28T19:51:46.860677+0000 mon.a (mon.0) 490 : cluster [WRN] Health check update: Degraded data redundancy: 38/486 objects degraded (7.819%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:47 smithi139 bash[43301]: cluster 2022-04-28T19:51:46.255693+0000 mon.a (mon.0) 489 : cluster [DBG] osdmap e487: 8 total, 8 up, 8 in 2022-04-28T19:51:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:47 smithi139 bash[43301]: cluster 2022-04-28T19:51:46.860677+0000 mon.a (mon.0) 490 : cluster [WRN] Health check update: Degraded data redundancy: 38/486 objects degraded (7.819%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:51:48.290 INFO:tasks.rados.rados.0.smithi059.stdout:2297: done (2 left) 2022-04-28T19:51:48.339 INFO:tasks.rados.rados.0.smithi059.stdout:2298: read oid 27 snap -1 2022-04-28T19:51:48.340 INFO:tasks.rados.rados.0.smithi059.stdout:2298: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:51:48.340 INFO:tasks.rados.rados.0.smithi059.stdout:2299: rmattr oid 45 current snap is 213 2022-04-28T19:51:48.340 INFO:tasks.rados.rados.0.smithi059.stdout:2296: finishing write tid 1 to smithi05955964-10 2022-04-28T19:51:48.340 INFO:tasks.rados.rados.0.smithi059.stdout:2295: done (3 left) 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2300: copy_from oid 36 from oid 12 current snap is 213 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2296: finishing write tid 3 to smithi05955964-10 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2296: finishing write tid 4 to smithi05955964-10 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1070 (ObjNum 733 snap 213 seq_num 733) dirty exists 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2296: left oid 10 (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2296: done (3 left) 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2301: rollback oid 47 current snap is 213 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 211 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2302: rollback oid 41 current snap is 213 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 199 2022-04-28T19:51:48.341 INFO:tasks.rados.rados.0.smithi059.stdout:2303: snap_create 2022-04-28T19:51:48.342 INFO:tasks.rados.rados.0.smithi059.stdout:2302: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:51:48.342 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 819 (ObjNum -156545328 snap 21893 seq_num 3497984) dirty dne 2022-04-28T19:51:48.343 INFO:tasks.rados.rados.0.smithi059.stdout:2301: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:51:48.343 INFO:tasks.rados.rados.0.smithi059.stdout:2301: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:51:48.343 INFO:tasks.rados.rados.0.smithi059.stdout:2301: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:51:48.343 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1074 (ObjNum 718 snap 209 seq_num 718) dirty exists 2022-04-28T19:51:48.343 INFO:tasks.rados.rados.0.smithi059.stdout:2298: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:51:48.404 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 921 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:48.404 INFO:tasks.rados.rados.0.smithi059.stdout:2300: finishing copy_from to smithi05955964-36 2022-04-28T19:51:48.404 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 882 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:51:48.405 INFO:tasks.rados.rados.0.smithi059.stdout:2300: finishing copy_from racing read to smithi05955964-36 2022-04-28T19:51:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[59745]: cluster 2022-04-28T19:51:46.902822+0000 mgr.y (mgr.24961) 148 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 334 MiB data, 673 MiB used, 710 GiB / 715 GiB avail; 582 KiB/s wr, 2 op/s; 13 MiB/s, 2 objects/s recovering 2022-04-28T19:51:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[59745]: cluster 2022-04-28T19:51:47.275322+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2022-04-28T19:51:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[59745]: cluster 2022-04-28T19:51:47.863780+0000 mon.a (mon.0) 492 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/486 objects degraded (7.819%), 5 pgs degraded) 2022-04-28T19:51:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[59745]: cluster 2022-04-28T19:51:47.863852+0000 mon.a (mon.0) 493 : cluster [INF] Cluster is now healthy 2022-04-28T19:51:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[61152]: cluster 2022-04-28T19:51:46.902822+0000 mgr.y (mgr.24961) 148 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 334 MiB data, 673 MiB used, 710 GiB / 715 GiB avail; 582 KiB/s wr, 2 op/s; 13 MiB/s, 2 objects/s recovering 2022-04-28T19:51:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[61152]: cluster 2022-04-28T19:51:47.275322+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2022-04-28T19:51:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[61152]: cluster 2022-04-28T19:51:47.863780+0000 mon.a (mon.0) 492 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/486 objects degraded (7.819%), 5 pgs degraded) 2022-04-28T19:51:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:48 smithi059 bash[61152]: cluster 2022-04-28T19:51:47.863852+0000 mon.a (mon.0) 493 : cluster [INF] Cluster is now healthy 2022-04-28T19:51:48.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:48 smithi139 bash[43301]: cluster 2022-04-28T19:51:46.902822+0000 mgr.y (mgr.24961) 148 : cluster [DBG] pgmap v216: 89 pgs: 89 active+clean; 334 MiB data, 673 MiB used, 710 GiB / 715 GiB avail; 582 KiB/s wr, 2 op/s; 13 MiB/s, 2 objects/s recovering 2022-04-28T19:51:48.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:48 smithi139 bash[43301]: cluster 2022-04-28T19:51:47.275322+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e488: 8 total, 8 up, 8 in 2022-04-28T19:51:48.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:48 smithi139 bash[43301]: cluster 2022-04-28T19:51:47.863780+0000 mon.a (mon.0) 492 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/486 objects degraded (7.819%), 5 pgs degraded) 2022-04-28T19:51:48.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:48 smithi139 bash[43301]: cluster 2022-04-28T19:51:47.863852+0000 mon.a (mon.0) 493 : cluster [INF] Cluster is now healthy 2022-04-28T19:51:49.295 INFO:tasks.rados.rados.0.smithi059.stdout:2298: done (5 left) 2022-04-28T19:51:49.295 INFO:tasks.rados.rados.0.smithi059.stdout:2299: done (4 left) 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2300: done (3 left) 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2301: done (2 left) 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2302: done (1 left) 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2303: done (0 left) 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2304: copy_from oid 22 from oid 29 current snap is 214 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2305: read oid 10 snap -1 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2305: expect (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 214 2022-04-28T19:51:49.296 INFO:tasks.rados.rados.0.smithi059.stdout:2306: seq_num 734 ranges {11804672=294912} 2022-04-28T19:51:49.364 INFO:tasks.rados.rados.0.smithi059.stdout:2306: writing smithi05955964-34 from 11804672 to 12099584 tid 1 2022-04-28T19:51:49.364 INFO:tasks.rados.rados.0.smithi059.stdout:2307: rollback oid 16 current snap is 214 2022-04-28T19:51:49.365 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 211 2022-04-28T19:51:49.365 INFO:tasks.rados.rados.0.smithi059.stdout:2305: expect (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:2304: finishing copy_from to smithi05955964-22 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:2304: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:2304: done (3 left) 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:2305: done (2 left) 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:2308: read oid 47 snap -1 2022-04-28T19:51:49.382 INFO:tasks.rados.rados.0.smithi059.stdout:2308: expect (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2309: copy_from oid 38 from oid 45 current snap is 214 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2307: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2307: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2307: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 950 (ObjNum 672 snap 195 seq_num 672) dirty exists 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2306: finishing write tid 1 to smithi05955964-34 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2306: finishing write tid 3 to smithi05955964-34 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2306: finishing write tid 4 to smithi05955964-34 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1003 (ObjNum 734 snap 214 seq_num 734) dirty exists 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2306: left oid 34 (ObjNum 734 snap 214 seq_num 734) 2022-04-28T19:51:49.383 INFO:tasks.rados.rados.0.smithi059.stdout:2306: done (3 left) 2022-04-28T19:51:49.384 INFO:tasks.rados.rados.0.smithi059.stdout:2307: done (2 left) 2022-04-28T19:51:49.384 INFO:tasks.rados.rados.0.smithi059.stdout:2310: rollback oid 34 current snap is 214 2022-04-28T19:51:49.384 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 34 to 211 2022-04-28T19:51:49.384 INFO:tasks.rados.rados.0.smithi059.stdout:2311: setattr oid 20 current snap is 214 2022-04-28T19:51:49.384 INFO:tasks.rados.rados.0.smithi059.stdout:2312: setattr oid 26 current snap is 214 2022-04-28T19:51:49.384 INFO:tasks.rados.rados.0.smithi059.stdout:2308: expect (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:51:49.385 INFO:tasks.rados.rados.0.smithi059.stdout:2310: finishing rollback tid 0 to smithi05955964-34 2022-04-28T19:51:49.385 INFO:tasks.rados.rados.0.smithi059.stdout:2310: finishing rollback tid 1 to smithi05955964-34 2022-04-28T19:51:49.386 INFO:tasks.rados.rados.0.smithi059.stdout:2310: finishing rollback tid 2 to smithi05955964-34 2022-04-28T19:51:49.386 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1006 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:51:49.387 INFO:tasks.rados.rados.0.smithi059.stdout:2308: done (4 left) 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2310: done (3 left) 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2313: read oid 49 snap -1 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2313: expect deleted 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2314: rmattr oid 14 current snap is 214 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2314: done (4 left) 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2315: copy_from oid 17 from oid 3 current snap is 214 2022-04-28T19:51:49.388 INFO:tasks.rados.rados.0.smithi059.stdout:2316: delete oid 31 current snap is 214 2022-04-28T19:51:49.389 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 923 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:49.390 INFO:tasks.rados.rados.0.smithi059.stdout:2311: done (5 left) 2022-04-28T19:51:49.390 INFO:tasks.rados.rados.0.smithi059.stdout:2313: done (4 left) 2022-04-28T19:51:49.390 INFO:tasks.rados.rados.0.smithi059.stdout:2316: done (3 left) 2022-04-28T19:51:49.391 INFO:tasks.rados.rados.0.smithi059.stdout:2317: rollback oid 14 current snap is 214 2022-04-28T19:51:49.391 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 213 2022-04-28T19:51:49.391 INFO:tasks.rados.rados.0.smithi059.stdout:2318: read oid 22 snap -1 2022-04-28T19:51:49.391 INFO:tasks.rados.rados.0.smithi059.stdout:2318: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:51:49.391 INFO:tasks.rados.rados.0.smithi059.stdout:2319: snap_create 2022-04-28T19:51:49.397 INFO:tasks.rados.rados.0.smithi059.stdout:2309: finishing copy_from to smithi05955964-38 2022-04-28T19:51:49.397 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 921 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:51:49.398 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1071 (ObjNum 729 snap 211 seq_num 729) dirty exists 2022-04-28T19:51:49.398 INFO:tasks.rados.rados.0.smithi059.stdout:2309: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:51:49.399 INFO:tasks.rados.rados.0.smithi059.stdout:2317: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:51:49.399 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 954 (ObjNum 1314938711 snap 1903318620 seq_num 1834570087) dirty dne 2022-04-28T19:51:49.404 INFO:tasks.rados.rados.0.smithi059.stdout:2318: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:51:49.499 INFO:tasks.rados.rados.0.smithi059.stdout:2315: finishing copy_from to smithi05955964-17 2022-04-28T19:51:49.499 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 912 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:51:49.499 INFO:tasks.rados.rados.0.smithi059.stdout:2315: finishing copy_from racing read to smithi05955964-17 2022-04-28T19:51:49.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:49 smithi059 bash[59745]: cluster 2022-04-28T19:51:48.289122+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2022-04-28T19:51:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:49 smithi059 bash[61152]: cluster 2022-04-28T19:51:48.289122+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2022-04-28T19:51:49.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:49 smithi139 bash[43301]: cluster 2022-04-28T19:51:48.289122+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e489: 8 total, 8 up, 8 in 2022-04-28T19:51:50.304 INFO:tasks.rados.rados.0.smithi059.stdout:2309: done (5 left) 2022-04-28T19:51:50.304 INFO:tasks.rados.rados.0.smithi059.stdout:2312: done (4 left) 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2315: done (3 left) 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2317: done (2 left) 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2318: done (1 left) 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2319: done (0 left) 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2320: rollback oid 45 current snap is 215 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 199 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2321: read oid 14 snap -1 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2321: expect deleted 2022-04-28T19:51:50.305 INFO:tasks.rados.rados.0.smithi059.stdout:2322: read oid 3 snap -1 2022-04-28T19:51:50.306 INFO:tasks.rados.rados.0.smithi059.stdout:2322: expect (ObjNum 709 snap 203 seq_num 709) 2022-04-28T19:51:50.306 INFO:tasks.rados.rados.0.smithi059.stdout:2323: snap_create 2022-04-28T19:51:50.314 INFO:tasks.rados.rados.0.smithi059.stdout:2320: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:51:50.315 INFO:tasks.rados.rados.0.smithi059.stdout:2320: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:51:50.315 INFO:tasks.rados.rados.0.smithi059.stdout:2320: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:51:50.315 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 927 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:51:50.336 INFO:tasks.rados.rados.0.smithi059.stdout:2322: expect (ObjNum 709 snap 203 seq_num 709) 2022-04-28T19:51:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:50 smithi059 bash[59745]: cluster 2022-04-28T19:51:48.903616+0000 mgr.y (mgr.24961) 149 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 308 MiB data, 666 MiB used, 710 GiB / 715 GiB avail; 0 B/s rd, 582 KiB/s wr, 4 op/s; 13 MiB/s, 2 objects/s recovering 2022-04-28T19:51:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:50 smithi059 bash[59745]: cluster 2022-04-28T19:51:49.292950+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2022-04-28T19:51:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:50 smithi059 bash[61152]: cluster 2022-04-28T19:51:48.903616+0000 mgr.y (mgr.24961) 149 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 308 MiB data, 666 MiB used, 710 GiB / 715 GiB avail; 0 B/s rd, 582 KiB/s wr, 4 op/s; 13 MiB/s, 2 objects/s recovering 2022-04-28T19:51:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:50 smithi059 bash[61152]: cluster 2022-04-28T19:51:49.292950+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2022-04-28T19:51:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:50 smithi139 bash[43301]: cluster 2022-04-28T19:51:48.903616+0000 mgr.y (mgr.24961) 149 : cluster [DBG] pgmap v219: 89 pgs: 89 active+clean; 308 MiB data, 666 MiB used, 710 GiB / 715 GiB avail; 0 B/s rd, 582 KiB/s wr, 4 op/s; 13 MiB/s, 2 objects/s recovering 2022-04-28T19:51:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:50 smithi139 bash[43301]: cluster 2022-04-28T19:51:49.292950+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e490: 8 total, 8 up, 8 in 2022-04-28T19:51:51.316 INFO:tasks.rados.rados.0.smithi059.stdout:2320: done (3 left) 2022-04-28T19:51:51.385 INFO:tasks.rados.rados.0.smithi059.stdout:2321: done (2 left) 2022-04-28T19:51:51.386 INFO:tasks.rados.rados.0.smithi059.stdout:2322: done (1 left) 2022-04-28T19:51:51.386 INFO:tasks.rados.rados.0.smithi059.stdout:2323: done (0 left) 2022-04-28T19:51:51.386 INFO:tasks.rados.rados.0.smithi059.stdout:2324: snap_create 2022-04-28T19:51:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:51 smithi059 bash[59745]: cluster 2022-04-28T19:51:50.301508+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2022-04-28T19:51:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:51 smithi059 bash[61152]: cluster 2022-04-28T19:51:50.301508+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2022-04-28T19:51:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:51 smithi139 bash[43301]: cluster 2022-04-28T19:51:50.301508+0000 mon.a (mon.0) 496 : cluster [DBG] osdmap e491: 8 total, 8 up, 8 in 2022-04-28T19:51:51.876 INFO:tasks.rados.rados.0.smithi059.stdout:2324: done (0 left) 2022-04-28T19:51:51.876 INFO:tasks.rados.rados.0.smithi059.stdout:2325: read oid 2 snap -1 2022-04-28T19:51:51.877 INFO:tasks.rados.rados.0.smithi059.stdout:2325: expect deleted 2022-04-28T19:51:51.877 INFO:tasks.rados.rados.0.smithi059.stdout:2326: snap_create 2022-04-28T19:51:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:51.630Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:51.630Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:51.630Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:51:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[61152]: cluster 2022-04-28T19:51:50.904300+0000 mgr.y (mgr.24961) 150 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 310 MiB data, 663 MiB used, 710 GiB / 715 GiB avail; 2.9 MiB/s rd, 844 KiB/s wr, 13 op/s 2022-04-28T19:51:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[61152]: cluster 2022-04-28T19:51:51.314628+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2022-04-28T19:51:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[61152]: cluster 2022-04-28T19:51:51.873519+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2022-04-28T19:51:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[61152]: audit 2022-04-28T19:51:52.040393+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[61152]: audit 2022-04-28T19:51:52.254472+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[59745]: cluster 2022-04-28T19:51:50.904300+0000 mgr.y (mgr.24961) 150 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 310 MiB data, 663 MiB used, 710 GiB / 715 GiB avail; 2.9 MiB/s rd, 844 KiB/s wr, 13 op/s 2022-04-28T19:51:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[59745]: cluster 2022-04-28T19:51:51.314628+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2022-04-28T19:51:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[59745]: cluster 2022-04-28T19:51:51.873519+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2022-04-28T19:51:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[59745]: audit 2022-04-28T19:51:52.040393+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:52 smithi059 bash[59745]: audit 2022-04-28T19:51:52.254472+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:52 smithi139 bash[43301]: cluster 2022-04-28T19:51:50.904300+0000 mgr.y (mgr.24961) 150 : cluster [DBG] pgmap v222: 89 pgs: 89 active+clean; 310 MiB data, 663 MiB used, 710 GiB / 715 GiB avail; 2.9 MiB/s rd, 844 KiB/s wr, 13 op/s 2022-04-28T19:51:52.690 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:52 smithi139 bash[43301]: cluster 2022-04-28T19:51:51.314628+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e492: 8 total, 8 up, 8 in 2022-04-28T19:51:52.690 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:52 smithi139 bash[43301]: cluster 2022-04-28T19:51:51.873519+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e493: 8 total, 8 up, 8 in 2022-04-28T19:51:52.690 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:52 smithi139 bash[43301]: audit 2022-04-28T19:51:52.040393+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:52.690 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:52 smithi139 bash[43301]: audit 2022-04-28T19:51:52.254472+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:52.884 INFO:tasks.rados.rados.0.smithi059.stdout:2325: done (1 left) 2022-04-28T19:51:52.884 INFO:tasks.rados.rados.0.smithi059.stdout:2326: done (0 left) 2022-04-28T19:51:52.885 INFO:tasks.rados.rados.0.smithi059.stdout:2327: snap_create 2022-04-28T19:51:53.584 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[61152]: audit 2022-04-28T19:51:52.040875+0000 mgr.y (mgr.24961) 151 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:53.643 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[61152]: cephadm 2022-04-28T19:51:52.043270+0000 mgr.y (mgr.24961) 152 : cephadm [INF] Upgrade: osd.3 is safe to restart 2022-04-28T19:51:53.644 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[61152]: cluster 2022-04-28T19:51:52.883841+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2022-04-28T19:51:53.645 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[59745]: audit 2022-04-28T19:51:52.040875+0000 mgr.y (mgr.24961) 151 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:53.645 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[59745]: cephadm 2022-04-28T19:51:52.043270+0000 mgr.y (mgr.24961) 152 : cephadm [INF] Upgrade: osd.3 is safe to restart 2022-04-28T19:51:53.645 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[59745]: cluster 2022-04-28T19:51:52.883841+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2022-04-28T19:51:53.646 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:53.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:51:53.646 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:53.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:51:53.647 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:51:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:51:53.524Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:51:53.648 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:53 smithi139 bash[43301]: audit 2022-04-28T19:51:52.040875+0000 mgr.y (mgr.24961) 151 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2022-04-28T19:51:53.648 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:53 smithi139 bash[43301]: cephadm 2022-04-28T19:51:52.043270+0000 mgr.y (mgr.24961) 152 : cephadm [INF] Upgrade: osd.3 is safe to restart 2022-04-28T19:51:53.648 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:53 smithi139 bash[43301]: cluster 2022-04-28T19:51:52.883841+0000 mon.a (mon.0) 501 : cluster [DBG] osdmap e494: 8 total, 8 up, 8 in 2022-04-28T19:51:53.891 INFO:tasks.rados.rados.0.smithi059.stdout:2327: done (0 left) 2022-04-28T19:51:53.892 INFO:tasks.rados.rados.0.smithi059.stdout:2328: copy_from oid 41 from oid 34 current snap is 219 2022-04-28T19:51:53.892 INFO:tasks.rados.rados.0.smithi059.stdout:2329: read oid 14 snap 199 2022-04-28T19:51:53.892 INFO:tasks.rados.rados.0.smithi059.stdout:2329: expect deleted 2022-04-28T19:51:53.892 INFO:tasks.rados.rados.0.smithi059.stdout:2330: delete oid 13 current snap is 219 2022-04-28T19:51:53.895 INFO:tasks.rados.rados.0.smithi059.stdout:2330: done (2 left) 2022-04-28T19:51:53.895 INFO:tasks.rados.rados.0.smithi059.stdout:2331: rmattr oid 25 current snap is 219 2022-04-28T19:51:53.895 INFO:tasks.rados.rados.0.smithi059.stdout:2331: done (2 left) 2022-04-28T19:51:53.895 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 219 2022-04-28T19:51:53.896 INFO:tasks.rados.rados.0.smithi059.stdout:2332: seq_num 737 ranges {1769472=352256} 2022-04-28T19:51:53.909 INFO:tasks.rados.rados.0.smithi059.stdout:2332: writing smithi05955964-12 from 1769472 to 2121728 tid 1 2022-04-28T19:51:53.909 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 219 2022-04-28T19:51:53.909 INFO:tasks.rados.rados.0.smithi059.stdout:2333: seq_num 738 ranges {0=368640} 2022-04-28T19:51:53.912 INFO:tasks.rados.rados.0.smithi059.stdout:2333: writing smithi05955964-19 from 0 to 368640 tid 1 2022-04-28T19:51:53.912 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 219 2022-04-28T19:51:53.912 INFO:tasks.rados.rados.0.smithi059.stdout:2334: seq_num 739 ranges {1613824=385024} 2022-04-28T19:51:53.924 INFO:tasks.rados.rados.0.smithi059.stdout:2334: writing smithi05955964-35 from 1613824 to 1998848 tid 1 2022-04-28T19:51:53.924 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 219 2022-04-28T19:51:53.924 INFO:tasks.rados.rados.0.smithi059.stdout:2335: seq_num 740 ranges {7569408=401408} 2022-04-28T19:51:53.971 INFO:tasks.rados.rados.0.smithi059.stdout:2335: writing smithi05955964-17 from 7569408 to 7970816 tid 1 2022-04-28T19:51:53.971 INFO:tasks.rados.rados.0.smithi059.stdout:2336: read oid 26 snap 217 2022-04-28T19:51:53.971 INFO:tasks.rados.rados.0.smithi059.stdout:2336: expect (ObjNum 729 snap 211 seq_num 729) 2022-04-28T19:51:53.971 INFO:tasks.rados.rados.0.smithi059.stdout:2337: copy_from oid 2 from oid 25 current snap is 219 2022-04-28T19:51:53.971 INFO:tasks.rados.rados.0.smithi059.stdout:2332: finishing write tid 1 to smithi05955964-12 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2332: finishing write tid 3 to smithi05955964-12 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2332: finishing write tid 4 to smithi05955964-12 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 939 (ObjNum 737 snap 219 seq_num 737) dirty exists 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2332: left oid 12 (ObjNum 737 snap 219 seq_num 737) 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2333: finishing write tid 1 to smithi05955964-19 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2333: finishing write tid 3 to smithi05955964-19 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2333: finishing write tid 4 to smithi05955964-19 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1074 (ObjNum 738 snap 219 seq_num 738) dirty exists 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2333: left oid 19 (ObjNum 738 snap 219 seq_num 738) 2022-04-28T19:51:53.972 INFO:tasks.rados.rados.0.smithi059.stdout:2334: finishing write tid 1 to smithi05955964-35 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2334: finishing write tid 3 to smithi05955964-35 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2334: finishing write tid 4 to smithi05955964-35 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 934 (ObjNum 739 snap 219 seq_num 739) dirty exists 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2334: left oid 35 (ObjNum 739 snap 219 seq_num 739) 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2328: finishing copy_from to smithi05955964-41 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1006 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2328: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2328: done (7 left) 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2329: done (6 left) 2022-04-28T19:51:53.973 INFO:tasks.rados.rados.0.smithi059.stdout:2332: done (5 left) 2022-04-28T19:51:53.974 INFO:tasks.rados.rados.0.smithi059.stdout:2333: done (4 left) 2022-04-28T19:51:53.974 INFO:tasks.rados.rados.0.smithi059.stdout:2334: done (3 left) 2022-04-28T19:51:53.974 INFO:tasks.rados.rados.0.smithi059.stdout:2338: rmattr oid 4 current snap is 219 2022-04-28T19:51:53.974 INFO:tasks.rados.rados.0.smithi059.stdout:2338: done (3 left) 2022-04-28T19:51:53.974 INFO:tasks.rados.rados.0.smithi059.stdout:2339: rmattr oid 46 current snap is 219 2022-04-28T19:51:53.974 INFO:tasks.rados.rados.0.smithi059.stdout:2340: snap_create 2022-04-28T19:51:53.976 INFO:tasks.rados.rados.0.smithi059.stdout:2335: finishing write tid 1 to smithi05955964-17 2022-04-28T19:51:53.976 INFO:tasks.rados.rados.0.smithi059.stdout:2335: finishing write tid 3 to smithi05955964-17 2022-04-28T19:51:53.977 INFO:tasks.rados.rados.0.smithi059.stdout:2336: expect (ObjNum 729 snap 211 seq_num 729) 2022-04-28T19:51:54.041 INFO:tasks.rados.rados.0.smithi059.stdout:2335: finishing write tid 4 to smithi05955964-17 2022-04-28T19:51:54.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 915 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:51:54.041 INFO:tasks.rados.rados.0.smithi059.stdout:2335: left oid 17 (ObjNum 740 snap 219 seq_num 740) 2022-04-28T19:51:54.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1036 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:51:54.041 INFO:tasks.rados.rados.0.smithi059.stdout:2337: finishing copy_from to smithi05955964-2 2022-04-28T19:51:54.042 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 988 (ObjNum 719 snap 209 seq_num 719) dirty exists 2022-04-28T19:51:54.042 INFO:tasks.rados.rados.0.smithi059.stdout:2337: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:51:54.089 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:51:54.409 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:51:54.409 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (10m) 17s ago 11m 14.7M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:51:54.409 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (11m) 2m ago 11m 25.8M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:51:54.410 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (9m) 17s ago 9m 14.8M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:51:54.410 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (9m) 2m ago 9m 17.6M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:51:54.410 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (3m) 2m ago 15m 375M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:51:54.410 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (3m) 17s ago 17m 414M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:51:54.410 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (2m) 17s ago 17m 60.3M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:51:54.410 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (2m) 2m ago 15m 19.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:51:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (2m) 17s ago 15m 49.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:51:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (11m) 17s ago 11m 11.1M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:51:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (11m) 2m ago 11m 10.7M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:51:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (79s) 17s ago 14m 279M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:51:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (49s) 17s ago 14m 186M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:51:54.411 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (19s) 17s ago 13m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:51:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (13m) 17s ago 13m 240M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 2344bb9dfbae 2022-04-28T19:51:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (13m) 2m ago 13m 196M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:51:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (12m) 2m ago 12m 214M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:51:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (12m) 2m ago 12m 162M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:51:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (11m) 2m ago 12m 138M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:51:54.412 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (3m) 2m ago 11m 36.6M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:51:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:54 smithi059 bash[59745]: cluster 2022-04-28T19:51:52.904945+0000 mgr.y (mgr.24961) 153 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 320 MiB data, 668 MiB used, 710 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2022-04-28T19:51:54.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:54 smithi059 bash[59745]: cluster 2022-04-28T19:51:53.890657+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2022-04-28T19:51:54.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:54 smithi059 bash[61152]: cluster 2022-04-28T19:51:52.904945+0000 mgr.y (mgr.24961) 153 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 320 MiB data, 668 MiB used, 710 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2022-04-28T19:51:54.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:54 smithi059 bash[61152]: cluster 2022-04-28T19:51:53.890657+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2022-04-28T19:51:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:54 smithi139 bash[43301]: cluster 2022-04-28T19:51:52.904945+0000 mgr.y (mgr.24961) 153 : cluster [DBG] pgmap v226: 89 pgs: 89 active+clean; 320 MiB data, 668 MiB used, 710 GiB / 715 GiB avail; 4.1 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2022-04-28T19:51:54.644 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:54 smithi139 bash[43301]: cluster 2022-04-28T19:51:53.890657+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e495: 8 total, 8 up, 8 in 2022-04-28T19:51:54.802 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:51:54.802 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:51:54.802 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:51:54.802 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:54.803 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:51:54.803 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:51:54.803 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:54.803 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:51:54.803 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 5, 2022-04-28T19:51:54.804 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:51:54.804 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:54.804 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:51:54.804 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:51:54.804 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:51:54.805 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:51:54.805 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 7, 2022-04-28T19:51:54.805 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 8 2022-04-28T19:51:54.805 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:51:54.806 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2335: done (4 left) 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2336: done (3 left) 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2337: done (2 left) 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2339: done (1 left) 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2340: done (0 left) 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2341: read oid 19 snap -1 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:2341: expect (ObjNum 738 snap 219 seq_num 738) 2022-04-28T19:51:54.901 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 220 2022-04-28T19:51:54.902 INFO:tasks.rados.rados.0.smithi059.stdout:2342: seq_num 741 ranges {11804672=417792} 2022-04-28T19:51:54.976 INFO:tasks.rados.rados.0.smithi059.stdout:2342: writing smithi05955964-41 from 11804672 to 12222464 tid 1 2022-04-28T19:51:54.976 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 220 2022-04-28T19:51:54.976 INFO:tasks.rados.rados.0.smithi059.stdout:2343: seq_num 742 ranges {11313152=434176} 2022-04-28T19:51:55.043 INFO:tasks.rados.rados.0.smithi059.stdout:2343: writing smithi05955964-7 from 11313152 to 11747328 tid 1 2022-04-28T19:51:55.043 INFO:tasks.rados.rados.0.smithi059.stdout:2344: snap_create 2022-04-28T19:51:55.043 INFO:tasks.rados.rados.0.smithi059.stdout:2341: expect (ObjNum 738 snap 219 seq_num 738) 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2342: finishing write tid 1 to smithi05955964-41 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2342: finishing write tid 3 to smithi05955964-41 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2342: finishing write tid 4 to smithi05955964-41 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1008 (ObjNum 741 snap 220 seq_num 741) dirty exists 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2342: left oid 41 (ObjNum 741 snap 220 seq_num 741) 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2343: finishing write tid 1 to smithi05955964-7 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2343: finishing write tid 3 to smithi05955964-7 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:2343: finishing write tid 4 to smithi05955964-7 2022-04-28T19:51:55.058 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 942 (ObjNum 742 snap 220 seq_num 742) dirty exists 2022-04-28T19:51:55.059 INFO:tasks.rados.rados.0.smithi059.stdout:2343: left oid 7 (ObjNum 742 snap 220 seq_num 742) 2022-04-28T19:51:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.076424+0000 mgr.y (mgr.24961) 154 : audit [DBG] from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.391737+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.392267+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.392703+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.393177+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.393631+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.394077+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.396349+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.396813+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.397283+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.397743+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.398239+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.467239+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.469845+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "container_image"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: audit 2022-04-28T19:51:54.801200+0000 mon.a (mon.0) 516 : audit [DBG] from='client.? 172.21.15.59:0/3714267603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[59745]: cluster 2022-04-28T19:51:54.899541+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.076424+0000 mgr.y (mgr.24961) 154 : audit [DBG] from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.391737+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.392267+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.392703+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.393177+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.393631+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.394077+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.396349+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.396813+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.397283+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.397743+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.398239+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.467239+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.469845+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "container_image"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: audit 2022-04-28T19:51:54.801200+0000 mon.a (mon.0) 516 : audit [DBG] from='client.? 172.21.15.59:0/3714267603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:55.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:55 smithi059 bash[61152]: cluster 2022-04-28T19:51:54.899541+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2022-04-28T19:51:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.076424+0000 mgr.y (mgr.24961) 154 : audit [DBG] from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.391737+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.392267+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.392703+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.393177+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.393631+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.394077+0000 mon.a (mon.0) 508 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.396349+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.396813+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.397283+0000 mon.a (mon.0) 511 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.397743+0000 mon.a (mon.0) 512 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.398239+0000 mon.a (mon.0) 513 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:51:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.467239+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:51:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.469845+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "container_image"}]: dispatch 2022-04-28T19:51:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: audit 2022-04-28T19:51:54.801200+0000 mon.a (mon.0) 516 : audit [DBG] from='client.? 172.21.15.59:0/3714267603' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:51:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:55 smithi139 bash[43301]: cluster 2022-04-28T19:51:54.899541+0000 mon.a (mon.0) 517 : cluster [DBG] osdmap e496: 8 total, 8 up, 8 in 2022-04-28T19:51:55.908 INFO:tasks.rados.rados.0.smithi059.stdout:2341: done (3 left) 2022-04-28T19:51:55.908 INFO:tasks.rados.rados.0.smithi059.stdout:2342: done (2 left) 2022-04-28T19:51:55.908 INFO:tasks.rados.rados.0.smithi059.stdout:2343: done (1 left) 2022-04-28T19:51:55.909 INFO:tasks.rados.rados.0.smithi059.stdout:2344: done (0 left) 2022-04-28T19:51:55.909 INFO:tasks.rados.rados.0.smithi059.stdout:2345: read oid 29 snap -1 2022-04-28T19:51:55.909 INFO:tasks.rados.rados.0.smithi059.stdout:2345: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:51:55.909 INFO:tasks.rados.rados.0.smithi059.stdout:2346: snap_create 2022-04-28T19:51:55.922 INFO:tasks.rados.rados.0.smithi059.stdout:2345: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:51:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[61152]: audit 2022-04-28T19:51:54.389245+0000 mgr.y (mgr.24961) 155 : audit [DBG] from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[61152]: cephadm 2022-04-28T19:51:54.457381+0000 mgr.y (mgr.24961) 156 : cephadm [INF] Upgrade: Updating osd.3 2022-04-28T19:51:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[61152]: cephadm 2022-04-28T19:51:54.469249+0000 mgr.y (mgr.24961) 157 : cephadm [INF] Deploying daemon osd.3 on smithi059 2022-04-28T19:51:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[61152]: cluster 2022-04-28T19:51:54.905515+0000 mgr.y (mgr.24961) 158 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 320 MiB data, 668 MiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2022-04-28T19:51:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[61152]: cluster 2022-04-28T19:51:55.902848+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2022-04-28T19:51:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[59745]: audit 2022-04-28T19:51:54.389245+0000 mgr.y (mgr.24961) 155 : audit [DBG] from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[59745]: cephadm 2022-04-28T19:51:54.457381+0000 mgr.y (mgr.24961) 156 : cephadm [INF] Upgrade: Updating osd.3 2022-04-28T19:51:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[59745]: cephadm 2022-04-28T19:51:54.469249+0000 mgr.y (mgr.24961) 157 : cephadm [INF] Deploying daemon osd.3 on smithi059 2022-04-28T19:51:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[59745]: cluster 2022-04-28T19:51:54.905515+0000 mgr.y (mgr.24961) 158 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 320 MiB data, 668 MiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2022-04-28T19:51:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:56 smithi059 bash[59745]: cluster 2022-04-28T19:51:55.902848+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2022-04-28T19:51:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:56 smithi139 bash[43301]: audit 2022-04-28T19:51:54.389245+0000 mgr.y (mgr.24961) 155 : audit [DBG] from='client.44250 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:51:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:56 smithi139 bash[43301]: cephadm 2022-04-28T19:51:54.457381+0000 mgr.y (mgr.24961) 156 : cephadm [INF] Upgrade: Updating osd.3 2022-04-28T19:51:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:56 smithi139 bash[43301]: cephadm 2022-04-28T19:51:54.469249+0000 mgr.y (mgr.24961) 157 : cephadm [INF] Deploying daemon osd.3 on smithi059 2022-04-28T19:51:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:56 smithi139 bash[43301]: cluster 2022-04-28T19:51:54.905515+0000 mgr.y (mgr.24961) 158 : cluster [DBG] pgmap v229: 89 pgs: 89 active+clean; 320 MiB data, 668 MiB used, 710 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2022-04-28T19:51:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:56 smithi139 bash[43301]: cluster 2022-04-28T19:51:55.902848+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e497: 8 total, 8 up, 8 in 2022-04-28T19:51:56.879 INFO:tasks.rados.rados.0.smithi059.stdout:2345: done (1 left) 2022-04-28T19:51:56.879 INFO:tasks.rados.rados.0.smithi059.stdout:2346: done (0 left) 2022-04-28T19:51:56.879 INFO:tasks.rados.rados.0.smithi059.stdout:2347: rollback oid 30 current snap is 222 2022-04-28T19:51:56.879 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 30 to 217 2022-04-28T19:51:56.879 INFO:tasks.rados.rados.0.smithi059.stdout:2348: delete oid 21 current snap is 222 2022-04-28T19:51:56.881 INFO:tasks.rados.rados.0.smithi059.stdout:2348: done (1 left) 2022-04-28T19:51:56.882 INFO:tasks.rados.rados.0.smithi059.stdout:2349: snap_create 2022-04-28T19:51:56.886 INFO:tasks.rados.rados.0.smithi059.stdout:2347: finishing rollback tid 1 to smithi05955964-30 2022-04-28T19:51:56.886 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 990 (ObjNum -133563424 snap 21893 seq_num 4159778832) dirty dne 2022-04-28T19:51:57.892 INFO:tasks.rados.rados.0.smithi059.stdout:2347: done (1 left) 2022-04-28T19:51:57.892 INFO:tasks.rados.rados.0.smithi059.stdout:2349: done (0 left) 2022-04-28T19:51:57.892 INFO:tasks.rados.rados.0.smithi059.stdout:2350: snap_create 2022-04-28T19:51:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:57 smithi059 bash[59745]: cluster 2022-04-28T19:51:56.873249+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2022-04-28T19:51:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:57 smithi059 bash[59745]: cluster 2022-04-28T19:51:56.906015+0000 mgr.y (mgr.24961) 159 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 341 MiB data, 681 MiB used, 710 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.2 MiB/s wr, 2 op/s 2022-04-28T19:51:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:57 smithi059 bash[61152]: cluster 2022-04-28T19:51:56.873249+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2022-04-28T19:51:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:57 smithi059 bash[61152]: cluster 2022-04-28T19:51:56.906015+0000 mgr.y (mgr.24961) 159 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 341 MiB data, 681 MiB used, 710 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.2 MiB/s wr, 2 op/s 2022-04-28T19:51:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:57 smithi139 bash[43301]: cluster 2022-04-28T19:51:56.873249+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e498: 8 total, 8 up, 8 in 2022-04-28T19:51:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:57 smithi139 bash[43301]: cluster 2022-04-28T19:51:56.906015+0000 mgr.y (mgr.24961) 159 : cluster [DBG] pgmap v232: 89 pgs: 89 active+clean; 341 MiB data, 681 MiB used, 710 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.2 MiB/s wr, 2 op/s 2022-04-28T19:51:58.764 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:58 smithi059 systemd[1]: Stopping Ceph osd.3 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:51:58.764 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:58 smithi059 bash[38013]: debug 2022-04-28T19:51:58.755+0000 7fde2e281700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:51:58.764 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:58 smithi059 bash[38013]: debug 2022-04-28T19:51:58.755+0000 7fde2e281700 -1 osd.3 499 *** Got signal Terminated *** 2022-04-28T19:51:58.764 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:58 smithi059 bash[38013]: debug 2022-04-28T19:51:58.755+0000 7fde2e281700 -1 osd.3 499 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:51:58.902 INFO:tasks.rados.rados.0.smithi059.stdout:2350: done (0 left) 2022-04-28T19:51:58.902 INFO:tasks.rados.rados.0.smithi059.stdout:2351: delete oid 36 current snap is 224 2022-04-28T19:51:58.905 INFO:tasks.rados.rados.0.smithi059.stdout:2351: done (0 left) 2022-04-28T19:51:58.905 INFO:tasks.rados.rados.0.smithi059.stdout:2352: delete oid 1 current snap is 224 2022-04-28T19:51:58.907 INFO:tasks.rados.rados.0.smithi059.stdout:2352: done (0 left) 2022-04-28T19:51:58.907 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 224 2022-04-28T19:51:58.907 INFO:tasks.rados.rados.0.smithi059.stdout:2353: seq_num 746 ranges {0=499712} 2022-04-28T19:51:58.912 INFO:tasks.rados.rados.0.smithi059.stdout:2353: writing smithi05955964-21 from 0 to 499712 tid 1 2022-04-28T19:51:58.912 INFO:tasks.rados.rados.0.smithi059.stdout:2354: copy_from oid 19 from oid 49 current snap is 224 2022-04-28T19:51:58.912 INFO:tasks.rados.rados.0.smithi059.stdout:2355: read oid 9 snap -1 2022-04-28T19:51:58.912 INFO:tasks.rados.rados.0.smithi059.stdout:2355: expect (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:51:58.912 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 224 2022-04-28T19:51:58.912 INFO:tasks.rados.rados.0.smithi059.stdout:2356: seq_num 747 ranges {0=516096} 2022-04-28T19:51:58.916 INFO:tasks.rados.rados.0.smithi059.stdout:2356: writing smithi05955964-5 from 0 to 516096 tid 1 2022-04-28T19:51:59.010 INFO:tasks.rados.rados.0.smithi059.stdout:2357: read oid 46 snap -1 2022-04-28T19:51:59.010 INFO:tasks.rados.rados.0.smithi059.stdout:2357: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:51:59.010 INFO:tasks.rados.rados.0.smithi059.stdout:2358: read oid 36 snap -1 2022-04-28T19:51:59.010 INFO:tasks.rados.rados.0.smithi059.stdout:2358: expect deleted 2022-04-28T19:51:59.011 INFO:tasks.rados.rados.0.smithi059.stdout:2359: rollback oid 10 current snap is 224 2022-04-28T19:51:59.011 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 217 2022-04-28T19:51:59.011 INFO:tasks.rados.rados.0.smithi059.stdout:2360: rmattr oid 38 current snap is 224 2022-04-28T19:51:59.011 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 224 2022-04-28T19:51:59.011 INFO:tasks.rados.rados.0.smithi059.stdout:2361: seq_num 748 ranges {1409024=532480} 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2361: writing smithi05955964-26 from 1409024 to 1941504 tid 1 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2362: read oid 4 snap -1 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2362: expect (ObjNum 628 snap 180 seq_num 628) 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 224 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2363: seq_num 749 ranges {8019968=548864} 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2363: writing smithi05955964-50 from 8019968 to 8568832 tid 1 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2364: snap_create 2022-04-28T19:51:59.013 INFO:tasks.rados.rados.0.smithi059.stdout:2353: finishing write tid 1 to smithi05955964-21 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2353: finishing write tid 3 to smithi05955964-21 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2356: finishing write tid 1 to smithi05955964-5 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2356: finishing write tid 3 to smithi05955964-5 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2353: finishing write tid 4 to smithi05955964-21 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 723 (ObjNum 746 snap 224 seq_num 746) dirty exists 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2353: left oid 21 (ObjNum 746 snap 224 seq_num 746) 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2356: finishing write tid 4 to smithi05955964-5 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1038 (ObjNum 747 snap 224 seq_num 747) dirty exists 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2356: left oid 5 (ObjNum 747 snap 224 seq_num 747) 2022-04-28T19:51:59.014 INFO:tasks.rados.rados.0.smithi059.stdout:2355: expect (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:51:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:58 smithi059 bash[61152]: cluster 2022-04-28T19:51:57.891723+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2022-04-28T19:51:59.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:58 smithi059 bash[59745]: cluster 2022-04-28T19:51:57.891723+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2022-04-28T19:51:59.339 INFO:tasks.rados.rados.0.smithi059.stdout:2357: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:51:59.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:58 smithi139 bash[43301]: cluster 2022-04-28T19:51:57.891723+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e499: 8 total, 8 up, 8 in 2022-04-28T19:51:59.741 INFO:tasks.rados.rados.0.smithi059.stdout:2363: finishing write tid 1 to smithi05955964-50 2022-04-28T19:51:59.741 INFO:tasks.rados.rados.0.smithi059.stdout:2363: finishing write tid 3 to smithi05955964-50 2022-04-28T19:51:59.742 INFO:tasks.rados.rados.0.smithi059.stdout:2363: finishing write tid 4 to smithi05955964-50 2022-04-28T19:51:59.742 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1093 (ObjNum 749 snap 224 seq_num 749) dirty exists 2022-04-28T19:51:59.742 INFO:tasks.rados.rados.0.smithi059.stdout:2363: left oid 50 (ObjNum 749 snap 224 seq_num 749) 2022-04-28T19:51:59.841 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[68163]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.3 2022-04-28T19:51:59.885 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[68163]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-3 2022-04-28T19:51:59.885 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[68163]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.3 2022-04-28T19:51:59.913 INFO:tasks.rados.rados.0.smithi059.stdout:2353: done (11 left) 2022-04-28T19:51:59.913 INFO:tasks.rados.rados.0.smithi059.stdout:2355: done (10 left) 2022-04-28T19:51:59.913 INFO:tasks.rados.rados.0.smithi059.stdout:2356: done (9 left) 2022-04-28T19:51:59.913 INFO:tasks.rados.rados.0.smithi059.stdout:2357: done (8 left) 2022-04-28T19:51:59.914 INFO:tasks.rados.rados.0.smithi059.stdout:2358: done (7 left) 2022-04-28T19:51:59.914 INFO:tasks.rados.rados.0.smithi059.stdout:2363: done (6 left) 2022-04-28T19:51:59.914 INFO:tasks.rados.rados.0.smithi059.stdout:2364: done (5 left) 2022-04-28T19:51:59.914 INFO:tasks.rados.rados.0.smithi059.stdout:2365: copy_from oid 48 from oid 46 current snap is 225 2022-04-28T19:51:59.914 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 225 2022-04-28T19:51:59.914 INFO:tasks.rados.rados.0.smithi059.stdout:2366: seq_num 750 ranges {7684096=565248} 2022-04-28T19:51:59.960 INFO:tasks.rados.rados.0.smithi059.stdout:2366: writing smithi05955964-39 from 7684096 to 8249344 tid 1 2022-04-28T19:51:59.960 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 225 2022-04-28T19:51:59.960 INFO:tasks.rados.rados.0.smithi059.stdout:2367: seq_num 751 ranges {1409024=581632} 2022-04-28T19:51:59.971 INFO:tasks.rados.rados.0.smithi059.stdout:2367: writing smithi05955964-20 from 1409024 to 1990656 tid 1 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2368: read oid 42 snap -1 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2368: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2369: snap_create 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2366: finishing write tid 1 to smithi05955964-39 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2366: finishing write tid 3 to smithi05955964-39 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2366: finishing write tid 4 to smithi05955964-39 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1096 (ObjNum 750 snap 225 seq_num 750) dirty exists 2022-04-28T19:51:59.972 INFO:tasks.rados.rados.0.smithi059.stdout:2366: left oid 39 (ObjNum 750 snap 225 seq_num 750) 2022-04-28T19:51:59.976 INFO:tasks.rados.rados.0.smithi059.stdout:2367: finishing write tid 1 to smithi05955964-20 2022-04-28T19:51:59.976 INFO:tasks.rados.rados.0.smithi059.stdout:2367: finishing write tid 3 to smithi05955964-20 2022-04-28T19:51:59.978 INFO:tasks.rados.rados.0.smithi059.stdout:2367: finishing write tid 4 to smithi05955964-20 2022-04-28T19:51:59.978 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 930 (ObjNum 751 snap 225 seq_num 751) dirty exists 2022-04-28T19:51:59.978 INFO:tasks.rados.rados.0.smithi059.stdout:2367: left oid 20 (ObjNum 751 snap 225 seq_num 751) 2022-04-28T19:52:00.001 INFO:tasks.rados.rados.0.smithi059.stdout:2368: expect (ObjNum 613 snap 172 seq_num 613) 2022-04-28T19:52:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:58.901543+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2022-04-28T19:52:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:58.906448+0000 mgr.y (mgr.24961) 160 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 343 MiB data, 698 MiB used, 710 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.7 MiB/s wr, 10 op/s 2022-04-28T19:52:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:58.920195+0000 mon.a (mon.0) 522 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:58.920248+0000 mon.a (mon.0) 523 : cluster [INF] osd.3 failed (root=default,host=smithi059) (connection refused reported by osd.2) 2022-04-28T19:52:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:58.921151+0000 mon.a (mon.0) 524 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.067769+0000 mon.a (mon.0) 525 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.068172+0000 mon.a (mon.0) 526 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.068372+0000 mon.a (mon.0) 527 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.068549+0000 mon.a (mon.0) 528 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.068869+0000 mon.a (mon.0) 529 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.069119+0000 mon.a (mon.0) 530 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.069308+0000 mon.a (mon.0) 531 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.069602+0000 mon.a (mon.0) 532 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.069920+0000 mon.a (mon.0) 533 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.070252+0000 mon.a (mon.0) 534 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.070573+0000 mon.a (mon.0) 535 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.070816+0000 mon.a (mon.0) 536 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.071211+0000 mon.a (mon.0) 537 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.071453+0000 mon.a (mon.0) 538 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.120579+0000 mon.a (mon.0) 539 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.124007+0000 mon.a (mon.0) 540 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.351437+0000 mon.a (mon.0) 541 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.471904+0000 mon.a (mon.0) 542 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.472052+0000 mon.a (mon.0) 543 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.472183+0000 mon.a (mon.0) 544 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:58.901543+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:58.906448+0000 mgr.y (mgr.24961) 160 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 343 MiB data, 698 MiB used, 710 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.7 MiB/s wr, 10 op/s 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:58.920195+0000 mon.a (mon.0) 522 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:58.920248+0000 mon.a (mon.0) 523 : cluster [INF] osd.3 failed (root=default,host=smithi059) (connection refused reported by osd.2) 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:58.921151+0000 mon.a (mon.0) 524 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.067769+0000 mon.a (mon.0) 525 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.068172+0000 mon.a (mon.0) 526 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.068372+0000 mon.a (mon.0) 527 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.068549+0000 mon.a (mon.0) 528 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.068869+0000 mon.a (mon.0) 529 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.069119+0000 mon.a (mon.0) 530 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.069308+0000 mon.a (mon.0) 531 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.069602+0000 mon.a (mon.0) 532 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.069920+0000 mon.a (mon.0) 533 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.070252+0000 mon.a (mon.0) 534 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.070573+0000 mon.a (mon.0) 535 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.070816+0000 mon.a (mon.0) 536 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.071211+0000 mon.a (mon.0) 537 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.071453+0000 mon.a (mon.0) 538 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.120579+0000 mon.a (mon.0) 539 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.124007+0000 mon.a (mon.0) 540 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.351437+0000 mon.a (mon.0) 541 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.471904+0000 mon.a (mon.0) 542 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.472052+0000 mon.a (mon.0) 543 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.472183+0000 mon.a (mon.0) 544 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.472343+0000 mon.a (mon.0) 545 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.472494+0000 mon.a (mon.0) 546 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.472658+0000 mon.a (mon.0) 547 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.472792+0000 mon.a (mon.0) 548 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473024+0000 mon.a (mon.0) 549 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473134+0000 mon.a (mon.0) 550 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473349+0000 mon.a (mon.0) 551 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473531+0000 mon.a (mon.0) 552 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473642+0000 mon.a (mon.0) 553 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473862+0000 mon.a (mon.0) 554 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.473969+0000 mon.a (mon.0) 555 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.524334+0000 mon.a (mon.0) 556 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.527970+0000 mon.a (mon.0) 557 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.472343+0000 mon.a (mon.0) 545 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.472494+0000 mon.a (mon.0) 546 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.472658+0000 mon.a (mon.0) 547 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.472792+0000 mon.a (mon.0) 548 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473024+0000 mon.a (mon.0) 549 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473134+0000 mon.a (mon.0) 550 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473349+0000 mon.a (mon.0) 551 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473531+0000 mon.a (mon.0) 552 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473642+0000 mon.a (mon.0) 553 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473862+0000 mon.a (mon.0) 554 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.473969+0000 mon.a (mon.0) 555 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.524334+0000 mon.a (mon.0) 556 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:51:59 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.527970+0000 mon.a (mon.0) 557 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:58.901543+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e500: 8 total, 8 up, 8 in 2022-04-28T19:52:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:58.906448+0000 mgr.y (mgr.24961) 160 : cluster [DBG] pgmap v235: 89 pgs: 89 active+clean; 343 MiB data, 698 MiB used, 710 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.7 MiB/s wr, 10 op/s 2022-04-28T19:52:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:58.920195+0000 mon.a (mon.0) 522 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:58.920248+0000 mon.a (mon.0) 523 : cluster [INF] osd.3 failed (root=default,host=smithi059) (connection refused reported by osd.2) 2022-04-28T19:52:00.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:58.921151+0000 mon.a (mon.0) 524 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.067769+0000 mon.a (mon.0) 525 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.068172+0000 mon.a (mon.0) 526 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.068372+0000 mon.a (mon.0) 527 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.068549+0000 mon.a (mon.0) 528 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.068869+0000 mon.a (mon.0) 529 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.069119+0000 mon.a (mon.0) 530 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.069308+0000 mon.a (mon.0) 531 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.069602+0000 mon.a (mon.0) 532 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.069920+0000 mon.a (mon.0) 533 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.070252+0000 mon.a (mon.0) 534 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.070573+0000 mon.a (mon.0) 535 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.070816+0000 mon.a (mon.0) 536 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.071211+0000 mon.a (mon.0) 537 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.071453+0000 mon.a (mon.0) 538 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.120579+0000 mon.a (mon.0) 539 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.124007+0000 mon.a (mon.0) 540 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.351437+0000 mon.a (mon.0) 541 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.471904+0000 mon.a (mon.0) 542 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.472052+0000 mon.a (mon.0) 543 : cluster [DBG] osd.3 reported immediately failed by osd.7 2022-04-28T19:52:00.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.472183+0000 mon.a (mon.0) 544 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.472343+0000 mon.a (mon.0) 545 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.472494+0000 mon.a (mon.0) 546 : cluster [DBG] osd.3 reported immediately failed by osd.6 2022-04-28T19:52:00.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.472658+0000 mon.a (mon.0) 547 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.472792+0000 mon.a (mon.0) 548 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473024+0000 mon.a (mon.0) 549 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473134+0000 mon.a (mon.0) 550 : cluster [DBG] osd.3 reported immediately failed by osd.5 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473349+0000 mon.a (mon.0) 551 : cluster [DBG] osd.3 reported immediately failed by osd.4 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473531+0000 mon.a (mon.0) 552 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473642+0000 mon.a (mon.0) 553 : cluster [DBG] osd.3 reported immediately failed by osd.1 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473862+0000 mon.a (mon.0) 554 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.473969+0000 mon.a (mon.0) 555 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.524334+0000 mon.a (mon.0) 556 : cluster [DBG] osd.3 reported immediately failed by osd.0 2022-04-28T19:52:00.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:51:59 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.527970+0000 mon.a (mon.0) 557 : cluster [DBG] osd.3 reported immediately failed by osd.2 2022-04-28T19:52:00.920 INFO:tasks.rados.rados.0.smithi059.stdout:2366: done (9 left) 2022-04-28T19:52:00.920 INFO:tasks.rados.rados.0.smithi059.stdout:2367: done (8 left) 2022-04-28T19:52:00.921 INFO:tasks.rados.rados.0.smithi059.stdout:2368: done (7 left) 2022-04-28T19:52:00.921 INFO:tasks.rados.rados.0.smithi059.stdout:2369: done (6 left) 2022-04-28T19:52:00.921 INFO:tasks.rados.rados.0.smithi059.stdout:2370: copy_from oid 24 from oid 45 current snap is 226 2022-04-28T19:52:00.921 INFO:tasks.rados.rados.0.smithi059.stdout:2371: read oid 32 snap -1 2022-04-28T19:52:00.921 INFO:tasks.rados.rados.0.smithi059.stdout:2371: expect (ObjNum 703 snap 203 seq_num 703) 2022-04-28T19:52:00.922 INFO:tasks.rados.rados.0.smithi059.stdout:2372: copy_from oid 30 from oid 22 current snap is 226 2022-04-28T19:52:00.922 INFO:tasks.rados.rados.0.smithi059.stdout:2373: snap_remove snap 223 2022-04-28T19:52:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:00 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.894818+0000 mon.a (mon.0) 558 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:00 smithi059 bash[61152]: cluster 2022-04-28T19:51:59.909808+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2022-04-28T19:52:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:00 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.894818+0000 mon.a (mon.0) 558 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:00 smithi059 bash[59745]: cluster 2022-04-28T19:51:59.909808+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2022-04-28T19:52:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:00 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.894818+0000 mon.a (mon.0) 558 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:00 smithi139 bash[43301]: cluster 2022-04-28T19:51:59.909808+0000 mon.a (mon.0) 559 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2022-04-28T19:52:01.891 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:01.630Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:01.891 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:01.630Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:01.891 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:01.630Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:52:01.922 INFO:tasks.rados.rados.0.smithi059.stdout:2373: done (9 left) 2022-04-28T19:52:01.922 INFO:tasks.rados.rados.0.smithi059.stdout:2374: rmattr oid 23 current snap is 226 2022-04-28T19:52:01.923 INFO:tasks.rados.rados.0.smithi059.stdout:2374: done (9 left) 2022-04-28T19:52:01.923 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 226 2022-04-28T19:52:01.923 INFO:tasks.rados.rados.0.smithi059.stdout:2375: seq_num 752 ranges {147456=598016} 2022-04-28T19:52:01.928 INFO:tasks.rados.rados.0.smithi059.stdout:2375: writing smithi05955964-8 from 147456 to 745472 tid 1 2022-04-28T19:52:01.928 INFO:tasks.rados.rados.0.smithi059.stdout:2376: copy_from oid 13 from oid 18 current snap is 226 2022-04-28T19:52:01.928 INFO:tasks.rados.rados.0.smithi059.stdout:2359: finishing rollback tid 0 to smithi05955964-10 2022-04-28T19:52:01.929 INFO:tasks.rados.rados.0.smithi059.stdout:2359: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:52:01.929 INFO:tasks.rados.rados.0.smithi059.stdout:2359: finishing rollback tid 2 to smithi05955964-10 2022-04-28T19:52:01.929 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1078 (ObjNum 733 snap 213 seq_num 733) dirty exists 2022-04-28T19:52:01.929 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 922 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:52:01.929 INFO:tasks.rados.rados.0.smithi059.stdout:2371: expect (ObjNum 703 snap 203 seq_num 703) 2022-04-28T19:52:02.013 INFO:tasks.rados.rados.0.smithi059.stdout:2362: expect (ObjNum 628 snap 180 seq_num 628) 2022-04-28T19:52:02.047 INFO:tasks.rados.rados.0.smithi059.stdout:2354: finishing copy_from to smithi05955964-19 2022-04-28T19:52:02.047 INFO:tasks.rados.rados.0.smithi059.stdout:2354: got expected ENOENT (src dne) 2022-04-28T19:52:02.047 INFO:tasks.rados.rados.0.smithi059.stdout:2354: finishing copy_from racing read to smithi05955964-19 2022-04-28T19:52:02.047 INFO:tasks.rados.rados.0.smithi059.stdout:2361: finishing write tid 1 to smithi05955964-26 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:2354: done (10 left) 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:2359: done (9 left) 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:2360: done (8 left) 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:2362: done (7 left) 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:2371: done (6 left) 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 226 2022-04-28T19:52:02.048 INFO:tasks.rados.rados.0.smithi059.stdout:2377: seq_num 753 ranges {1998848=614400} 2022-04-28T19:52:02.067 INFO:tasks.rados.rados.0.smithi059.stdout:2377: writing smithi05955964-35 from 1998848 to 2613248 tid 1 2022-04-28T19:52:02.067 INFO:tasks.rados.rados.0.smithi059.stdout:2378: delete oid 32 current snap is 226 2022-04-28T19:52:02.067 INFO:tasks.rados.rados.0.smithi059.stdout:2361: finishing write tid 3 to smithi05955964-26 2022-04-28T19:52:02.067 INFO:tasks.rados.rados.0.smithi059.stdout:2361: finishing write tid 4 to smithi05955964-26 2022-04-28T19:52:02.068 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1077 (ObjNum 748 snap 224 seq_num 748) dirty exists 2022-04-28T19:52:02.068 INFO:tasks.rados.rados.0.smithi059.stdout:2361: left oid 26 (ObjNum 748 snap 224 seq_num 748) 2022-04-28T19:52:02.068 INFO:tasks.rados.rados.0.smithi059.stdout:2370: finishing copy_from to smithi05955964-24 2022-04-28T19:52:02.068 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 927 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:52:02.068 INFO:tasks.rados.rados.0.smithi059.stdout:2370: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:52:02.068 INFO:tasks.rados.rados.0.smithi059.stdout:2372: finishing copy_from to smithi05955964-30 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:2372: finishing copy_from racing read to smithi05955964-30 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:2365: finishing copy_from to smithi05955964-48 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1036 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:2365: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:2375: finishing write tid 1 to smithi05955964-8 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:2375: finishing write tid 3 to smithi05955964-8 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:2375: finishing write tid 4 to smithi05955964-8 2022-04-28T19:52:02.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1041 (ObjNum 752 snap 226 seq_num 752) dirty exists 2022-04-28T19:52:02.070 INFO:tasks.rados.rados.0.smithi059.stdout:2375: left oid 8 (ObjNum 752 snap 226 seq_num 752) 2022-04-28T19:52:02.070 INFO:tasks.rados.rados.0.smithi059.stdout:2376: finishing copy_from to smithi05955964-13 2022-04-28T19:52:02.070 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 960 (ObjNum 715 snap 207 seq_num 715) dirty exists 2022-04-28T19:52:02.070 INFO:tasks.rados.rados.0.smithi059.stdout:2376: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:52:02.071 INFO:tasks.rados.rados.0.smithi059.stdout:2361: done (7 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2365: done (6 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2370: done (5 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2372: done (4 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2375: done (3 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2376: done (2 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2378: done (1 left) 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2379: rollback oid 42 current snap is 226 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 225 2022-04-28T19:52:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:2380: rollback oid 36 current snap is 226 2022-04-28T19:52:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 222 2022-04-28T19:52:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:2381: rollback oid 19 current snap is 226 2022-04-28T19:52:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 19 to 218 2022-04-28T19:52:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 226 2022-04-28T19:52:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:2382: seq_num 755 ranges {0=647168} 2022-04-28T19:52:02.076 INFO:tasks.rados.rados.0.smithi059.stdout:2382: writing smithi05955964-40 from 0 to 647168 tid 1 2022-04-28T19:52:02.076 INFO:tasks.rados.rados.0.smithi059.stdout:2383: snap_create 2022-04-28T19:52:02.076 INFO:tasks.rados.rados.0.smithi059.stdout:2377: finishing write tid 1 to smithi05955964-35 2022-04-28T19:52:02.076 INFO:tasks.rados.rados.0.smithi059.stdout:2380: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:52:02.076 INFO:tasks.rados.rados.0.smithi059.stdout:2377: finishing write tid 3 to smithi05955964-35 2022-04-28T19:52:02.076 INFO:tasks.rados.rados.0.smithi059.stdout:2380: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:52:02.077 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1028 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:52:02.080 INFO:tasks.rados.rados.0.smithi059.stdout:2379: finishing rollback tid 0 to smithi05955964-42 2022-04-28T19:52:02.080 INFO:tasks.rados.rados.0.smithi059.stdout:2377: finishing write tid 4 to smithi05955964-35 2022-04-28T19:52:02.080 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 936 (ObjNum 753 snap 226 seq_num 753) dirty exists 2022-04-28T19:52:02.080 INFO:tasks.rados.rados.0.smithi059.stdout:2377: left oid 35 (ObjNum 753 snap 226 seq_num 753) 2022-04-28T19:52:02.081 INFO:tasks.rados.rados.0.smithi059.stdout:2381: finishing rollback tid 0 to smithi05955964-19 2022-04-28T19:52:02.081 INFO:tasks.rados.rados.0.smithi059.stdout:2381: finishing rollback tid 1 to smithi05955964-19 2022-04-28T19:52:02.081 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1081 (ObjNum 0 snap 0 seq_num 4159778832) dirty dne 2022-04-28T19:52:02.081 INFO:tasks.rados.rados.0.smithi059.stdout:2382: finishing write tid 1 to smithi05955964-40 2022-04-28T19:52:02.082 INFO:tasks.rados.rados.0.smithi059.stdout:2379: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:52:02.082 INFO:tasks.rados.rados.0.smithi059.stdout:2379: finishing rollback tid 2 to smithi05955964-42 2022-04-28T19:52:02.082 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1012 (ObjNum 613 snap 172 seq_num 613) dirty exists 2022-04-28T19:52:02.082 INFO:tasks.rados.rados.0.smithi059.stdout:2382: finishing write tid 3 to smithi05955964-40 2022-04-28T19:52:02.084 INFO:tasks.rados.rados.0.smithi059.stdout:2382: finishing write tid 4 to smithi05955964-40 2022-04-28T19:52:02.084 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1030 (ObjNum 755 snap 226 seq_num 755) dirty exists 2022-04-28T19:52:02.084 INFO:tasks.rados.rados.0.smithi059.stdout:2382: left oid 40 (ObjNum 755 snap 226 seq_num 755) 2022-04-28T19:52:02.341 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:01 smithi059 systemd[1]: Stopped Ceph osd.3 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:52:02.341 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:02 smithi059 systemd[1]: Started Ceph osd.3 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:52:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[59745]: cluster 2022-04-28T19:52:00.907147+0000 mgr.y (mgr.24961) 161 : cluster [DBG] pgmap v237: 89 pgs: 4 peering, 13 stale+active+clean, 72 active+clean; 344 MiB data, 699 MiB used, 710 GiB / 715 GiB avail; 8.5 MiB/s rd, 571 KiB/s wr, 10 op/s 2022-04-28T19:52:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[59745]: cluster 2022-04-28T19:52:00.915871+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e502: 8 total, 7 up, 8 in 2022-04-28T19:52:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[61152]: cluster 2022-04-28T19:52:00.907147+0000 mgr.y (mgr.24961) 161 : cluster [DBG] pgmap v237: 89 pgs: 4 peering, 13 stale+active+clean, 72 active+clean; 344 MiB data, 699 MiB used, 710 GiB / 715 GiB avail; 8.5 MiB/s rd, 571 KiB/s wr, 10 op/s 2022-04-28T19:52:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:01 smithi059 bash[61152]: cluster 2022-04-28T19:52:00.915871+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e502: 8 total, 7 up, 8 in 2022-04-28T19:52:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:01 smithi139 bash[43301]: cluster 2022-04-28T19:52:00.907147+0000 mgr.y (mgr.24961) 161 : cluster [DBG] pgmap v237: 89 pgs: 4 peering, 13 stale+active+clean, 72 active+clean; 344 MiB data, 699 MiB used, 710 GiB / 715 GiB avail; 8.5 MiB/s rd, 571 KiB/s wr, 10 op/s 2022-04-28T19:52:02.438 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:01 smithi139 bash[43301]: cluster 2022-04-28T19:52:00.915871+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e502: 8 total, 7 up, 8 in 2022-04-28T19:52:02.938 INFO:tasks.rados.rados.0.smithi059.stdout:2377: done (5 left) 2022-04-28T19:52:02.938 INFO:tasks.rados.rados.0.smithi059.stdout:2379: done (4 left) 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2380: done (3 left) 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2381: done (2 left) 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2382: done (1 left) 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2383: done (0 left) 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2384: read oid 19 snap -1 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2384: expect deleted 2022-04-28T19:52:02.939 INFO:tasks.rados.rados.0.smithi059.stdout:2385: snap_remove snap 190 2022-04-28T19:52:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: cluster 2022-04-28T19:52:01.919455+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e503: 8 total, 7 up, 8 in 2022-04-28T19:52:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: audit 2022-04-28T19:52:02.088698+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: audit 2022-04-28T19:52:02.089799+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: audit 2022-04-28T19:52:02.091008+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:52:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: audit 2022-04-28T19:52:02.091801+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:52:03.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: cephadm 2022-04-28T19:52:02.092529+0000 mgr.y (mgr.24961) 162 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[61152]: audit 2022-04-28T19:52:02.093246+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: cluster 2022-04-28T19:52:01.919455+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e503: 8 total, 7 up, 8 in 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: audit 2022-04-28T19:52:02.088698+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: audit 2022-04-28T19:52:02.089799+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: audit 2022-04-28T19:52:02.091008+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: audit 2022-04-28T19:52:02.091801+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:52:03.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: cephadm 2022-04-28T19:52:02.092529+0000 mgr.y (mgr.24961) 162 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:52:03.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:02 smithi059 bash[59745]: audit 2022-04-28T19:52:02.093246+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:52:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: cluster 2022-04-28T19:52:01.919455+0000 mon.a (mon.0) 561 : cluster [DBG] osdmap e503: 8 total, 7 up, 8 in 2022-04-28T19:52:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: audit 2022-04-28T19:52:02.088698+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: audit 2022-04-28T19:52:02.089799+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: audit 2022-04-28T19:52:02.091008+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:52:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: audit 2022-04-28T19:52:02.091801+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:52:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: cephadm 2022-04-28T19:52:02.092529+0000 mgr.y (mgr.24961) 162 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:52:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:02 smithi139 bash[43301]: audit 2022-04-28T19:52:02.093246+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:52:03.841 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2022-04-28T19:52:03.872 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2022-04-28T19:52:03.872 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2022-04-28T19:52:03.872 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2022-04-28T19:52:03.872 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2022-04-28T19:52:03.872 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2022-04-28T19:52:03.872 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[68412]: --> ceph-volume lvm activate successful for osd ID: 3 2022-04-28T19:52:03.873 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:03.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:03.873 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:03.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:03.873 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:03.524Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:52:03.948 INFO:tasks.rados.rados.0.smithi059.stdout:2385: done (1 left) 2022-04-28T19:52:03.993 INFO:tasks.rados.rados.0.smithi059.stdout:2386: snap_create 2022-04-28T19:52:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:03 smithi139 bash[43301]: cluster 2022-04-28T19:52:02.908339+0000 mgr.y (mgr.24961) 163 : cluster [DBG] pgmap v240: 89 pgs: 5 activating+undersized, 5 active+undersized+degraded, 4 peering, 21 active+undersized, 54 active+clean; 353 MiB data, 715 MiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 2.4 MiB/s wr, 12 op/s; 42/555 objects degraded (7.568%) 2022-04-28T19:52:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:03 smithi139 bash[43301]: cluster 2022-04-28T19:52:02.937739+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e504: 8 total, 7 up, 8 in 2022-04-28T19:52:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:03 smithi139 bash[43301]: cluster 2022-04-28T19:52:03.086720+0000 mon.a (mon.0) 568 : cluster [WRN] Health check failed: Degraded data redundancy: 42/555 objects degraded (7.568%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:04.442 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[59745]: cluster 2022-04-28T19:52:02.908339+0000 mgr.y (mgr.24961) 163 : cluster [DBG] pgmap v240: 89 pgs: 5 activating+undersized, 5 active+undersized+degraded, 4 peering, 21 active+undersized, 54 active+clean; 353 MiB data, 715 MiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 2.4 MiB/s wr, 12 op/s; 42/555 objects degraded (7.568%) 2022-04-28T19:52:04.442 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[59745]: cluster 2022-04-28T19:52:02.937739+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e504: 8 total, 7 up, 8 in 2022-04-28T19:52:04.442 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[59745]: cluster 2022-04-28T19:52:03.086720+0000 mon.a (mon.0) 568 : cluster [WRN] Health check failed: Degraded data redundancy: 42/555 objects degraded (7.568%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:04.443 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[61152]: cluster 2022-04-28T19:52:02.908339+0000 mgr.y (mgr.24961) 163 : cluster [DBG] pgmap v240: 89 pgs: 5 activating+undersized, 5 active+undersized+degraded, 4 peering, 21 active+undersized, 54 active+clean; 353 MiB data, 715 MiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 2.4 MiB/s wr, 12 op/s; 42/555 objects degraded (7.568%) 2022-04-28T19:52:04.443 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[61152]: cluster 2022-04-28T19:52:02.937739+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e504: 8 total, 7 up, 8 in 2022-04-28T19:52:04.443 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:03 smithi059 bash[61152]: cluster 2022-04-28T19:52:03.086720+0000 mon.a (mon.0) 568 : cluster [WRN] Health check failed: Degraded data redundancy: 42/555 objects degraded (7.568%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:04.959 INFO:tasks.rados.rados.0.smithi059.stdout:2384: done (1 left) 2022-04-28T19:52:04.959 INFO:tasks.rados.rados.0.smithi059.stdout:2386: done (0 left) 2022-04-28T19:52:04.960 INFO:tasks.rados.rados.0.smithi059.stdout:2387: copy_from oid 50 from oid 43 current snap is 228 2022-04-28T19:52:04.960 INFO:tasks.rados.rados.0.smithi059.stdout:2388: snap_create 2022-04-28T19:52:04.963 INFO:tasks.rados.rados.0.smithi059.stdout:2387: finishing copy_from to smithi05955964-50 2022-04-28T19:52:04.963 INFO:tasks.rados.rados.0.smithi059.stdout:2387: got expected ENOENT (src dne) 2022-04-28T19:52:04.963 INFO:tasks.rados.rados.0.smithi059.stdout:2387: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:52:05.244 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:04 smithi059 bash[61152]: cluster 2022-04-28T19:52:03.941848+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e505: 8 total, 7 up, 8 in 2022-04-28T19:52:05.244 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:04 smithi059 bash[59745]: cluster 2022-04-28T19:52:03.941848+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e505: 8 total, 7 up, 8 in 2022-04-28T19:52:05.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:04 smithi139 bash[43301]: cluster 2022-04-28T19:52:03.941848+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e505: 8 total, 7 up, 8 in 2022-04-28T19:52:05.591 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:05 smithi059 bash[68412]: debug 2022-04-28T19:52:05.355+0000 7f928a494700 1 -- 172.21.15.59:0/2841667263 <== mon.1 v2:172.21.15.59:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b0a483c4e0 con 0x55b0a48b4c00 2022-04-28T19:52:05.969 INFO:tasks.rados.rados.0.smithi059.stdout:2387: done (1 left) 2022-04-28T19:52:05.969 INFO:tasks.rados.rados.0.smithi059.stdout:2388: done (0 left) 2022-04-28T19:52:05.969 INFO:tasks.rados.rados.0.smithi059.stdout:2389: rollback oid 44 current snap is 229 2022-04-28T19:52:05.969 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 44 to 226 2022-04-28T19:52:05.969 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 229 2022-04-28T19:52:05.969 INFO:tasks.rados.rados.0.smithi059.stdout:2390: seq_num 756 ranges {5726208=671744} 2022-04-28T19:52:06.007 INFO:tasks.rados.rados.0.smithi059.stdout:2390: writing smithi05955964-42 from 5726208 to 6397952 tid 1 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:2391: rollback oid 5 current snap is 229 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 218 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:2389: finishing rollback tid 0 to smithi05955964-44 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:2389: finishing rollback tid 1 to smithi05955964-44 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:2389: finishing rollback tid 2 to smithi05955964-44 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 731 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:2389: done (2 left) 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 229 2022-04-28T19:52:06.008 INFO:tasks.rados.rados.0.smithi059.stdout:2392: seq_num 757 ranges {0=688128} 2022-04-28T19:52:06.011 INFO:tasks.rados.rados.0.smithi059.stdout:2392: writing smithi05955964-37 from 0 to 688128 tid 1 2022-04-28T19:52:06.011 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 229 2022-04-28T19:52:06.011 INFO:tasks.rados.rados.0.smithi059.stdout:2393: seq_num 758 ranges {7684096=704512} 2022-04-28T19:52:06.062 INFO:tasks.rados.rados.0.smithi059.stdout:2393: writing smithi05955964-9 from 7684096 to 8388608 tid 1 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2394: read oid 19 snap -1 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2394: expect deleted 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2395: delete oid 36 current snap is 229 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2391: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2391: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1054 (ObjNum 0 snap 0 seq_num 0) dirty dne 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2390: finishing write tid 1 to smithi05955964-42 2022-04-28T19:52:06.063 INFO:tasks.rados.rados.0.smithi059.stdout:2390: finishing write tid 3 to smithi05955964-42 2022-04-28T19:52:06.064 INFO:tasks.rados.rados.0.smithi059.stdout:2392: finishing write tid 1 to smithi05955964-37 2022-04-28T19:52:06.064 INFO:tasks.rados.rados.0.smithi059.stdout:2392: finishing write tid 3 to smithi05955964-37 2022-04-28T19:52:06.064 INFO:tasks.rados.rados.0.smithi059.stdout:2390: finishing write tid 4 to smithi05955964-42 2022-04-28T19:52:06.064 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1021 (ObjNum 756 snap 229 seq_num 756) dirty exists 2022-04-28T19:52:06.064 INFO:tasks.rados.rados.0.smithi059.stdout:2390: left oid 42 (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:52:06.065 INFO:tasks.rados.rados.0.smithi059.stdout:2392: finishing write tid 4 to smithi05955964-37 2022-04-28T19:52:06.065 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1023 (ObjNum 757 snap 229 seq_num 757) dirty exists 2022-04-28T19:52:06.065 INFO:tasks.rados.rados.0.smithi059.stdout:2392: left oid 37 (ObjNum 757 snap 229 seq_num 757) 2022-04-28T19:52:06.066 INFO:tasks.rados.rados.0.smithi059.stdout:2390: done (5 left) 2022-04-28T19:52:06.066 INFO:tasks.rados.rados.0.smithi059.stdout:2391: done (4 left) 2022-04-28T19:52:06.066 INFO:tasks.rados.rados.0.smithi059.stdout:2392: done (3 left) 2022-04-28T19:52:06.067 INFO:tasks.rados.rados.0.smithi059.stdout:2394: done (2 left) 2022-04-28T19:52:06.067 INFO:tasks.rados.rados.0.smithi059.stdout:2395: done (1 left) 2022-04-28T19:52:06.067 INFO:tasks.rados.rados.0.smithi059.stdout:2396: read oid 20 snap -1 2022-04-28T19:52:06.067 INFO:tasks.rados.rados.0.smithi059.stdout:2396: expect (ObjNum 751 snap 225 seq_num 751) 2022-04-28T19:52:06.067 INFO:tasks.rados.rados.0.smithi059.stdout:2397: delete oid 41 current snap is 229 2022-04-28T19:52:06.069 INFO:tasks.rados.rados.0.smithi059.stdout:2393: finishing write tid 1 to smithi05955964-9 2022-04-28T19:52:06.070 INFO:tasks.rados.rados.0.smithi059.stdout:2393: finishing write tid 3 to smithi05955964-9 2022-04-28T19:52:06.074 INFO:tasks.rados.rados.0.smithi059.stdout:2393: finishing write tid 4 to smithi05955964-9 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 733 (ObjNum 758 snap 229 seq_num 758) dirty exists 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2393: left oid 9 (ObjNum 758 snap 229 seq_num 758) 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2393: done (2 left) 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2397: done (1 left) 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2398: read oid 13 snap -1 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2398: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2399: read oid 32 snap -1 2022-04-28T19:52:06.075 INFO:tasks.rados.rados.0.smithi059.stdout:2399: expect deleted 2022-04-28T19:52:06.076 INFO:tasks.rados.rados.0.smithi059.stdout:2400: read oid 47 snap -1 2022-04-28T19:52:06.076 INFO:tasks.rados.rados.0.smithi059.stdout:2400: expect (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:52:06.076 INFO:tasks.rados.rados.0.smithi059.stdout:2401: read oid 38 snap 224 2022-04-28T19:52:06.076 INFO:tasks.rados.rados.0.smithi059.stdout:2401: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:52:06.076 INFO:tasks.rados.rados.0.smithi059.stdout:2402: setattr oid 40 current snap is 229 2022-04-28T19:52:06.077 INFO:tasks.rados.rados.0.smithi059.stdout:2399: done (5 left) 2022-04-28T19:52:06.077 INFO:tasks.rados.rados.0.smithi059.stdout:2403: setattr oid 7 current snap is 229 2022-04-28T19:52:06.077 INFO:tasks.rados.rados.0.smithi059.stdout:2396: expect (ObjNum 751 snap 225 seq_num 751) 2022-04-28T19:52:06.192 INFO:tasks.rados.rados.0.smithi059.stdout:2400: expect (ObjNum 718 snap 209 seq_num 718) 2022-04-28T19:52:06.193 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1032 (ObjNum 755 snap 226 seq_num 755) dirty exists 2022-04-28T19:52:06.193 INFO:tasks.rados.rados.0.smithi059.stdout:2401: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:52:06.235 INFO:tasks.rados.rados.0.smithi059.stdout:2398: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:52:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:05 smithi059 bash[59745]: cluster 2022-04-28T19:52:04.908935+0000 mgr.y (mgr.24961) 164 : cluster [DBG] pgmap v243: 89 pgs: 5 activating+undersized, 5 active+undersized+degraded, 4 peering, 21 active+undersized, 54 active+clean; 353 MiB data, 715 MiB used, 710 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.3 MiB/s wr, 9 op/s; 42/555 objects degraded (7.568%) 2022-04-28T19:52:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:05 smithi059 bash[59745]: cluster 2022-04-28T19:52:04.958901+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e506: 8 total, 7 up, 8 in 2022-04-28T19:52:06.341 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:05 smithi059 bash[68412]: debug 2022-04-28T19:52:05.899+0000 7f9291cef200 -1 Falling back to public interface 2022-04-28T19:52:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:05 smithi059 bash[61152]: cluster 2022-04-28T19:52:04.908935+0000 mgr.y (mgr.24961) 164 : cluster [DBG] pgmap v243: 89 pgs: 5 activating+undersized, 5 active+undersized+degraded, 4 peering, 21 active+undersized, 54 active+clean; 353 MiB data, 715 MiB used, 710 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.3 MiB/s wr, 9 op/s; 42/555 objects degraded (7.568%) 2022-04-28T19:52:06.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:05 smithi059 bash[61152]: cluster 2022-04-28T19:52:04.958901+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e506: 8 total, 7 up, 8 in 2022-04-28T19:52:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:05 smithi139 bash[43301]: cluster 2022-04-28T19:52:04.908935+0000 mgr.y (mgr.24961) 164 : cluster [DBG] pgmap v243: 89 pgs: 5 activating+undersized, 5 active+undersized+degraded, 4 peering, 21 active+undersized, 54 active+clean; 353 MiB data, 715 MiB used, 710 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.3 MiB/s wr, 9 op/s; 42/555 objects degraded (7.568%) 2022-04-28T19:52:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:05 smithi139 bash[43301]: cluster 2022-04-28T19:52:04.958901+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e506: 8 total, 7 up, 8 in 2022-04-28T19:52:06.372 INFO:tasks.rados.rados.0.smithi059.stdout:2396: done (5 left) 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2398: done (4 left) 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2400: done (3 left) 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2401: done (2 left) 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2402: done (1 left) 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2404: rollback oid 38 current snap is 229 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 221 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2405: copy_from oid 8 from oid 48 current snap is 229 2022-04-28T19:52:06.375 INFO:tasks.rados.rados.0.smithi059.stdout:2406: read oid 9 snap 224 2022-04-28T19:52:06.376 INFO:tasks.rados.rados.0.smithi059.stdout:2406: expect (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:52:06.376 INFO:tasks.rados.rados.0.smithi059.stdout:2407: snap_remove snap 225 2022-04-28T19:52:06.981 INFO:tasks.rados.rados.0.smithi059.stdout:2407: done (4 left) 2022-04-28T19:52:06.981 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 229 2022-04-28T19:52:06.981 INFO:tasks.rados.rados.0.smithi059.stdout:2408: seq_num 761 ranges {0=753664} 2022-04-28T19:52:06.985 INFO:tasks.rados.rados.0.smithi059.stdout:2408: writing smithi05955964-32 from 0 to 753664 tid 1 2022-04-28T19:52:06.985 INFO:tasks.rados.rados.0.smithi059.stdout:2404: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2409: read oid 15 snap 211 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2409: expect deleted 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2410: read oid 36 snap -1 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2410: expect deleted 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2411: copy_from oid 11 from oid 4 current snap is 229 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 948 (ObjNum 742 snap 220 seq_num 742) dirty exists 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2404: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:2404: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:52:06.986 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 930 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:52:06.987 INFO:tasks.rados.rados.0.smithi059.stdout:2406: expect (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:52:07.341 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[57771]: debug 2022-04-28T19:52:06.875+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2022-04-28T19:52:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: cluster 2022-04-28T19:52:05.967699+0000 mon.a (mon.0) 571 : cluster [DBG] osdmap e507: 8 total, 7 up, 8 in 2022-04-28T19:52:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.569144+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.858399+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.866641+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.874804+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.876680+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.878172+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[59745]: audit 2022-04-28T19:52:06.880314+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: cluster 2022-04-28T19:52:05.967699+0000 mon.a (mon.0) 571 : cluster [DBG] osdmap e507: 8 total, 7 up, 8 in 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.569144+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.858399+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.866641+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.874804+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.876680+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.878172+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:06 smithi059 bash[61152]: audit 2022-04-28T19:52:06.880314+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:2405: finishing copy_from to smithi05955964-8 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1036 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:2405: finishing copy_from racing read to smithi05955964-8 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:2403: done (7 left) 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:2404: done (6 left) 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:2405: done (5 left) 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:2406: done (4 left) 2022-04-28T19:52:07.351 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 229 2022-04-28T19:52:07.352 INFO:tasks.rados.rados.0.smithi059.stdout:2412: seq_num 762 ranges {5554176=770048} 2022-04-28T19:52:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: cluster 2022-04-28T19:52:05.967699+0000 mon.a (mon.0) 571 : cluster [DBG] osdmap e507: 8 total, 7 up, 8 in 2022-04-28T19:52:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.569144+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.858399+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.866641+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.874804+0000 mon.a (mon.0) 575 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.876680+0000 mon.a (mon.0) 576 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.878172+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:06 smithi139 bash[43301]: audit 2022-04-28T19:52:06.880314+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:07.387 INFO:tasks.rados.rados.0.smithi059.stdout:2412: writing smithi05955964-8 from 5554176 to 6324224 tid 1 2022-04-28T19:52:07.387 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 229 2022-04-28T19:52:07.387 INFO:tasks.rados.rados.0.smithi059.stdout:2413: seq_num 763 ranges {0=786432} 2022-04-28T19:52:07.391 INFO:tasks.rados.rados.0.smithi059.stdout:2413: writing smithi05955964-1 from 0 to 786432 tid 1 2022-04-28T19:52:07.391 INFO:tasks.rados.rados.0.smithi059.stdout:2414: delete oid 46 current snap is 229 2022-04-28T19:52:07.391 INFO:tasks.rados.rados.0.smithi059.stdout:2408: finishing write tid 1 to smithi05955964-32 2022-04-28T19:52:07.391 INFO:tasks.rados.rados.0.smithi059.stdout:2408: finishing write tid 3 to smithi05955964-32 2022-04-28T19:52:07.391 INFO:tasks.rados.rados.0.smithi059.stdout:2408: finishing write tid 4 to smithi05955964-32 2022-04-28T19:52:07.392 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1112 (ObjNum 761 snap 229 seq_num 761) dirty exists 2022-04-28T19:52:07.392 INFO:tasks.rados.rados.0.smithi059.stdout:2408: left oid 32 (ObjNum 761 snap 229 seq_num 761) 2022-04-28T19:52:07.392 INFO:tasks.rados.rados.0.smithi059.stdout:2411: finishing copy_from to smithi05955964-11 2022-04-28T19:52:07.392 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 823 (ObjNum 628 snap 180 seq_num 628) dirty exists 2022-04-28T19:52:07.392 INFO:tasks.rados.rados.0.smithi059.stdout:2411: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:52:07.396 INFO:tasks.rados.rados.0.smithi059.stdout:2412: finishing write tid 1 to smithi05955964-8 2022-04-28T19:52:07.396 INFO:tasks.rados.rados.0.smithi059.stdout:2412: finishing write tid 3 to smithi05955964-8 2022-04-28T19:52:07.397 INFO:tasks.rados.rados.0.smithi059.stdout:2413: finishing write tid 1 to smithi05955964-1 2022-04-28T19:52:07.397 INFO:tasks.rados.rados.0.smithi059.stdout:2413: finishing write tid 3 to smithi05955964-1 2022-04-28T19:52:07.398 INFO:tasks.rados.rados.0.smithi059.stdout:2413: finishing write tid 4 to smithi05955964-1 2022-04-28T19:52:07.399 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1034 (ObjNum 763 snap 229 seq_num 763) dirty exists 2022-04-28T19:52:07.399 INFO:tasks.rados.rados.0.smithi059.stdout:2413: left oid 1 (ObjNum 763 snap 229 seq_num 763) 2022-04-28T19:52:07.399 INFO:tasks.rados.rados.0.smithi059.stdout:2412: finishing write tid 4 to smithi05955964-8 2022-04-28T19:52:07.399 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1062 (ObjNum 762 snap 229 seq_num 762) dirty exists 2022-04-28T19:52:07.399 INFO:tasks.rados.rados.0.smithi059.stdout:2412: left oid 8 (ObjNum 762 snap 229 seq_num 762) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2408: done (6 left) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2409: done (5 left) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2410: done (4 left) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2411: done (3 left) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2412: done (2 left) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2413: done (1 left) 2022-04-28T19:52:07.401 INFO:tasks.rados.rados.0.smithi059.stdout:2414: done (0 left) 2022-04-28T19:52:07.402 INFO:tasks.rados.rados.0.smithi059.stdout:2415: snap_remove snap 214 2022-04-28T19:52:07.989 INFO:tasks.rados.rados.0.smithi059.stdout:2415: done (0 left) 2022-04-28T19:52:07.989 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 229 2022-04-28T19:52:07.990 INFO:tasks.rados.rados.0.smithi059.stdout:2416: seq_num 765 ranges {5914624=417792,6332416=401408} 2022-04-28T19:52:08.029 INFO:tasks.rados.rados.0.smithi059.stdout:2416: writing smithi05955964-25 from 5914624 to 6332416 tid 1 2022-04-28T19:52:08.031 INFO:tasks.rados.rados.0.smithi059.stdout:2416: writing smithi05955964-25 from 6332416 to 6733824 tid 2 2022-04-28T19:52:08.031 INFO:tasks.rados.rados.0.smithi059.stdout:2417: read oid 15 snap -1 2022-04-28T19:52:08.031 INFO:tasks.rados.rados.0.smithi059.stdout:2417: expect deleted 2022-04-28T19:52:08.031 INFO:tasks.rados.rados.0.smithi059.stdout:2418: snap_create 2022-04-28T19:52:08.039 INFO:tasks.rados.rados.0.smithi059.stdout:2416: finishing write tid 1 to smithi05955964-25 2022-04-28T19:52:08.039 INFO:tasks.rados.rados.0.smithi059.stdout:2416: finishing write tid 2 to smithi05955964-25 2022-04-28T19:52:08.041 INFO:tasks.rados.rados.0.smithi059.stdout:2416: finishing write tid 4 to smithi05955964-25 2022-04-28T19:52:08.044 INFO:tasks.rados.rados.0.smithi059.stdout:2416: finishing write tid 5 to smithi05955964-25 2022-04-28T19:52:08.044 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1068 (ObjNum 765 snap 229 seq_num 765) dirty exists 2022-04-28T19:52:08.044 INFO:tasks.rados.rados.0.smithi059.stdout:2416: left oid 25 (ObjNum 765 snap 229 seq_num 765) 2022-04-28T19:52:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[59745]: audit 2022-04-28T19:52:06.880615+0000 mgr.y (mgr.24961) 165 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[59745]: cephadm 2022-04-28T19:52:06.882244+0000 mgr.y (mgr.24961) 166 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2022-04-28T19:52:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[59745]: cluster 2022-04-28T19:52:06.909685+0000 mgr.y (mgr.24961) 167 : cluster [DBG] pgmap v246: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 366 MiB data, 723 MiB used, 710 GiB / 715 GiB avail; 767 B/s rd, 2.8 MiB/s wr, 4 op/s; 55/564 objects degraded (9.752%) 2022-04-28T19:52:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[59745]: cluster 2022-04-28T19:52:06.980451+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e508: 8 total, 7 up, 8 in 2022-04-28T19:52:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[61152]: audit 2022-04-28T19:52:06.880615+0000 mgr.y (mgr.24961) 165 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[61152]: cephadm 2022-04-28T19:52:06.882244+0000 mgr.y (mgr.24961) 166 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2022-04-28T19:52:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[61152]: cluster 2022-04-28T19:52:06.909685+0000 mgr.y (mgr.24961) 167 : cluster [DBG] pgmap v246: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 366 MiB data, 723 MiB used, 710 GiB / 715 GiB avail; 767 B/s rd, 2.8 MiB/s wr, 4 op/s; 55/564 objects degraded (9.752%) 2022-04-28T19:52:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:07 smithi059 bash[61152]: cluster 2022-04-28T19:52:06.980451+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e508: 8 total, 7 up, 8 in 2022-04-28T19:52:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:07 smithi139 bash[43301]: audit 2022-04-28T19:52:06.880615+0000 mgr.y (mgr.24961) 165 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:07 smithi139 bash[43301]: cephadm 2022-04-28T19:52:06.882244+0000 mgr.y (mgr.24961) 166 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2022-04-28T19:52:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:07 smithi139 bash[43301]: cluster 2022-04-28T19:52:06.909685+0000 mgr.y (mgr.24961) 167 : cluster [DBG] pgmap v246: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 366 MiB data, 723 MiB used, 710 GiB / 715 GiB avail; 767 B/s rd, 2.8 MiB/s wr, 4 op/s; 55/564 objects degraded (9.752%) 2022-04-28T19:52:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:07 smithi139 bash[43301]: cluster 2022-04-28T19:52:06.980451+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e508: 8 total, 7 up, 8 in 2022-04-28T19:52:08.996 INFO:tasks.rados.rados.0.smithi059.stdout:2416: done (2 left) 2022-04-28T19:52:08.997 INFO:tasks.rados.rados.0.smithi059.stdout:2417: done (1 left) 2022-04-28T19:52:08.997 INFO:tasks.rados.rados.0.smithi059.stdout:2418: done (0 left) 2022-04-28T19:52:08.997 INFO:tasks.rados.rados.0.smithi059.stdout:2419: copy_from oid 10 from oid 15 current snap is 230 2022-04-28T19:52:08.997 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 230 2022-04-28T19:52:08.997 INFO:tasks.rados.rados.0.smithi059.stdout:2420: seq_num 766 ranges {688128=434176,1122304=401408} 2022-04-28T19:52:09.008 INFO:tasks.rados.rados.0.smithi059.stdout:2420: writing smithi05955964-37 from 688128 to 1122304 tid 1 2022-04-28T19:52:09.013 INFO:tasks.rados.rados.0.smithi059.stdout:2420: writing smithi05955964-37 from 1122304 to 1523712 tid 2 2022-04-28T19:52:09.013 INFO:tasks.rados.rados.0.smithi059.stdout:2421: copy_from oid 6 from oid 9 current snap is 230 2022-04-28T19:52:09.013 INFO:tasks.rados.rados.0.smithi059.stdout:2419: finishing copy_from to smithi05955964-10 2022-04-28T19:52:09.013 INFO:tasks.rados.rados.0.smithi059.stdout:2419: got expected ENOENT (src dne) 2022-04-28T19:52:09.013 INFO:tasks.rados.rados.0.smithi059.stdout:2419: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:52:09.013 INFO:tasks.rados.rados.0.smithi059.stdout:2419: done (2 left) 2022-04-28T19:52:09.014 INFO:tasks.rados.rados.0.smithi059.stdout:2422: snap_remove snap 211 2022-04-28T19:52:09.308 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:08 smithi059 bash[59745]: cluster 2022-04-28T19:52:07.988183+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e509: 8 total, 7 up, 8 in 2022-04-28T19:52:09.309 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:08 smithi059 bash[61152]: cluster 2022-04-28T19:52:07.988183+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e509: 8 total, 7 up, 8 in 2022-04-28T19:52:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:08 smithi139 bash[43301]: cluster 2022-04-28T19:52:07.988183+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e509: 8 total, 7 up, 8 in 2022-04-28T19:52:09.591 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:09 smithi059 bash[68412]: debug 2022-04-28T19:52:09.303+0000 7f9291cef200 -1 osd.3 499 log_to_monitors {default=true} 2022-04-28T19:52:10.009 INFO:tasks.rados.rados.0.smithi059.stdout:2420: finishing write tid 1 to smithi05955964-37 2022-04-28T19:52:10.009 INFO:tasks.rados.rados.0.smithi059.stdout:2420: finishing write tid 2 to smithi05955964-37 2022-04-28T19:52:10.009 INFO:tasks.rados.rados.0.smithi059.stdout:2420: finishing write tid 4 to smithi05955964-37 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2420: finishing write tid 5 to smithi05955964-37 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1031 (ObjNum 766 snap 230 seq_num 766) dirty exists 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2420: left oid 37 (ObjNum 766 snap 230 seq_num 766) 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2421: finishing copy_from to smithi05955964-6 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 733 (ObjNum 758 snap 229 seq_num 758) dirty exists 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2421: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2420: done (2 left) 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2421: done (1 left) 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2422: done (0 left) 2022-04-28T19:52:10.010 INFO:tasks.rados.rados.0.smithi059.stdout:2423: delete oid 49 current snap is 230 2022-04-28T19:52:10.020 INFO:tasks.rados.rados.0.smithi059.stdout:2423: done (0 left) 2022-04-28T19:52:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:2424: rmattr oid 13 current snap is 230 2022-04-28T19:52:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:2424: done (0 left) 2022-04-28T19:52:10.021 INFO:tasks.rados.rados.0.smithi059.stdout:2425: setattr oid 39 current snap is 230 2022-04-28T19:52:10.023 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 230 2022-04-28T19:52:10.023 INFO:tasks.rados.rados.0.smithi059.stdout:2426: seq_num 768 ranges {0=466944,466944=401408} 2022-04-28T19:52:10.026 INFO:tasks.rados.rados.0.smithi059.stdout:2426: writing smithi05955964-43 from 0 to 466944 tid 1 2022-04-28T19:52:10.030 INFO:tasks.rados.rados.0.smithi059.stdout:2426: writing smithi05955964-43 from 466944 to 868352 tid 2 2022-04-28T19:52:10.030 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 230 2022-04-28T19:52:10.030 INFO:tasks.rados.rados.0.smithi059.stdout:2427: seq_num 769 ranges {32768=483328,516096=401408} 2022-04-28T19:52:10.034 INFO:tasks.rados.rados.0.smithi059.stdout:2427: writing smithi05955964-47 from 32768 to 516096 tid 1 2022-04-28T19:52:10.037 INFO:tasks.rados.rados.0.smithi059.stdout:2427: writing smithi05955964-47 from 516096 to 917504 tid 2 2022-04-28T19:52:10.038 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 230 2022-04-28T19:52:10.038 INFO:tasks.rados.rados.0.smithi059.stdout:2428: seq_num 770 ranges {8388608=499712,8888320=401408} 2022-04-28T19:52:10.089 INFO:tasks.rados.rados.0.smithi059.stdout:2428: writing smithi05955964-9 from 8388608 to 8888320 tid 1 2022-04-28T19:52:10.092 INFO:tasks.rados.rados.0.smithi059.stdout:2428: writing smithi05955964-9 from 8888320 to 9289728 tid 2 2022-04-28T19:52:10.092 INFO:tasks.rados.rados.0.smithi059.stdout:2429: rollback oid 48 current snap is 230 2022-04-28T19:52:10.092 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 226 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1122 (ObjNum 750 snap 225 seq_num 750) dirty exists 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2425: done (4 left) 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2430: delete oid 50 current snap is 230 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2426: finishing write tid 1 to smithi05955964-43 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2427: finishing write tid 1 to smithi05955964-47 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2426: finishing write tid 2 to smithi05955964-43 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2426: finishing write tid 4 to smithi05955964-43 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2427: finishing write tid 2 to smithi05955964-47 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2427: finishing write tid 4 to smithi05955964-47 2022-04-28T19:52:10.093 INFO:tasks.rados.rados.0.smithi059.stdout:2427: finishing write tid 5 to smithi05955964-47 2022-04-28T19:52:10.094 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1082 (ObjNum 769 snap 230 seq_num 769) dirty exists 2022-04-28T19:52:10.094 INFO:tasks.rados.rados.0.smithi059.stdout:2427: left oid 47 (ObjNum 769 snap 230 seq_num 769) 2022-04-28T19:52:10.094 INFO:tasks.rados.rados.0.smithi059.stdout:2426: finishing write tid 5 to smithi05955964-43 2022-04-28T19:52:10.094 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1129 (ObjNum 768 snap 230 seq_num 768) dirty exists 2022-04-28T19:52:10.094 INFO:tasks.rados.rados.0.smithi059.stdout:2426: left oid 43 (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:52:10.095 INFO:tasks.rados.rados.0.smithi059.stdout:2428: finishing write tid 1 to smithi05955964-9 2022-04-28T19:52:10.096 INFO:tasks.rados.rados.0.smithi059.stdout:2428: finishing write tid 2 to smithi05955964-9 2022-04-28T19:52:10.096 INFO:tasks.rados.rados.0.smithi059.stdout:2428: finishing write tid 4 to smithi05955964-9 2022-04-28T19:52:10.097 INFO:tasks.rados.rados.0.smithi059.stdout:2429: finishing rollback tid 0 to smithi05955964-48 2022-04-28T19:52:10.097 INFO:tasks.rados.rados.0.smithi059.stdout:2426: done (4 left) 2022-04-28T19:52:10.098 INFO:tasks.rados.rados.0.smithi059.stdout:2427: done (3 left) 2022-04-28T19:52:10.098 INFO:tasks.rados.rados.0.smithi059.stdout:2430: done (2 left) 2022-04-28T19:52:10.098 INFO:tasks.rados.rados.0.smithi059.stdout:2431: snap_create 2022-04-28T19:52:10.098 INFO:tasks.rados.rados.0.smithi059.stdout:2428: finishing write tid 5 to smithi05955964-9 2022-04-28T19:52:10.098 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 736 (ObjNum 770 snap 230 seq_num 770) dirty exists 2022-04-28T19:52:10.098 INFO:tasks.rados.rados.0.smithi059.stdout:2428: left oid 9 (ObjNum 770 snap 230 seq_num 770) 2022-04-28T19:52:10.099 INFO:tasks.rados.rados.0.smithi059.stdout:2429: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:52:10.099 INFO:tasks.rados.rados.0.smithi059.stdout:2429: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:52:10.099 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1085 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:10.341 INFO:journalctl@ceph.osd.3.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[68412]: debug 2022-04-28T19:52:10.043+0000 7f9284c0c700 -1 osd.3 499 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:52:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[61152]: cluster 2022-04-28T19:52:08.910518+0000 mgr.y (mgr.24961) 168 : cluster [DBG] pgmap v249: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 353 MiB data, 708 MiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 4.8 MiB/s wr, 14 op/s; 55/546 objects degraded (10.073%) 2022-04-28T19:52:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[61152]: cluster 2022-04-28T19:52:08.985167+0000 mon.a (mon.0) 581 : cluster [WRN] Health check update: Degraded data redundancy: 55/546 objects degraded (10.073%), 7 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[61152]: cluster 2022-04-28T19:52:08.993324+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e510: 8 total, 7 up, 8 in 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[61152]: audit 2022-04-28T19:52:09.311792+0000 mon.c (mon.1) 9 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[61152]: audit 2022-04-28T19:52:09.312117+0000 mon.a (mon.0) 583 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[59745]: cluster 2022-04-28T19:52:08.910518+0000 mgr.y (mgr.24961) 168 : cluster [DBG] pgmap v249: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 353 MiB data, 708 MiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 4.8 MiB/s wr, 14 op/s; 55/546 objects degraded (10.073%) 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[59745]: cluster 2022-04-28T19:52:08.985167+0000 mon.a (mon.0) 581 : cluster [WRN] Health check update: Degraded data redundancy: 55/546 objects degraded (10.073%), 7 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[59745]: cluster 2022-04-28T19:52:08.993324+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e510: 8 total, 7 up, 8 in 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[59745]: audit 2022-04-28T19:52:09.311792+0000 mon.c (mon.1) 9 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:52:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:10 smithi059 bash[59745]: audit 2022-04-28T19:52:09.312117+0000 mon.a (mon.0) 583 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:52:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:10 smithi139 bash[43301]: cluster 2022-04-28T19:52:08.910518+0000 mgr.y (mgr.24961) 168 : cluster [DBG] pgmap v249: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 353 MiB data, 708 MiB used, 710 GiB / 715 GiB avail; 2.7 MiB/s rd, 4.8 MiB/s wr, 14 op/s; 55/546 objects degraded (10.073%) 2022-04-28T19:52:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:10 smithi139 bash[43301]: cluster 2022-04-28T19:52:08.985167+0000 mon.a (mon.0) 581 : cluster [WRN] Health check update: Degraded data redundancy: 55/546 objects degraded (10.073%), 7 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:10 smithi139 bash[43301]: cluster 2022-04-28T19:52:08.993324+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e510: 8 total, 7 up, 8 in 2022-04-28T19:52:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:10 smithi139 bash[43301]: audit 2022-04-28T19:52:09.311792+0000 mon.c (mon.1) 9 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:52:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:10 smithi139 bash[43301]: audit 2022-04-28T19:52:09.312117+0000 mon.a (mon.0) 583 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2022-04-28T19:52:11.029 INFO:tasks.rados.rados.0.smithi059.stdout:2428: done (2 left) 2022-04-28T19:52:11.061 INFO:tasks.rados.rados.0.smithi059.stdout:2429: done (1 left) 2022-04-28T19:52:11.061 INFO:tasks.rados.rados.0.smithi059.stdout:2431: done (0 left) 2022-04-28T19:52:11.061 INFO:tasks.rados.rados.0.smithi059.stdout:2432: copy_from oid 8 from oid 3 current snap is 231 2022-04-28T19:52:11.061 INFO:tasks.rados.rados.0.smithi059.stdout:2433: setattr oid 13 current snap is 231 2022-04-28T19:52:11.062 INFO:tasks.rados.rados.0.smithi059.stdout:2434: delete oid 23 current snap is 231 2022-04-28T19:52:11.062 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 961 (ObjNum 715 snap 207 seq_num 715) dirty exists 2022-04-28T19:52:11.075 INFO:tasks.rados.rados.0.smithi059.stdout:2432: finishing copy_from to smithi05955964-8 2022-04-28T19:52:11.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 912 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:52:11.076 INFO:tasks.rados.rados.0.smithi059.stdout:2432: finishing copy_from racing read to smithi05955964-8 2022-04-28T19:52:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[61152]: audit 2022-04-28T19:52:10.005865+0000 mon.a (mon.0) 584 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-04-28T19:52:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[61152]: cluster 2022-04-28T19:52:10.006125+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e511: 8 total, 7 up, 8 in 2022-04-28T19:52:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[61152]: audit 2022-04-28T19:52:10.010177+0000 mon.c (mon.1) 10 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:52:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[61152]: audit 2022-04-28T19:52:10.017197+0000 mon.a (mon.0) 586 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:52:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[59745]: audit 2022-04-28T19:52:10.005865+0000 mon.a (mon.0) 584 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-04-28T19:52:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[59745]: cluster 2022-04-28T19:52:10.006125+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e511: 8 total, 7 up, 8 in 2022-04-28T19:52:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[59745]: audit 2022-04-28T19:52:10.010177+0000 mon.c (mon.1) 10 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:52:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[59745]: audit 2022-04-28T19:52:10.017197+0000 mon.a (mon.0) 586 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:52:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:11 smithi139 bash[43301]: audit 2022-04-28T19:52:10.005865+0000 mon.a (mon.0) 584 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2022-04-28T19:52:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:11 smithi139 bash[43301]: cluster 2022-04-28T19:52:10.006125+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e511: 8 total, 7 up, 8 in 2022-04-28T19:52:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:11 smithi139 bash[43301]: audit 2022-04-28T19:52:10.010177+0000 mon.c (mon.1) 10 : audit [INF] from='osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:52:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:11 smithi139 bash[43301]: audit 2022-04-28T19:52:10.017197+0000 mon.a (mon.0) 586 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi059", "root=default"]}]: dispatch 2022-04-28T19:52:12.036 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:11.631Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:12.036 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:11.631Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:12.036 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:11.631Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:52:12.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[59745]: cluster 2022-04-28T19:52:10.911357+0000 mgr.y (mgr.24961) 169 : cluster [DBG] pgmap v252: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 363 MiB data, 726 MiB used, 711 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.4 MiB/s wr, 21 op/s; 55/564 objects degraded (9.752%) 2022-04-28T19:52:12.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[59745]: cluster 2022-04-28T19:52:11.012663+0000 mon.a (mon.0) 587 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:52:12.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[59745]: cluster 2022-04-28T19:52:11.020431+0000 mon.a (mon.0) 588 : cluster [INF] osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104] boot 2022-04-28T19:52:12.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[59745]: cluster 2022-04-28T19:52:11.020735+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2022-04-28T19:52:12.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[59745]: audit 2022-04-28T19:52:11.022311+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:52:12.047 INFO:tasks.rados.rados.0.smithi059.stdout:2432: done (2 left) 2022-04-28T19:52:12.047 INFO:tasks.rados.rados.0.smithi059.stdout:2433: done (1 left) 2022-04-28T19:52:12.047 INFO:tasks.rados.rados.0.smithi059.stdout:2434: done (0 left) 2022-04-28T19:52:12.047 INFO:tasks.rados.rados.0.smithi059.stdout:2435: copy_from oid 14 from oid 16 current snap is 231 2022-04-28T19:52:12.047 INFO:tasks.rados.rados.0.smithi059.stdout:2436: snap_create 2022-04-28T19:52:12.113 INFO:tasks.rados.rados.0.smithi059.stdout:2435: finishing copy_from to smithi05955964-14 2022-04-28T19:52:12.114 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 950 (ObjNum 672 snap 195 seq_num 672) dirty exists 2022-04-28T19:52:12.114 INFO:tasks.rados.rados.0.smithi059.stdout:2435: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:52:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[61152]: cluster 2022-04-28T19:52:10.911357+0000 mgr.y (mgr.24961) 169 : cluster [DBG] pgmap v252: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 363 MiB data, 726 MiB used, 711 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.4 MiB/s wr, 21 op/s; 55/564 objects degraded (9.752%) 2022-04-28T19:52:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[61152]: cluster 2022-04-28T19:52:11.012663+0000 mon.a (mon.0) 587 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:52:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[61152]: cluster 2022-04-28T19:52:11.020431+0000 mon.a (mon.0) 588 : cluster [INF] osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104] boot 2022-04-28T19:52:12.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[61152]: cluster 2022-04-28T19:52:11.020735+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2022-04-28T19:52:12.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:12 smithi059 bash[61152]: audit 2022-04-28T19:52:11.022311+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:52:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:12 smithi139 bash[43301]: cluster 2022-04-28T19:52:10.911357+0000 mgr.y (mgr.24961) 169 : cluster [DBG] pgmap v252: 89 pgs: 7 active+undersized+degraded, 28 active+undersized, 54 active+clean; 363 MiB data, 726 MiB used, 711 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.4 MiB/s wr, 21 op/s; 55/564 objects degraded (9.752%) 2022-04-28T19:52:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:12 smithi139 bash[43301]: cluster 2022-04-28T19:52:11.012663+0000 mon.a (mon.0) 587 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:52:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:12 smithi139 bash[43301]: cluster 2022-04-28T19:52:11.020431+0000 mon.a (mon.0) 588 : cluster [INF] osd.3 [v2:172.21.15.59:6826/3357327104,v1:172.21.15.59:6827/3357327104] boot 2022-04-28T19:52:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:12 smithi139 bash[43301]: cluster 2022-04-28T19:52:11.020735+0000 mon.a (mon.0) 589 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2022-04-28T19:52:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:12 smithi139 bash[43301]: audit 2022-04-28T19:52:11.022311+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2022-04-28T19:52:13.050 INFO:tasks.rados.rados.0.smithi059.stdout:2435: done (1 left) 2022-04-28T19:52:13.050 INFO:tasks.rados.rados.0.smithi059.stdout:2436: done (0 left) 2022-04-28T19:52:13.050 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 232 2022-04-28T19:52:13.050 INFO:tasks.rados.rados.0.smithi059.stdout:2437: seq_num 773 ranges {2121728=548864,2670592=401408} 2022-04-28T19:52:13.065 INFO:tasks.rados.rados.0.smithi059.stdout:2437: writing smithi05955964-12 from 2121728 to 2670592 tid 1 2022-04-28T19:52:13.068 INFO:tasks.rados.rados.0.smithi059.stdout:2437: writing smithi05955964-12 from 2670592 to 3072000 tid 2 2022-04-28T19:52:13.068 INFO:tasks.rados.rados.0.smithi059.stdout:2438: read oid 33 snap -1 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2438: expect deleted 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2439: rmattr oid 38 current snap is 232 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2439: done (2 left) 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2440: rollback oid 44 current snap is 232 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 44 to 227 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2441: rollback oid 4 current snap is 232 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 199 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2442: rollback oid 41 current snap is 232 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 230 2022-04-28T19:52:13.069 INFO:tasks.rados.rados.0.smithi059.stdout:2443: read oid 29 snap -1 2022-04-28T19:52:13.070 INFO:tasks.rados.rados.0.smithi059.stdout:2443: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:13.070 INFO:tasks.rados.rados.0.smithi059.stdout:2444: rollback oid 40 current snap is 232 2022-04-28T19:52:13.070 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 215 2022-04-28T19:52:13.070 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 232 2022-04-28T19:52:13.070 INFO:tasks.rados.rados.0.smithi059.stdout:2445: seq_num 774 ranges {11804672=565248,12369920=401408} 2022-04-28T19:52:13.138 INFO:tasks.rados.rados.0.smithi059.stdout:2445: writing smithi05955964-34 from 11804672 to 12369920 tid 1 2022-04-28T19:52:13.141 INFO:tasks.rados.rados.0.smithi059.stdout:2445: writing smithi05955964-34 from 12369920 to 12771328 tid 2 2022-04-28T19:52:13.141 INFO:tasks.rados.rados.0.smithi059.stdout:2446: read oid 22 snap -1 2022-04-28T19:52:13.141 INFO:tasks.rados.rados.0.smithi059.stdout:2446: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2447: snap_create 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2437: finishing write tid 1 to smithi05955964-12 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2437: finishing write tid 2 to smithi05955964-12 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2437: finishing write tid 4 to smithi05955964-12 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2440: finishing rollback tid 0 to smithi05955964-44 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2442: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1009 (ObjNum 288 snap 0 seq_num 33) dirty dne 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2440: finishing rollback tid 1 to smithi05955964-44 2022-04-28T19:52:13.142 INFO:tasks.rados.rados.0.smithi059.stdout:2440: finishing rollback tid 2 to smithi05955964-44 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 739 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2437: finishing write tid 5 to smithi05955964-12 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 962 (ObjNum 773 snap 232 seq_num 773) dirty exists 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2437: left oid 12 (ObjNum 773 snap 232 seq_num 773) 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2444: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2444: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1036 (ObjNum 2 snap 0 seq_num 5) dirty dne 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2441: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2441: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:52:13.143 INFO:tasks.rados.rados.0.smithi059.stdout:2441: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:52:13.144 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 933 (ObjNum 628 snap 180 seq_num 628) dirty exists 2022-04-28T19:52:13.144 INFO:tasks.rados.rados.0.smithi059.stdout:2443: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:13.237 INFO:tasks.rados.rados.0.smithi059.stdout:2445: finishing write tid 1 to smithi05955964-34 2022-04-28T19:52:13.237 INFO:tasks.rados.rados.0.smithi059.stdout:2445: finishing write tid 2 to smithi05955964-34 2022-04-28T19:52:13.237 INFO:tasks.rados.rados.0.smithi059.stdout:2445: finishing write tid 4 to smithi05955964-34 2022-04-28T19:52:13.238 INFO:tasks.rados.rados.0.smithi059.stdout:2445: finishing write tid 5 to smithi05955964-34 2022-04-28T19:52:13.238 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1041 (ObjNum 774 snap 232 seq_num 774) dirty exists 2022-04-28T19:52:13.238 INFO:tasks.rados.rados.0.smithi059.stdout:2445: left oid 34 (ObjNum 774 snap 232 seq_num 774) 2022-04-28T19:52:13.238 INFO:tasks.rados.rados.0.smithi059.stdout:2446: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:13 smithi059 bash[61152]: cluster 2022-04-28T19:52:12.030779+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2022-04-28T19:52:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:13 smithi059 bash[59745]: cluster 2022-04-28T19:52:12.030779+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2022-04-28T19:52:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:13 smithi139 bash[43301]: cluster 2022-04-28T19:52:12.030779+0000 mon.a (mon.0) 591 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2022-04-28T19:52:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:13.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:13.524Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:13.525Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:52:14.054 INFO:tasks.rados.rados.0.smithi059.stdout:2437: done (9 left) 2022-04-28T19:52:14.054 INFO:tasks.rados.rados.0.smithi059.stdout:2438: done (8 left) 2022-04-28T19:52:14.054 INFO:tasks.rados.rados.0.smithi059.stdout:2440: done (7 left) 2022-04-28T19:52:14.054 INFO:tasks.rados.rados.0.smithi059.stdout:2441: done (6 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2442: done (5 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2443: done (4 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2444: done (3 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2445: done (2 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2446: done (1 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2447: done (0 left) 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2448: rollback oid 6 current snap is 233 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 227 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 233 2022-04-28T19:52:14.055 INFO:tasks.rados.rados.0.smithi059.stdout:2449: seq_num 775 ranges {9289728=589824,9879552=401408} 2022-04-28T19:52:14.112 INFO:tasks.rados.rados.0.smithi059.stdout:2449: writing smithi05955964-9 from 9289728 to 9879552 tid 1 2022-04-28T19:52:14.114 INFO:tasks.rados.rados.0.smithi059.stdout:2449: writing smithi05955964-9 from 9879552 to 10280960 tid 2 2022-04-28T19:52:14.114 INFO:tasks.rados.rados.0.smithi059.stdout:2450: copy_from oid 19 from oid 31 current snap is 233 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:2448: finishing rollback tid 0 to smithi05955964-6 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:2448: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:2448: finishing rollback tid 2 to smithi05955964-6 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 936 (ObjNum 728 snap 211 seq_num 728) dirty exists 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:2448: done (2 left) 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:2451: rollback oid 34 current snap is 233 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 34 to 218 2022-04-28T19:52:14.115 INFO:tasks.rados.rados.0.smithi059.stdout:2452: delete oid 7 current snap is 233 2022-04-28T19:52:14.120 INFO:tasks.rados.rados.0.smithi059.stdout:2449: finishing write tid 1 to smithi05955964-9 2022-04-28T19:52:14.120 INFO:tasks.rados.rados.0.smithi059.stdout:2449: finishing write tid 2 to smithi05955964-9 2022-04-28T19:52:14.120 INFO:tasks.rados.rados.0.smithi059.stdout:2449: finishing write tid 4 to smithi05955964-9 2022-04-28T19:52:14.122 INFO:tasks.rados.rados.0.smithi059.stdout:2449: finishing write tid 5 to smithi05955964-9 2022-04-28T19:52:14.122 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 742 (ObjNum 775 snap 233 seq_num 775) dirty exists 2022-04-28T19:52:14.122 INFO:tasks.rados.rados.0.smithi059.stdout:2449: left oid 9 (ObjNum 775 snap 233 seq_num 775) 2022-04-28T19:52:14.128 INFO:tasks.rados.rados.0.smithi059.stdout:2451: finishing rollback tid 0 to smithi05955964-34 2022-04-28T19:52:14.128 INFO:tasks.rados.rados.0.smithi059.stdout:2451: finishing rollback tid 1 to smithi05955964-34 2022-04-28T19:52:14.129 INFO:tasks.rados.rados.0.smithi059.stdout:2451: finishing rollback tid 2 to smithi05955964-34 2022-04-28T19:52:14.129 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1045 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:52:14.132 INFO:tasks.rados.rados.0.smithi059.stdout:2450: finishing copy_from to smithi05955964-19 2022-04-28T19:52:14.132 INFO:tasks.rados.rados.0.smithi059.stdout:2450: got expected ENOENT (src dne) 2022-04-28T19:52:14.132 INFO:tasks.rados.rados.0.smithi059.stdout:2450: finishing copy_from racing read to smithi05955964-19 2022-04-28T19:52:14.133 INFO:tasks.rados.rados.0.smithi059.stdout:2449: done (3 left) 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2450: done (2 left) 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2451: done (1 left) 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2452: done (0 left) 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2453: read oid 30 snap -1 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2453: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2454: copy_from oid 22 from oid 27 current snap is 233 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2455: read oid 1 snap -1 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2455: expect (ObjNum 763 snap 229 seq_num 763) 2022-04-28T19:52:14.134 INFO:tasks.rados.rados.0.smithi059.stdout:2456: read oid 18 snap -1 2022-04-28T19:52:14.135 INFO:tasks.rados.rados.0.smithi059.stdout:2456: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:52:14.135 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 233 2022-04-28T19:52:14.135 INFO:tasks.rados.rados.0.smithi059.stdout:2457: seq_num 777 ranges {0=622592,622592=401408} 2022-04-28T19:52:14.137 INFO:tasks.rados.rados.0.smithi059.stdout:2457: writing smithi05955964-33 from 0 to 622592 tid 1 2022-04-28T19:52:14.140 INFO:tasks.rados.rados.0.smithi059.stdout:2457: writing smithi05955964-33 from 622592 to 1024000 tid 2 2022-04-28T19:52:14.140 INFO:tasks.rados.rados.0.smithi059.stdout:2458: copy_from oid 31 from oid 38 current snap is 233 2022-04-28T19:52:14.140 INFO:tasks.rados.rados.0.smithi059.stdout:2455: expect (ObjNum 763 snap 229 seq_num 763) 2022-04-28T19:52:14.161 INFO:tasks.rados.rados.0.smithi059.stdout:2455: done (5 left) 2022-04-28T19:52:14.161 INFO:tasks.rados.rados.0.smithi059.stdout:2459: snap_remove snap 232 2022-04-28T19:52:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:14 smithi059 bash[59745]: cluster 2022-04-28T19:52:12.912467+0000 mgr.y (mgr.24961) 170 : cluster [DBG] pgmap v255: 89 pgs: 5 active+undersized+degraded, 21 active+undersized, 63 active+clean; 362 MiB data, 763 MiB used, 710 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.0 MiB/s wr, 27 op/s; 52/582 objects degraded (8.935%); 1.1 MiB/s, 0 objects/s recovering 2022-04-28T19:52:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:14 smithi059 bash[59745]: cluster 2022-04-28T19:52:13.049326+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2022-04-28T19:52:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:14 smithi059 bash[61152]: cluster 2022-04-28T19:52:12.912467+0000 mgr.y (mgr.24961) 170 : cluster [DBG] pgmap v255: 89 pgs: 5 active+undersized+degraded, 21 active+undersized, 63 active+clean; 362 MiB data, 763 MiB used, 710 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.0 MiB/s wr, 27 op/s; 52/582 objects degraded (8.935%); 1.1 MiB/s, 0 objects/s recovering 2022-04-28T19:52:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:14 smithi059 bash[61152]: cluster 2022-04-28T19:52:13.049326+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2022-04-28T19:52:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:14 smithi139 bash[43301]: cluster 2022-04-28T19:52:12.912467+0000 mgr.y (mgr.24961) 170 : cluster [DBG] pgmap v255: 89 pgs: 5 active+undersized+degraded, 21 active+undersized, 63 active+clean; 362 MiB data, 763 MiB used, 710 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.0 MiB/s wr, 27 op/s; 52/582 objects degraded (8.935%); 1.1 MiB/s, 0 objects/s recovering 2022-04-28T19:52:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:14 smithi139 bash[43301]: cluster 2022-04-28T19:52:13.049326+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2022-04-28T19:52:15.070 INFO:tasks.rados.rados.0.smithi059.stdout:2459: done (5 left) 2022-04-28T19:52:15.070 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 233 2022-04-28T19:52:15.070 INFO:tasks.rados.rados.0.smithi059.stdout:2460: seq_num 778 ranges {4292608=638976,4931584=401408} 2022-04-28T19:52:15.098 INFO:tasks.rados.rados.0.smithi059.stdout:2460: writing smithi05955964-45 from 4292608 to 4931584 tid 1 2022-04-28T19:52:15.100 INFO:tasks.rados.rados.0.smithi059.stdout:2460: writing smithi05955964-45 from 4931584 to 5332992 tid 2 2022-04-28T19:52:15.100 INFO:tasks.rados.rados.0.smithi059.stdout:2461: delete oid 7 current snap is 233 2022-04-28T19:52:15.101 INFO:tasks.rados.rados.0.smithi059.stdout:2453: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:15.197 INFO:tasks.rados.rados.0.smithi059.stdout:2454: finishing copy_from to smithi05955964-22 2022-04-28T19:52:15.197 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1066 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:52:15.197 INFO:tasks.rados.rados.0.smithi059.stdout:2454: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:52:15.197 INFO:tasks.rados.rados.0.smithi059.stdout:2457: finishing write tid 1 to smithi05955964-33 2022-04-28T19:52:15.197 INFO:tasks.rados.rados.0.smithi059.stdout:2457: finishing write tid 2 to smithi05955964-33 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2457: finishing write tid 4 to smithi05955964-33 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2453: done (6 left) 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2454: done (5 left) 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2461: done (4 left) 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2462: rollback oid 2 current snap is 233 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 2 to 219 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2457: finishing write tid 5 to smithi05955964-33 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1136 (ObjNum 777 snap 233 seq_num 777) dirty exists 2022-04-28T19:52:15.198 INFO:tasks.rados.rados.0.smithi059.stdout:2457: left oid 33 (ObjNum 777 snap 233 seq_num 777) 2022-04-28T19:52:15.199 INFO:tasks.rados.rados.0.smithi059.stdout:2456: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:52:15.314 INFO:tasks.rados.rados.0.smithi059.stdout:2458: finishing copy_from to smithi05955964-31 2022-04-28T19:52:15.314 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 930 (ObjNum 493 snap 141 seq_num 493) dirty exists 2022-04-28T19:52:15.314 INFO:tasks.rados.rados.0.smithi059.stdout:2456: done (4 left) 2022-04-28T19:52:15.314 INFO:tasks.rados.rados.0.smithi059.stdout:2457: done (3 left) 2022-04-28T19:52:15.314 INFO:tasks.rados.rados.0.smithi059.stdout:2463: snap_create 2022-04-28T19:52:15.314 INFO:tasks.rados.rados.0.smithi059.stdout:2458: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2460: finishing write tid 1 to smithi05955964-45 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2460: finishing write tid 2 to smithi05955964-45 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2460: finishing write tid 4 to smithi05955964-45 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2460: finishing write tid 5 to smithi05955964-45 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 940 (ObjNum 778 snap 233 seq_num 778) dirty exists 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2460: left oid 45 (ObjNum 778 snap 233 seq_num 778) 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2462: finishing rollback tid 0 to smithi05955964-2 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2462: finishing rollback tid 1 to smithi05955964-2 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:2462: finishing rollback tid 2 to smithi05955964-2 2022-04-28T19:52:15.315 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 991 (ObjNum 719 snap 209 seq_num 719) dirty exists 2022-04-28T19:52:15.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:15 smithi059 bash[59745]: cluster 2022-04-28T19:52:14.050684+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2022-04-28T19:52:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:15 smithi059 bash[61152]: cluster 2022-04-28T19:52:14.050684+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2022-04-28T19:52:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:15 smithi139 bash[43301]: cluster 2022-04-28T19:52:14.050684+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2022-04-28T19:52:16.083 INFO:tasks.rados.rados.0.smithi059.stdout:2458: done (3 left) 2022-04-28T19:52:16.083 INFO:tasks.rados.rados.0.smithi059.stdout:2460: done (2 left) 2022-04-28T19:52:16.083 INFO:tasks.rados.rados.0.smithi059.stdout:2462: done (1 left) 2022-04-28T19:52:16.083 INFO:tasks.rados.rados.0.smithi059.stdout:2463: done (0 left) 2022-04-28T19:52:16.083 INFO:tasks.rados.rados.0.smithi059.stdout:2464: delete oid 44 current snap is 234 2022-04-28T19:52:16.089 INFO:tasks.rados.rados.0.smithi059.stdout:2464: done (0 left) 2022-04-28T19:52:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:2465: read oid 11 snap -1 2022-04-28T19:52:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:2465: expect (ObjNum 628 snap 180 seq_num 628) 2022-04-28T19:52:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:2466: copy_from oid 15 from oid 43 current snap is 234 2022-04-28T19:52:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 234 2022-04-28T19:52:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:2467: seq_num 781 ranges {753664=688128,1441792=401408} 2022-04-28T19:52:16.098 INFO:tasks.rados.rados.0.smithi059.stdout:2467: writing smithi05955964-32 from 753664 to 1441792 tid 1 2022-04-28T19:52:16.101 INFO:tasks.rados.rados.0.smithi059.stdout:2467: writing smithi05955964-32 from 1441792 to 1843200 tid 2 2022-04-28T19:52:16.101 INFO:tasks.rados.rados.0.smithi059.stdout:2468: delete oid 37 current snap is 234 2022-04-28T19:52:16.101 INFO:tasks.rados.rados.0.smithi059.stdout:2465: expect (ObjNum 628 snap 180 seq_num 628) 2022-04-28T19:52:16.134 INFO:tasks.rados.rados.0.smithi059.stdout:2466: finishing copy_from to smithi05955964-15 2022-04-28T19:52:16.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1129 (ObjNum 768 snap 230 seq_num 768) dirty exists 2022-04-28T19:52:16.135 INFO:tasks.rados.rados.0.smithi059.stdout:2466: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:52:16.135 INFO:tasks.rados.rados.0.smithi059.stdout:2467: finishing write tid 1 to smithi05955964-32 2022-04-28T19:52:16.135 INFO:tasks.rados.rados.0.smithi059.stdout:2467: finishing write tid 2 to smithi05955964-32 2022-04-28T19:52:16.135 INFO:tasks.rados.rados.0.smithi059.stdout:2467: finishing write tid 4 to smithi05955964-32 2022-04-28T19:52:16.135 INFO:tasks.rados.rados.0.smithi059.stdout:2467: finishing write tid 5 to smithi05955964-32 2022-04-28T19:52:16.136 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1140 (ObjNum 781 snap 234 seq_num 781) dirty exists 2022-04-28T19:52:16.136 INFO:tasks.rados.rados.0.smithi059.stdout:2467: left oid 32 (ObjNum 781 snap 234 seq_num 781) 2022-04-28T19:52:16.136 INFO:tasks.rados.rados.0.smithi059.stdout:2465: done (3 left) 2022-04-28T19:52:16.136 INFO:tasks.rados.rados.0.smithi059.stdout:2466: done (2 left) 2022-04-28T19:52:16.136 INFO:tasks.rados.rados.0.smithi059.stdout:2467: done (1 left) 2022-04-28T19:52:16.137 INFO:tasks.rados.rados.0.smithi059.stdout:2468: done (0 left) 2022-04-28T19:52:16.137 INFO:tasks.rados.rados.0.smithi059.stdout:2469: setattr oid 50 current snap is 234 2022-04-28T19:52:16.138 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 234 2022-04-28T19:52:16.138 INFO:tasks.rados.rados.0.smithi059.stdout:2470: seq_num 783 ranges {2473984=720896,3194880=401408} 2022-04-28T19:52:16.159 INFO:tasks.rados.rados.0.smithi059.stdout:2470: writing smithi05955964-29 from 2473984 to 3194880 tid 1 2022-04-28T19:52:16.162 INFO:tasks.rados.rados.0.smithi059.stdout:2470: writing smithi05955964-29 from 3194880 to 3596288 tid 2 2022-04-28T19:52:16.162 INFO:tasks.rados.rados.0.smithi059.stdout:2471: setattr oid 36 current snap is 234 2022-04-28T19:52:16.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1141 (ObjNum -132758080 snap 21893 seq_num 4161652416) dirty exists 2022-04-28T19:52:16.163 INFO:tasks.rados.rados.0.smithi059.stdout:2469: done (2 left) 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:2472: rollback oid 17 current snap is 234 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 227 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:2473: read oid 34 snap -1 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:2473: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:2474: rollback oid 22 current snap is 234 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 231 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 234 2022-04-28T19:52:16.164 INFO:tasks.rados.rados.0.smithi059.stdout:2475: seq_num 784 ranges {499712=737280,1236992=401408} 2022-04-28T19:52:16.171 INFO:tasks.rados.rados.0.smithi059.stdout:2475: writing smithi05955964-21 from 499712 to 1236992 tid 1 2022-04-28T19:52:16.174 INFO:tasks.rados.rados.0.smithi059.stdout:2475: writing smithi05955964-21 from 1236992 to 1638400 tid 2 2022-04-28T19:52:16.174 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 234 2022-04-28T19:52:16.175 INFO:tasks.rados.rados.0.smithi059.stdout:2476: seq_num 785 ranges {0=753664,753664=401408} 2022-04-28T19:52:16.179 INFO:tasks.rados.rados.0.smithi059.stdout:2476: writing smithi05955964-41 from 0 to 753664 tid 1 2022-04-28T19:52:16.182 INFO:tasks.rados.rados.0.smithi059.stdout:2476: writing smithi05955964-41 from 753664 to 1155072 tid 2 2022-04-28T19:52:16.182 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1037 (ObjNum 0 snap 32762 seq_num 4161522704) dirty exists 2022-04-28T19:52:16.182 INFO:tasks.rados.rados.0.smithi059.stdout:2472: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2470: finishing write tid 1 to smithi05955964-29 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2470: finishing write tid 2 to smithi05955964-29 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2474: finishing rollback tid 0 to smithi05955964-22 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2474: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2474: finishing rollback tid 2 to smithi05955964-22 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1092 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2472: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2470: finishing write tid 4 to smithi05955964-29 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:2472: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:52:16.183 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 964 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2470: finishing write tid 5 to smithi05955964-29 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1084 (ObjNum 783 snap 234 seq_num 783) dirty exists 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2470: left oid 29 (ObjNum 783 snap 234 seq_num 783) 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2475: finishing write tid 1 to smithi05955964-21 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2475: finishing write tid 2 to smithi05955964-21 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2477: read oid 14 snap -1 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2477: expect (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2470: done (7 left) 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2471: done (6 left) 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2472: done (5 left) 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2474: done (4 left) 2022-04-28T19:52:16.184 INFO:tasks.rados.rados.0.smithi059.stdout:2478: setattr oid 26 current snap is 234 2022-04-28T19:52:16.185 INFO:tasks.rados.rados.0.smithi059.stdout:2475: finishing write tid 4 to smithi05955964-21 2022-04-28T19:52:16.185 INFO:tasks.rados.rados.0.smithi059.stdout:2475: finishing write tid 5 to smithi05955964-21 2022-04-28T19:52:16.185 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 726 (ObjNum 784 snap 234 seq_num 784) dirty exists 2022-04-28T19:52:16.185 INFO:tasks.rados.rados.0.smithi059.stdout:2475: left oid 21 (ObjNum 784 snap 234 seq_num 784) 2022-04-28T19:52:16.185 INFO:tasks.rados.rados.0.smithi059.stdout:2475: done (4 left) 2022-04-28T19:52:16.186 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 234 2022-04-28T19:52:16.186 INFO:tasks.rados.rados.0.smithi059.stdout:2479: seq_num 786 ranges {0=770048,770048=401408} 2022-04-28T19:52:16.187 INFO:tasks.rados.rados.0.smithi059.stdout:2479: writing smithi05955964-49 from 0 to 770048 tid 1 2022-04-28T19:52:16.190 INFO:tasks.rados.rados.0.smithi059.stdout:2479: writing smithi05955964-49 from 770048 to 1171456 tid 2 2022-04-28T19:52:16.190 INFO:tasks.rados.rados.0.smithi059.stdout:2480: delete oid 29 current snap is 234 2022-04-28T19:52:16.190 INFO:tasks.rados.rados.0.smithi059.stdout:2476: finishing write tid 1 to smithi05955964-41 2022-04-28T19:52:16.191 INFO:tasks.rados.rados.0.smithi059.stdout:2476: finishing write tid 2 to smithi05955964-41 2022-04-28T19:52:16.193 INFO:tasks.rados.rados.0.smithi059.stdout:2480: done (5 left) 2022-04-28T19:52:16.193 INFO:tasks.rados.rados.0.smithi059.stdout:2481: copy_from oid 33 from oid 30 current snap is 234 2022-04-28T19:52:16.194 INFO:tasks.rados.rados.0.smithi059.stdout:2482: snap_remove snap 216 2022-04-28T19:52:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:16 smithi059 bash[59745]: cluster 2022-04-28T19:52:14.913349+0000 mgr.y (mgr.24961) 171 : cluster [DBG] pgmap v258: 89 pgs: 5 active+undersized+degraded, 21 active+undersized, 63 active+clean; 362 MiB data, 763 MiB used, 710 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 16 op/s; 52/582 objects degraded (8.935%); 1.1 MiB/s, 0 objects/s recovering 2022-04-28T19:52:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:16 smithi059 bash[59745]: cluster 2022-04-28T19:52:15.069348+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2022-04-28T19:52:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:16 smithi059 bash[61152]: cluster 2022-04-28T19:52:14.913349+0000 mgr.y (mgr.24961) 171 : cluster [DBG] pgmap v258: 89 pgs: 5 active+undersized+degraded, 21 active+undersized, 63 active+clean; 362 MiB data, 763 MiB used, 710 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 16 op/s; 52/582 objects degraded (8.935%); 1.1 MiB/s, 0 objects/s recovering 2022-04-28T19:52:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:16 smithi059 bash[61152]: cluster 2022-04-28T19:52:15.069348+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2022-04-28T19:52:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:16 smithi139 bash[43301]: cluster 2022-04-28T19:52:14.913349+0000 mgr.y (mgr.24961) 171 : cluster [DBG] pgmap v258: 89 pgs: 5 active+undersized+degraded, 21 active+undersized, 63 active+clean; 362 MiB data, 763 MiB used, 710 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.7 MiB/s wr, 16 op/s; 52/582 objects degraded (8.935%); 1.1 MiB/s, 0 objects/s recovering 2022-04-28T19:52:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:16 smithi139 bash[43301]: cluster 2022-04-28T19:52:15.069348+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2022-04-28T19:52:17.098 INFO:tasks.rados.rados.0.smithi059.stdout:2482: done (6 left) 2022-04-28T19:52:17.098 INFO:tasks.rados.rados.0.smithi059.stdout:2483: setattr oid 11 current snap is 234 2022-04-28T19:52:17.100 INFO:tasks.rados.rados.0.smithi059.stdout:2484: snap_create 2022-04-28T19:52:17.100 INFO:tasks.rados.rados.0.smithi059.stdout:2476: finishing write tid 4 to smithi05955964-41 2022-04-28T19:52:17.100 INFO:tasks.rados.rados.0.smithi059.stdout:2476: finishing write tid 5 to smithi05955964-41 2022-04-28T19:52:17.100 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1012 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:52:17.100 INFO:tasks.rados.rados.0.smithi059.stdout:2476: left oid 41 (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:52:17.100 INFO:tasks.rados.rados.0.smithi059.stdout:2479: finishing write tid 1 to smithi05955964-49 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2479: finishing write tid 2 to smithi05955964-49 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2479: finishing write tid 4 to smithi05955964-49 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2479: finishing write tid 5 to smithi05955964-49 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1040 (ObjNum 786 snap 234 seq_num 786) dirty exists 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2479: left oid 49 (ObjNum 786 snap 234 seq_num 786) 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2481: finishing copy_from to smithi05955964-33 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2481: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:52:17.101 INFO:tasks.rados.rados.0.smithi059.stdout:2473: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:52:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[59745]: cluster 2022-04-28T19:52:16.082635+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2022-04-28T19:52:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[59745]: cluster 2022-04-28T19:52:16.871566+0000 mon.a (mon.0) 596 : cluster [WRN] Health check update: Degraded data redundancy: 52/582 objects degraded (8.935%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[59745]: audit 2022-04-28T19:52:16.891579+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:52:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[59745]: audit 2022-04-28T19:52:16.892274+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T19:52:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[59745]: audit 2022-04-28T19:52:16.892959+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T19:52:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[61152]: cluster 2022-04-28T19:52:16.082635+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2022-04-28T19:52:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[61152]: cluster 2022-04-28T19:52:16.871566+0000 mon.a (mon.0) 596 : cluster [WRN] Health check update: Degraded data redundancy: 52/582 objects degraded (8.935%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[61152]: audit 2022-04-28T19:52:16.891579+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:52:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[61152]: audit 2022-04-28T19:52:16.892274+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T19:52:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:17 smithi059 bash[61152]: audit 2022-04-28T19:52:16.892959+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T19:52:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:17 smithi139 bash[43301]: cluster 2022-04-28T19:52:16.082635+0000 mon.a (mon.0) 595 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2022-04-28T19:52:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:17 smithi139 bash[43301]: cluster 2022-04-28T19:52:16.871566+0000 mon.a (mon.0) 596 : cluster [WRN] Health check update: Degraded data redundancy: 52/582 objects degraded (8.935%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:17 smithi139 bash[43301]: audit 2022-04-28T19:52:16.891579+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:52:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:17 smithi139 bash[43301]: audit 2022-04-28T19:52:16.892274+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T19:52:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:17 smithi139 bash[43301]: audit 2022-04-28T19:52:16.892959+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T19:52:17.941 INFO:tasks.rados.rados.0.smithi059.stdout:2477: expect (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:52:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:18 smithi139 bash[43301]: cluster 2022-04-28T19:52:16.914045+0000 mgr.y (mgr.24961) 172 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 391 MiB data, 801 MiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.9 MiB/s wr, 8 op/s; 8.2 MiB/s, 3 objects/s recovering 2022-04-28T19:52:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:18 smithi139 bash[43301]: cluster 2022-04-28T19:52:17.094595+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2022-04-28T19:52:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:18 smithi139 bash[43301]: audit 2022-04-28T19:52:17.120256+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:52:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:18 smithi139 bash[43301]: audit 2022-04-28T19:52:17.136339+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:52:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:18 smithi139 bash[43301]: cluster 2022-04-28T19:52:17.874633+0000 mon.a (mon.0) 603 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/582 objects degraded (8.935%), 5 pgs degraded) 2022-04-28T19:52:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:18 smithi139 bash[43301]: cluster 2022-04-28T19:52:17.874718+0000 mon.a (mon.0) 604 : cluster [INF] Cluster is now healthy 2022-04-28T19:52:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[61152]: cluster 2022-04-28T19:52:16.914045+0000 mgr.y (mgr.24961) 172 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 391 MiB data, 801 MiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.9 MiB/s wr, 8 op/s; 8.2 MiB/s, 3 objects/s recovering 2022-04-28T19:52:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[61152]: cluster 2022-04-28T19:52:17.094595+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2022-04-28T19:52:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[61152]: audit 2022-04-28T19:52:17.120256+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:52:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[61152]: audit 2022-04-28T19:52:17.136339+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:52:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[61152]: cluster 2022-04-28T19:52:17.874633+0000 mon.a (mon.0) 603 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/582 objects degraded (8.935%), 5 pgs degraded) 2022-04-28T19:52:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[61152]: cluster 2022-04-28T19:52:17.874718+0000 mon.a (mon.0) 604 : cluster [INF] Cluster is now healthy 2022-04-28T19:52:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[59745]: cluster 2022-04-28T19:52:16.914045+0000 mgr.y (mgr.24961) 172 : cluster [DBG] pgmap v261: 89 pgs: 89 active+clean; 391 MiB data, 801 MiB used, 710 GiB / 715 GiB avail; 2.3 MiB/s rd, 3.9 MiB/s wr, 8 op/s; 8.2 MiB/s, 3 objects/s recovering 2022-04-28T19:52:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[59745]: cluster 2022-04-28T19:52:17.094595+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2022-04-28T19:52:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[59745]: audit 2022-04-28T19:52:17.120256+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:52:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[59745]: audit 2022-04-28T19:52:17.136339+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:52:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[59745]: cluster 2022-04-28T19:52:17.874633+0000 mon.a (mon.0) 603 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/582 objects degraded (8.935%), 5 pgs degraded) 2022-04-28T19:52:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:18 smithi059 bash[59745]: cluster 2022-04-28T19:52:17.874718+0000 mon.a (mon.0) 604 : cluster [INF] Cluster is now healthy 2022-04-28T19:52:18.763 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1094 (ObjNum 748 snap 224 seq_num 748) dirty exists 2022-04-28T19:52:18.763 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 934 (ObjNum 628 snap 180 seq_num 628) dirty exists 2022-04-28T19:52:18.764 INFO:tasks.rados.rados.0.smithi059.stdout:2473: done (7 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2476: done (6 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2477: done (5 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2478: done (4 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2479: done (3 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2481: done (2 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2483: done (1 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2484: done (0 left) 2022-04-28T19:52:18.766 INFO:tasks.rados.rados.0.smithi059.stdout:2485: snap_remove snap 218 2022-04-28T19:52:19.123 INFO:tasks.rados.rados.0.smithi059.stdout:2485: done (0 left) 2022-04-28T19:52:19.154 INFO:tasks.rados.rados.0.smithi059.stdout:2486: delete oid 29 current snap is 235 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2486: done (0 left) 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2487: copy_from oid 14 from oid 22 current snap is 235 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2488: delete oid 40 current snap is 235 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2488: done (1 left) 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2489: delete oid 21 current snap is 235 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2489: done (1 left) 2022-04-28T19:52:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:2490: rmattr oid 28 current snap is 235 2022-04-28T19:52:19.157 INFO:tasks.rados.rados.0.smithi059.stdout:2490: done (1 left) 2022-04-28T19:52:19.157 INFO:tasks.rados.rados.0.smithi059.stdout:2491: read oid 7 snap 227 2022-04-28T19:52:19.157 INFO:tasks.rados.rados.0.smithi059.stdout:2491: expect (ObjNum 742 snap 220 seq_num 742) 2022-04-28T19:52:19.157 INFO:tasks.rados.rados.0.smithi059.stdout:2492: delete oid 11 current snap is 235 2022-04-28T19:52:19.157 INFO:tasks.rados.rados.0.smithi059.stdout:2492: done (2 left) 2022-04-28T19:52:19.157 INFO:tasks.rados.rados.0.smithi059.stdout:2493: snap_remove snap 221 2022-04-28T19:52:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:19 smithi139 bash[43301]: cluster 2022-04-28T19:52:18.098459+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2022-04-28T19:52:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:19 smithi059 bash[61152]: cluster 2022-04-28T19:52:18.098459+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2022-04-28T19:52:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:19 smithi059 bash[59745]: cluster 2022-04-28T19:52:18.098459+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2022-04-28T19:52:20.130 INFO:tasks.rados.rados.0.smithi059.stdout:2493: done (2 left) 2022-04-28T19:52:20.130 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 235 2022-04-28T19:52:20.130 INFO:tasks.rados.rados.0.smithi059.stdout:2494: seq_num 792 ranges {786432=466944,1253376=761856,2015232=40960} 2022-04-28T19:52:20.137 INFO:tasks.rados.rados.0.smithi059.stdout:2494: writing smithi05955964-1 from 786432 to 1253376 tid 1 2022-04-28T19:52:20.141 INFO:tasks.rados.rados.0.smithi059.stdout:2494: writing smithi05955964-1 from 1253376 to 2015232 tid 2 2022-04-28T19:52:20.142 INFO:tasks.rados.rados.0.smithi059.stdout:2494: writing smithi05955964-1 from 2015232 to 2056192 tid 3 2022-04-28T19:52:20.142 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 235 2022-04-28T19:52:20.142 INFO:tasks.rados.rados.0.smithi059.stdout:2495: seq_num 793 ranges {6733824=483328,7217152=647168,7864320=155648} 2022-04-28T19:52:20.183 INFO:tasks.rados.rados.0.smithi059.stdout:2495: writing smithi05955964-25 from 6733824 to 7217152 tid 1 2022-04-28T19:52:20.187 INFO:tasks.rados.rados.0.smithi059.stdout:2495: writing smithi05955964-25 from 7217152 to 7864320 tid 2 2022-04-28T19:52:20.188 INFO:tasks.rados.rados.0.smithi059.stdout:2495: writing smithi05955964-25 from 7864320 to 8019968 tid 3 2022-04-28T19:52:20.188 INFO:tasks.rados.rados.0.smithi059.stdout:2496: read oid 41 snap -1 2022-04-28T19:52:20.188 INFO:tasks.rados.rados.0.smithi059.stdout:2496: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:52:20.189 INFO:tasks.rados.rados.0.smithi059.stdout:2497: snap_remove snap 231 2022-04-28T19:52:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:20 smithi139 bash[43301]: cluster 2022-04-28T19:52:18.914902+0000 mgr.y (mgr.24961) 173 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 399 MiB data, 808 MiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 6.3 MiB/s wr, 26 op/s; 8.2 MiB/s, 3 objects/s recovering 2022-04-28T19:52:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:20 smithi139 bash[43301]: cluster 2022-04-28T19:52:19.122902+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2022-04-28T19:52:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:20 smithi059 bash[61152]: cluster 2022-04-28T19:52:18.914902+0000 mgr.y (mgr.24961) 173 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 399 MiB data, 808 MiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 6.3 MiB/s wr, 26 op/s; 8.2 MiB/s, 3 objects/s recovering 2022-04-28T19:52:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:20 smithi059 bash[61152]: cluster 2022-04-28T19:52:19.122902+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2022-04-28T19:52:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:20 smithi059 bash[59745]: cluster 2022-04-28T19:52:18.914902+0000 mgr.y (mgr.24961) 173 : cluster [DBG] pgmap v264: 89 pgs: 89 active+clean; 399 MiB data, 808 MiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 6.3 MiB/s wr, 26 op/s; 8.2 MiB/s, 3 objects/s recovering 2022-04-28T19:52:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:20 smithi059 bash[59745]: cluster 2022-04-28T19:52:19.122902+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2022-04-28T19:52:21.142 INFO:tasks.rados.rados.0.smithi059.stdout:2487: finishing copy_from to smithi05955964-14 2022-04-28T19:52:21.142 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1092 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:21.143 INFO:tasks.rados.rados.0.smithi059.stdout:2487: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:52:21.143 INFO:tasks.rados.rados.0.smithi059.stdout:2491: expect (ObjNum 742 snap 220 seq_num 742) 2022-04-28T19:52:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:21 smithi059 bash[61152]: cluster 2022-04-28T19:52:20.129454+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2022-04-28T19:52:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:21 smithi059 bash[59745]: cluster 2022-04-28T19:52:20.129454+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2022-04-28T19:52:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:21 smithi139 bash[43301]: cluster 2022-04-28T19:52:20.129454+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2022-04-28T19:52:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:21.631Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:21.631Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:21.631Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:52:22.172 INFO:tasks.rados.rados.0.smithi059.stdout:2494: finishing write tid 1 to smithi05955964-1 2022-04-28T19:52:22.172 INFO:tasks.rados.rados.0.smithi059.stdout:2494: finishing write tid 2 to smithi05955964-1 2022-04-28T19:52:22.172 INFO:tasks.rados.rados.0.smithi059.stdout:2494: finishing write tid 3 to smithi05955964-1 2022-04-28T19:52:22.172 INFO:tasks.rados.rados.0.smithi059.stdout:2494: finishing write tid 5 to smithi05955964-1 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2494: finishing write tid 6 to smithi05955964-1 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1044 (ObjNum 792 snap 235 seq_num 792) dirty exists 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2494: left oid 1 (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2495: finishing write tid 1 to smithi05955964-25 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2495: finishing write tid 2 to smithi05955964-25 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2495: finishing write tid 3 to smithi05955964-25 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2495: finishing write tid 5 to smithi05955964-25 2022-04-28T19:52:22.173 INFO:tasks.rados.rados.0.smithi059.stdout:2496: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:52:22.198 INFO:tasks.rados.rados.0.smithi059.stdout:2495: finishing write tid 6 to smithi05955964-25 2022-04-28T19:52:22.198 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1115 (ObjNum 793 snap 235 seq_num 793) dirty exists 2022-04-28T19:52:22.198 INFO:tasks.rados.rados.0.smithi059.stdout:2495: left oid 25 (ObjNum 793 snap 235 seq_num 793) 2022-04-28T19:52:22.198 INFO:tasks.rados.rados.0.smithi059.stdout:2487: done (5 left) 2022-04-28T19:52:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2491: done (4 left) 2022-04-28T19:52:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2494: done (3 left) 2022-04-28T19:52:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2495: done (2 left) 2022-04-28T19:52:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2496: done (1 left) 2022-04-28T19:52:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2497: done (0 left) 2022-04-28T19:52:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2498: delete oid 15 current snap is 235 2022-04-28T19:52:22.201 INFO:tasks.rados.rados.0.smithi059.stdout:2498: done (0 left) 2022-04-28T19:52:22.201 INFO:tasks.rados.rados.0.smithi059.stdout:2499: delete oid 39 current snap is 235 2022-04-28T19:52:22.207 INFO:tasks.rados.rados.0.smithi059.stdout:2499: done (0 left) 2022-04-28T19:52:22.207 INFO:tasks.rados.rados.0.smithi059.stdout:2500: copy_from oid 18 from oid 48 current snap is 235 2022-04-28T19:52:22.207 INFO:tasks.rados.rados.0.smithi059.stdout:2501: read oid 29 snap -1 2022-04-28T19:52:22.208 INFO:tasks.rados.rados.0.smithi059.stdout:2501: expect deleted 2022-04-28T19:52:22.208 INFO:tasks.rados.rados.0.smithi059.stdout:2502: read oid 12 snap -1 2022-04-28T19:52:22.208 INFO:tasks.rados.rados.0.smithi059.stdout:2502: expect (ObjNum 773 snap 232 seq_num 773) 2022-04-28T19:52:22.208 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 235 2022-04-28T19:52:22.208 INFO:tasks.rados.rados.0.smithi059.stdout:2503: seq_num 796 ranges {0=540672,540672=704512,1245184=98304} 2022-04-28T19:52:22.216 INFO:tasks.rados.rados.0.smithi059.stdout:2503: writing smithi05955964-21 from 0 to 540672 tid 1 2022-04-28T19:52:22.222 INFO:tasks.rados.rados.0.smithi059.stdout:2503: writing smithi05955964-21 from 540672 to 1245184 tid 2 2022-04-28T19:52:22.224 INFO:tasks.rados.rados.0.smithi059.stdout:2503: writing smithi05955964-21 from 1245184 to 1343488 tid 3 2022-04-28T19:52:22.224 INFO:tasks.rados.rados.0.smithi059.stdout:2504: read oid 17 snap -1 2022-04-28T19:52:22.224 INFO:tasks.rados.rados.0.smithi059.stdout:2504: expect (ObjNum 740 snap 219 seq_num 740) 2022-04-28T19:52:22.225 INFO:tasks.rados.rados.0.smithi059.stdout:2505: delete oid 24 current snap is 235 2022-04-28T19:52:22.225 INFO:tasks.rados.rados.0.smithi059.stdout:2502: expect (ObjNum 773 snap 232 seq_num 773) 2022-04-28T19:52:22.369 INFO:tasks.rados.rados.0.smithi059.stdout:2503: finishing write tid 1 to smithi05955964-21 2022-04-28T19:52:22.369 INFO:tasks.rados.rados.0.smithi059.stdout:2503: finishing write tid 2 to smithi05955964-21 2022-04-28T19:52:22.369 INFO:tasks.rados.rados.0.smithi059.stdout:2503: finishing write tid 3 to smithi05955964-21 2022-04-28T19:52:22.369 INFO:tasks.rados.rados.0.smithi059.stdout:2503: finishing write tid 5 to smithi05955964-21 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:2503: finishing write tid 6 to smithi05955964-21 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 731 (ObjNum 796 snap 235 seq_num 796) dirty exists 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:2503: left oid 21 (ObjNum 796 snap 235 seq_num 796) 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:2500: finishing copy_from to smithi05955964-18 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1085 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:2500: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:52:22.370 INFO:tasks.rados.rados.0.smithi059.stdout:2504: expect (ObjNum 740 snap 219 seq_num 740) 2022-04-28T19:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[61152]: cluster 2022-04-28T19:52:20.915688+0000 mgr.y (mgr.24961) 174 : cluster [DBG] pgmap v267: 89 pgs: 89 active+clean; 407 MiB data, 816 MiB used, 710 GiB / 715 GiB avail; 23 MiB/s rd, 3.8 MiB/s wr, 32 op/s 2022-04-28T19:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[61152]: cluster 2022-04-28T19:52:21.138818+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2022-04-28T19:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[61152]: cluster 2022-04-28T19:52:21.883137+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2022-04-28T19:52:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[61152]: audit 2022-04-28T19:52:21.897726+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[59745]: cluster 2022-04-28T19:52:20.915688+0000 mgr.y (mgr.24961) 174 : cluster [DBG] pgmap v267: 89 pgs: 89 active+clean; 407 MiB data, 816 MiB used, 710 GiB / 715 GiB avail; 23 MiB/s rd, 3.8 MiB/s wr, 32 op/s 2022-04-28T19:52:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[59745]: cluster 2022-04-28T19:52:21.138818+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2022-04-28T19:52:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[59745]: cluster 2022-04-28T19:52:21.883137+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2022-04-28T19:52:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:22 smithi059 bash[59745]: audit 2022-04-28T19:52:21.897726+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:22 smithi139 bash[43301]: cluster 2022-04-28T19:52:20.915688+0000 mgr.y (mgr.24961) 174 : cluster [DBG] pgmap v267: 89 pgs: 89 active+clean; 407 MiB data, 816 MiB used, 710 GiB / 715 GiB avail; 23 MiB/s rd, 3.8 MiB/s wr, 32 op/s 2022-04-28T19:52:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:22 smithi139 bash[43301]: cluster 2022-04-28T19:52:21.138818+0000 mon.a (mon.0) 608 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2022-04-28T19:52:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:22 smithi139 bash[43301]: cluster 2022-04-28T19:52:21.883137+0000 mon.a (mon.0) 609 : cluster [DBG] osdmap e523: 8 total, 8 up, 8 in 2022-04-28T19:52:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:22 smithi139 bash[43301]: audit 2022-04-28T19:52:21.897726+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:22.946 INFO:tasks.rados.rados.0.smithi059.stdout:2500: done (5 left) 2022-04-28T19:52:22.946 INFO:tasks.rados.rados.0.smithi059.stdout:2501: done (4 left) 2022-04-28T19:52:22.946 INFO:tasks.rados.rados.0.smithi059.stdout:2502: done (3 left) 2022-04-28T19:52:22.946 INFO:tasks.rados.rados.0.smithi059.stdout:2503: done (2 left) 2022-04-28T19:52:22.947 INFO:tasks.rados.rados.0.smithi059.stdout:2504: done (1 left) 2022-04-28T19:52:22.947 INFO:tasks.rados.rados.0.smithi059.stdout:2505: done (0 left) 2022-04-28T19:52:22.947 INFO:tasks.rados.rados.0.smithi059.stdout:2506: delete oid 19 current snap is 235 2022-04-28T19:52:22.948 INFO:tasks.rados.rados.0.smithi059.stdout:2506: done (0 left) 2022-04-28T19:52:22.948 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 235 2022-04-28T19:52:22.949 INFO:tasks.rados.rados.0.smithi059.stdout:2507: seq_num 799 ranges {5554176=589824,6144000=802816} 2022-04-28T19:52:22.993 INFO:tasks.rados.rados.0.smithi059.stdout:2507: writing smithi05955964-18 from 5554176 to 6144000 tid 1 2022-04-28T19:52:22.997 INFO:tasks.rados.rados.0.smithi059.stdout:2507: writing smithi05955964-18 from 6144000 to 6946816 tid 2 2022-04-28T19:52:22.998 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 235 2022-04-28T19:52:22.998 INFO:tasks.rados.rados.0.smithi059.stdout:2508: seq_num 800 ranges {2473984=606208,3080192=688128,3768320=114688} 2022-04-28T19:52:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:2508: writing smithi05955964-14 from 2473984 to 3080192 tid 1 2022-04-28T19:52:23.019 INFO:tasks.rados.rados.0.smithi059.stdout:2508: writing smithi05955964-14 from 3080192 to 3768320 tid 2 2022-04-28T19:52:23.020 INFO:tasks.rados.rados.0.smithi059.stdout:2508: writing smithi05955964-14 from 3768320 to 3883008 tid 3 2022-04-28T19:52:23.020 INFO:tasks.rados.rados.0.smithi059.stdout:2509: delete oid 35 current snap is 235 2022-04-28T19:52:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:2507: finishing write tid 1 to smithi05955964-18 2022-04-28T19:52:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:2507: finishing write tid 2 to smithi05955964-18 2022-04-28T19:52:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:2507: finishing write tid 4 to smithi05955964-18 2022-04-28T19:52:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:2507: finishing write tid 5 to smithi05955964-18 2022-04-28T19:52:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1088 (ObjNum 799 snap 235 seq_num 799) dirty exists 2022-04-28T19:52:23.022 INFO:tasks.rados.rados.0.smithi059.stdout:2507: left oid 18 (ObjNum 799 snap 235 seq_num 799) 2022-04-28T19:52:23.022 INFO:tasks.rados.rados.0.smithi059.stdout:2508: finishing write tid 1 to smithi05955964-14 2022-04-28T19:52:23.023 INFO:tasks.rados.rados.0.smithi059.stdout:2507: done (2 left) 2022-04-28T19:52:23.023 INFO:tasks.rados.rados.0.smithi059.stdout:2509: done (1 left) 2022-04-28T19:52:23.023 INFO:tasks.rados.rados.0.smithi059.stdout:2510: setattr oid 35 current snap is 235 2022-04-28T19:52:23.024 INFO:tasks.rados.rados.0.smithi059.stdout:2508: finishing write tid 2 to smithi05955964-14 2022-04-28T19:52:23.024 INFO:tasks.rados.rados.0.smithi059.stdout:2508: finishing write tid 3 to smithi05955964-14 2022-04-28T19:52:23.025 INFO:tasks.rados.rados.0.smithi059.stdout:2508: finishing write tid 5 to smithi05955964-14 2022-04-28T19:52:23.027 INFO:tasks.rados.rados.0.smithi059.stdout:2511: copy_from oid 26 from oid 1 current snap is 235 2022-04-28T19:52:23.027 INFO:tasks.rados.rados.0.smithi059.stdout:2508: finishing write tid 6 to smithi05955964-14 2022-04-28T19:52:23.028 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1101 (ObjNum 800 snap 235 seq_num 800) dirty exists 2022-04-28T19:52:23.028 INFO:tasks.rados.rados.0.smithi059.stdout:2508: left oid 14 (ObjNum 800 snap 235 seq_num 800) 2022-04-28T19:52:23.028 INFO:tasks.rados.rados.0.smithi059.stdout:2508: done (2 left) 2022-04-28T19:52:23.028 INFO:tasks.rados.rados.0.smithi059.stdout:2512: copy_from oid 44 from oid 13 current snap is 235 2022-04-28T19:52:23.028 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 235 2022-04-28T19:52:23.028 INFO:tasks.rados.rados.0.smithi059.stdout:2513: seq_num 802 ranges {1409024=638976,2048000=458752,2506752=344064} 2022-04-28T19:52:23.038 INFO:tasks.rados.rados.0.smithi059.stdout:2513: writing smithi05955964-31 from 1409024 to 2048000 tid 1 2022-04-28T19:52:23.041 INFO:tasks.rados.rados.0.smithi059.stdout:2513: writing smithi05955964-31 from 2048000 to 2506752 tid 2 2022-04-28T19:52:23.043 INFO:tasks.rados.rados.0.smithi059.stdout:2513: writing smithi05955964-31 from 2506752 to 2850816 tid 3 2022-04-28T19:52:23.044 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 235 2022-04-28T19:52:23.044 INFO:tasks.rados.rados.0.smithi059.stdout:2514: seq_num 803 ranges {0=655360,655360=745472,1400832=57344} 2022-04-28T19:52:23.047 INFO:tasks.rados.rados.0.smithi059.stdout:2514: writing smithi05955964-28 from 0 to 655360 tid 1 2022-04-28T19:52:23.051 INFO:tasks.rados.rados.0.smithi059.stdout:2514: writing smithi05955964-28 from 655360 to 1400832 tid 2 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:2514: writing smithi05955964-28 from 1400832 to 1458176 tid 3 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:2515: read oid 5 snap -1 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:2515: expect deleted 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:2516: rmattr oid 27 current snap is 235 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 938 (ObjNum 188341632 snap 32762 seq_num 4162380672) dirty exists 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:2511: finishing copy_from to smithi05955964-26 2022-04-28T19:52:23.053 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1044 (ObjNum 792 snap 235 seq_num 792) dirty exists 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2511: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2513: finishing write tid 1 to smithi05955964-31 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2510: done (6 left) 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2511: done (5 left) 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2517: rollback oid 25 current snap is 235 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 25 to 233 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2512: finishing copy_from to smithi05955964-44 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 961 (ObjNum 715 snap 207 seq_num 715) dirty exists 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2512: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:52:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:2513: finishing write tid 2 to smithi05955964-31 2022-04-28T19:52:23.055 INFO:tasks.rados.rados.0.smithi059.stdout:2513: finishing write tid 3 to smithi05955964-31 2022-04-28T19:52:23.055 INFO:tasks.rados.rados.0.smithi059.stdout:2512: done (5 left) 2022-04-28T19:52:23.055 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 235 2022-04-28T19:52:23.055 INFO:tasks.rados.rados.0.smithi059.stdout:2518: seq_num 804 ranges {1196032=671744,1867776=622592,2490368=180224} 2022-04-28T19:52:23.062 INFO:tasks.rados.rados.0.smithi059.stdout:2518: writing smithi05955964-4 from 1196032 to 1867776 tid 1 2022-04-28T19:52:23.066 INFO:tasks.rados.rados.0.smithi059.stdout:2518: writing smithi05955964-4 from 1867776 to 2490368 tid 2 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2518: writing smithi05955964-4 from 2490368 to 2670592 tid 3 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2513: finishing write tid 5 to smithi05955964-31 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2519: read oid 20 snap -1 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2519: expect (ObjNum 751 snap 225 seq_num 751) 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2520: read oid 47 snap -1 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2520: expect (ObjNum 769 snap 230 seq_num 769) 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2521: copy_from oid 44 from oid 35 current snap is 235 2022-04-28T19:52:23.068 INFO:tasks.rados.rados.0.smithi059.stdout:2513: finishing write tid 6 to smithi05955964-31 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1134 (ObjNum 802 snap 235 seq_num 802) dirty exists 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2513: left oid 31 (ObjNum 802 snap 235 seq_num 802) 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2514: finishing write tid 1 to smithi05955964-28 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2513: done (8 left) 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2522: read oid 1 snap 234 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2522: expect (ObjNum 763 snap 229 seq_num 763) 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2523: read oid 29 snap -1 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2523: expect deleted 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:2524: snap_create 2022-04-28T19:52:23.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1086 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2514: finishing write tid 2 to smithi05955964-28 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2514: finishing write tid 3 to smithi05955964-28 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2514: finishing write tid 5 to smithi05955964-28 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2517: finishing rollback tid 0 to smithi05955964-25 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2517: finishing rollback tid 1 to smithi05955964-25 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2517: finishing rollback tid 2 to smithi05955964-25 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1122 (ObjNum 765 snap 229 seq_num 765) dirty exists 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2514: finishing write tid 6 to smithi05955964-28 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 954 (ObjNum 803 snap 235 seq_num 803) dirty exists 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2514: left oid 28 (ObjNum 803 snap 235 seq_num 803) 2022-04-28T19:52:23.070 INFO:tasks.rados.rados.0.smithi059.stdout:2518: finishing write tid 1 to smithi05955964-4 2022-04-28T19:52:23.071 INFO:tasks.rados.rados.0.smithi059.stdout:2518: finishing write tid 2 to smithi05955964-4 2022-04-28T19:52:23.071 INFO:tasks.rados.rados.0.smithi059.stdout:2518: finishing write tid 3 to smithi05955964-4 2022-04-28T19:52:23.071 INFO:tasks.rados.rados.0.smithi059.stdout:2518: finishing write tid 5 to smithi05955964-4 2022-04-28T19:52:23.072 INFO:tasks.rados.rados.0.smithi059.stdout:2518: finishing write tid 6 to smithi05955964-4 2022-04-28T19:52:23.072 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 940 (ObjNum 804 snap 235 seq_num 804) dirty exists 2022-04-28T19:52:23.072 INFO:tasks.rados.rados.0.smithi059.stdout:2518: left oid 4 (ObjNum 804 snap 235 seq_num 804) 2022-04-28T19:52:23.075 INFO:tasks.rados.rados.0.smithi059.stdout:2521: finishing copy_from to smithi05955964-44 2022-04-28T19:52:23.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 938 (ObjNum 1767987791 snap 1633115986 seq_num 1783715182) dirty exists 2022-04-28T19:52:23.075 INFO:tasks.rados.rados.0.smithi059.stdout:2521: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:52:23.075 INFO:tasks.rados.rados.0.smithi059.stdout:2519: expect (ObjNum 751 snap 225 seq_num 751) 2022-04-28T19:52:23.153 INFO:tasks.rados.rados.0.smithi059.stdout:2520: expect (ObjNum 769 snap 230 seq_num 769) 2022-04-28T19:52:23.173 INFO:tasks.rados.rados.0.smithi059.stdout:2522: expect (ObjNum 763 snap 229 seq_num 763) 2022-04-28T19:52:23.299 INFO:tasks.rados.rados.0.smithi059.stdout:2514: done (10 left) 2022-04-28T19:52:23.299 INFO:tasks.rados.rados.0.smithi059.stdout:2515: done (9 left) 2022-04-28T19:52:23.299 INFO:tasks.rados.rados.0.smithi059.stdout:2516: done (8 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2517: done (7 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2518: done (6 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2519: done (5 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2520: done (4 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2521: done (3 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2522: done (2 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2523: done (1 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2524: done (0 left) 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2525: read oid 46 snap 235 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2525: expect deleted 2022-04-28T19:52:23.300 INFO:tasks.rados.rados.0.smithi059.stdout:2526: snap_create 2022-04-28T19:52:23.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:23 smithi139 bash[43301]: audit 2022-04-28T19:52:21.897955+0000 mgr.y (mgr.24961) 175 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:23.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:23 smithi139 bash[43301]: cephadm 2022-04-28T19:52:21.899275+0000 mgr.y (mgr.24961) 176 : cephadm [INF] Upgrade: osd.4 is safe to restart 2022-04-28T19:52:23.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:23 smithi139 bash[43301]: audit 2022-04-28T19:52:22.286235+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[59745]: audit 2022-04-28T19:52:21.897955+0000 mgr.y (mgr.24961) 175 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[59745]: cephadm 2022-04-28T19:52:21.899275+0000 mgr.y (mgr.24961) 176 : cephadm [INF] Upgrade: osd.4 is safe to restart 2022-04-28T19:52:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[59745]: audit 2022-04-28T19:52:22.286235+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[61152]: audit 2022-04-28T19:52:21.897955+0000 mgr.y (mgr.24961) 175 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2022-04-28T19:52:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[61152]: cephadm 2022-04-28T19:52:21.899275+0000 mgr.y (mgr.24961) 176 : cephadm [INF] Upgrade: osd.4 is safe to restart 2022-04-28T19:52:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[61152]: audit 2022-04-28T19:52:22.286235+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:23.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:23.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:23.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:23.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:23.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:23.525Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:52:24.308 INFO:tasks.rados.rados.0.smithi059.stdout:2525: done (1 left) 2022-04-28T19:52:24.309 INFO:tasks.rados.rados.0.smithi059.stdout:2526: done (0 left) 2022-04-28T19:52:24.309 INFO:tasks.rados.rados.0.smithi059.stdout:2527: rollback oid 18 current snap is 237 2022-04-28T19:52:24.309 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 226 2022-04-28T19:52:24.309 INFO:tasks.rados.rados.0.smithi059.stdout:2528: copy_from oid 24 from oid 30 current snap is 237 2022-04-28T19:52:24.309 INFO:tasks.rados.rados.0.smithi059.stdout:2529: copy_from oid 10 from oid 22 current snap is 237 2022-04-28T19:52:24.309 INFO:tasks.rados.rados.0.smithi059.stdout:2530: snap_remove snap 228 2022-04-28T19:52:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[61152]: cluster 2022-04-28T19:52:22.916672+0000 mgr.y (mgr.24961) 177 : cluster [DBG] pgmap v270: 89 pgs: 89 active+clean; 414 MiB data, 831 MiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2022-04-28T19:52:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[61152]: cluster 2022-04-28T19:52:23.297129+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2022-04-28T19:52:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[61152]: audit 2022-04-28T19:52:24.052423+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[61152]: audit 2022-04-28T19:52:24.056613+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "container_image"}]: dispatch 2022-04-28T19:52:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[59745]: cluster 2022-04-28T19:52:22.916672+0000 mgr.y (mgr.24961) 177 : cluster [DBG] pgmap v270: 89 pgs: 89 active+clean; 414 MiB data, 831 MiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2022-04-28T19:52:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[59745]: cluster 2022-04-28T19:52:23.297129+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2022-04-28T19:52:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[59745]: audit 2022-04-28T19:52:24.052423+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:24 smithi059 bash[59745]: audit 2022-04-28T19:52:24.056613+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "container_image"}]: dispatch 2022-04-28T19:52:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:24 smithi139 bash[43301]: cluster 2022-04-28T19:52:22.916672+0000 mgr.y (mgr.24961) 177 : cluster [DBG] pgmap v270: 89 pgs: 89 active+clean; 414 MiB data, 831 MiB used, 710 GiB / 715 GiB avail; 25 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2022-04-28T19:52:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:24 smithi139 bash[43301]: cluster 2022-04-28T19:52:23.297129+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e524: 8 total, 8 up, 8 in 2022-04-28T19:52:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:24 smithi139 bash[43301]: audit 2022-04-28T19:52:24.052423+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:24 smithi139 bash[43301]: audit 2022-04-28T19:52:24.056613+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "container_image"}]: dispatch 2022-04-28T19:52:25.145 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:52:25.320 INFO:tasks.rados.rados.0.smithi059.stdout:2530: done (3 left) 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2531: read oid 16 snap -1 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2531: expect (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2532: setattr oid 8 current snap is 237 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2527: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2527: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2527: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1091 (ObjNum 715 snap 207 seq_num 715) dirty exists 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2529: finishing copy_from to smithi05955964-10 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1092 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2529: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:52:25.321 INFO:tasks.rados.rados.0.smithi059.stdout:2528: finishing copy_from to smithi05955964-24 2022-04-28T19:52:25.322 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:25.322 INFO:tasks.rados.rados.0.smithi059.stdout:2528: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:52:25.323 INFO:tasks.rados.rados.0.smithi059.stdout:2527: done (4 left) 2022-04-28T19:52:25.323 INFO:tasks.rados.rados.0.smithi059.stdout:2528: done (3 left) 2022-04-28T19:52:25.323 INFO:tasks.rados.rados.0.smithi059.stdout:2529: done (2 left) 2022-04-28T19:52:25.323 INFO:tasks.rados.rados.0.smithi059.stdout:2533: snap_create 2022-04-28T19:52:25.333 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1124 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:52:25.382 INFO:tasks.rados.rados.0.smithi059.stdout:2531: expect (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:52:25.491 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (11m) 18s ago 11m 14.7M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (11m) 2m ago 11m 25.8M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (9m) 18s ago 9m 14.9M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (9m) 2m ago 9m 17.6M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (3m) 2m ago 15m 375M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (4m) 18s ago 17m 415M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:52:25.492 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (3m) 18s ago 17m 66.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (2m) 2m ago 15m 19.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (2m) 18s ago 16m 52.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (11m) 18s ago 11m 11.2M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (11m) 2m ago 11m 10.7M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (110s) 18s ago 15m 294M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (80s) 18s ago 14m 215M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (50s) 18s ago 14m 256M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (21s) 18s ago 14m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (13m) 2m ago 13m 196M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 5976654882b1 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (13m) 2m ago 13m 214M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:52:25.493 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (12m) 2m ago 12m 162M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:52:25.494 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (12m) 2m ago 12m 138M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:52:25.494 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (4m) 2m ago 12m 36.6M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:52:25.498 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:25 smithi139 bash[43301]: cephadm 2022-04-28T19:52:24.043479+0000 mgr.y (mgr.24961) 178 : cephadm [INF] Upgrade: Updating osd.4 2022-04-28T19:52:25.498 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:25 smithi139 bash[43301]: cephadm 2022-04-28T19:52:24.055754+0000 mgr.y (mgr.24961) 179 : cephadm [INF] Deploying daemon osd.4 on smithi139 2022-04-28T19:52:25.498 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:25 smithi139 bash[43301]: cluster 2022-04-28T19:52:24.308037+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2022-04-28T19:52:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:25 smithi059 bash[61152]: cephadm 2022-04-28T19:52:24.043479+0000 mgr.y (mgr.24961) 178 : cephadm [INF] Upgrade: Updating osd.4 2022-04-28T19:52:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:25 smithi059 bash[61152]: cephadm 2022-04-28T19:52:24.055754+0000 mgr.y (mgr.24961) 179 : cephadm [INF] Deploying daemon osd.4 on smithi139 2022-04-28T19:52:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:25 smithi059 bash[61152]: cluster 2022-04-28T19:52:24.308037+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2022-04-28T19:52:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:25 smithi059 bash[59745]: cephadm 2022-04-28T19:52:24.043479+0000 mgr.y (mgr.24961) 178 : cephadm [INF] Upgrade: Updating osd.4 2022-04-28T19:52:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:25 smithi059 bash[59745]: cephadm 2022-04-28T19:52:24.055754+0000 mgr.y (mgr.24961) 179 : cephadm [INF] Deploying daemon osd.4 on smithi139 2022-04-28T19:52:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:25 smithi059 bash[59745]: cluster 2022-04-28T19:52:24.308037+0000 mon.a (mon.0) 615 : cluster [DBG] osdmap e525: 8 total, 8 up, 8 in 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:52:25.859 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 4, 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 4 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 6, 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 9 2022-04-28T19:52:25.860 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:52:25.861 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:52:26.328 INFO:tasks.rados.rados.0.smithi059.stdout:2531: done (2 left) 2022-04-28T19:52:26.328 INFO:tasks.rados.rados.0.smithi059.stdout:2532: done (1 left) 2022-04-28T19:52:26.329 INFO:tasks.rados.rados.0.smithi059.stdout:2533: done (0 left) 2022-04-28T19:52:26.329 INFO:tasks.rados.rados.0.smithi059.stdout:2534: snap_remove snap 215 2022-04-28T19:52:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: cluster 2022-04-28T19:52:24.917239+0000 mgr.y (mgr.24961) 180 : cluster [DBG] pgmap v273: 89 pgs: 89 active+clean; 418 MiB data, 834 MiB used, 710 GiB / 715 GiB avail; 21 MiB/s rd, 3.7 MiB/s wr, 21 op/s 2022-04-28T19:52:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.131668+0000 mgr.y (mgr.24961) 181 : audit [DBG] from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: cluster 2022-04-28T19:52:25.318699+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2022-04-28T19:52:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.471480+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.472268+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.472982+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.473713+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.474311+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.474897+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.477697+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.478375+0000 mon.a (mon.0) 624 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.479041+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.479706+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.480325+0000 mon.a (mon.0) 627 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[59745]: audit 2022-04-28T19:52:25.858382+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.59:0/1850743035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: cluster 2022-04-28T19:52:24.917239+0000 mgr.y (mgr.24961) 180 : cluster [DBG] pgmap v273: 89 pgs: 89 active+clean; 418 MiB data, 834 MiB used, 710 GiB / 715 GiB avail; 21 MiB/s rd, 3.7 MiB/s wr, 21 op/s 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.131668+0000 mgr.y (mgr.24961) 181 : audit [DBG] from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: cluster 2022-04-28T19:52:25.318699+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.471480+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.472268+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.472982+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.473713+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.474311+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.474897+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.477697+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.478375+0000 mon.a (mon.0) 624 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.479041+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.479706+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.480325+0000 mon.a (mon.0) 627 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:26 smithi059 bash[61152]: audit 2022-04-28T19:52:25.858382+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.59:0/1850743035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: cluster 2022-04-28T19:52:24.917239+0000 mgr.y (mgr.24961) 180 : cluster [DBG] pgmap v273: 89 pgs: 89 active+clean; 418 MiB data, 834 MiB used, 710 GiB / 715 GiB avail; 21 MiB/s rd, 3.7 MiB/s wr, 21 op/s 2022-04-28T19:52:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.131668+0000 mgr.y (mgr.24961) 181 : audit [DBG] from='client.54217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: cluster 2022-04-28T19:52:25.318699+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e526: 8 total, 8 up, 8 in 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.471480+0000 mon.a (mon.0) 617 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.472268+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.472982+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.473713+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.474311+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.474897+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.477697+0000 mon.a (mon.0) 623 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.478375+0000 mon.a (mon.0) 624 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.479041+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.479706+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.480325+0000 mon.a (mon.0) 627 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:26 smithi139 bash[43301]: audit 2022-04-28T19:52:25.858382+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.59:0/1850743035' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:26.892 INFO:tasks.rados.rados.0.smithi059.stdout:2534: done (0 left) 2022-04-28T19:52:26.892 INFO:tasks.rados.rados.0.smithi059.stdout:2535: snap_remove snap 199 2022-04-28T19:52:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:27 smithi059 bash[61152]: audit 2022-04-28T19:52:25.468179+0000 mgr.y (mgr.24961) 182 : audit [DBG] from='client.44274 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:27 smithi059 bash[61152]: cluster 2022-04-28T19:52:26.325027+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2022-04-28T19:52:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:27 smithi059 bash[61152]: cluster 2022-04-28T19:52:26.888488+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2022-04-28T19:52:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:27 smithi059 bash[59745]: audit 2022-04-28T19:52:25.468179+0000 mgr.y (mgr.24961) 182 : audit [DBG] from='client.44274 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:27 smithi059 bash[59745]: cluster 2022-04-28T19:52:26.325027+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2022-04-28T19:52:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:27 smithi059 bash[59745]: cluster 2022-04-28T19:52:26.888488+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2022-04-28T19:52:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:27 smithi139 bash[43301]: audit 2022-04-28T19:52:25.468179+0000 mgr.y (mgr.24961) 182 : audit [DBG] from='client.44274 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:27 smithi139 bash[43301]: cluster 2022-04-28T19:52:26.325027+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e527: 8 total, 8 up, 8 in 2022-04-28T19:52:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:27 smithi139 bash[43301]: cluster 2022-04-28T19:52:26.888488+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e528: 8 total, 8 up, 8 in 2022-04-28T19:52:27.909 INFO:tasks.rados.rados.0.smithi059.stdout:2535: done (0 left) 2022-04-28T19:52:27.909 INFO:tasks.rados.rados.0.smithi059.stdout:2536: rollback oid 18 current snap is 238 2022-04-28T19:52:27.909 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 213 2022-04-28T19:52:27.909 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 238 2022-04-28T19:52:27.910 INFO:tasks.rados.rados.0.smithi059.stdout:2537: seq_num 805 ranges {2473984=688128,3162112=507904,3670016=294912} 2022-04-28T19:52:27.929 INFO:tasks.rados.rados.0.smithi059.stdout:2537: writing smithi05955964-10 from 2473984 to 3162112 tid 1 2022-04-28T19:52:28.017 INFO:tasks.rados.rados.0.smithi059.stdout:2537: writing smithi05955964-10 from 3162112 to 3670016 tid 2 2022-04-28T19:52:28.017 INFO:tasks.rados.rados.0.smithi059.stdout:2537: writing smithi05955964-10 from 3670016 to 3964928 tid 3 2022-04-28T19:52:28.017 INFO:tasks.rados.rados.0.smithi059.stdout:2538: snap_create 2022-04-28T19:52:28.017 INFO:tasks.rados.rados.0.smithi059.stdout:2536: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:52:28.017 INFO:tasks.rados.rados.0.smithi059.stdout:2536: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:52:28.018 INFO:tasks.rados.rados.0.smithi059.stdout:2536: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:52:28.018 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1098 (ObjNum 715 snap 207 seq_num 715) dirty exists 2022-04-28T19:52:28.018 INFO:tasks.rados.rados.0.smithi059.stdout:2537: finishing write tid 1 to smithi05955964-10 2022-04-28T19:52:28.018 INFO:tasks.rados.rados.0.smithi059.stdout:2537: finishing write tid 2 to smithi05955964-10 2022-04-28T19:52:28.019 INFO:tasks.rados.rados.0.smithi059.stdout:2537: finishing write tid 3 to smithi05955964-10 2022-04-28T19:52:28.019 INFO:tasks.rados.rados.0.smithi059.stdout:2537: finishing write tid 5 to smithi05955964-10 2022-04-28T19:52:28.019 INFO:tasks.rados.rados.0.smithi059.stdout:2537: finishing write tid 6 to smithi05955964-10 2022-04-28T19:52:28.019 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1096 (ObjNum 805 snap 238 seq_num 805) dirty exists 2022-04-28T19:52:28.020 INFO:tasks.rados.rados.0.smithi059.stdout:2537: left oid 10 (ObjNum 805 snap 238 seq_num 805) 2022-04-28T19:52:28.311 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 systemd[1]: Stopping Ceph osd.4 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:52:28.311 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[25976]: debug 2022-04-28T19:52:28.150+0000 7f4c1a91a700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:52:28.311 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[25976]: debug 2022-04-28T19:52:28.150+0000 7f4c1a91a700 -1 osd.4 529 *** Got signal Terminated *** 2022-04-28T19:52:28.311 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[25976]: debug 2022-04-28T19:52:28.150+0000 7f4c1a91a700 -1 osd.4 529 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:52:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:28 smithi059 bash[61152]: cluster 2022-04-28T19:52:26.917649+0000 mgr.y (mgr.24961) 183 : cluster [DBG] pgmap v277: 89 pgs: 89 active+clean; 426 MiB data, 858 MiB used, 710 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2022-04-28T19:52:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:28 smithi059 bash[61152]: cluster 2022-04-28T19:52:27.907912+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2022-04-28T19:52:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:28 smithi059 bash[59745]: cluster 2022-04-28T19:52:26.917649+0000 mgr.y (mgr.24961) 183 : cluster [DBG] pgmap v277: 89 pgs: 89 active+clean; 426 MiB data, 858 MiB used, 710 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2022-04-28T19:52:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:28 smithi059 bash[59745]: cluster 2022-04-28T19:52:27.907912+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2022-04-28T19:52:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[43301]: cluster 2022-04-28T19:52:26.917649+0000 mgr.y (mgr.24961) 183 : cluster [DBG] pgmap v277: 89 pgs: 89 active+clean; 426 MiB data, 858 MiB used, 710 GiB / 715 GiB avail; 3.9 MiB/s rd, 4.3 MiB/s wr, 14 op/s 2022-04-28T19:52:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[43301]: cluster 2022-04-28T19:52:27.907912+0000 mon.a (mon.0) 630 : cluster [DBG] osdmap e529: 8 total, 8 up, 8 in 2022-04-28T19:52:28.921 INFO:tasks.rados.rados.0.smithi059.stdout:2536: done (2 left) 2022-04-28T19:52:28.921 INFO:tasks.rados.rados.0.smithi059.stdout:2537: done (1 left) 2022-04-28T19:52:28.921 INFO:tasks.rados.rados.0.smithi059.stdout:2538: done (0 left) 2022-04-28T19:52:28.921 INFO:tasks.rados.rados.0.smithi059.stdout:2539: rollback oid 20 current snap is 239 2022-04-28T19:52:28.921 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 235 2022-04-28T19:52:28.922 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 239 2022-04-28T19:52:28.922 INFO:tasks.rados.rados.0.smithi059.stdout:2540: seq_num 806 ranges {0=704512,704512=442368,1146880=360448} 2022-04-28T19:52:28.923 INFO:tasks.rados.rados.0.smithi059.stdout:2540: writing smithi05955964-35 from 0 to 704512 tid 1 2022-04-28T19:52:28.925 INFO:tasks.rados.rados.0.smithi059.stdout:2540: writing smithi05955964-35 from 704512 to 1146880 tid 2 2022-04-28T19:52:28.927 INFO:tasks.rados.rados.0.smithi059.stdout:2540: writing smithi05955964-35 from 1146880 to 1507328 tid 3 2022-04-28T19:52:28.927 INFO:tasks.rados.rados.0.smithi059.stdout:2541: delete oid 5 current snap is 239 2022-04-28T19:52:28.932 INFO:tasks.rados.rados.0.smithi059.stdout:2541: done (2 left) 2022-04-28T19:52:28.932 INFO:tasks.rados.rados.0.smithi059.stdout:2542: setattr oid 15 current snap is 239 2022-04-28T19:52:28.934 INFO:tasks.rados.rados.0.smithi059.stdout:2539: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:52:28.935 INFO:tasks.rados.rados.0.smithi059.stdout:2543: rmattr oid 43 current snap is 239 2022-04-28T19:52:28.935 INFO:tasks.rados.rados.0.smithi059.stdout:2543: done (3 left) 2022-04-28T19:52:28.935 INFO:tasks.rados.rados.0.smithi059.stdout:2544: rmattr oid 1 current snap is 239 2022-04-28T19:52:28.935 INFO:tasks.rados.rados.0.smithi059.stdout:2544: done (3 left) 2022-04-28T19:52:28.935 INFO:tasks.rados.rados.0.smithi059.stdout:2545: read oid 2 snap 224 2022-04-28T19:52:28.936 INFO:tasks.rados.rados.0.smithi059.stdout:2545: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:52:28.936 INFO:tasks.rados.rados.0.smithi059.stdout:2546: read oid 46 snap -1 2022-04-28T19:52:28.936 INFO:tasks.rados.rados.0.smithi059.stdout:2546: expect deleted 2022-04-28T19:52:28.936 INFO:tasks.rados.rados.0.smithi059.stdout:2547: delete oid 4 current snap is 239 2022-04-28T19:52:28.936 INFO:tasks.rados.rados.0.smithi059.stdout:2539: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:52:28.937 INFO:tasks.rados.rados.0.smithi059.stdout:2539: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:52:28.937 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 944 (ObjNum 751 snap 225 seq_num 751) dirty exists 2022-04-28T19:52:28.937 INFO:tasks.rados.rados.0.smithi059.stdout:2539: done (5 left) 2022-04-28T19:52:28.938 INFO:tasks.rados.rados.0.smithi059.stdout:2546: done (4 left) 2022-04-28T19:52:28.938 INFO:tasks.rados.rados.0.smithi059.stdout:2547: done (3 left) 2022-04-28T19:52:28.938 INFO:tasks.rados.rados.0.smithi059.stdout:2548: snap_create 2022-04-28T19:52:29.334 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[45276]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.4 2022-04-28T19:52:29.334 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[45276]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-4 2022-04-28T19:52:29.334 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:28 smithi139 bash[45276]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.4 2022-04-28T19:52:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.499881+0000 mon.a (mon.0) 631 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.500217+0000 mon.a (mon.0) 632 : cluster [INF] osd.4 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.500830+0000 mon.a (mon.0) 633 : cluster [DBG] osd.4 reported immediately failed by osd.6 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.501266+0000 mon.a (mon.0) 634 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.501737+0000 mon.a (mon.0) 635 : cluster [DBG] osd.4 reported immediately failed by osd.2 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.502148+0000 mon.a (mon.0) 636 : cluster [DBG] osd.4 reported immediately failed by osd.5 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.502448+0000 mon.a (mon.0) 637 : cluster [DBG] osd.4 reported immediately failed by osd.6 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.502665+0000 mon.a (mon.0) 638 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.502865+0000 mon.a (mon.0) 639 : cluster [DBG] osd.4 reported immediately failed by osd.2 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.503144+0000 mon.a (mon.0) 640 : cluster [DBG] osd.4 reported immediately failed by osd.5 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.503541+0000 mon.a (mon.0) 641 : cluster [DBG] osd.4 reported immediately failed by osd.1 2022-04-28T19:52:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.503949+0000 mon.a (mon.0) 642 : cluster [DBG] osd.4 reported immediately failed by osd.0 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.504215+0000 mon.a (mon.0) 643 : cluster [DBG] osd.4 reported immediately failed by osd.1 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.504658+0000 mon.a (mon.0) 644 : cluster [DBG] osd.4 reported immediately failed by osd.3 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.504929+0000 mon.a (mon.0) 645 : cluster [DBG] osd.4 reported immediately failed by osd.0 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.505376+0000 mon.a (mon.0) 646 : cluster [DBG] osd.4 reported immediately failed by osd.3 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.904962+0000 mon.a (mon.0) 647 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.917975+0000 mon.a (mon.0) 648 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2022-04-28T19:52:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[59745]: cluster 2022-04-28T19:52:28.922153+0000 mgr.y (mgr.24961) 184 : cluster [DBG] pgmap v280: 89 pgs: 15 stale+active+clean, 74 active+clean; 433 MiB data, 866 MiB used, 710 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.5 MiB/s wr, 21 op/s 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.499881+0000 mon.a (mon.0) 631 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.500217+0000 mon.a (mon.0) 632 : cluster [INF] osd.4 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.500830+0000 mon.a (mon.0) 633 : cluster [DBG] osd.4 reported immediately failed by osd.6 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.501266+0000 mon.a (mon.0) 634 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.501737+0000 mon.a (mon.0) 635 : cluster [DBG] osd.4 reported immediately failed by osd.2 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.502148+0000 mon.a (mon.0) 636 : cluster [DBG] osd.4 reported immediately failed by osd.5 2022-04-28T19:52:29.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.502448+0000 mon.a (mon.0) 637 : cluster [DBG] osd.4 reported immediately failed by osd.6 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.502665+0000 mon.a (mon.0) 638 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.502865+0000 mon.a (mon.0) 639 : cluster [DBG] osd.4 reported immediately failed by osd.2 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.503144+0000 mon.a (mon.0) 640 : cluster [DBG] osd.4 reported immediately failed by osd.5 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.503541+0000 mon.a (mon.0) 641 : cluster [DBG] osd.4 reported immediately failed by osd.1 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.503949+0000 mon.a (mon.0) 642 : cluster [DBG] osd.4 reported immediately failed by osd.0 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.504215+0000 mon.a (mon.0) 643 : cluster [DBG] osd.4 reported immediately failed by osd.1 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.504658+0000 mon.a (mon.0) 644 : cluster [DBG] osd.4 reported immediately failed by osd.3 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.504929+0000 mon.a (mon.0) 645 : cluster [DBG] osd.4 reported immediately failed by osd.0 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.505376+0000 mon.a (mon.0) 646 : cluster [DBG] osd.4 reported immediately failed by osd.3 2022-04-28T19:52:29.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.904962+0000 mon.a (mon.0) 647 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:29.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.917975+0000 mon.a (mon.0) 648 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2022-04-28T19:52:29.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:29 smithi059 bash[61152]: cluster 2022-04-28T19:52:28.922153+0000 mgr.y (mgr.24961) 184 : cluster [DBG] pgmap v280: 89 pgs: 15 stale+active+clean, 74 active+clean; 433 MiB data, 866 MiB used, 710 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.5 MiB/s wr, 21 op/s 2022-04-28T19:52:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.499881+0000 mon.a (mon.0) 631 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.500217+0000 mon.a (mon.0) 632 : cluster [INF] osd.4 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:52:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.500830+0000 mon.a (mon.0) 633 : cluster [DBG] osd.4 reported immediately failed by osd.6 2022-04-28T19:52:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.501266+0000 mon.a (mon.0) 634 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.501737+0000 mon.a (mon.0) 635 : cluster [DBG] osd.4 reported immediately failed by osd.2 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.502148+0000 mon.a (mon.0) 636 : cluster [DBG] osd.4 reported immediately failed by osd.5 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.502448+0000 mon.a (mon.0) 637 : cluster [DBG] osd.4 reported immediately failed by osd.6 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.502665+0000 mon.a (mon.0) 638 : cluster [DBG] osd.4 reported immediately failed by osd.7 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.502865+0000 mon.a (mon.0) 639 : cluster [DBG] osd.4 reported immediately failed by osd.2 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.503144+0000 mon.a (mon.0) 640 : cluster [DBG] osd.4 reported immediately failed by osd.5 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.503541+0000 mon.a (mon.0) 641 : cluster [DBG] osd.4 reported immediately failed by osd.1 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.503949+0000 mon.a (mon.0) 642 : cluster [DBG] osd.4 reported immediately failed by osd.0 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.504215+0000 mon.a (mon.0) 643 : cluster [DBG] osd.4 reported immediately failed by osd.1 2022-04-28T19:52:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.504658+0000 mon.a (mon.0) 644 : cluster [DBG] osd.4 reported immediately failed by osd.3 2022-04-28T19:52:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.504929+0000 mon.a (mon.0) 645 : cluster [DBG] osd.4 reported immediately failed by osd.0 2022-04-28T19:52:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.505376+0000 mon.a (mon.0) 646 : cluster [DBG] osd.4 reported immediately failed by osd.3 2022-04-28T19:52:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.904962+0000 mon.a (mon.0) 647 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.917975+0000 mon.a (mon.0) 648 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2022-04-28T19:52:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:29 smithi139 bash[43301]: cluster 2022-04-28T19:52:28.922153+0000 mgr.y (mgr.24961) 184 : cluster [DBG] pgmap v280: 89 pgs: 15 stale+active+clean, 74 active+clean; 433 MiB data, 866 MiB used, 710 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.5 MiB/s wr, 21 op/s 2022-04-28T19:52:29.924 INFO:tasks.rados.rados.0.smithi059.stdout:2548: done (3 left) 2022-04-28T19:52:29.924 INFO:tasks.rados.rados.0.smithi059.stdout:2549: snap_remove snap 220 2022-04-28T19:52:30.934 INFO:tasks.rados.rados.0.smithi059.stdout:2549: done (3 left) 2022-04-28T19:52:30.934 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 240 2022-04-28T19:52:30.934 INFO:tasks.rados.rados.0.smithi059.stdout:2550: seq_num 809 ranges {0=753664,753664=499712,1253376=303104} 2022-04-28T19:52:30.938 INFO:tasks.rados.rados.0.smithi059.stdout:2550: writing smithi05955964-4 from 0 to 753664 tid 1 2022-04-28T19:52:30.941 INFO:tasks.rados.rados.0.smithi059.stdout:2550: writing smithi05955964-4 from 753664 to 1253376 tid 2 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2550: writing smithi05955964-4 from 1253376 to 1556480 tid 3 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2551: delete oid 38 current snap is 240 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1135 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty exists 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2540: finishing write tid 1 to smithi05955964-35 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2540: finishing write tid 2 to smithi05955964-35 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2540: finishing write tid 3 to smithi05955964-35 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2540: finishing write tid 5 to smithi05955964-35 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2540: finishing write tid 6 to smithi05955964-35 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 942 (ObjNum 806 snap 239 seq_num 806) dirty exists 2022-04-28T19:52:30.944 INFO:tasks.rados.rados.0.smithi059.stdout:2540: left oid 35 (ObjNum 806 snap 239 seq_num 806) 2022-04-28T19:52:30.945 INFO:tasks.rados.rados.0.smithi059.stdout:2545: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:52:31.264 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:30 smithi139 bash[43301]: cluster 2022-04-28T19:52:29.921716+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2022-04-28T19:52:31.264 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:31 smithi139 systemd[1]: Stopped Ceph osd.4 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:52:31.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:30 smithi059 bash[61152]: cluster 2022-04-28T19:52:29.921716+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2022-04-28T19:52:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:30 smithi059 bash[59745]: cluster 2022-04-28T19:52:29.921716+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2022-04-28T19:52:31.364 INFO:tasks.rados.rados.0.smithi059.stdout:2550: finishing write tid 1 to smithi05955964-4 2022-04-28T19:52:31.364 INFO:tasks.rados.rados.0.smithi059.stdout:2550: finishing write tid 2 to smithi05955964-4 2022-04-28T19:52:31.364 INFO:tasks.rados.rados.0.smithi059.stdout:2550: finishing write tid 3 to smithi05955964-4 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2550: finishing write tid 5 to smithi05955964-4 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2550: finishing write tid 6 to smithi05955964-4 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 978 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2550: left oid 4 (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2540: done (4 left) 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2542: done (3 left) 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2545: done (2 left) 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2550: done (1 left) 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2551: done (0 left) 2022-04-28T19:52:31.365 INFO:tasks.rados.rados.0.smithi059.stdout:2552: delete oid 44 current snap is 240 2022-04-28T19:52:31.369 INFO:tasks.rados.rados.0.smithi059.stdout:2552: done (0 left) 2022-04-28T19:52:31.369 INFO:tasks.rados.rados.0.smithi059.stdout:2553: snap_remove snap 234 2022-04-28T19:52:31.618 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:31 smithi139 systemd[1]: Started Ceph osd.4 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:52:31.891 INFO:tasks.rados.rados.0.smithi059.stdout:2553: done (0 left) 2022-04-28T19:52:31.891 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 240 2022-04-28T19:52:31.891 INFO:tasks.rados.rados.0.smithi059.stdout:2554: seq_num 812 ranges {5332992=802816,6135808=548864,6684672=253952} 2022-04-28T19:52:31.927 INFO:tasks.rados.rados.0.smithi059.stdout:2554: writing smithi05955964-45 from 5332992 to 6135808 tid 1 2022-04-28T19:52:31.931 INFO:tasks.rados.rados.0.smithi059.stdout:2554: writing smithi05955964-45 from 6135808 to 6684672 tid 2 2022-04-28T19:52:31.931 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:31.631Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:31.931 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:31.631Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:31.932 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:31.632Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:52:31.932 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: cluster 2022-04-28T19:52:30.923241+0000 mgr.y (mgr.24961) 185 : cluster [DBG] pgmap v282: 89 pgs: 3 active+undersized+degraded, 3 peering, 9 stale+active+clean, 4 active+undersized, 70 active+clean; 436 MiB data, 860 MiB used, 710 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.7 MiB/s wr, 18 op/s; 28/717 objects degraded (3.905%) 2022-04-28T19:52:31.932 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: cluster 2022-04-28T19:52:30.931851+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2022-04-28T19:52:31.932 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: audit 2022-04-28T19:52:31.302811+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:31.933 INFO:tasks.rados.rados.0.smithi059.stdout:2554: writing smithi05955964-45 from 6684672 to 6938624 tid 3 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:2555: read oid 30 snap -1 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:2555: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:2556: rmattr oid 8 current snap is 240 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:2557: copy_from oid 5 from oid 38 current snap is 240 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:2558: rmattr oid 46 current snap is 240 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:2558: done (4 left) 2022-04-28T19:52:31.934 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 240 2022-04-28T19:52:31.935 INFO:tasks.rados.rados.0.smithi059.stdout:2559: seq_num 813 ranges {0=417792,417792=434176,851968=442368,1294336=327680} 2022-04-28T19:52:31.936 INFO:tasks.rados.rados.0.smithi059.stdout:2559: writing smithi05955964-36 from 0 to 417792 tid 1 2022-04-28T19:52:31.939 INFO:tasks.rados.rados.0.smithi059.stdout:2559: writing smithi05955964-36 from 417792 to 851968 tid 2 2022-04-28T19:52:31.942 INFO:tasks.rados.rados.0.smithi059.stdout:2559: writing smithi05955964-36 from 851968 to 1294336 tid 3 2022-04-28T19:52:31.945 INFO:tasks.rados.rados.0.smithi059.stdout:2559: writing smithi05955964-36 from 1294336 to 1622016 tid 4 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2560: delete oid 41 current snap is 240 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2554: finishing write tid 1 to smithi05955964-45 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2554: finishing write tid 2 to smithi05955964-45 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2554: finishing write tid 3 to smithi05955964-45 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2559: finishing write tid 1 to smithi05955964-36 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2554: finishing write tid 5 to smithi05955964-45 2022-04-28T19:52:31.946 INFO:tasks.rados.rados.0.smithi059.stdout:2559: finishing write tid 2 to smithi05955964-36 2022-04-28T19:52:31.948 INFO:tasks.rados.rados.0.smithi059.stdout:2554: finishing write tid 6 to smithi05955964-45 2022-04-28T19:52:31.948 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 949 (ObjNum 812 snap 240 seq_num 812) dirty exists 2022-04-28T19:52:31.948 INFO:tasks.rados.rados.0.smithi059.stdout:2554: left oid 45 (ObjNum 812 snap 240 seq_num 812) 2022-04-28T19:52:31.949 INFO:tasks.rados.rados.0.smithi059.stdout:2559: finishing write tid 3 to smithi05955964-36 2022-04-28T19:52:31.950 INFO:tasks.rados.rados.0.smithi059.stdout:2554: done (5 left) 2022-04-28T19:52:31.950 INFO:tasks.rados.rados.0.smithi059.stdout:2560: done (4 left) 2022-04-28T19:52:31.950 INFO:tasks.rados.rados.0.smithi059.stdout:2561: snap_remove snap 217 2022-04-28T19:52:32.291 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: cluster 2022-04-28T19:52:30.923241+0000 mgr.y (mgr.24961) 185 : cluster [DBG] pgmap v282: 89 pgs: 3 active+undersized+degraded, 3 peering, 9 stale+active+clean, 4 active+undersized, 70 active+clean; 436 MiB data, 860 MiB used, 710 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.7 MiB/s wr, 18 op/s; 28/717 objects degraded (3.905%) 2022-04-28T19:52:32.292 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: cluster 2022-04-28T19:52:30.931851+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2022-04-28T19:52:32.292 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: audit 2022-04-28T19:52:31.302811+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:32.292 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: audit 2022-04-28T19:52:31.304049+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:32.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: audit 2022-04-28T19:52:31.306844+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:52:32.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: audit 2022-04-28T19:52:31.308045+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:52:32.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: cephadm 2022-04-28T19:52:31.309153+0000 mgr.y (mgr.24961) 186 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:52:32.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: audit 2022-04-28T19:52:31.310413+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:52:32.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: cluster 2022-04-28T19:52:31.886870+0000 mon.a (mon.0) 656 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2022-04-28T19:52:32.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:31 smithi139 bash[43301]: cluster 2022-04-28T19:52:31.918601+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: Degraded data redundancy: 28/717 objects degraded (3.905%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: audit 2022-04-28T19:52:31.304049+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:32.391 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: audit 2022-04-28T19:52:31.306844+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:52:32.391 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: audit 2022-04-28T19:52:31.308045+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:52:32.392 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: cephadm 2022-04-28T19:52:31.309153+0000 mgr.y (mgr.24961) 186 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:52:32.392 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: audit 2022-04-28T19:52:31.310413+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:52:32.392 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: cluster 2022-04-28T19:52:31.886870+0000 mon.a (mon.0) 656 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2022-04-28T19:52:32.392 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[59745]: cluster 2022-04-28T19:52:31.918601+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: Degraded data redundancy: 28/717 objects degraded (3.905%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:32.393 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: cluster 2022-04-28T19:52:30.923241+0000 mgr.y (mgr.24961) 185 : cluster [DBG] pgmap v282: 89 pgs: 3 active+undersized+degraded, 3 peering, 9 stale+active+clean, 4 active+undersized, 70 active+clean; 436 MiB data, 860 MiB used, 710 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.7 MiB/s wr, 18 op/s; 28/717 objects degraded (3.905%) 2022-04-28T19:52:32.393 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: cluster 2022-04-28T19:52:30.931851+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2022-04-28T19:52:32.393 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: audit 2022-04-28T19:52:31.302811+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:32.393 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: audit 2022-04-28T19:52:31.304049+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:32.394 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: audit 2022-04-28T19:52:31.306844+0000 mon.a (mon.0) 653 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:52:32.394 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: audit 2022-04-28T19:52:31.308045+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:52:32.394 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: cephadm 2022-04-28T19:52:31.309153+0000 mgr.y (mgr.24961) 186 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:52:32.395 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: audit 2022-04-28T19:52:31.310413+0000 mon.a (mon.0) 655 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:52:32.395 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: cluster 2022-04-28T19:52:31.886870+0000 mon.a (mon.0) 656 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2022-04-28T19:52:32.395 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:31 smithi059 bash[61152]: cluster 2022-04-28T19:52:31.918601+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: Degraded data redundancy: 28/717 objects degraded (3.905%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:32.950 INFO:tasks.rados.rados.0.smithi059.stdout:2561: done (4 left) 2022-04-28T19:52:32.950 INFO:tasks.rados.rados.0.smithi059.stdout:2562: read oid 2 snap -1 2022-04-28T19:52:32.950 INFO:tasks.rados.rados.0.smithi059.stdout:2562: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:52:32.950 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 240 2022-04-28T19:52:32.950 INFO:tasks.rados.rados.0.smithi059.stdout:2563: seq_num 815 ranges {0=458752,458752=655360,1114112=409600,1523712=139264} 2022-04-28T19:52:32.952 INFO:tasks.rados.rados.0.smithi059.stdout:2563: writing smithi05955964-40 from 0 to 458752 tid 1 2022-04-28T19:52:32.956 INFO:tasks.rados.rados.0.smithi059.stdout:2563: writing smithi05955964-40 from 458752 to 1114112 tid 2 2022-04-28T19:52:32.959 INFO:tasks.rados.rados.0.smithi059.stdout:2563: writing smithi05955964-40 from 1114112 to 1523712 tid 3 2022-04-28T19:52:32.963 INFO:tasks.rados.rados.0.smithi059.stdout:2563: writing smithi05955964-40 from 1523712 to 1662976 tid 4 2022-04-28T19:52:32.964 INFO:tasks.rados.rados.0.smithi059.stdout:2564: rollback oid 41 current snap is 240 2022-04-28T19:52:32.964 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 41 to 239 2022-04-28T19:52:32.964 INFO:tasks.rados.rados.0.smithi059.stdout:2559: finishing write tid 4 to smithi05955964-36 2022-04-28T19:52:32.964 INFO:tasks.rados.rados.0.smithi059.stdout:2559: finishing write tid 6 to smithi05955964-36 2022-04-28T19:52:32.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1163 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:52:32.964 INFO:tasks.rados.rados.0.smithi059.stdout:2557: finishing copy_from to smithi05955964-5 2022-04-28T19:52:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:2557: got expected ENOENT (src dne) 2022-04-28T19:52:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:2559: finishing write tid 7 to smithi05955964-36 2022-04-28T19:52:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1049 (ObjNum 813 snap 240 seq_num 813) dirty exists 2022-04-28T19:52:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:2559: left oid 36 (ObjNum 813 snap 240 seq_num 813) 2022-04-28T19:52:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:2557: finishing copy_from racing read to smithi05955964-5 2022-04-28T19:52:32.965 INFO:tasks.rados.rados.0.smithi059.stdout:2556: done (6 left) 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:2557: done (5 left) 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:2559: done (4 left) 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:2565: rmattr oid 47 current snap is 240 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:2565: done (4 left) 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:2566: read oid 38 snap 222 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:2566: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:52:32.966 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 240 2022-04-28T19:52:32.967 INFO:tasks.rados.rados.0.smithi059.stdout:2567: seq_num 816 ranges {3964928=475136,4440064=540672,4980736=614400,5595136=49152} 2022-04-28T19:52:32.986 INFO:tasks.rados.rados.0.smithi059.stdout:2567: writing smithi05955964-10 from 3964928 to 4440064 tid 1 2022-04-28T19:52:32.989 INFO:tasks.rados.rados.0.smithi059.stdout:2567: writing smithi05955964-10 from 4440064 to 4980736 tid 2 2022-04-28T19:52:32.993 INFO:tasks.rados.rados.0.smithi059.stdout:2567: writing smithi05955964-10 from 4980736 to 5595136 tid 3 2022-04-28T19:52:32.993 INFO:tasks.rados.rados.0.smithi059.stdout:2567: writing smithi05955964-10 from 5595136 to 5644288 tid 4 2022-04-28T19:52:32.993 INFO:tasks.rados.rados.0.smithi059.stdout:2568: setattr oid 29 current snap is 240 2022-04-28T19:52:32.994 INFO:tasks.rados.rados.0.smithi059.stdout:2555: expect (ObjNum 600 snap 169 seq_num 600) 2022-04-28T19:52:33.086 INFO:tasks.rados.rados.0.smithi059.stdout:2563: finishing write tid 1 to smithi05955964-40 2022-04-28T19:52:33.086 INFO:tasks.rados.rados.0.smithi059.stdout:2563: finishing write tid 2 to smithi05955964-40 2022-04-28T19:52:33.087 INFO:tasks.rados.rados.0.smithi059.stdout:2564: finishing rollback tid 1 to smithi05955964-41 2022-04-28T19:52:33.087 INFO:tasks.rados.rados.0.smithi059.stdout:2564: finishing rollback tid 2 to smithi05955964-41 2022-04-28T19:52:33.087 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1015 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:52:33.087 INFO:tasks.rados.rados.0.smithi059.stdout:2563: finishing write tid 3 to smithi05955964-40 2022-04-28T19:52:33.087 INFO:tasks.rados.rados.0.smithi059.stdout:2566: expect (ObjNum 493 snap 141 seq_num 493) 2022-04-28T19:52:33.118 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2022-04-28T19:52:33.118 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2022-04-28T19:52:33.118 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2022-04-28T19:52:33.119 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2022-04-28T19:52:33.119 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2022-04-28T19:52:33.119 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2022-04-28T19:52:33.119 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:32 smithi139 bash[45529]: --> ceph-volume lvm activate successful for osd ID: 4 2022-04-28T19:52:33.125 INFO:tasks.rados.rados.0.smithi059.stdout:2563: finishing write tid 4 to smithi05955964-40 2022-04-28T19:52:33.125 INFO:tasks.rados.rados.0.smithi059.stdout:2562: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:52:33.537 INFO:tasks.rados.rados.0.smithi059.stdout:2563: finishing write tid 6 to smithi05955964-40 2022-04-28T19:52:33.537 INFO:tasks.rados.rados.0.smithi059.stdout:2563: finishing write tid 7 to smithi05955964-40 2022-04-28T19:52:33.537 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1054 (ObjNum 815 snap 240 seq_num 815) dirty exists 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2563: left oid 40 (ObjNum 815 snap 240 seq_num 815) 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: finishing write tid 1 to smithi05955964-10 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: finishing write tid 2 to smithi05955964-10 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: finishing write tid 3 to smithi05955964-10 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: finishing write tid 4 to smithi05955964-10 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: finishing write tid 6 to smithi05955964-10 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: finishing write tid 7 to smithi05955964-10 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1101 (ObjNum 816 snap 240 seq_num 816) dirty exists 2022-04-28T19:52:33.538 INFO:tasks.rados.rados.0.smithi059.stdout:2567: left oid 10 (ObjNum 816 snap 240 seq_num 816) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2555: done (6 left) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2562: done (5 left) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2563: done (4 left) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2564: done (3 left) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2566: done (2 left) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2567: done (1 left) 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 240 2022-04-28T19:52:33.539 INFO:tasks.rados.rados.0.smithi059.stdout:2569: seq_num 817 ranges {5644288=491520,6135808=417792,6553600=425984,6979584=360448} 2022-04-28T19:52:33.584 INFO:tasks.rados.rados.0.smithi059.stdout:2569: writing smithi05955964-10 from 5644288 to 6135808 tid 1 2022-04-28T19:52:33.584 INFO:tasks.rados.rados.0.smithi059.stdout:2569: writing smithi05955964-10 from 6135808 to 6553600 tid 2 2022-04-28T19:52:33.586 INFO:tasks.rados.rados.0.smithi059.stdout:2569: writing smithi05955964-10 from 6553600 to 6979584 tid 3 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2569: writing smithi05955964-10 from 6979584 to 7340032 tid 4 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2570: read oid 21 snap -1 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2570: expect (ObjNum 796 snap 235 seq_num 796) 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2571: delete oid 3 current snap is 240 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1176 (ObjNum -132563456 snap 21893 seq_num 4159778832) dirty exists 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2569: finishing write tid 1 to smithi05955964-10 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2569: finishing write tid 2 to smithi05955964-10 2022-04-28T19:52:33.589 INFO:tasks.rados.rados.0.smithi059.stdout:2569: finishing write tid 3 to smithi05955964-10 2022-04-28T19:52:33.591 INFO:tasks.rados.rados.0.smithi059.stdout:2569: finishing write tid 4 to smithi05955964-10 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:2569: finishing write tid 6 to smithi05955964-10 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:2568: done (3 left) 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:2571: done (2 left) 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:2572: read oid 39 snap -1 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:2572: expect deleted 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 240 2022-04-28T19:52:33.592 INFO:tasks.rados.rados.0.smithi059.stdout:2573: seq_num 819 ranges {1843200=524288,2367488=589824,2957312=614400} 2022-04-28T19:52:33.605 INFO:tasks.rados.rados.0.smithi059.stdout:2573: writing smithi05955964-32 from 1843200 to 2367488 tid 1 2022-04-28T19:52:33.608 INFO:tasks.rados.rados.0.smithi059.stdout:2573: writing smithi05955964-32 from 2367488 to 2957312 tid 2 2022-04-28T19:52:33.612 INFO:tasks.rados.rados.0.smithi059.stdout:2573: writing smithi05955964-32 from 2957312 to 3571712 tid 3 2022-04-28T19:52:33.613 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 240 2022-04-28T19:52:33.613 INFO:tasks.rados.rados.0.smithi059.stdout:2574: seq_num 820 ranges {14106624=540672,14647296=475136,15122432=598016,15720448=131072} 2022-04-28T19:52:33.695 INFO:tasks.rados.rados.0.smithi059.stdout:2574: writing smithi05955964-6 from 14106624 to 14647296 tid 1 2022-04-28T19:52:33.698 INFO:tasks.rados.rados.0.smithi059.stdout:2574: writing smithi05955964-6 from 14647296 to 15122432 tid 2 2022-04-28T19:52:33.702 INFO:tasks.rados.rados.0.smithi059.stdout:2574: writing smithi05955964-6 from 15122432 to 15720448 tid 3 2022-04-28T19:52:33.703 INFO:tasks.rados.rados.0.smithi059.stdout:2574: writing smithi05955964-6 from 15720448 to 15851520 tid 4 2022-04-28T19:52:33.703 INFO:tasks.rados.rados.0.smithi059.stdout:2575: rollback oid 7 current snap is 240 2022-04-28T19:52:33.703 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 7 to 237 2022-04-28T19:52:33.703 INFO:tasks.rados.rados.0.smithi059.stdout:2569: finishing write tid 7 to smithi05955964-10 2022-04-28T19:52:33.704 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1106 (ObjNum 817 snap 240 seq_num 817) dirty exists 2022-04-28T19:52:33.704 INFO:tasks.rados.rados.0.smithi059.stdout:2569: left oid 10 (ObjNum 817 snap 240 seq_num 817) 2022-04-28T19:52:33.704 INFO:tasks.rados.rados.0.smithi059.stdout:2570: expect (ObjNum 796 snap 235 seq_num 796) 2022-04-28T19:52:33.734 INFO:tasks.rados.rados.0.smithi059.stdout:2573: finishing write tid 1 to smithi05955964-32 2022-04-28T19:52:33.734 INFO:tasks.rados.rados.0.smithi059.stdout:2573: finishing write tid 2 to smithi05955964-32 2022-04-28T19:52:33.734 INFO:tasks.rados.rados.0.smithi059.stdout:2573: finishing write tid 3 to smithi05955964-32 2022-04-28T19:52:33.734 INFO:tasks.rados.rados.0.smithi059.stdout:2573: finishing write tid 5 to smithi05955964-32 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2573: finishing write tid 6 to smithi05955964-32 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1202 (ObjNum 819 snap 240 seq_num 819) dirty exists 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2573: left oid 32 (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2569: done (5 left) 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2570: done (4 left) 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2572: done (3 left) 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2573: done (2 left) 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2576: rmattr oid 49 current snap is 240 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:2576: done (2 left) 2022-04-28T19:52:33.735 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 240 2022-04-28T19:52:33.736 INFO:tasks.rados.rados.0.smithi059.stdout:2577: seq_num 821 ranges {1990656=557056,2547712=761856,3309568=442368} 2022-04-28T19:52:33.752 INFO:tasks.rados.rados.0.smithi059.stdout:2577: writing smithi05955964-20 from 1990656 to 2547712 tid 1 2022-04-28T19:52:33.757 INFO:tasks.rados.rados.0.smithi059.stdout:2577: writing smithi05955964-20 from 2547712 to 3309568 tid 2 2022-04-28T19:52:33.761 INFO:tasks.rados.rados.0.smithi059.stdout:2577: writing smithi05955964-20 from 3309568 to 3751936 tid 3 2022-04-28T19:52:33.761 INFO:tasks.rados.rados.0.smithi059.stdout:2578: copy_from oid 19 from oid 35 current snap is 240 2022-04-28T19:52:33.761 INFO:tasks.rados.rados.0.smithi059.stdout:2574: finishing write tid 1 to smithi05955964-6 2022-04-28T19:52:33.761 INFO:tasks.rados.rados.0.smithi059.stdout:2574: finishing write tid 2 to smithi05955964-6 2022-04-28T19:52:33.761 INFO:tasks.rados.rados.0.smithi059.stdout:2574: finishing write tid 3 to smithi05955964-6 2022-04-28T19:52:33.761 INFO:tasks.rados.rados.0.smithi059.stdout:2575: finishing rollback tid 1 to smithi05955964-7 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 964 (ObjNum 0 snap 0 seq_num 188341408) dirty dne 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2574: finishing write tid 4 to smithi05955964-6 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2574: finishing write tid 6 to smithi05955964-6 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2574: finishing write tid 7 to smithi05955964-6 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 957 (ObjNum 820 snap 240 seq_num 820) dirty exists 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2574: left oid 6 (ObjNum 820 snap 240 seq_num 820) 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2577: finishing write tid 1 to smithi05955964-20 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2577: finishing write tid 2 to smithi05955964-20 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2574: done (3 left) 2022-04-28T19:52:33.762 INFO:tasks.rados.rados.0.smithi059.stdout:2575: done (2 left) 2022-04-28T19:52:33.763 INFO:tasks.rados.rados.0.smithi059.stdout:2579: read oid 4 snap -1 2022-04-28T19:52:33.763 INFO:tasks.rados.rados.0.smithi059.stdout:2579: expect (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:52:33.763 INFO:tasks.rados.rados.0.smithi059.stdout:2580: setattr oid 39 current snap is 240 2022-04-28T19:52:33.763 INFO:tasks.rados.rados.0.smithi059.stdout:2581: rmattr oid 10 current snap is 240 2022-04-28T19:52:33.763 INFO:tasks.rados.rados.0.smithi059.stdout:2582: read oid 40 snap 236 2022-04-28T19:52:33.764 INFO:tasks.rados.rados.0.smithi059.stdout:2582: expect deleted 2022-04-28T19:52:33.764 INFO:tasks.rados.rados.0.smithi059.stdout:2583: snap_create 2022-04-28T19:52:33.765 INFO:tasks.rados.rados.0.smithi059.stdout:2577: finishing write tid 3 to smithi05955964-20 2022-04-28T19:52:33.765 INFO:tasks.rados.rados.0.smithi059.stdout:2577: finishing write tid 5 to smithi05955964-20 2022-04-28T19:52:33.766 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1107 (ObjNum 817 snap 240 seq_num 817) dirty exists 2022-04-28T19:52:33.767 INFO:tasks.rados.rados.0.smithi059.stdout:2577: finishing write tid 6 to smithi05955964-20 2022-04-28T19:52:33.767 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 962 (ObjNum 821 snap 240 seq_num 821) dirty exists 2022-04-28T19:52:33.767 INFO:tasks.rados.rados.0.smithi059.stdout:2577: left oid 20 (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:52:33.769 INFO:tasks.rados.rados.0.smithi059.stdout:2579: expect (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:52:33.805 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1203 (ObjNum -132562256 snap 21893 seq_num 4161449392) dirty exists 2022-04-28T19:52:33.805 INFO:tasks.rados.rados.0.smithi059.stdout:2578: finishing copy_from to smithi05955964-19 2022-04-28T19:52:33.805 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 942 (ObjNum 806 snap 239 seq_num 806) dirty exists 2022-04-28T19:52:33.805 INFO:tasks.rados.rados.0.smithi059.stdout:2578: finishing copy_from racing read to smithi05955964-19 2022-04-28T19:52:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:33.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:33.914 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:33.525Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:33.914 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:33.525Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2577: done (6 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2578: done (5 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2579: done (4 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2580: done (3 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2581: done (2 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2582: done (1 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2583: done (0 left) 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 241 2022-04-28T19:52:33.951 INFO:tasks.rados.rados.0.smithi059.stdout:2584: seq_num 822 ranges {0=573440,573440=696320,1269760=507904} 2022-04-28T19:52:33.957 INFO:tasks.rados.rados.0.smithi059.stdout:2584: writing smithi05955964-15 from 0 to 573440 tid 1 2022-04-28T19:52:33.961 INFO:tasks.rados.rados.0.smithi059.stdout:2584: writing smithi05955964-15 from 573440 to 1269760 tid 2 2022-04-28T19:52:33.964 INFO:tasks.rados.rados.0.smithi059.stdout:2584: writing smithi05955964-15 from 1269760 to 1777664 tid 3 2022-04-28T19:52:33.964 INFO:tasks.rados.rados.0.smithi059.stdout:2585: delete oid 23 current snap is 241 2022-04-28T19:52:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:2585: done (1 left) 2022-04-28T19:52:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 241 2022-04-28T19:52:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:2586: seq_num 824 ranges {1458176=606208,2064384=458752,2523136=573440,3096576=172032} 2022-04-28T19:52:33.979 INFO:tasks.rados.rados.0.smithi059.stdout:2586: writing smithi05955964-28 from 1458176 to 2064384 tid 1 2022-04-28T19:52:33.982 INFO:tasks.rados.rados.0.smithi059.stdout:2586: writing smithi05955964-28 from 2064384 to 2523136 tid 2 2022-04-28T19:52:33.985 INFO:tasks.rados.rados.0.smithi059.stdout:2586: writing smithi05955964-28 from 2523136 to 3096576 tid 3 2022-04-28T19:52:33.986 INFO:tasks.rados.rados.0.smithi059.stdout:2586: writing smithi05955964-28 from 3096576 to 3268608 tid 4 2022-04-28T19:52:33.986 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 241 2022-04-28T19:52:33.987 INFO:tasks.rados.rados.0.smithi059.stdout:2587: seq_num 825 ranges {0=622592,622592=745472,1368064=458752} 2022-04-28T19:52:33.990 INFO:tasks.rados.rados.0.smithi059.stdout:2587: writing smithi05955964-37 from 0 to 622592 tid 1 2022-04-28T19:52:33.994 INFO:tasks.rados.rados.0.smithi059.stdout:2587: writing smithi05955964-37 from 622592 to 1368064 tid 2 2022-04-28T19:52:33.998 INFO:tasks.rados.rados.0.smithi059.stdout:2587: writing smithi05955964-37 from 1368064 to 1826816 tid 3 2022-04-28T19:52:33.998 INFO:tasks.rados.rados.0.smithi059.stdout:2588: read oid 42 snap -1 2022-04-28T19:52:33.998 INFO:tasks.rados.rados.0.smithi059.stdout:2588: expect (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:52:33.998 INFO:tasks.rados.rados.0.smithi059.stdout:2589: copy_from oid 14 from oid 44 current snap is 241 2022-04-28T19:52:33.998 INFO:tasks.rados.rados.0.smithi059.stdout:2584: finishing write tid 1 to smithi05955964-15 2022-04-28T19:52:33.998 INFO:tasks.rados.rados.0.smithi059.stdout:2584: finishing write tid 2 to smithi05955964-15 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2584: finishing write tid 3 to smithi05955964-15 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2584: finishing write tid 5 to smithi05955964-15 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2584: finishing write tid 6 to smithi05955964-15 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1139 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2584: left oid 15 (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2586: finishing write tid 1 to smithi05955964-28 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2586: finishing write tid 2 to smithi05955964-28 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2586: finishing write tid 3 to smithi05955964-28 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2584: done (4 left) 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2590: copy_from oid 13 from oid 17 current snap is 241 2022-04-28T19:52:33.999 INFO:tasks.rados.rados.0.smithi059.stdout:2586: finishing write tid 4 to smithi05955964-28 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:2586: finishing write tid 6 to smithi05955964-28 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:2586: finishing write tid 7 to smithi05955964-28 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 959 (ObjNum 824 snap 241 seq_num 824) dirty exists 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:2586: left oid 28 (ObjNum 824 snap 241 seq_num 824) 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:2587: finishing write tid 1 to smithi05955964-37 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:2586: done (4 left) 2022-04-28T19:52:34.000 INFO:tasks.rados.rados.0.smithi059.stdout:2591: snap_remove snap 213 2022-04-28T19:52:34.338 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:33 smithi139 bash[43301]: cluster 2022-04-28T19:52:32.924602+0000 mgr.y (mgr.24961) 187 : cluster [DBG] pgmap v285: 89 pgs: 15 active+undersized+degraded, 3 peering, 22 active+undersized, 49 active+clean; 384 MiB data, 818 MiB used, 710 GiB / 715 GiB avail; 6.0 MiB/s rd, 913 KiB/s wr, 15 op/s; 73/678 objects degraded (10.767%) 2022-04-28T19:52:34.338 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:33 smithi139 bash[43301]: cluster 2022-04-28T19:52:32.945577+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2022-04-28T19:52:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[59745]: cluster 2022-04-28T19:52:32.924602+0000 mgr.y (mgr.24961) 187 : cluster [DBG] pgmap v285: 89 pgs: 15 active+undersized+degraded, 3 peering, 22 active+undersized, 49 active+clean; 384 MiB data, 818 MiB used, 710 GiB / 715 GiB avail; 6.0 MiB/s rd, 913 KiB/s wr, 15 op/s; 73/678 objects degraded (10.767%) 2022-04-28T19:52:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[59745]: cluster 2022-04-28T19:52:32.945577+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2022-04-28T19:52:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[61152]: cluster 2022-04-28T19:52:32.924602+0000 mgr.y (mgr.24961) 187 : cluster [DBG] pgmap v285: 89 pgs: 15 active+undersized+degraded, 3 peering, 22 active+undersized, 49 active+clean; 384 MiB data, 818 MiB used, 710 GiB / 715 GiB avail; 6.0 MiB/s rd, 913 KiB/s wr, 15 op/s; 73/678 objects degraded (10.767%) 2022-04-28T19:52:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:33 smithi059 bash[61152]: cluster 2022-04-28T19:52:32.945577+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2022-04-28T19:52:34.618 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:34 smithi139 bash[45529]: debug 2022-04-28T19:52:34.458+0000 7fc6416f5700 1 -- 172.21.15.139:0/2894486439 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564222cf04e0 con 0x564222d68c00 2022-04-28T19:52:34.968 INFO:tasks.rados.rados.0.smithi059.stdout:2591: done (4 left) 2022-04-28T19:52:34.968 INFO:tasks.rados.rados.0.smithi059.stdout:2592: delete oid 31 current snap is 241 2022-04-28T19:52:34.971 INFO:tasks.rados.rados.0.smithi059.stdout:2589: finishing copy_from to smithi05955964-14 2022-04-28T19:52:34.971 INFO:tasks.rados.rados.0.smithi059.stdout:2589: got expected ENOENT (src dne) 2022-04-28T19:52:34.971 INFO:tasks.rados.rados.0.smithi059.stdout:2589: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:52:34.971 INFO:tasks.rados.rados.0.smithi059.stdout:2587: finishing write tid 2 to smithi05955964-37 2022-04-28T19:52:34.971 INFO:tasks.rados.rados.0.smithi059.stdout:2587: finishing write tid 3 to smithi05955964-37 2022-04-28T19:52:34.971 INFO:tasks.rados.rados.0.smithi059.stdout:2587: finishing write tid 5 to smithi05955964-37 2022-04-28T19:52:34.972 INFO:tasks.rados.rados.0.smithi059.stdout:2588: expect (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:52:35.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:34 smithi059 bash[61152]: cluster 2022-04-28T19:52:33.949552+0000 mon.a (mon.0) 659 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2022-04-28T19:52:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:34 smithi059 bash[59745]: cluster 2022-04-28T19:52:33.949552+0000 mon.a (mon.0) 659 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2022-04-28T19:52:35.365 INFO:tasks.rados.rados.0.smithi059.stdout:2587: finishing write tid 6 to smithi05955964-37 2022-04-28T19:52:35.365 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1116 (ObjNum 825 snap 241 seq_num 825) dirty exists 2022-04-28T19:52:35.365 INFO:tasks.rados.rados.0.smithi059.stdout:2587: left oid 37 (ObjNum 825 snap 241 seq_num 825) 2022-04-28T19:52:35.365 INFO:tasks.rados.rados.0.smithi059.stdout:2590: finishing copy_from to smithi05955964-13 2022-04-28T19:52:35.365 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 964 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:52:35.365 INFO:tasks.rados.rados.0.smithi059.stdout:2590: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:52:35.368 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:35 smithi139 bash[45529]: debug 2022-04-28T19:52:35.014+0000 7fc648f50200 -1 Falling back to public interface 2022-04-28T19:52:35.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:34 smithi139 bash[43301]: cluster 2022-04-28T19:52:33.949552+0000 mon.a (mon.0) 659 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2022-04-28T19:52:35.369 INFO:tasks.rados.rados.0.smithi059.stdout:2587: done (4 left) 2022-04-28T19:52:35.369 INFO:tasks.rados.rados.0.smithi059.stdout:2588: done (3 left) 2022-04-28T19:52:35.369 INFO:tasks.rados.rados.0.smithi059.stdout:2589: done (2 left) 2022-04-28T19:52:35.369 INFO:tasks.rados.rados.0.smithi059.stdout:2590: done (1 left) 2022-04-28T19:52:35.369 INFO:tasks.rados.rados.0.smithi059.stdout:2592: done (0 left) 2022-04-28T19:52:35.369 INFO:tasks.rados.rados.0.smithi059.stdout:2593: delete oid 21 current snap is 241 2022-04-28T19:52:35.372 INFO:tasks.rados.rados.0.smithi059.stdout:2593: done (0 left) 2022-04-28T19:52:35.372 INFO:tasks.rados.rados.0.smithi059.stdout:2594: copy_from oid 13 from oid 17 current snap is 241 2022-04-28T19:52:35.372 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 241 2022-04-28T19:52:35.372 INFO:tasks.rados.rados.0.smithi059.stdout:2595: seq_num 828 ranges {2056192=671744,2727936=802816,3530752=401408} 2022-04-28T19:52:35.394 INFO:tasks.rados.rados.0.smithi059.stdout:2595: writing smithi05955964-26 from 2056192 to 2727936 tid 1 2022-04-28T19:52:35.399 INFO:tasks.rados.rados.0.smithi059.stdout:2595: writing smithi05955964-26 from 2727936 to 3530752 tid 2 2022-04-28T19:52:35.402 INFO:tasks.rados.rados.0.smithi059.stdout:2595: writing smithi05955964-26 from 3530752 to 3932160 tid 3 2022-04-28T19:52:35.402 INFO:tasks.rados.rados.0.smithi059.stdout:2596: snap_create 2022-04-28T19:52:35.402 INFO:tasks.rados.rados.0.smithi059.stdout:2595: finishing write tid 1 to smithi05955964-26 2022-04-28T19:52:35.402 INFO:tasks.rados.rados.0.smithi059.stdout:2595: finishing write tid 2 to smithi05955964-26 2022-04-28T19:52:35.405 INFO:tasks.rados.rados.0.smithi059.stdout:2595: finishing write tid 3 to smithi05955964-26 2022-04-28T19:52:35.405 INFO:tasks.rados.rados.0.smithi059.stdout:2595: finishing write tid 5 to smithi05955964-26 2022-04-28T19:52:35.406 INFO:tasks.rados.rados.0.smithi059.stdout:2595: finishing write tid 6 to smithi05955964-26 2022-04-28T19:52:35.407 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1109 (ObjNum 828 snap 241 seq_num 828) dirty exists 2022-04-28T19:52:35.407 INFO:tasks.rados.rados.0.smithi059.stdout:2595: left oid 26 (ObjNum 828 snap 241 seq_num 828) 2022-04-28T19:52:35.411 INFO:tasks.rados.rados.0.smithi059.stdout:2594: finishing copy_from to smithi05955964-13 2022-04-28T19:52:35.411 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 964 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:52:35.412 INFO:tasks.rados.rados.0.smithi059.stdout:2594: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:52:35.979 INFO:tasks.rados.rados.0.smithi059.stdout:2594: done (2 left) 2022-04-28T19:52:35.979 INFO:tasks.rados.rados.0.smithi059.stdout:2595: done (1 left) 2022-04-28T19:52:35.979 INFO:tasks.rados.rados.0.smithi059.stdout:2596: done (0 left) 2022-04-28T19:52:35.979 INFO:tasks.rados.rados.0.smithi059.stdout:2597: copy_from oid 24 from oid 35 current snap is 242 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2598: read oid 32 snap 230 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2598: expect (ObjNum 761 snap 229 seq_num 761) 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2599: read oid 38 snap -1 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2599: expect deleted 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2600: read oid 50 snap 227 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2600: expect (ObjNum 749 snap 224 seq_num 749) 2022-04-28T19:52:35.980 INFO:tasks.rados.rados.0.smithi059.stdout:2601: setattr oid 9 current snap is 242 2022-04-28T19:52:35.981 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 242 2022-04-28T19:52:35.981 INFO:tasks.rados.rados.0.smithi059.stdout:2602: seq_num 829 ranges {7569408=688128,8257536=729088,8986624=475136} 2022-04-28T19:52:36.031 INFO:tasks.rados.rados.0.smithi059.stdout:2602: writing smithi05955964-8 from 7569408 to 8257536 tid 1 2022-04-28T19:52:36.035 INFO:tasks.rados.rados.0.smithi059.stdout:2602: writing smithi05955964-8 from 8257536 to 8986624 tid 2 2022-04-28T19:52:36.038 INFO:tasks.rados.rados.0.smithi059.stdout:2602: writing smithi05955964-8 from 8986624 to 9461760 tid 3 2022-04-28T19:52:36.038 INFO:tasks.rados.rados.0.smithi059.stdout:2603: copy_from oid 46 from oid 10 current snap is 242 2022-04-28T19:52:36.038 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 963 (ObjNum 775 snap 233 seq_num 775) dirty exists 2022-04-28T19:52:36.038 INFO:tasks.rados.rados.0.smithi059.stdout:2599: done (6 left) 2022-04-28T19:52:36.038 INFO:tasks.rados.rados.0.smithi059.stdout:2601: done (5 left) 2022-04-28T19:52:36.038 INFO:tasks.rados.rados.0.smithi059.stdout:2604: rollback oid 25 current snap is 242 2022-04-28T19:52:36.039 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 25 to 224 2022-04-28T19:52:36.039 INFO:tasks.rados.rados.0.smithi059.stdout:2598: expect (ObjNum 761 snap 229 seq_num 761) 2022-04-28T19:52:36.061 INFO:tasks.rados.rados.0.smithi059.stdout:2597: finishing copy_from to smithi05955964-24 2022-04-28T19:52:36.061 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 942 (ObjNum 806 snap 239 seq_num 806) dirty exists 2022-04-28T19:52:36.061 INFO:tasks.rados.rados.0.smithi059.stdout:2597: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:52:36.061 INFO:tasks.rados.rados.0.smithi059.stdout:2600: expect (ObjNum 749 snap 224 seq_num 749) 2022-04-28T19:52:36.235 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:35 smithi059 bash[61152]: cluster 2022-04-28T19:52:34.925679+0000 mgr.y (mgr.24961) 188 : cluster [DBG] pgmap v288: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 396 MiB data, 827 MiB used, 710 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.0 MiB/s wr, 25 op/s; 90/702 objects degraded (12.821%) 2022-04-28T19:52:36.235 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:35 smithi059 bash[61152]: cluster 2022-04-28T19:52:34.966611+0000 mon.a (mon.0) 660 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2022-04-28T19:52:36.235 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:35 smithi059 bash[59745]: cluster 2022-04-28T19:52:34.925679+0000 mgr.y (mgr.24961) 188 : cluster [DBG] pgmap v288: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 396 MiB data, 827 MiB used, 710 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.0 MiB/s wr, 25 op/s; 90/702 objects degraded (12.821%) 2022-04-28T19:52:36.235 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:35 smithi059 bash[59745]: cluster 2022-04-28T19:52:34.966611+0000 mon.a (mon.0) 660 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2022-04-28T19:52:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:35 smithi139 bash[43301]: cluster 2022-04-28T19:52:34.925679+0000 mgr.y (mgr.24961) 188 : cluster [DBG] pgmap v288: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 396 MiB data, 827 MiB used, 710 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.0 MiB/s wr, 25 op/s; 90/702 objects degraded (12.821%) 2022-04-28T19:52:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:35 smithi139 bash[43301]: cluster 2022-04-28T19:52:34.966611+0000 mon.a (mon.0) 660 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2022-04-28T19:52:36.590 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[57771]: debug 2022-04-28T19:52:36.227+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2022-04-28T19:52:36.833 INFO:tasks.rados.rados.0.smithi059.stdout:2602: finishing write tid 1 to smithi05955964-8 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2602: finishing write tid 2 to smithi05955964-8 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2602: finishing write tid 3 to smithi05955964-8 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2602: finishing write tid 5 to smithi05955964-8 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2602: finishing write tid 6 to smithi05955964-8 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1189 (ObjNum 829 snap 242 seq_num 829) dirty exists 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2602: left oid 8 (ObjNum 829 snap 242 seq_num 829) 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2604: finishing rollback tid 0 to smithi05955964-25 2022-04-28T19:52:36.834 INFO:tasks.rados.rados.0.smithi059.stdout:2604: finishing rollback tid 1 to smithi05955964-25 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2604: finishing rollback tid 2 to smithi05955964-25 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1193 (ObjNum 719 snap 209 seq_num 719) dirty exists 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2603: finishing copy_from to smithi05955964-46 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1107 (ObjNum 817 snap 240 seq_num 817) dirty exists 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2603: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2597: done (5 left) 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2598: done (4 left) 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2600: done (3 left) 2022-04-28T19:52:36.835 INFO:tasks.rados.rados.0.smithi059.stdout:2602: done (2 left) 2022-04-28T19:52:36.836 INFO:tasks.rados.rados.0.smithi059.stdout:2603: done (1 left) 2022-04-28T19:52:36.836 INFO:tasks.rados.rados.0.smithi059.stdout:2604: done (0 left) 2022-04-28T19:52:36.836 INFO:tasks.rados.rados.0.smithi059.stdout:2605: setattr oid 39 current snap is 242 2022-04-28T19:52:36.838 INFO:tasks.rados.rados.0.smithi059.stdout:2606: snap_create 2022-04-28T19:52:36.844 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1211 (ObjNum 1 snap 32762 seq_num 4162028288) dirty exists 2022-04-28T19:52:36.890 INFO:tasks.rados.rados.0.smithi059.stdout:2605: done (1 left) 2022-04-28T19:52:36.890 INFO:tasks.rados.rados.0.smithi059.stdout:2606: done (0 left) 2022-04-28T19:52:36.890 INFO:tasks.rados.rados.0.smithi059.stdout:2607: read oid 18 snap -1 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2607: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2608: read oid 38 snap -1 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2608: expect deleted 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2609: read oid 10 snap -1 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2609: expect (ObjNum 817 snap 240 seq_num 817) 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2610: read oid 42 snap -1 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2610: expect (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2611: copy_from oid 1 from oid 44 current snap is 243 2022-04-28T19:52:36.891 INFO:tasks.rados.rados.0.smithi059.stdout:2612: delete oid 6 current snap is 243 2022-04-28T19:52:36.900 INFO:tasks.rados.rados.0.smithi059.stdout:2611: finishing copy_from to smithi05955964-1 2022-04-28T19:52:36.900 INFO:tasks.rados.rados.0.smithi059.stdout:2611: got expected ENOENT (src dne) 2022-04-28T19:52:36.900 INFO:tasks.rados.rados.0.smithi059.stdout:2611: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:52:36.905 INFO:tasks.rados.rados.0.smithi059.stdout:2608: done (5 left) 2022-04-28T19:52:36.906 INFO:tasks.rados.rados.0.smithi059.stdout:2611: done (4 left) 2022-04-28T19:52:36.906 INFO:tasks.rados.rados.0.smithi059.stdout:2612: done (3 left) 2022-04-28T19:52:36.906 INFO:tasks.rados.rados.0.smithi059.stdout:2613: rollback oid 49 current snap is 243 2022-04-28T19:52:36.906 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 242 2022-04-28T19:52:36.906 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 243 2022-04-28T19:52:36.906 INFO:tasks.rados.rados.0.smithi059.stdout:2614: seq_num 831 ranges {1507328=720896,2228224=499712,2727936=704512} 2022-04-28T19:52:36.919 INFO:tasks.rados.rados.0.smithi059.stdout:2614: writing smithi05955964-35 from 1507328 to 2228224 tid 1 2022-04-28T19:52:36.922 INFO:tasks.rados.rados.0.smithi059.stdout:2614: writing smithi05955964-35 from 2228224 to 2727936 tid 2 2022-04-28T19:52:36.927 INFO:tasks.rados.rados.0.smithi059.stdout:2614: writing smithi05955964-35 from 2727936 to 3432448 tid 3 2022-04-28T19:52:36.927 INFO:tasks.rados.rados.0.smithi059.stdout:2615: read oid 20 snap 240 2022-04-28T19:52:36.927 INFO:tasks.rados.rados.0.smithi059.stdout:2615: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:52:36.927 INFO:tasks.rados.rados.0.smithi059.stdout:2616: snap_remove snap 239 2022-04-28T19:52:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: cluster 2022-04-28T19:52:35.975199+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2022-04-28T19:52:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.215271+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.221137+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.227609+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.229908+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.231273+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.232432+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.234578+0000 mon.a (mon.0) 668 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: audit 2022-04-28T19:52:36.234838+0000 mgr.y (mgr.24961) 189 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: cephadm 2022-04-28T19:52:36.236067+0000 mgr.y (mgr.24961) 190 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[59745]: cluster 2022-04-28T19:52:36.888506+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: cluster 2022-04-28T19:52:35.975199+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.215271+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.221137+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.227609+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.229908+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.231273+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.232432+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.234578+0000 mon.a (mon.0) 668 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: audit 2022-04-28T19:52:36.234838+0000 mgr.y (mgr.24961) 189 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: cephadm 2022-04-28T19:52:36.236067+0000 mgr.y (mgr.24961) 190 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2022-04-28T19:52:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:36 smithi059 bash[61152]: cluster 2022-04-28T19:52:36.888506+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2022-04-28T19:52:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: cluster 2022-04-28T19:52:35.975199+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2022-04-28T19:52:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.215271+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.221137+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.227609+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.229908+0000 mon.a (mon.0) 665 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.231273+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.232432+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.234578+0000 mon.a (mon.0) 668 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: audit 2022-04-28T19:52:36.234838+0000 mgr.y (mgr.24961) 189 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: cephadm 2022-04-28T19:52:36.236067+0000 mgr.y (mgr.24961) 190 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2022-04-28T19:52:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:36 smithi139 bash[43301]: cluster 2022-04-28T19:52:36.888506+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2022-04-28T19:52:37.910 INFO:tasks.rados.rados.0.smithi059.stdout:2616: done (6 left) 2022-04-28T19:52:37.911 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 243 2022-04-28T19:52:37.911 INFO:tasks.rados.rados.0.smithi059.stdout:2617: seq_num 832 ranges {7970816=737280,8708096=786432,9494528=417792} 2022-04-28T19:52:37.964 INFO:tasks.rados.rados.0.smithi059.stdout:2617: writing smithi05955964-13 from 7970816 to 8708096 tid 1 2022-04-28T19:52:37.968 INFO:tasks.rados.rados.0.smithi059.stdout:2617: writing smithi05955964-13 from 8708096 to 9494528 tid 2 2022-04-28T19:52:37.971 INFO:tasks.rados.rados.0.smithi059.stdout:2617: writing smithi05955964-13 from 9494528 to 9912320 tid 3 2022-04-28T19:52:37.971 INFO:tasks.rados.rados.0.smithi059.stdout:2618: snap_create 2022-04-28T19:52:37.971 INFO:tasks.rados.rados.0.smithi059.stdout:2607: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:52:38.089 INFO:tasks.rados.rados.0.smithi059.stdout:2613: finishing rollback tid 0 to smithi05955964-49 2022-04-28T19:52:38.089 INFO:tasks.rados.rados.0.smithi059.stdout:2613: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:52:38.089 INFO:tasks.rados.rados.0.smithi059.stdout:2613: finishing rollback tid 2 to smithi05955964-49 2022-04-28T19:52:38.089 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1057 (ObjNum 786 snap 234 seq_num 786) dirty exists 2022-04-28T19:52:38.090 INFO:tasks.rados.rados.0.smithi059.stdout:2614: finishing write tid 1 to smithi05955964-35 2022-04-28T19:52:38.090 INFO:tasks.rados.rados.0.smithi059.stdout:2609: expect (ObjNum 817 snap 240 seq_num 817) 2022-04-28T19:52:38.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:37 smithi059 bash[59745]: cluster 2022-04-28T19:52:36.926305+0000 mgr.y (mgr.24961) 191 : cluster [DBG] pgmap v292: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 399 MiB data, 855 MiB used, 710 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.4 MiB/s wr, 19 op/s; 91/708 objects degraded (12.853%) 2022-04-28T19:52:38.378 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:37 smithi059 bash[59745]: cluster 2022-04-28T19:52:37.894130+0000 mon.a (mon.0) 670 : cluster [WRN] Health check update: Degraded data redundancy: 91/708 objects degraded (12.853%), 16 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:38.378 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:37 smithi059 bash[59745]: cluster 2022-04-28T19:52:37.908671+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2022-04-28T19:52:38.378 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:37 smithi059 bash[61152]: cluster 2022-04-28T19:52:36.926305+0000 mgr.y (mgr.24961) 191 : cluster [DBG] pgmap v292: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 399 MiB data, 855 MiB used, 710 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.4 MiB/s wr, 19 op/s; 91/708 objects degraded (12.853%) 2022-04-28T19:52:38.378 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:37 smithi059 bash[61152]: cluster 2022-04-28T19:52:37.894130+0000 mon.a (mon.0) 670 : cluster [WRN] Health check update: Degraded data redundancy: 91/708 objects degraded (12.853%), 16 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:38.379 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:37 smithi059 bash[61152]: cluster 2022-04-28T19:52:37.908671+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2022-04-28T19:52:38.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:37 smithi139 bash[43301]: cluster 2022-04-28T19:52:36.926305+0000 mgr.y (mgr.24961) 191 : cluster [DBG] pgmap v292: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 399 MiB data, 855 MiB used, 710 GiB / 715 GiB avail; 7.4 MiB/s rd, 3.4 MiB/s wr, 19 op/s; 91/708 objects degraded (12.853%) 2022-04-28T19:52:38.379 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:37 smithi139 bash[43301]: cluster 2022-04-28T19:52:37.894130+0000 mon.a (mon.0) 670 : cluster [WRN] Health check update: Degraded data redundancy: 91/708 objects degraded (12.853%), 16 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:38.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:37 smithi139 bash[43301]: cluster 2022-04-28T19:52:37.908671+0000 mon.a (mon.0) 671 : cluster [DBG] osdmap e539: 8 total, 7 up, 8 in 2022-04-28T19:52:38.494 INFO:tasks.rados.rados.0.smithi059.stdout:2614: finishing write tid 2 to smithi05955964-35 2022-04-28T19:52:38.494 INFO:tasks.rados.rados.0.smithi059.stdout:2614: finishing write tid 3 to smithi05955964-35 2022-04-28T19:52:38.494 INFO:tasks.rados.rados.0.smithi059.stdout:2614: finishing write tid 5 to smithi05955964-35 2022-04-28T19:52:38.495 INFO:tasks.rados.rados.0.smithi059.stdout:2614: finishing write tid 6 to smithi05955964-35 2022-04-28T19:52:38.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 946 (ObjNum 831 snap 243 seq_num 831) dirty exists 2022-04-28T19:52:38.495 INFO:tasks.rados.rados.0.smithi059.stdout:2614: left oid 35 (ObjNum 831 snap 243 seq_num 831) 2022-04-28T19:52:38.495 INFO:tasks.rados.rados.0.smithi059.stdout:2610: expect (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:52:38.868 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[45529]: debug 2022-04-28T19:52:38.446+0000 7fc648f50200 -1 osd.4 529 log_to_monitors {default=true} 2022-04-28T19:52:38.880 INFO:tasks.rados.rados.0.smithi059.stdout:2615: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:52:39.032 INFO:tasks.rados.rados.0.smithi059.stdout:2617: finishing write tid 1 to smithi05955964-13 2022-04-28T19:52:39.032 INFO:tasks.rados.rados.0.smithi059.stdout:2617: finishing write tid 2 to smithi05955964-13 2022-04-28T19:52:39.032 INFO:tasks.rados.rados.0.smithi059.stdout:2617: finishing write tid 3 to smithi05955964-13 2022-04-28T19:52:39.032 INFO:tasks.rados.rados.0.smithi059.stdout:2617: finishing write tid 5 to smithi05955964-13 2022-04-28T19:52:39.032 INFO:tasks.rados.rados.0.smithi059.stdout:2617: finishing write tid 6 to smithi05955964-13 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 968 (ObjNum 832 snap 243 seq_num 832) dirty exists 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2617: left oid 13 (ObjNum 832 snap 243 seq_num 832) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2607: done (7 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2609: done (6 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2610: done (5 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2613: done (4 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2614: done (3 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2615: done (2 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2617: done (1 left) 2022-04-28T19:52:39.033 INFO:tasks.rados.rados.0.smithi059.stdout:2618: done (0 left) 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2619: rollback oid 12 current snap is 244 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 12 to 229 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2620: rmattr oid 23 current snap is 244 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2620: done (1 left) 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2621: read oid 2 snap -1 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2621: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2622: read oid 34 snap -1 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2622: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:52:39.034 INFO:tasks.rados.rados.0.smithi059.stdout:2623: setattr oid 22 current snap is 244 2022-04-28T19:52:39.035 INFO:tasks.rados.rados.0.smithi059.stdout:2624: rollback oid 4 current snap is 244 2022-04-28T19:52:39.035 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 243 2022-04-28T19:52:39.035 INFO:tasks.rados.rados.0.smithi059.stdout:2625: snap_create 2022-04-28T19:52:39.038 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1111 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:39.038 INFO:tasks.rados.rados.0.smithi059.stdout:2619: finishing rollback tid 0 to smithi05955964-12 2022-04-28T19:52:39.039 INFO:tasks.rados.rados.0.smithi059.stdout:2619: finishing rollback tid 1 to smithi05955964-12 2022-04-28T19:52:39.041 INFO:tasks.rados.rados.0.smithi059.stdout:2619: finishing rollback tid 2 to smithi05955964-12 2022-04-28T19:52:39.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1143 (ObjNum 737 snap 219 seq_num 737) dirty exists 2022-04-28T19:52:39.045 INFO:tasks.rados.rados.0.smithi059.stdout:2624: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:52:39.046 INFO:tasks.rados.rados.0.smithi059.stdout:2624: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:52:39.046 INFO:tasks.rados.rados.0.smithi059.stdout:2624: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:52:39.046 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 982 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:52:39.052 INFO:tasks.rados.rados.0.smithi059.stdout:2621: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:52:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[61152]: audit 2022-04-28T19:52:38.452949+0000 mon.b (mon.2) 3 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:52:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[61152]: audit 2022-04-28T19:52:38.455060+0000 mon.a (mon.0) 672 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:52:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[61152]: audit 2022-04-28T19:52:38.911979+0000 mon.a (mon.0) 673 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-04-28T19:52:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[61152]: cluster 2022-04-28T19:52:38.912130+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e540: 8 total, 7 up, 8 in 2022-04-28T19:52:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[61152]: audit 2022-04-28T19:52:38.914233+0000 mon.b (mon.2) 4 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:52:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[61152]: audit 2022-04-28T19:52:38.916552+0000 mon.a (mon.0) 675 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:52:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[59745]: audit 2022-04-28T19:52:38.452949+0000 mon.b (mon.2) 3 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:52:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[59745]: audit 2022-04-28T19:52:38.455060+0000 mon.a (mon.0) 672 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:52:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[59745]: audit 2022-04-28T19:52:38.911979+0000 mon.a (mon.0) 673 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-04-28T19:52:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[59745]: cluster 2022-04-28T19:52:38.912130+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e540: 8 total, 7 up, 8 in 2022-04-28T19:52:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[59745]: audit 2022-04-28T19:52:38.914233+0000 mon.b (mon.2) 4 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:52:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:38 smithi059 bash[59745]: audit 2022-04-28T19:52:38.916552+0000 mon.a (mon.0) 675 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:52:39.368 INFO:journalctl@ceph.osd.4.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[45529]: debug 2022-04-28T19:52:38.934+0000 7fc63ae6b700 -1 osd.4 529 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:52:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[43301]: audit 2022-04-28T19:52:38.452949+0000 mon.b (mon.2) 3 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:52:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[43301]: audit 2022-04-28T19:52:38.455060+0000 mon.a (mon.0) 672 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2022-04-28T19:52:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[43301]: audit 2022-04-28T19:52:38.911979+0000 mon.a (mon.0) 673 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2022-04-28T19:52:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[43301]: cluster 2022-04-28T19:52:38.912130+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e540: 8 total, 7 up, 8 in 2022-04-28T19:52:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[43301]: audit 2022-04-28T19:52:38.914233+0000 mon.b (mon.2) 4 : audit [INF] from='osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:52:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:38 smithi139 bash[43301]: audit 2022-04-28T19:52:38.916552+0000 mon.a (mon.0) 675 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:52:39.475 INFO:tasks.rados.rados.0.smithi059.stdout:2622: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:52:40.307 INFO:tasks.rados.rados.0.smithi059.stdout:2619: done (5 left) 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:2621: done (4 left) 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:2622: done (3 left) 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:2623: done (2 left) 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:2624: done (1 left) 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:2625: done (0 left) 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:2626: rollback oid 30 current snap is 245 2022-04-28T19:52:40.308 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 30 to 242 2022-04-28T19:52:40.309 INFO:tasks.rados.rados.0.smithi059.stdout:2627: snap_remove snap 243 2022-04-28T19:52:40.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[59745]: cluster 2022-04-28T19:52:38.927030+0000 mgr.y (mgr.24961) 192 : cluster [DBG] pgmap v295: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 422 MiB data, 876 MiB used, 710 GiB / 715 GiB avail; 26 MiB/s rd, 8.3 MiB/s wr, 35 op/s; 94/723 objects degraded (13.001%) 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[59745]: cluster 2022-04-28T19:52:39.908195+0000 mon.a (mon.0) 676 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[59745]: cluster 2022-04-28T19:52:39.920069+0000 mon.a (mon.0) 677 : cluster [INF] osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394] boot 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[59745]: cluster 2022-04-28T19:52:39.920157+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[59745]: audit 2022-04-28T19:52:39.920310+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[61152]: cluster 2022-04-28T19:52:38.927030+0000 mgr.y (mgr.24961) 192 : cluster [DBG] pgmap v295: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 422 MiB data, 876 MiB used, 710 GiB / 715 GiB avail; 26 MiB/s rd, 8.3 MiB/s wr, 35 op/s; 94/723 objects degraded (13.001%) 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[61152]: cluster 2022-04-28T19:52:39.908195+0000 mon.a (mon.0) 676 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:52:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[61152]: cluster 2022-04-28T19:52:39.920069+0000 mon.a (mon.0) 677 : cluster [INF] osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394] boot 2022-04-28T19:52:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[61152]: cluster 2022-04-28T19:52:39.920157+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2022-04-28T19:52:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:39 smithi059 bash[61152]: audit 2022-04-28T19:52:39.920310+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:52:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:39 smithi139 bash[43301]: cluster 2022-04-28T19:52:38.927030+0000 mgr.y (mgr.24961) 192 : cluster [DBG] pgmap v295: 89 pgs: 16 active+undersized+degraded, 24 active+undersized, 49 active+clean; 422 MiB data, 876 MiB used, 710 GiB / 715 GiB avail; 26 MiB/s rd, 8.3 MiB/s wr, 35 op/s; 94/723 objects degraded (13.001%) 2022-04-28T19:52:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:39 smithi139 bash[43301]: cluster 2022-04-28T19:52:39.908195+0000 mon.a (mon.0) 676 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:52:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:39 smithi139 bash[43301]: cluster 2022-04-28T19:52:39.920069+0000 mon.a (mon.0) 677 : cluster [INF] osd.4 [v2:172.21.15.139:6800/3795640394,v1:172.21.15.139:6801/3795640394] boot 2022-04-28T19:52:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:39 smithi139 bash[43301]: cluster 2022-04-28T19:52:39.920157+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2022-04-28T19:52:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:39 smithi139 bash[43301]: audit 2022-04-28T19:52:39.920310+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2022-04-28T19:52:40.926 INFO:tasks.rados.rados.0.smithi059.stdout:2627: done (1 left) 2022-04-28T19:52:40.927 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 245 2022-04-28T19:52:40.927 INFO:tasks.rados.rados.0.smithi059.stdout:2628: seq_num 833 ranges {1171456=761856,1933312=671744,2605056=532480} 2022-04-28T19:52:40.938 INFO:tasks.rados.rados.0.smithi059.stdout:2628: writing smithi05955964-49 from 1171456 to 1933312 tid 1 2022-04-28T19:52:40.942 INFO:tasks.rados.rados.0.smithi059.stdout:2628: writing smithi05955964-49 from 1933312 to 2605056 tid 2 2022-04-28T19:52:40.946 INFO:tasks.rados.rados.0.smithi059.stdout:2628: writing smithi05955964-49 from 2605056 to 3137536 tid 3 2022-04-28T19:52:40.946 INFO:tasks.rados.rados.0.smithi059.stdout:2629: snap_remove snap 222 2022-04-28T19:52:41.928 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:41.632Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:41.929 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:41.632Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:41.929 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:41.632Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:52:41.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[59745]: cluster 2022-04-28T19:52:40.924578+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2022-04-28T19:52:41.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[59745]: cluster 2022-04-28T19:52:40.932959+0000 mgr.y (mgr.24961) 193 : cluster [DBG] pgmap v298: 89 pgs: 3 peering, 14 active+undersized+degraded, 23 active+undersized, 49 active+clean; 424 MiB data, 900 MiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 8.1 MiB/s wr, 43 op/s; 71/738 objects degraded (9.621%) 2022-04-28T19:52:41.934 INFO:tasks.rados.rados.0.smithi059.stdout:2629: done (2 left) 2022-04-28T19:52:41.934 INFO:tasks.rados.rados.0.smithi059.stdout:2630: copy_from oid 12 from oid 33 current snap is 245 2022-04-28T19:52:41.934 INFO:tasks.rados.rados.0.smithi059.stdout:2626: finishing rollback tid 0 to smithi05955964-30 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2626: finishing rollback tid 1 to smithi05955964-30 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2626: finishing rollback tid 2 to smithi05955964-30 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1126 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2628: finishing write tid 1 to smithi05955964-49 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2628: finishing write tid 2 to smithi05955964-49 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2628: finishing write tid 3 to smithi05955964-49 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2628: finishing write tid 5 to smithi05955964-49 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2628: finishing write tid 6 to smithi05955964-49 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1061 (ObjNum 833 snap 245 seq_num 833) dirty exists 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2628: left oid 49 (ObjNum 833 snap 245 seq_num 833) 2022-04-28T19:52:41.935 INFO:tasks.rados.rados.0.smithi059.stdout:2626: done (2 left) 2022-04-28T19:52:41.936 INFO:tasks.rados.rados.0.smithi059.stdout:2628: done (1 left) 2022-04-28T19:52:41.936 INFO:tasks.rados.rados.0.smithi059.stdout:2631: delete oid 47 current snap is 245 2022-04-28T19:52:41.944 INFO:tasks.rados.rados.0.smithi059.stdout:2631: done (1 left) 2022-04-28T19:52:41.944 INFO:tasks.rados.rados.0.smithi059.stdout:2632: copy_from oid 30 from oid 39 current snap is 245 2022-04-28T19:52:41.945 INFO:tasks.rados.rados.0.smithi059.stdout:2633: setattr oid 10 current snap is 245 2022-04-28T19:52:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:2634: rollback oid 21 current snap is 245 2022-04-28T19:52:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 219 2022-04-28T19:52:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:2635: rmattr oid 24 current snap is 245 2022-04-28T19:52:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 245 2022-04-28T19:52:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:2636: seq_num 835 ranges {1507328=794624,2301952=442368,2744320=712704,3457024=49152} 2022-04-28T19:52:41.960 INFO:tasks.rados.rados.0.smithi059.stdout:2636: writing smithi05955964-19 from 1507328 to 2301952 tid 1 2022-04-28T19:52:41.963 INFO:tasks.rados.rados.0.smithi059.stdout:2636: writing smithi05955964-19 from 2301952 to 2744320 tid 2 2022-04-28T19:52:41.967 INFO:tasks.rados.rados.0.smithi059.stdout:2636: writing smithi05955964-19 from 2744320 to 3457024 tid 3 2022-04-28T19:52:41.968 INFO:tasks.rados.rados.0.smithi059.stdout:2636: writing smithi05955964-19 from 3457024 to 3506176 tid 4 2022-04-28T19:52:41.968 INFO:tasks.rados.rados.0.smithi059.stdout:2637: delete oid 38 current snap is 245 2022-04-28T19:52:41.968 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1110 (ObjNum 817 snap 240 seq_num 817) dirty exists 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 983 (ObjNum 806 snap 239 seq_num 806) dirty exists 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2634: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2634: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 734 (ObjNum 717 snap 208 seq_num 717) dirty exists 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2630: finishing copy_from to smithi05955964-12 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2632: finishing copy_from to smithi05955964-30 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1211 (ObjNum 1 snap 21893 seq_num 4162143696) dirty exists 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2630: finishing copy_from racing read to smithi05955964-12 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2632: finishing copy_from racing read to smithi05955964-30 2022-04-28T19:52:41.969 INFO:tasks.rados.rados.0.smithi059.stdout:2636: finishing write tid 1 to smithi05955964-19 2022-04-28T19:52:41.970 INFO:tasks.rados.rados.0.smithi059.stdout:2630: done (6 left) 2022-04-28T19:52:41.970 INFO:tasks.rados.rados.0.smithi059.stdout:2632: done (5 left) 2022-04-28T19:52:41.970 INFO:tasks.rados.rados.0.smithi059.stdout:2633: done (4 left) 2022-04-28T19:52:41.971 INFO:tasks.rados.rados.0.smithi059.stdout:2634: done (3 left) 2022-04-28T19:52:41.971 INFO:tasks.rados.rados.0.smithi059.stdout:2635: done (2 left) 2022-04-28T19:52:41.971 INFO:tasks.rados.rados.0.smithi059.stdout:2637: done (1 left) 2022-04-28T19:52:41.971 INFO:tasks.rados.rados.0.smithi059.stdout:2638: copy_from oid 23 from oid 33 current snap is 245 2022-04-28T19:52:41.971 INFO:tasks.rados.rados.0.smithi059.stdout:2639: delete oid 40 current snap is 245 2022-04-28T19:52:41.973 INFO:tasks.rados.rados.0.smithi059.stdout:2636: finishing write tid 2 to smithi05955964-19 2022-04-28T19:52:41.973 INFO:tasks.rados.rados.0.smithi059.stdout:2636: finishing write tid 3 to smithi05955964-19 2022-04-28T19:52:41.973 INFO:tasks.rados.rados.0.smithi059.stdout:2636: finishing write tid 4 to smithi05955964-19 2022-04-28T19:52:41.974 INFO:tasks.rados.rados.0.smithi059.stdout:2639: done (2 left) 2022-04-28T19:52:41.974 INFO:tasks.rados.rados.0.smithi059.stdout:2640: setattr oid 31 current snap is 245 2022-04-28T19:52:41.974 INFO:tasks.rados.rados.0.smithi059.stdout:2636: finishing write tid 6 to smithi05955964-19 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2641: rollback oid 28 current snap is 245 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 28 to 242 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2642: copy_from oid 21 from oid 48 current snap is 245 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2643: rollback oid 24 current snap is 245 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 226 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2644: read oid 32 snap -1 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2644: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2645: delete oid 46 current snap is 245 2022-04-28T19:52:41.976 INFO:tasks.rados.rados.0.smithi059.stdout:2636: finishing write tid 7 to smithi05955964-19 2022-04-28T19:52:41.977 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1117 (ObjNum 835 snap 245 seq_num 835) dirty exists 2022-04-28T19:52:41.977 INFO:tasks.rados.rados.0.smithi059.stdout:2636: left oid 19 (ObjNum 835 snap 245 seq_num 835) 2022-04-28T19:52:41.979 INFO:tasks.rados.rados.0.smithi059.stdout:2643: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:52:41.981 INFO:tasks.rados.rados.0.smithi059.stdout:2641: finishing rollback tid 0 to smithi05955964-28 2022-04-28T19:52:41.981 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1144 (ObjNum -132831808 snap 21893 seq_num 4161970432) dirty exists 2022-04-28T19:52:41.981 INFO:tasks.rados.rados.0.smithi059.stdout:2643: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:52:41.981 INFO:tasks.rados.rados.0.smithi059.stdout:2641: finishing rollback tid 1 to smithi05955964-28 2022-04-28T19:52:41.981 INFO:tasks.rados.rados.0.smithi059.stdout:2643: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:52:41.981 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 986 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2641: finishing rollback tid 2 to smithi05955964-28 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 962 (ObjNum 824 snap 241 seq_num 824) dirty exists 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2636: done (7 left) 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2640: done (6 left) 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2641: done (5 left) 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2643: done (4 left) 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2645: done (3 left) 2022-04-28T19:52:41.982 INFO:tasks.rados.rados.0.smithi059.stdout:2646: setattr oid 3 current snap is 245 2022-04-28T19:52:41.983 INFO:tasks.rados.rados.0.smithi059.stdout:2647: snap_remove snap 227 2022-04-28T19:52:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[61152]: cluster 2022-04-28T19:52:40.924578+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2022-04-28T19:52:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:41 smithi059 bash[61152]: cluster 2022-04-28T19:52:40.932959+0000 mgr.y (mgr.24961) 193 : cluster [DBG] pgmap v298: 89 pgs: 3 peering, 14 active+undersized+degraded, 23 active+undersized, 49 active+clean; 424 MiB data, 900 MiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 8.1 MiB/s wr, 43 op/s; 71/738 objects degraded (9.621%) 2022-04-28T19:52:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:41 smithi139 bash[43301]: cluster 2022-04-28T19:52:40.924578+0000 mon.a (mon.0) 680 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2022-04-28T19:52:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:41 smithi139 bash[43301]: cluster 2022-04-28T19:52:40.932959+0000 mgr.y (mgr.24961) 193 : cluster [DBG] pgmap v298: 89 pgs: 3 peering, 14 active+undersized+degraded, 23 active+undersized, 49 active+clean; 424 MiB data, 900 MiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 8.1 MiB/s wr, 43 op/s; 71/738 objects degraded (9.621%) 2022-04-28T19:52:42.945 INFO:tasks.rados.rados.0.smithi059.stdout:2647: done (4 left) 2022-04-28T19:52:42.946 INFO:tasks.rados.rados.0.smithi059.stdout:2648: read oid 10 snap 224 2022-04-28T19:52:42.946 INFO:tasks.rados.rados.0.smithi059.stdout:2648: expect (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:52:42.946 INFO:tasks.rados.rados.0.smithi059.stdout:2649: read oid 16 snap -1 2022-04-28T19:52:42.946 INFO:tasks.rados.rados.0.smithi059.stdout:2649: expect (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:52:42.946 INFO:tasks.rados.rados.0.smithi059.stdout:2650: snap_create 2022-04-28T19:52:42.947 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 965 (ObjNum 0 snap 0 seq_num 4163629568) dirty exists 2022-04-28T19:52:42.947 INFO:tasks.rados.rados.0.smithi059.stdout:2638: finishing copy_from to smithi05955964-23 2022-04-28T19:52:42.947 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 973 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:52:42.947 INFO:tasks.rados.rados.0.smithi059.stdout:2638: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:52:42.947 INFO:tasks.rados.rados.0.smithi059.stdout:2644: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:52:43.074 INFO:tasks.rados.rados.0.smithi059.stdout:2642: finishing copy_from to smithi05955964-21 2022-04-28T19:52:43.074 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1085 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:43.075 INFO:tasks.rados.rados.0.smithi059.stdout:2642: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:52:43.075 INFO:tasks.rados.rados.0.smithi059.stdout:2648: expect (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:52:43.080 INFO:tasks.rados.rados.0.smithi059.stdout:2649: expect (ObjNum 672 snap 195 seq_num 672) 2022-04-28T19:52:43.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:42 smithi059 bash[61152]: cluster 2022-04-28T19:52:41.932350+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2022-04-28T19:52:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:42 smithi059 bash[59745]: cluster 2022-04-28T19:52:41.932350+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2022-04-28T19:52:43.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:42 smithi139 bash[43301]: cluster 2022-04-28T19:52:41.932350+0000 mon.a (mon.0) 681 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2022-04-28T19:52:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:43.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:43.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:43.526Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:52:43.944 INFO:tasks.rados.rados.0.smithi059.stdout:2638: done (6 left) 2022-04-28T19:52:43.944 INFO:tasks.rados.rados.0.smithi059.stdout:2642: done (5 left) 2022-04-28T19:52:43.945 INFO:tasks.rados.rados.0.smithi059.stdout:2644: done (4 left) 2022-04-28T19:52:43.945 INFO:tasks.rados.rados.0.smithi059.stdout:2646: done (3 left) 2022-04-28T19:52:43.945 INFO:tasks.rados.rados.0.smithi059.stdout:2648: done (2 left) 2022-04-28T19:52:43.945 INFO:tasks.rados.rados.0.smithi059.stdout:2649: done (1 left) 2022-04-28T19:52:43.945 INFO:tasks.rados.rados.0.smithi059.stdout:2650: done (0 left) 2022-04-28T19:52:43.945 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 246 2022-04-28T19:52:43.946 INFO:tasks.rados.rados.0.smithi059.stdout:2651: seq_num 839 ranges {0=458752,458752=425984,884736=688128,1572864=491520} 2022-04-28T19:52:43.951 INFO:tasks.rados.rados.0.smithi059.stdout:2651: writing smithi05955964-40 from 0 to 458752 tid 1 2022-04-28T19:52:43.956 INFO:tasks.rados.rados.0.smithi059.stdout:2651: writing smithi05955964-40 from 458752 to 884736 tid 2 2022-04-28T19:52:43.960 INFO:tasks.rados.rados.0.smithi059.stdout:2651: writing smithi05955964-40 from 884736 to 1572864 tid 3 2022-04-28T19:52:43.963 INFO:tasks.rados.rados.0.smithi059.stdout:2651: writing smithi05955964-40 from 1572864 to 2064384 tid 4 2022-04-28T19:52:43.963 INFO:tasks.rados.rados.0.smithi059.stdout:2652: snap_create 2022-04-28T19:52:43.963 INFO:tasks.rados.rados.0.smithi059.stdout:2651: finishing write tid 1 to smithi05955964-40 2022-04-28T19:52:43.963 INFO:tasks.rados.rados.0.smithi059.stdout:2651: finishing write tid 2 to smithi05955964-40 2022-04-28T19:52:43.963 INFO:tasks.rados.rados.0.smithi059.stdout:2651: finishing write tid 3 to smithi05955964-40 2022-04-28T19:52:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:2651: finishing write tid 4 to smithi05955964-40 2022-04-28T19:52:43.965 INFO:tasks.rados.rados.0.smithi059.stdout:2651: finishing write tid 6 to smithi05955964-40 2022-04-28T19:52:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:2651: finishing write tid 7 to smithi05955964-40 2022-04-28T19:52:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1067 (ObjNum 839 snap 246 seq_num 839) dirty exists 2022-04-28T19:52:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:2651: left oid 40 (ObjNum 839 snap 246 seq_num 839) 2022-04-28T19:52:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[59745]: cluster 2022-04-28T19:52:42.934259+0000 mgr.y (mgr.24961) 194 : cluster [DBG] pgmap v300: 89 pgs: 5 activating, 3 peering, 4 active+undersized+degraded, 12 active+undersized, 65 active+clean; 427 MiB data, 912 MiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 24 op/s; 36/753 objects degraded (4.781%); 2.3 MiB/s, 1 objects/s recovering 2022-04-28T19:52:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[59745]: cluster 2022-04-28T19:52:42.944603+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2022-04-28T19:52:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[59745]: cluster 2022-04-28T19:52:43.929297+0000 mon.a (mon.0) 683 : cluster [WRN] Health check update: Degraded data redundancy: 36/753 objects degraded (4.781%), 4 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[61152]: cluster 2022-04-28T19:52:42.934259+0000 mgr.y (mgr.24961) 194 : cluster [DBG] pgmap v300: 89 pgs: 5 activating, 3 peering, 4 active+undersized+degraded, 12 active+undersized, 65 active+clean; 427 MiB data, 912 MiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 24 op/s; 36/753 objects degraded (4.781%); 2.3 MiB/s, 1 objects/s recovering 2022-04-28T19:52:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[61152]: cluster 2022-04-28T19:52:42.944603+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2022-04-28T19:52:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:43 smithi059 bash[61152]: cluster 2022-04-28T19:52:43.929297+0000 mon.a (mon.0) 683 : cluster [WRN] Health check update: Degraded data redundancy: 36/753 objects degraded (4.781%), 4 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:43 smithi139 bash[43301]: cluster 2022-04-28T19:52:42.934259+0000 mgr.y (mgr.24961) 194 : cluster [DBG] pgmap v300: 89 pgs: 5 activating, 3 peering, 4 active+undersized+degraded, 12 active+undersized, 65 active+clean; 427 MiB data, 912 MiB used, 711 GiB / 715 GiB avail; 19 MiB/s rd, 1.4 MiB/s wr, 24 op/s; 36/753 objects degraded (4.781%); 2.3 MiB/s, 1 objects/s recovering 2022-04-28T19:52:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:43 smithi139 bash[43301]: cluster 2022-04-28T19:52:42.944603+0000 mon.a (mon.0) 682 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2022-04-28T19:52:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:43 smithi139 bash[43301]: cluster 2022-04-28T19:52:43.929297+0000 mon.a (mon.0) 683 : cluster [WRN] Health check update: Degraded data redundancy: 36/753 objects degraded (4.781%), 4 pgs degraded (PG_DEGRADED) 2022-04-28T19:52:44.956 INFO:tasks.rados.rados.0.smithi059.stdout:2651: done (1 left) 2022-04-28T19:52:44.956 INFO:tasks.rados.rados.0.smithi059.stdout:2652: done (0 left) 2022-04-28T19:52:44.956 INFO:tasks.rados.rados.0.smithi059.stdout:2653: snap_create 2022-04-28T19:52:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:44 smithi059 bash[61152]: cluster 2022-04-28T19:52:43.942466+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2022-04-28T19:52:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:44 smithi059 bash[59745]: cluster 2022-04-28T19:52:43.942466+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2022-04-28T19:52:45.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:44 smithi139 bash[43301]: cluster 2022-04-28T19:52:43.942466+0000 mon.a (mon.0) 684 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2022-04-28T19:52:45.976 INFO:tasks.rados.rados.0.smithi059.stdout:2653: done (0 left) 2022-04-28T19:52:45.976 INFO:tasks.rados.rados.0.smithi059.stdout:2654: setattr oid 37 current snap is 248 2022-04-28T19:52:45.980 INFO:tasks.rados.rados.0.smithi059.stdout:2655: copy_from oid 13 from oid 1 current snap is 248 2022-04-28T19:52:45.980 INFO:tasks.rados.rados.0.smithi059.stdout:2656: delete oid 27 current snap is 248 2022-04-28T19:52:45.985 INFO:tasks.rados.rados.0.smithi059.stdout:2656: done (2 left) 2022-04-28T19:52:45.985 INFO:tasks.rados.rados.0.smithi059.stdout:2657: copy_from oid 26 from oid 36 current snap is 248 2022-04-28T19:52:45.985 INFO:tasks.rados.rados.0.smithi059.stdout:2658: snap_remove snap 224 2022-04-28T19:52:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[61152]: cluster 2022-04-28T19:52:44.935237+0000 mgr.y (mgr.24961) 195 : cluster [DBG] pgmap v303: 89 pgs: 5 activating, 3 peering, 81 active+clean; 436 MiB data, 957 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 27 op/s; 2/771 objects degraded (0.259%); 11 MiB/s, 3 objects/s recovering 2022-04-28T19:52:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[61152]: cluster 2022-04-28T19:52:44.946600+0000 mon.a (mon.0) 685 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/753 objects degraded (4.781%), 4 pgs degraded) 2022-04-28T19:52:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[61152]: cluster 2022-04-28T19:52:44.946675+0000 mon.a (mon.0) 686 : cluster [INF] Cluster is now healthy 2022-04-28T19:52:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[61152]: cluster 2022-04-28T19:52:44.955018+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2022-04-28T19:52:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[59745]: cluster 2022-04-28T19:52:44.935237+0000 mgr.y (mgr.24961) 195 : cluster [DBG] pgmap v303: 89 pgs: 5 activating, 3 peering, 81 active+clean; 436 MiB data, 957 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 27 op/s; 2/771 objects degraded (0.259%); 11 MiB/s, 3 objects/s recovering 2022-04-28T19:52:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[59745]: cluster 2022-04-28T19:52:44.946600+0000 mon.a (mon.0) 685 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/753 objects degraded (4.781%), 4 pgs degraded) 2022-04-28T19:52:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[59745]: cluster 2022-04-28T19:52:44.946675+0000 mon.a (mon.0) 686 : cluster [INF] Cluster is now healthy 2022-04-28T19:52:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:45 smithi059 bash[59745]: cluster 2022-04-28T19:52:44.955018+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2022-04-28T19:52:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:45 smithi139 bash[43301]: cluster 2022-04-28T19:52:44.935237+0000 mgr.y (mgr.24961) 195 : cluster [DBG] pgmap v303: 89 pgs: 5 activating, 3 peering, 81 active+clean; 436 MiB data, 957 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 27 op/s; 2/771 objects degraded (0.259%); 11 MiB/s, 3 objects/s recovering 2022-04-28T19:52:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:45 smithi139 bash[43301]: cluster 2022-04-28T19:52:44.946600+0000 mon.a (mon.0) 685 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/753 objects degraded (4.781%), 4 pgs degraded) 2022-04-28T19:52:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:45 smithi139 bash[43301]: cluster 2022-04-28T19:52:44.946675+0000 mon.a (mon.0) 686 : cluster [INF] Cluster is now healthy 2022-04-28T19:52:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:45 smithi139 bash[43301]: cluster 2022-04-28T19:52:44.955018+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2022-04-28T19:52:46.987 INFO:tasks.rados.rados.0.smithi059.stdout:2658: done (3 left) 2022-04-28T19:52:46.987 INFO:tasks.rados.rados.0.smithi059.stdout:2659: rmattr oid 3 current snap is 248 2022-04-28T19:52:46.987 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1212 (ObjNum 825 snap 241 seq_num 825) dirty exists 2022-04-28T19:52:46.987 INFO:tasks.rados.rados.0.smithi059.stdout:2655: finishing copy_from to smithi05955964-13 2022-04-28T19:52:46.988 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1044 (ObjNum 792 snap 235 seq_num 792) dirty exists 2022-04-28T19:52:46.988 INFO:tasks.rados.rados.0.smithi059.stdout:2657: finishing copy_from to smithi05955964-26 2022-04-28T19:52:46.988 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1049 (ObjNum 813 snap 240 seq_num 813) dirty exists 2022-04-28T19:52:46.988 INFO:tasks.rados.rados.0.smithi059.stdout:2655: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:52:46.988 INFO:tasks.rados.rados.0.smithi059.stdout:2657: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:52:46.988 INFO:tasks.rados.rados.0.smithi059.stdout:2654: done (3 left) 2022-04-28T19:52:46.989 INFO:tasks.rados.rados.0.smithi059.stdout:2655: done (2 left) 2022-04-28T19:52:46.989 INFO:tasks.rados.rados.0.smithi059.stdout:2657: done (1 left) 2022-04-28T19:52:46.989 INFO:tasks.rados.rados.0.smithi059.stdout:2660: setattr oid 48 current snap is 248 2022-04-28T19:52:46.990 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 248 2022-04-28T19:52:46.990 INFO:tasks.rados.rados.0.smithi059.stdout:2661: seq_num 841 ranges {1622016=491520,2113536=598016,2711552=704512,3416064=303104} 2022-04-28T19:52:47.001 INFO:tasks.rados.rados.0.smithi059.stdout:2661: writing smithi05955964-26 from 1622016 to 2113536 tid 1 2022-04-28T19:52:47.005 INFO:tasks.rados.rados.0.smithi059.stdout:2661: writing smithi05955964-26 from 2113536 to 2711552 tid 2 2022-04-28T19:52:47.009 INFO:tasks.rados.rados.0.smithi059.stdout:2661: writing smithi05955964-26 from 2711552 to 3416064 tid 3 2022-04-28T19:52:47.011 INFO:tasks.rados.rados.0.smithi059.stdout:2661: writing smithi05955964-26 from 3416064 to 3719168 tid 4 2022-04-28T19:52:47.011 INFO:tasks.rados.rados.0.smithi059.stdout:2662: read oid 32 snap -1 2022-04-28T19:52:47.011 INFO:tasks.rados.rados.0.smithi059.stdout:2662: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:52:47.011 INFO:tasks.rados.rados.0.smithi059.stdout:2663: snap_remove snap 230 2022-04-28T19:52:47.012 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 967 (ObjNum -133136176 snap 21893 seq_num 0) dirty exists 2022-04-28T19:52:47.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:46 smithi059 bash[61152]: cluster 2022-04-28T19:52:45.974987+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2022-04-28T19:52:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:46 smithi059 bash[59745]: cluster 2022-04-28T19:52:45.974987+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2022-04-28T19:52:47.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:46 smithi139 bash[43301]: cluster 2022-04-28T19:52:45.974987+0000 mon.a (mon.0) 688 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2022-04-28T19:52:47.991 INFO:tasks.rados.rados.0.smithi059.stdout:2659: done (4 left) 2022-04-28T19:52:47.991 INFO:tasks.rados.rados.0.smithi059.stdout:2663: done (3 left) 2022-04-28T19:52:47.991 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 248 2022-04-28T19:52:47.992 INFO:tasks.rados.rados.0.smithi059.stdout:2664: seq_num 842 ranges {3883008=507904,4390912=475136,4866048=516096,5382144=499712,5881856=114688} 2022-04-28T19:52:48.017 INFO:tasks.rados.rados.0.smithi059.stdout:2664: writing smithi05955964-14 from 3883008 to 4390912 tid 1 2022-04-28T19:52:48.019 INFO:tasks.rados.rados.0.smithi059.stdout:2664: writing smithi05955964-14 from 4390912 to 4866048 tid 2 2022-04-28T19:52:48.023 INFO:tasks.rados.rados.0.smithi059.stdout:2664: writing smithi05955964-14 from 4866048 to 5382144 tid 3 2022-04-28T19:52:48.026 INFO:tasks.rados.rados.0.smithi059.stdout:2664: writing smithi05955964-14 from 5382144 to 5881856 tid 4 2022-04-28T19:52:48.027 INFO:tasks.rados.rados.0.smithi059.stdout:2664: writing smithi05955964-14 from 5881856 to 5996544 tid 5 2022-04-28T19:52:48.027 INFO:tasks.rados.rados.0.smithi059.stdout:2665: snap_remove snap 240 2022-04-28T19:52:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:47 smithi059 bash[61152]: cluster 2022-04-28T19:52:46.936145+0000 mgr.y (mgr.24961) 196 : cluster [DBG] pgmap v306: 89 pgs: 89 active+clean; 448 MiB data, 974 MiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 4.3 MiB/s wr, 20 op/s; 12 MiB/s, 3 objects/s recovering 2022-04-28T19:52:48.407 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:47 smithi059 bash[61152]: cluster 2022-04-28T19:52:46.985124+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2022-04-28T19:52:48.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:47 smithi059 bash[59745]: cluster 2022-04-28T19:52:46.936145+0000 mgr.y (mgr.24961) 196 : cluster [DBG] pgmap v306: 89 pgs: 89 active+clean; 448 MiB data, 974 MiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 4.3 MiB/s wr, 20 op/s; 12 MiB/s, 3 objects/s recovering 2022-04-28T19:52:48.408 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:47 smithi059 bash[59745]: cluster 2022-04-28T19:52:46.985124+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2022-04-28T19:52:48.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:47 smithi139 bash[43301]: cluster 2022-04-28T19:52:46.936145+0000 mgr.y (mgr.24961) 196 : cluster [DBG] pgmap v306: 89 pgs: 89 active+clean; 448 MiB data, 974 MiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 4.3 MiB/s wr, 20 op/s; 12 MiB/s, 3 objects/s recovering 2022-04-28T19:52:48.409 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:47 smithi139 bash[43301]: cluster 2022-04-28T19:52:46.985124+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2022-04-28T19:52:49.007 INFO:tasks.rados.rados.0.smithi059.stdout:2665: done (4 left) 2022-04-28T19:52:49.007 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 248 2022-04-28T19:52:49.007 INFO:tasks.rados.rados.0.smithi059.stdout:2666: seq_num 843 ranges {5914624=524288,6438912=761856,7200768=671744,7872512=172032} 2022-04-28T19:52:49.050 INFO:tasks.rados.rados.0.smithi059.stdout:2666: writing smithi05955964-25 from 5914624 to 6438912 tid 1 2022-04-28T19:52:49.055 INFO:tasks.rados.rados.0.smithi059.stdout:2666: writing smithi05955964-25 from 6438912 to 7200768 tid 2 2022-04-28T19:52:49.058 INFO:tasks.rados.rados.0.smithi059.stdout:2666: writing smithi05955964-25 from 7200768 to 7872512 tid 3 2022-04-28T19:52:49.060 INFO:tasks.rados.rados.0.smithi059.stdout:2666: writing smithi05955964-25 from 7872512 to 8044544 tid 4 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2667: snap_create 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1121 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: finishing write tid 1 to smithi05955964-26 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: finishing write tid 2 to smithi05955964-26 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: finishing write tid 3 to smithi05955964-26 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: finishing write tid 4 to smithi05955964-26 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: finishing write tid 6 to smithi05955964-26 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: finishing write tid 7 to smithi05955964-26 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1126 (ObjNum 841 snap 248 seq_num 841) dirty exists 2022-04-28T19:52:49.061 INFO:tasks.rados.rados.0.smithi059.stdout:2661: left oid 26 (ObjNum 841 snap 248 seq_num 841) 2022-04-28T19:52:49.062 INFO:tasks.rados.rados.0.smithi059.stdout:2662: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 1 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 2 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 3 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 4 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 5 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 7 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:2664: finishing write tid 8 to smithi05955964-14 2022-04-28T19:52:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1133 (ObjNum 842 snap 248 seq_num 842) dirty exists 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2664: left oid 14 (ObjNum 842 snap 248 seq_num 842) 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: finishing write tid 1 to smithi05955964-25 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: finishing write tid 2 to smithi05955964-25 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: finishing write tid 3 to smithi05955964-25 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: finishing write tid 4 to smithi05955964-25 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: finishing write tid 6 to smithi05955964-25 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: finishing write tid 7 to smithi05955964-25 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1244 (ObjNum 843 snap 248 seq_num 843) dirty exists 2022-04-28T19:52:49.197 INFO:tasks.rados.rados.0.smithi059.stdout:2666: left oid 25 (ObjNum 843 snap 248 seq_num 843) 2022-04-28T19:52:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:48 smithi059 bash[59745]: cluster 2022-04-28T19:52:47.990180+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2022-04-28T19:52:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:48 smithi059 bash[61152]: cluster 2022-04-28T19:52:47.990180+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2022-04-28T19:52:49.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:48 smithi139 bash[43301]: cluster 2022-04-28T19:52:47.990180+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2022-04-28T19:52:50.013 INFO:tasks.rados.rados.0.smithi059.stdout:2660: done (5 left) 2022-04-28T19:52:50.013 INFO:tasks.rados.rados.0.smithi059.stdout:2661: done (4 left) 2022-04-28T19:52:50.013 INFO:tasks.rados.rados.0.smithi059.stdout:2662: done (3 left) 2022-04-28T19:52:50.013 INFO:tasks.rados.rados.0.smithi059.stdout:2664: done (2 left) 2022-04-28T19:52:50.014 INFO:tasks.rados.rados.0.smithi059.stdout:2666: done (1 left) 2022-04-28T19:52:50.014 INFO:tasks.rados.rados.0.smithi059.stdout:2667: done (0 left) 2022-04-28T19:52:50.014 INFO:tasks.rados.rados.0.smithi059.stdout:2668: read oid 30 snap -1 2022-04-28T19:52:50.014 INFO:tasks.rados.rados.0.smithi059.stdout:2668: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:52:50.014 INFO:tasks.rados.rados.0.smithi059.stdout:2669: setattr oid 11 current snap is 249 2022-04-28T19:52:50.015 INFO:tasks.rados.rados.0.smithi059.stdout:2668: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:52:50.022 INFO:tasks.rados.rados.0.smithi059.stdout:2668: done (1 left) 2022-04-28T19:52:50.022 INFO:tasks.rados.rados.0.smithi059.stdout:2670: rmattr oid 25 current snap is 249 2022-04-28T19:52:50.022 INFO:tasks.rados.rados.0.smithi059.stdout:2670: done (1 left) 2022-04-28T19:52:50.022 INFO:tasks.rados.rados.0.smithi059.stdout:2671: snap_create 2022-04-28T19:52:50.027 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 987 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:52:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:50 smithi059 bash[61152]: cluster 2022-04-28T19:52:48.937026+0000 mgr.y (mgr.24961) 197 : cluster [DBG] pgmap v309: 89 pgs: 89 active+clean; 448 MiB data, 977 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 10 op/s; 3.3 MiB/s, 1 objects/s recovering 2022-04-28T19:52:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:50 smithi059 bash[61152]: cluster 2022-04-28T19:52:49.005576+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2022-04-28T19:52:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:50 smithi059 bash[59745]: cluster 2022-04-28T19:52:48.937026+0000 mgr.y (mgr.24961) 197 : cluster [DBG] pgmap v309: 89 pgs: 89 active+clean; 448 MiB data, 977 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 10 op/s; 3.3 MiB/s, 1 objects/s recovering 2022-04-28T19:52:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:50 smithi059 bash[59745]: cluster 2022-04-28T19:52:49.005576+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2022-04-28T19:52:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:50 smithi139 bash[43301]: cluster 2022-04-28T19:52:48.937026+0000 mgr.y (mgr.24961) 197 : cluster [DBG] pgmap v309: 89 pgs: 89 active+clean; 448 MiB data, 977 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 10 op/s; 3.3 MiB/s, 1 objects/s recovering 2022-04-28T19:52:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:50 smithi139 bash[43301]: cluster 2022-04-28T19:52:49.005576+0000 mon.a (mon.0) 691 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2022-04-28T19:52:51.024 INFO:tasks.rados.rados.0.smithi059.stdout:2669: done (1 left) 2022-04-28T19:52:51.102 INFO:tasks.rados.rados.0.smithi059.stdout:2671: done (0 left) 2022-04-28T19:52:51.102 INFO:tasks.rados.rados.0.smithi059.stdout:2672: snap_remove snap 247 2022-04-28T19:52:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:51 smithi059 bash[61152]: cluster 2022-04-28T19:52:50.008790+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2022-04-28T19:52:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:51 smithi059 bash[59745]: cluster 2022-04-28T19:52:50.008790+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2022-04-28T19:52:51.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:51 smithi139 bash[43301]: cluster 2022-04-28T19:52:50.008790+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2022-04-28T19:52:51.897 INFO:tasks.rados.rados.0.smithi059.stdout:2672: done (0 left) 2022-04-28T19:52:51.897 INFO:tasks.rados.rados.0.smithi059.stdout:2673: setattr oid 32 current snap is 250 2022-04-28T19:52:51.899 INFO:tasks.rados.rados.0.smithi059.stdout:2674: copy_from oid 38 from oid 50 current snap is 250 2022-04-28T19:52:51.899 INFO:tasks.rados.rados.0.smithi059.stdout:2675: read oid 1 snap 245 2022-04-28T19:52:51.899 INFO:tasks.rados.rados.0.smithi059.stdout:2675: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:52:51.899 INFO:tasks.rados.rados.0.smithi059.stdout:2676: read oid 40 snap 244 2022-04-28T19:52:51.899 INFO:tasks.rados.rados.0.smithi059.stdout:2676: expect (ObjNum 815 snap 240 seq_num 815) 2022-04-28T19:52:51.899 INFO:tasks.rados.rados.0.smithi059.stdout:2677: rollback oid 6 current snap is 250 2022-04-28T19:52:51.900 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 242 2022-04-28T19:52:51.900 INFO:tasks.rados.rados.0.smithi059.stdout:2678: snap_remove snap 235 2022-04-28T19:52:52.029 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:51.632Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:52.051 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:51.632Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:52.051 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:51.632Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:52:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[59745]: cluster 2022-04-28T19:52:50.937982+0000 mgr.y (mgr.24961) 198 : cluster [DBG] pgmap v312: 89 pgs: 89 active+clean; 455 MiB data, 993 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2022-04-28T19:52:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[59745]: cluster 2022-04-28T19:52:51.023210+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2022-04-28T19:52:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[59745]: audit 2022-04-28T19:52:51.242784+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[59745]: audit 2022-04-28T19:52:51.243208+0000 mgr.y (mgr.24961) 199 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[59745]: cephadm 2022-04-28T19:52:51.245577+0000 mgr.y (mgr.24961) 200 : cephadm [INF] Upgrade: osd.5 is safe to restart 2022-04-28T19:52:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[59745]: cluster 2022-04-28T19:52:51.895469+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2022-04-28T19:52:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[61152]: cluster 2022-04-28T19:52:50.937982+0000 mgr.y (mgr.24961) 198 : cluster [DBG] pgmap v312: 89 pgs: 89 active+clean; 455 MiB data, 993 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2022-04-28T19:52:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[61152]: cluster 2022-04-28T19:52:51.023210+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2022-04-28T19:52:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[61152]: audit 2022-04-28T19:52:51.242784+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[61152]: audit 2022-04-28T19:52:51.243208+0000 mgr.y (mgr.24961) 199 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[61152]: cephadm 2022-04-28T19:52:51.245577+0000 mgr.y (mgr.24961) 200 : cephadm [INF] Upgrade: osd.5 is safe to restart 2022-04-28T19:52:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:52 smithi059 bash[61152]: cluster 2022-04-28T19:52:51.895469+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2022-04-28T19:52:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:52 smithi139 bash[43301]: cluster 2022-04-28T19:52:50.937982+0000 mgr.y (mgr.24961) 198 : cluster [DBG] pgmap v312: 89 pgs: 89 active+clean; 455 MiB data, 993 MiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.9 MiB/s wr, 18 op/s 2022-04-28T19:52:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:52 smithi139 bash[43301]: cluster 2022-04-28T19:52:51.023210+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2022-04-28T19:52:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:52 smithi139 bash[43301]: audit 2022-04-28T19:52:51.242784+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:52 smithi139 bash[43301]: audit 2022-04-28T19:52:51.243208+0000 mgr.y (mgr.24961) 199 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2022-04-28T19:52:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:52 smithi139 bash[43301]: cephadm 2022-04-28T19:52:51.245577+0000 mgr.y (mgr.24961) 200 : cephadm [INF] Upgrade: osd.5 is safe to restart 2022-04-28T19:52:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:52 smithi139 bash[43301]: cluster 2022-04-28T19:52:51.895469+0000 mon.a (mon.0) 695 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2022-04-28T19:52:52.913 INFO:tasks.rados.rados.0.smithi059.stdout:2678: done (5 left) 2022-04-28T19:52:52.913 INFO:tasks.rados.rados.0.smithi059.stdout:2679: snap_create 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1237 (ObjNum 819 snap 240 seq_num 819) dirty exists 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:2677: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:2674: finishing copy_from to smithi05955964-38 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1141 (ObjNum 0 snap 21893 seq_num 4163589616) dirty exists 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:2677: finishing rollback tid 2 to smithi05955964-6 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 969 (ObjNum 820 snap 240 seq_num 820) dirty exists 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:2674: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:52:52.914 INFO:tasks.rados.rados.0.smithi059.stdout:2675: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:52:52.974 INFO:tasks.rados.rados.0.smithi059.stdout:2676: expect (ObjNum 815 snap 240 seq_num 815) 2022-04-28T19:52:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[61152]: audit 2022-04-28T19:52:52.325907+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[61152]: cluster 2022-04-28T19:52:52.908971+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2022-04-28T19:52:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[59745]: audit 2022-04-28T19:52:52.325907+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:53.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[59745]: cluster 2022-04-28T19:52:52.908971+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2022-04-28T19:52:53.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:53.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:52:53.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:53.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:52:53.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:52:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:52:53.526Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:52:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:53 smithi139 bash[43301]: audit 2022-04-28T19:52:52.325907+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:53.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:53 smithi139 bash[43301]: cluster 2022-04-28T19:52:52.908971+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2022-04-28T19:52:53.919 INFO:tasks.rados.rados.0.smithi059.stdout:2673: done (5 left) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2674: done (4 left) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2675: done (3 left) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2676: done (2 left) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2677: done (1 left) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2679: done (0 left) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2680: read oid 39 snap -1 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2680: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2681: copy_from oid 26 from oid 40 current snap is 251 2022-04-28T19:52:53.920 INFO:tasks.rados.rados.0.smithi059.stdout:2682: delete oid 11 current snap is 251 2022-04-28T19:52:53.921 INFO:tasks.rados.rados.0.smithi059.stdout:2682: done (2 left) 2022-04-28T19:52:53.921 INFO:tasks.rados.rados.0.smithi059.stdout:2683: snap_remove snap 237 2022-04-28T19:52:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[61152]: cluster 2022-04-28T19:52:52.938469+0000 mgr.y (mgr.24961) 201 : cluster [DBG] pgmap v316: 89 pgs: 89 active+clean; 457 MiB data, 998 MiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 1.9 MiB/s wr, 23 op/s 2022-04-28T19:52:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[61152]: audit 2022-04-28T19:52:53.841999+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[61152]: audit 2022-04-28T19:52:53.846682+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "container_image"}]: dispatch 2022-04-28T19:52:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[61152]: cluster 2022-04-28T19:52:53.917432+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2022-04-28T19:52:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[59745]: cluster 2022-04-28T19:52:52.938469+0000 mgr.y (mgr.24961) 201 : cluster [DBG] pgmap v316: 89 pgs: 89 active+clean; 457 MiB data, 998 MiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 1.9 MiB/s wr, 23 op/s 2022-04-28T19:52:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[59745]: audit 2022-04-28T19:52:53.841999+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[59745]: audit 2022-04-28T19:52:53.846682+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "container_image"}]: dispatch 2022-04-28T19:52:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:54 smithi059 bash[59745]: cluster 2022-04-28T19:52:53.917432+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2022-04-28T19:52:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:54 smithi139 bash[43301]: cluster 2022-04-28T19:52:52.938469+0000 mgr.y (mgr.24961) 201 : cluster [DBG] pgmap v316: 89 pgs: 89 active+clean; 457 MiB data, 998 MiB used, 711 GiB / 715 GiB avail; 45 KiB/s rd, 1.9 MiB/s wr, 23 op/s 2022-04-28T19:52:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:54 smithi139 bash[43301]: audit 2022-04-28T19:52:53.841999+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:52:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:54 smithi139 bash[43301]: audit 2022-04-28T19:52:53.846682+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "container_image"}]: dispatch 2022-04-28T19:52:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:54 smithi139 bash[43301]: cluster 2022-04-28T19:52:53.917432+0000 mon.a (mon.0) 700 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2022-04-28T19:52:54.923 INFO:tasks.rados.rados.0.smithi059.stdout:2683: done (2 left) 2022-04-28T19:52:54.924 INFO:tasks.rados.rados.0.smithi059.stdout:2684: rollback oid 27 current snap is 251 2022-04-28T19:52:54.924 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 242 2022-04-28T19:52:54.924 INFO:tasks.rados.rados.0.smithi059.stdout:2685: copy_from oid 22 from oid 45 current snap is 251 2022-04-28T19:52:54.924 INFO:tasks.rados.rados.0.smithi059.stdout:2680: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:52:54.932 INFO:tasks.rados.rados.0.smithi059.stdout:2681: finishing copy_from to smithi05955964-26 2022-04-28T19:52:54.932 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1067 (ObjNum 839 snap 246 seq_num 839) dirty exists 2022-04-28T19:52:54.932 INFO:tasks.rados.rados.0.smithi059.stdout:2681: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:52:54.932 INFO:tasks.rados.rados.0.smithi059.stdout:2680: done (3 left) 2022-04-28T19:52:54.933 INFO:tasks.rados.rados.0.smithi059.stdout:2681: done (2 left) 2022-04-28T19:52:54.933 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 251 2022-04-28T19:52:54.933 INFO:tasks.rados.rados.0.smithi059.stdout:2686: seq_num 845 ranges {2473984=557056,3031040=581632,3612672=688128,4300800=335872} 2022-04-28T19:52:54.950 INFO:tasks.rados.rados.0.smithi059.stdout:2686: writing smithi05955964-33 from 2473984 to 3031040 tid 1 2022-04-28T19:52:54.953 INFO:tasks.rados.rados.0.smithi059.stdout:2686: writing smithi05955964-33 from 3031040 to 3612672 tid 2 2022-04-28T19:52:54.957 INFO:tasks.rados.rados.0.smithi059.stdout:2686: writing smithi05955964-33 from 3612672 to 4300800 tid 3 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:2686: writing smithi05955964-33 from 4300800 to 4636672 tid 4 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:2687: rmattr oid 17 current snap is 251 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:2684: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:2684: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1115 (ObjNum 578 snap 165 seq_num 578) dirty exists 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:2684: done (3 left) 2022-04-28T19:52:54.960 INFO:tasks.rados.rados.0.smithi059.stdout:2688: delete oid 12 current snap is 251 2022-04-28T19:52:54.961 INFO:tasks.rados.rados.0.smithi059.stdout:2686: finishing write tid 1 to smithi05955964-33 2022-04-28T19:52:54.967 INFO:tasks.rados.rados.0.smithi059.stdout:2686: finishing write tid 2 to smithi05955964-33 2022-04-28T19:52:54.968 INFO:tasks.rados.rados.0.smithi059.stdout:2686: finishing write tid 3 to smithi05955964-33 2022-04-28T19:52:54.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 1045 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:52:54.971 INFO:tasks.rados.rados.0.smithi059.stdout:2686: finishing write tid 4 to smithi05955964-33 2022-04-28T19:52:54.971 INFO:tasks.rados.rados.0.smithi059.stdout:2687: done (3 left) 2022-04-28T19:52:54.971 INFO:tasks.rados.rados.0.smithi059.stdout:2688: done (2 left) 2022-04-28T19:52:54.971 INFO:tasks.rados.rados.0.smithi059.stdout:2689: snap_remove snap 226 2022-04-28T19:52:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:55 smithi059 bash[61152]: cephadm 2022-04-28T19:52:53.833007+0000 mgr.y (mgr.24961) 202 : cephadm [INF] Upgrade: Updating osd.5 2022-04-28T19:52:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:55 smithi059 bash[61152]: cephadm 2022-04-28T19:52:53.845590+0000 mgr.y (mgr.24961) 203 : cephadm [INF] Deploying daemon osd.5 on smithi139 2022-04-28T19:52:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:55 smithi059 bash[61152]: cluster 2022-04-28T19:52:54.923027+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2022-04-28T19:52:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:55 smithi059 bash[59745]: cephadm 2022-04-28T19:52:53.833007+0000 mgr.y (mgr.24961) 202 : cephadm [INF] Upgrade: Updating osd.5 2022-04-28T19:52:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:55 smithi059 bash[59745]: cephadm 2022-04-28T19:52:53.845590+0000 mgr.y (mgr.24961) 203 : cephadm [INF] Deploying daemon osd.5 on smithi139 2022-04-28T19:52:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:55 smithi059 bash[59745]: cluster 2022-04-28T19:52:54.923027+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2022-04-28T19:52:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:55 smithi139 bash[43301]: cephadm 2022-04-28T19:52:53.833007+0000 mgr.y (mgr.24961) 202 : cephadm [INF] Upgrade: Updating osd.5 2022-04-28T19:52:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:55 smithi139 bash[43301]: cephadm 2022-04-28T19:52:53.845590+0000 mgr.y (mgr.24961) 203 : cephadm [INF] Deploying daemon osd.5 on smithi139 2022-04-28T19:52:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:55 smithi139 bash[43301]: cluster 2022-04-28T19:52:54.923027+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2022-04-28T19:52:55.928 INFO:tasks.rados.rados.0.smithi059.stdout:2689: done (2 left) 2022-04-28T19:52:55.929 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 251 2022-04-28T19:52:55.929 INFO:tasks.rados.rados.0.smithi059.stdout:2690: seq_num 847 ranges {3268608=589824,3858432=753664,4612096=655360,5267456=196608} 2022-04-28T19:52:55.951 INFO:tasks.rados.rados.0.smithi059.stdout:2690: writing smithi05955964-28 from 3268608 to 3858432 tid 1 2022-04-28T19:52:55.956 INFO:tasks.rados.rados.0.smithi059.stdout:2690: writing smithi05955964-28 from 3858432 to 4612096 tid 2 2022-04-28T19:52:55.960 INFO:tasks.rados.rados.0.smithi059.stdout:2690: writing smithi05955964-28 from 4612096 to 5267456 tid 3 2022-04-28T19:52:55.961 INFO:tasks.rados.rados.0.smithi059.stdout:2690: writing smithi05955964-28 from 5267456 to 5464064 tid 4 2022-04-28T19:52:55.961 INFO:tasks.rados.rados.0.smithi059.stdout:2691: rmattr oid 48 current snap is 251 2022-04-28T19:52:55.961 INFO:tasks.rados.rados.0.smithi059.stdout:2685: finishing copy_from to smithi05955964-22 2022-04-28T19:52:55.961 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 949 (ObjNum 812 snap 240 seq_num 812) dirty exists 2022-04-28T19:52:55.961 INFO:tasks.rados.rados.0.smithi059.stdout:2686: finishing write tid 6 to smithi05955964-33 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2692: rollback oid 50 current snap is 251 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 241 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2685: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2686: finishing write tid 7 to smithi05955964-33 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1247 (ObjNum 845 snap 251 seq_num 845) dirty exists 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2686: left oid 33 (ObjNum 845 snap 251 seq_num 845) 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2690: finishing write tid 1 to smithi05955964-28 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2690: finishing write tid 2 to smithi05955964-28 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2685: done (4 left) 2022-04-28T19:52:55.962 INFO:tasks.rados.rados.0.smithi059.stdout:2686: done (3 left) 2022-04-28T19:52:55.963 INFO:tasks.rados.rados.0.smithi059.stdout:2693: snap_remove snap 233 2022-04-28T19:52:56.205 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:52:56.543 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:52:56.543 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (11m) 49s ago 12m 14.7M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:52:56.543 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (12m) 20s ago 12m 26.2M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (10m) 49s ago 10m 14.9M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (10m) 20s ago 10m 18.8M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (4m) 20s ago 16m 376M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (4m) 49s ago 18m 415M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (3m) 49s ago 18m 66.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (3m) 20s ago 16m 45.7M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (3m) 49s ago 16m 52.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (12m) 49s ago 12m 11.2M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (12m) 20s ago 12m 10.9M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:52:56.544 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (2m) 49s ago 15m 294M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (111s) 49s ago 15m 215M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (81s) 49s ago 15m 256M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (52s) 49s ago 14m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (22s) 20s ago 14m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (13m) 20s ago 13m 344M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 e68cffb4766c 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (13m) 20s ago 13m 230M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (13m) 20s ago 13m 191M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:52:56.545 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (5m) 20s ago 12m 40.0M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:52:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:56 smithi059 bash[61152]: cluster 2022-04-28T19:52:54.939057+0000 mgr.y (mgr.24961) 204 : cluster [DBG] pgmap v319: 89 pgs: 89 active+clean; 472 MiB data, 999 MiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 531 KiB/s wr, 9 op/s 2022-04-28T19:52:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:56 smithi059 bash[61152]: cluster 2022-04-28T19:52:55.926265+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2022-04-28T19:52:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:56 smithi059 bash[59745]: cluster 2022-04-28T19:52:54.939057+0000 mgr.y (mgr.24961) 204 : cluster [DBG] pgmap v319: 89 pgs: 89 active+clean; 472 MiB data, 999 MiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 531 KiB/s wr, 9 op/s 2022-04-28T19:52:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:56 smithi059 bash[59745]: cluster 2022-04-28T19:52:55.926265+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2022-04-28T19:52:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:56 smithi139 bash[43301]: cluster 2022-04-28T19:52:54.939057+0000 mgr.y (mgr.24961) 204 : cluster [DBG] pgmap v319: 89 pgs: 89 active+clean; 472 MiB data, 999 MiB used, 711 GiB / 715 GiB avail; 2.8 MiB/s rd, 531 KiB/s wr, 9 op/s 2022-04-28T19:52:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:56 smithi139 bash[43301]: cluster 2022-04-28T19:52:55.926265+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2022-04-28T19:52:56.900 INFO:tasks.rados.rados.0.smithi059.stdout:2693: done (3 left) 2022-04-28T19:52:56.900 INFO:tasks.rados.rados.0.smithi059.stdout:2694: rollback oid 34 current snap is 251 2022-04-28T19:52:56.900 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 34 to 246 2022-04-28T19:52:56.900 INFO:tasks.rados.rados.0.smithi059.stdout:2695: setattr oid 49 current snap is 251 2022-04-28T19:52:56.901 INFO:tasks.rados.rados.0.smithi059.stdout:2696: delete oid 8 current snap is 251 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:2690: finishing write tid 3 to smithi05955964-28 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:2690: finishing write tid 4 to smithi05955964-28 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1139 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:2690: finishing write tid 6 to smithi05955964-28 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:2692: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:2692: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:52:56.902 INFO:tasks.rados.rados.0.smithi059.stdout:2692: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:52:56.903 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1255 (ObjNum 207135968 snap 32762 seq_num 4161867904) dirty exists 2022-04-28T19:52:56.903 INFO:tasks.rados.rados.0.smithi059.stdout:2690: finishing write tid 7 to smithi05955964-28 2022-04-28T19:52:56.903 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 967 (ObjNum 847 snap 251 seq_num 847) dirty exists 2022-04-28T19:52:56.903 INFO:tasks.rados.rados.0.smithi059.stdout:2690: left oid 28 (ObjNum 847 snap 251 seq_num 847) 2022-04-28T19:52:56.911 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1068 (ObjNum 833 snap 245 seq_num 833) dirty exists 2022-04-28T19:52:56.912 INFO:tasks.rados.rados.0.smithi059.stdout:2690: done (5 left) 2022-04-28T19:52:56.912 INFO:tasks.rados.rados.0.smithi059.stdout:2691: done (4 left) 2022-04-28T19:52:56.912 INFO:tasks.rados.rados.0.smithi059.stdout:2692: done (3 left) 2022-04-28T19:52:56.912 INFO:tasks.rados.rados.0.smithi059.stdout:2695: done (2 left) 2022-04-28T19:52:56.913 INFO:tasks.rados.rados.0.smithi059.stdout:2696: done (1 left) 2022-04-28T19:52:56.913 INFO:tasks.rados.rados.0.smithi059.stdout:2697: copy_from oid 8 from oid 41 current snap is 251 2022-04-28T19:52:56.913 INFO:tasks.rados.rados.0.smithi059.stdout:2698: snap_remove snap 249 2022-04-28T19:52:56.933 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:52:56.934 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:52:56.934 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:52:56.934 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:56.934 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:52:56.934 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:52:56.935 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:56.935 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:52:56.935 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 3, 2022-04-28T19:52:56.935 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 5 2022-04-28T19:52:56.935 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 5, 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 10 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:52:56.936 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:52:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.190683+0000 mgr.y (mgr.24961) 205 : audit [DBG] from='client.54238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:57.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.516652+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.517412+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.518063+0000 mon.a (mon.0) 705 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.518763+0000 mon.a (mon.0) 706 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.519391+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.520078+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.683 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.524636+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.683 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.525326+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.683 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.526005+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.683 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.526647+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.683 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.527310+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.683 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: cluster 2022-04-28T19:52:56.898866+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[59745]: audit 2022-04-28T19:52:56.933729+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.59:0/3880775661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.190683+0000 mgr.y (mgr.24961) 205 : audit [DBG] from='client.54238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.516652+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.517412+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.518063+0000 mon.a (mon.0) 705 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.518763+0000 mon.a (mon.0) 706 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.519391+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.684 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.520078+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.524636+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.525326+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.526005+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.526647+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.527310+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: cluster 2022-04-28T19:52:56.898866+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2022-04-28T19:52:57.685 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:57 smithi059 bash[61152]: audit 2022-04-28T19:52:56.933729+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.59:0/3880775661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:57.686 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.190683+0000 mgr.y (mgr.24961) 205 : audit [DBG] from='client.54238 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:57.686 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.516652+0000 mon.a (mon.0) 703 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.686 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.517412+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.686 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.518063+0000 mon.a (mon.0) 705 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.687 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.518763+0000 mon.a (mon.0) 706 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.687 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.519391+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.687 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.520078+0000 mon.a (mon.0) 708 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.687 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.524636+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:52:57.687 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.525326+0000 mon.a (mon.0) 710 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.688 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.526005+0000 mon.a (mon.0) 711 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.688 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.526647+0000 mon.a (mon.0) 712 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.688 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.527310+0000 mon.a (mon.0) 713 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:52:57.688 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: cluster 2022-04-28T19:52:56.898866+0000 mon.a (mon.0) 714 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2022-04-28T19:52:57.688 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[43301]: audit 2022-04-28T19:52:56.933729+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.59:0/3880775661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:52:57.918 INFO:tasks.rados.rados.0.smithi059.stdout:2698: done (2 left) 2022-04-28T19:52:57.919 INFO:tasks.rados.rados.0.smithi059.stdout:2699: rmattr oid 40 current snap is 251 2022-04-28T19:52:57.919 INFO:tasks.rados.rados.0.smithi059.stdout:2699: done (2 left) 2022-04-28T19:52:57.919 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 251 2022-04-28T19:52:57.919 INFO:tasks.rados.rados.0.smithi059.stdout:2700: seq_num 849 ranges {0=622592,622592=516096,1138688=663552,1802240=425984} 2022-04-28T19:52:57.922 INFO:tasks.rados.rados.0.smithi059.stdout:2700: writing smithi05955964-38 from 0 to 622592 tid 1 2022-04-28T19:52:57.925 INFO:tasks.rados.rados.0.smithi059.stdout:2700: writing smithi05955964-38 from 622592 to 1138688 tid 2 2022-04-28T19:52:57.929 INFO:tasks.rados.rados.0.smithi059.stdout:2700: writing smithi05955964-38 from 1138688 to 1802240 tid 3 2022-04-28T19:52:57.932 INFO:tasks.rados.rados.0.smithi059.stdout:2700: writing smithi05955964-38 from 1802240 to 2228224 tid 4 2022-04-28T19:52:57.934 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 251 2022-04-28T19:52:57.934 INFO:tasks.rados.rados.0.smithi059.stdout:2701: seq_num 850 ranges {3137536=638976,3776512=802816,4579328=475136,5054464=327680} 2022-04-28T19:52:57.956 INFO:tasks.rados.rados.0.smithi059.stdout:2701: writing smithi05955964-49 from 3137536 to 3776512 tid 1 2022-04-28T19:52:57.961 INFO:tasks.rados.rados.0.smithi059.stdout:2701: writing smithi05955964-49 from 3776512 to 4579328 tid 2 2022-04-28T19:52:57.963 INFO:tasks.rados.rados.0.smithi059.stdout:2701: writing smithi05955964-49 from 4579328 to 5054464 tid 3 2022-04-28T19:52:57.966 INFO:tasks.rados.rados.0.smithi059.stdout:2701: writing smithi05955964-49 from 5054464 to 5382144 tid 4 2022-04-28T19:52:57.966 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 251 2022-04-28T19:52:57.966 INFO:tasks.rados.rados.0.smithi059.stdout:2702: seq_num 851 ranges {0=655360,655360=688128,1343488=679936,2023424=237568} 2022-04-28T19:52:57.970 INFO:tasks.rados.rados.0.smithi059.stdout:2702: writing smithi05955964-29 from 0 to 655360 tid 1 2022-04-28T19:52:57.974 INFO:tasks.rados.rados.0.smithi059.stdout:2702: writing smithi05955964-29 from 655360 to 1343488 tid 2 2022-04-28T19:52:57.978 INFO:tasks.rados.rados.0.smithi059.stdout:2702: writing smithi05955964-29 from 1343488 to 2023424 tid 3 2022-04-28T19:52:57.980 INFO:tasks.rados.rados.0.smithi059.stdout:2702: writing smithi05955964-29 from 2023424 to 2260992 tid 4 2022-04-28T19:52:57.980 INFO:tasks.rados.rados.0.smithi059.stdout:2703: copy_from oid 16 from oid 4 current snap is 251 2022-04-28T19:52:57.980 INFO:tasks.rados.rados.0.smithi059.stdout:2694: finishing rollback tid 0 to smithi05955964-34 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2694: finishing rollback tid 1 to smithi05955964-34 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2694: finishing rollback tid 2 to smithi05955964-34 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1215 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2697: finishing copy_from to smithi05955964-8 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1015 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2697: finishing copy_from racing read to smithi05955964-8 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2700: finishing write tid 1 to smithi05955964-38 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2700: finishing write tid 2 to smithi05955964-38 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2700: finishing write tid 3 to smithi05955964-38 2022-04-28T19:52:57.981 INFO:tasks.rados.rados.0.smithi059.stdout:2700: finishing write tid 4 to smithi05955964-38 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2700: finishing write tid 6 to smithi05955964-38 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2700: finishing write tid 7 to smithi05955964-38 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1147 (ObjNum 849 snap 251 seq_num 849) dirty exists 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2700: left oid 38 (ObjNum 849 snap 251 seq_num 849) 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2694: done (5 left) 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2697: done (4 left) 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2700: done (3 left) 2022-04-28T19:52:57.982 INFO:tasks.rados.rados.0.smithi059.stdout:2704: snap_create 2022-04-28T19:52:57.983 INFO:tasks.rados.rados.0.smithi059.stdout:2702: finishing write tid 1 to smithi05955964-29 2022-04-28T19:52:57.990 INFO:tasks.rados.rados.0.smithi059.stdout:2702: finishing write tid 2 to smithi05955964-29 2022-04-28T19:52:57.990 INFO:tasks.rados.rados.0.smithi059.stdout:2702: finishing write tid 3 to smithi05955964-29 2022-04-28T19:52:57.990 INFO:tasks.rados.rados.0.smithi059.stdout:2702: finishing write tid 4 to smithi05955964-29 2022-04-28T19:52:57.992 INFO:tasks.rados.rados.0.smithi059.stdout:2702: finishing write tid 6 to smithi05955964-29 2022-04-28T19:52:57.993 INFO:tasks.rados.rados.0.smithi059.stdout:2702: finishing write tid 7 to smithi05955964-29 2022-04-28T19:52:57.994 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1281 (ObjNum 851 snap 251 seq_num 851) dirty exists 2022-04-28T19:52:57.994 INFO:tasks.rados.rados.0.smithi059.stdout:2702: left oid 29 (ObjNum 851 snap 251 seq_num 851) 2022-04-28T19:52:58.039 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:57 smithi139 systemd[1]: Stopping Ceph osd.5 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:52:58.040 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[27981]: debug 2022-04-28T19:52:57.918+0000 7f1bd31c8700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:52:58.040 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[27981]: debug 2022-04-28T19:52:57.918+0000 7f1bd31c8700 -1 osd.5 559 *** Got signal Terminated *** 2022-04-28T19:52:58.040 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:57 smithi139 bash[27981]: debug 2022-04-28T19:52:57.918+0000 7f1bd31c8700 -1 osd.5 559 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:52:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: audit 2022-04-28T19:52:56.511160+0000 mgr.y (mgr.24961) 206 : audit [DBG] from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:56.939514+0000 mgr.y (mgr.24961) 207 : cluster [DBG] pgmap v322: 89 pgs: 89 active+clean; 481 MiB data, 1007 MiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2022-04-28T19:52:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:57.917729+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2022-04-28T19:52:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.232106+0000 mon.a (mon.0) 716 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.232209+0000 mon.a (mon.0) 717 : cluster [INF] osd.5 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.232864+0000 mon.a (mon.0) 718 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.233143+0000 mon.a (mon.0) 719 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.233456+0000 mon.a (mon.0) 720 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.233746+0000 mon.a (mon.0) 721 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.234055+0000 mon.a (mon.0) 722 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.234315+0000 mon.a (mon.0) 723 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.234544+0000 mon.a (mon.0) 724 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.234849+0000 mon.a (mon.0) 725 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.235136+0000 mon.a (mon.0) 726 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.235359+0000 mon.a (mon.0) 727 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.235618+0000 mon.a (mon.0) 728 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.235923+0000 mon.a (mon.0) 729 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.236151+0000 mon.a (mon.0) 730 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.236426+0000 mon.a (mon.0) 731 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.236714+0000 mon.a (mon.0) 732 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.237033+0000 mon.a (mon.0) 733 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: audit 2022-04-28T19:52:56.511160+0000 mgr.y (mgr.24961) 206 : audit [DBG] from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:58.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:56.939514+0000 mgr.y (mgr.24961) 207 : cluster [DBG] pgmap v322: 89 pgs: 89 active+clean; 481 MiB data, 1007 MiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:57.917729+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.232106+0000 mon.a (mon.0) 716 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.232209+0000 mon.a (mon.0) 717 : cluster [INF] osd.5 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.232864+0000 mon.a (mon.0) 718 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.233143+0000 mon.a (mon.0) 719 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.233456+0000 mon.a (mon.0) 720 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.233746+0000 mon.a (mon.0) 721 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.234055+0000 mon.a (mon.0) 722 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.234315+0000 mon.a (mon.0) 723 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.234544+0000 mon.a (mon.0) 724 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.234849+0000 mon.a (mon.0) 725 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.235136+0000 mon.a (mon.0) 726 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.235359+0000 mon.a (mon.0) 727 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.235618+0000 mon.a (mon.0) 728 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.235923+0000 mon.a (mon.0) 729 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.236151+0000 mon.a (mon.0) 730 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.236426+0000 mon.a (mon.0) 731 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:58.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.236714+0000 mon.a (mon.0) 732 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:58.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:58 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.237033+0000 mon.a (mon.0) 733 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: audit 2022-04-28T19:52:56.511160+0000 mgr.y (mgr.24961) 206 : audit [DBG] from='client.44295 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:56.939514+0000 mgr.y (mgr.24961) 207 : cluster [DBG] pgmap v322: 89 pgs: 89 active+clean; 481 MiB data, 1007 MiB used, 711 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.1 MiB/s wr, 4 op/s 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:57.917729+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.232106+0000 mon.a (mon.0) 716 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.232209+0000 mon.a (mon.0) 717 : cluster [INF] osd.5 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.232864+0000 mon.a (mon.0) 718 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.233143+0000 mon.a (mon.0) 719 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.233456+0000 mon.a (mon.0) 720 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.233746+0000 mon.a (mon.0) 721 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.234055+0000 mon.a (mon.0) 722 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.234315+0000 mon.a (mon.0) 723 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.234544+0000 mon.a (mon.0) 724 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.234849+0000 mon.a (mon.0) 725 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.235136+0000 mon.a (mon.0) 726 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.235359+0000 mon.a (mon.0) 727 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.235618+0000 mon.a (mon.0) 728 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.235923+0000 mon.a (mon.0) 729 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.236151+0000 mon.a (mon.0) 730 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.236426+0000 mon.a (mon.0) 731 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:58.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.236714+0000 mon.a (mon.0) 732 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:58.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.237033+0000 mon.a (mon.0) 733 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:58.924 INFO:tasks.rados.rados.0.smithi059.stdout:2702: done (3 left) 2022-04-28T19:52:58.924 INFO:tasks.rados.rados.0.smithi059.stdout:2704: done (2 left) 2022-04-28T19:52:58.924 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 252 2022-04-28T19:52:58.924 INFO:tasks.rados.rados.0.smithi059.stdout:2705: seq_num 852 ranges {2260992=671744,2932736=622592,3555328=442368,3997696=540672} 2022-04-28T19:52:58.941 INFO:tasks.rados.rados.0.smithi059.stdout:2705: writing smithi05955964-29 from 2260992 to 2932736 tid 1 2022-04-28T19:52:58.944 INFO:tasks.rados.rados.0.smithi059.stdout:2705: writing smithi05955964-29 from 2932736 to 3555328 tid 2 2022-04-28T19:52:58.947 INFO:tasks.rados.rados.0.smithi059.stdout:2705: writing smithi05955964-29 from 3555328 to 3997696 tid 3 2022-04-28T19:52:58.951 INFO:tasks.rados.rados.0.smithi059.stdout:2705: writing smithi05955964-29 from 3997696 to 4538368 tid 4 2022-04-28T19:52:58.951 INFO:tasks.rados.rados.0.smithi059.stdout:2706: rollback oid 33 current snap is 252 2022-04-28T19:52:58.952 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 251 2022-04-28T19:52:58.952 INFO:tasks.rados.rados.0.smithi059.stdout:2705: finishing write tid 1 to smithi05955964-29 2022-04-28T19:52:58.952 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 252 2022-04-28T19:52:58.952 INFO:tasks.rados.rados.0.smithi059.stdout:2707: seq_num 853 ranges {1155072=696320,1851392=507904,2359296=647168,3006464=450560} 2022-04-28T19:52:58.961 INFO:tasks.rados.rados.0.smithi059.stdout:2707: writing smithi05955964-8 from 1155072 to 1851392 tid 1 2022-04-28T19:52:58.964 INFO:tasks.rados.rados.0.smithi059.stdout:2707: writing smithi05955964-8 from 1851392 to 2359296 tid 2 2022-04-28T19:52:58.967 INFO:tasks.rados.rados.0.smithi059.stdout:2707: writing smithi05955964-8 from 2359296 to 3006464 tid 3 2022-04-28T19:52:58.970 INFO:tasks.rados.rados.0.smithi059.stdout:2707: writing smithi05955964-8 from 3006464 to 3457024 tid 4 2022-04-28T19:52:58.970 INFO:tasks.rados.rados.0.smithi059.stdout:2708: snap_create 2022-04-28T19:52:58.970 INFO:tasks.rados.rados.0.smithi059.stdout:2705: finishing write tid 2 to smithi05955964-29 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2705: finishing write tid 3 to smithi05955964-29 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2705: finishing write tid 4 to smithi05955964-29 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2705: finishing write tid 6 to smithi05955964-29 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2705: finishing write tid 7 to smithi05955964-29 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1287 (ObjNum 852 snap 252 seq_num 852) dirty exists 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2705: left oid 29 (ObjNum 852 snap 252 seq_num 852) 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2707: finishing write tid 1 to smithi05955964-8 2022-04-28T19:52:58.971 INFO:tasks.rados.rados.0.smithi059.stdout:2707: finishing write tid 2 to smithi05955964-8 2022-04-28T19:52:58.972 INFO:tasks.rados.rados.0.smithi059.stdout:2707: finishing write tid 3 to smithi05955964-8 2022-04-28T19:52:58.974 INFO:tasks.rados.rados.0.smithi059.stdout:2707: finishing write tid 4 to smithi05955964-8 2022-04-28T19:52:58.974 INFO:tasks.rados.rados.0.smithi059.stdout:2707: finishing write tid 6 to smithi05955964-8 2022-04-28T19:52:58.976 INFO:tasks.rados.rados.0.smithi059.stdout:2707: finishing write tid 7 to smithi05955964-8 2022-04-28T19:52:58.976 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1293 (ObjNum 853 snap 252 seq_num 853) dirty exists 2022-04-28T19:52:58.976 INFO:tasks.rados.rados.0.smithi059.stdout:2707: left oid 8 (ObjNum 853 snap 252 seq_num 853) 2022-04-28T19:52:59.118 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[46919]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.5 2022-04-28T19:52:59.118 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[46919]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-5 2022-04-28T19:52:59.118 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:52:58 smithi139 bash[46919]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.5 2022-04-28T19:52:59.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.636180+0000 mon.a (mon.0) 734 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.636641+0000 mon.a (mon.0) 735 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.637175+0000 mon.a (mon.0) 736 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.637593+0000 mon.a (mon.0) 737 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.638020+0000 mon.a (mon.0) 738 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.638370+0000 mon.a (mon.0) 739 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.638653+0000 mon.a (mon.0) 740 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.638858+0000 mon.a (mon.0) 741 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.639056+0000 mon.a (mon.0) 742 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.639264+0000 mon.a (mon.0) 743 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.639559+0000 mon.a (mon.0) 744 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.639839+0000 mon.a (mon.0) 745 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.640041+0000 mon.a (mon.0) 746 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.640312+0000 mon.a (mon.0) 747 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.640497+0000 mon.a (mon.0) 748 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.640856+0000 mon.a (mon.0) 749 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.641043+0000 mon.a (mon.0) 750 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.913527+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.921356+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2022-04-28T19:52:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:52:59 smithi139 bash[43301]: cluster 2022-04-28T19:52:58.939922+0000 mgr.y (mgr.24961) 208 : cluster [DBG] pgmap v325: 89 pgs: 11 stale+active+clean, 78 active+clean; 480 MiB data, 1010 MiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 3.5 MiB/s wr, 8 op/s 2022-04-28T19:52:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.636180+0000 mon.a (mon.0) 734 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.636641+0000 mon.a (mon.0) 735 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.637175+0000 mon.a (mon.0) 736 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.637593+0000 mon.a (mon.0) 737 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.638020+0000 mon.a (mon.0) 738 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.638370+0000 mon.a (mon.0) 739 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.638653+0000 mon.a (mon.0) 740 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.638858+0000 mon.a (mon.0) 741 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.639056+0000 mon.a (mon.0) 742 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.639264+0000 mon.a (mon.0) 743 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.639559+0000 mon.a (mon.0) 744 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.639839+0000 mon.a (mon.0) 745 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.640041+0000 mon.a (mon.0) 746 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.640312+0000 mon.a (mon.0) 747 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.640497+0000 mon.a (mon.0) 748 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.640856+0000 mon.a (mon.0) 749 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.641043+0000 mon.a (mon.0) 750 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.913527+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.921356+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[59745]: cluster 2022-04-28T19:52:58.939922+0000 mgr.y (mgr.24961) 208 : cluster [DBG] pgmap v325: 89 pgs: 11 stale+active+clean, 78 active+clean; 480 MiB data, 1010 MiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 3.5 MiB/s wr, 8 op/s 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.636180+0000 mon.a (mon.0) 734 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.636641+0000 mon.a (mon.0) 735 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.637175+0000 mon.a (mon.0) 736 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.637593+0000 mon.a (mon.0) 737 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.638020+0000 mon.a (mon.0) 738 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.638370+0000 mon.a (mon.0) 739 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.638653+0000 mon.a (mon.0) 740 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.638858+0000 mon.a (mon.0) 741 : cluster [DBG] osd.5 reported immediately failed by osd.6 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.639056+0000 mon.a (mon.0) 742 : cluster [DBG] osd.5 reported immediately failed by osd.7 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.639264+0000 mon.a (mon.0) 743 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.639559+0000 mon.a (mon.0) 744 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.639839+0000 mon.a (mon.0) 745 : cluster [DBG] osd.5 reported immediately failed by osd.4 2022-04-28T19:52:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.640041+0000 mon.a (mon.0) 746 : cluster [DBG] osd.5 reported immediately failed by osd.2 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.640312+0000 mon.a (mon.0) 747 : cluster [DBG] osd.5 reported immediately failed by osd.3 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.640497+0000 mon.a (mon.0) 748 : cluster [DBG] osd.5 reported immediately failed by osd.1 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.640856+0000 mon.a (mon.0) 749 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.641043+0000 mon.a (mon.0) 750 : cluster [DBG] osd.5 reported immediately failed by osd.0 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.913527+0000 mon.a (mon.0) 751 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.921356+0000 mon.a (mon.0) 752 : cluster [DBG] osdmap e560: 8 total, 7 up, 8 in 2022-04-28T19:52:59.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:52:59 smithi059 bash[61152]: cluster 2022-04-28T19:52:58.939922+0000 mgr.y (mgr.24961) 208 : cluster [DBG] pgmap v325: 89 pgs: 11 stale+active+clean, 78 active+clean; 480 MiB data, 1010 MiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 3.5 MiB/s wr, 8 op/s 2022-04-28T19:52:59.936 INFO:tasks.rados.rados.0.smithi059.stdout:2705: done (5 left) 2022-04-28T19:53:00.010 INFO:tasks.rados.rados.0.smithi059.stdout:2707: done (4 left) 2022-04-28T19:53:00.010 INFO:tasks.rados.rados.0.smithi059.stdout:2708: done (3 left) 2022-04-28T19:53:00.010 INFO:tasks.rados.rados.0.smithi059.stdout:2709: snap_remove snap 248 2022-04-28T19:53:00.935 INFO:tasks.rados.rados.0.smithi059.stdout:2709: done (3 left) 2022-04-28T19:53:01.032 INFO:tasks.rados.rados.0.smithi059.stdout:2710: setattr oid 5 current snap is 253 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2711: read oid 27 snap 236 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2711: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2712: read oid 10 snap 219 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2712: expect (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 253 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2713: seq_num 854 ranges {3432448=712704,4145152=794624,4939776=450560,5390336=360448} 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2713: writing smithi05955964-35 from 3432448 to 4145152 tid 1 2022-04-28T19:53:01.035 INFO:tasks.rados.rados.0.smithi059.stdout:2713: writing smithi05955964-35 from 4145152 to 4939776 tid 2 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2713: writing smithi05955964-35 from 4939776 to 5390336 tid 3 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2713: writing smithi05955964-35 from 5390336 to 5750784 tid 4 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2714: copy_from oid 2 from oid 12 current snap is 253 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2703: finishing copy_from to smithi05955964-16 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 982 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2703: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2701: finishing write tid 1 to smithi05955964-49 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2701: finishing write tid 2 to smithi05955964-49 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2703: done (7 left) 2022-04-28T19:53:01.036 INFO:tasks.rados.rados.0.smithi059.stdout:2715: delete oid 42 current snap is 253 2022-04-28T19:53:01.037 INFO:tasks.rados.rados.0.smithi059.stdout:2701: finishing write tid 3 to smithi05955964-49 2022-04-28T19:53:01.038 INFO:tasks.rados.rados.0.smithi059.stdout:2701: finishing write tid 4 to smithi05955964-49 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2701: finishing write tid 6 to smithi05955964-49 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2701: finishing write tid 7 to smithi05955964-49 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1073 (ObjNum 850 snap 251 seq_num 850) dirty exists 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2701: left oid 49 (ObjNum 850 snap 251 seq_num 850) 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2706: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2706: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2706: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1269 (ObjNum 845 snap 251 seq_num 845) dirty exists 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1294 (ObjNum 1 snap 32762 seq_num 4162065360) dirty exists 2022-04-28T19:53:01.039 INFO:tasks.rados.rados.0.smithi059.stdout:2711: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:53:01.075 INFO:tasks.rados.rados.0.smithi059.stdout:2712: expect (ObjNum 733 snap 213 seq_num 733) 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2713: finishing write tid 1 to smithi05955964-35 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2713: finishing write tid 2 to smithi05955964-35 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2713: finishing write tid 3 to smithi05955964-35 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2713: finishing write tid 4 to smithi05955964-35 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2714: finishing copy_from to smithi05955964-2 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2714: got expected ENOENT (src dne) 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2713: finishing write tid 6 to smithi05955964-35 2022-04-28T19:53:01.082 INFO:tasks.rados.rados.0.smithi059.stdout:2714: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2713: finishing write tid 7 to smithi05955964-35 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 951 (ObjNum 854 snap 253 seq_num 854) dirty exists 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2713: left oid 35 (ObjNum 854 snap 253 seq_num 854) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2701: done (7 left) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2706: done (6 left) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2710: done (5 left) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2711: done (4 left) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2712: done (3 left) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2713: done (2 left) 2022-04-28T19:53:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:2714: done (1 left) 2022-04-28T19:53:01.084 INFO:tasks.rados.rados.0.smithi059.stdout:2715: done (0 left) 2022-04-28T19:53:01.084 INFO:tasks.rados.rados.0.smithi059.stdout:2716: copy_from oid 37 from oid 15 current snap is 253 2022-04-28T19:53:01.084 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 253 2022-04-28T19:53:01.084 INFO:tasks.rados.rados.0.smithi059.stdout:2717: seq_num 856 ranges {2310144=745472,3055616=557056,3612672=417792,4030464=434176,4464640=196608} 2022-04-28T19:53:01.105 INFO:tasks.rados.rados.0.smithi059.stdout:2717: writing smithi05955964-27 from 2310144 to 3055616 tid 1 2022-04-28T19:53:01.109 INFO:tasks.rados.rados.0.smithi059.stdout:2717: writing smithi05955964-27 from 3055616 to 3612672 tid 2 2022-04-28T19:53:01.111 INFO:tasks.rados.rados.0.smithi059.stdout:2717: writing smithi05955964-27 from 3612672 to 4030464 tid 3 2022-04-28T19:53:01.114 INFO:tasks.rados.rados.0.smithi059.stdout:2717: writing smithi05955964-27 from 4030464 to 4464640 tid 4 2022-04-28T19:53:01.115 INFO:tasks.rados.rados.0.smithi059.stdout:2717: writing smithi05955964-27 from 4464640 to 4661248 tid 5 2022-04-28T19:53:01.116 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 253 2022-04-28T19:53:01.116 INFO:tasks.rados.rados.0.smithi059.stdout:2718: seq_num 857 ranges {0=761856,761856=442368,1204224=630784,1835008=532480} 2022-04-28T19:53:01.120 INFO:tasks.rados.rados.0.smithi059.stdout:2718: writing smithi05955964-11 from 0 to 761856 tid 1 2022-04-28T19:53:01.122 INFO:tasks.rados.rados.0.smithi059.stdout:2718: writing smithi05955964-11 from 761856 to 1204224 tid 2 2022-04-28T19:53:01.126 INFO:tasks.rados.rados.0.smithi059.stdout:2718: writing smithi05955964-11 from 1204224 to 1835008 tid 3 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:2718: writing smithi05955964-11 from 1835008 to 2367488 tid 4 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:2719: copy_from oid 4 from oid 16 current snap is 253 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:2716: finishing copy_from to smithi05955964-37 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1139 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:2720: rollback oid 3 current snap is 253 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 238 2022-04-28T19:53:01.131 INFO:tasks.rados.rados.0.smithi059.stdout:2716: finishing copy_from racing read to smithi05955964-37 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 1 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 2 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 3 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 4 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 5 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 7 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: finishing write tid 8 to smithi05955964-27 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1122 (ObjNum 856 snap 253 seq_num 856) dirty exists 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2717: left oid 27 (ObjNum 856 snap 253 seq_num 856) 2022-04-28T19:53:01.132 INFO:tasks.rados.rados.0.smithi059.stdout:2718: finishing write tid 1 to smithi05955964-11 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2718: finishing write tid 2 to smithi05955964-11 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2718: finishing write tid 3 to smithi05955964-11 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2716: done (4 left) 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2717: done (3 left) 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2721: read oid 48 snap -1 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2721: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 253 2022-04-28T19:53:01.133 INFO:tasks.rados.rados.0.smithi059.stdout:2722: seq_num 858 ranges {0=778240,778240=729088,1507328=434176,1941504=442368} 2022-04-28T19:53:01.135 INFO:tasks.rados.rados.0.smithi059.stdout:2722: writing smithi05955964-5 from 0 to 778240 tid 1 2022-04-28T19:53:01.140 INFO:tasks.rados.rados.0.smithi059.stdout:2722: writing smithi05955964-5 from 778240 to 1507328 tid 2 2022-04-28T19:53:01.143 INFO:tasks.rados.rados.0.smithi059.stdout:2722: writing smithi05955964-5 from 1507328 to 1941504 tid 3 2022-04-28T19:53:01.146 INFO:tasks.rados.rados.0.smithi059.stdout:2722: writing smithi05955964-5 from 1941504 to 2383872 tid 4 2022-04-28T19:53:01.146 INFO:tasks.rados.rados.0.smithi059.stdout:2720: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:53:01.146 INFO:tasks.rados.rados.0.smithi059.stdout:2723: read oid 18 snap 229 2022-04-28T19:53:01.146 INFO:tasks.rados.rados.0.smithi059.stdout:2723: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:53:01.147 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 253 2022-04-28T19:53:01.147 INFO:tasks.rados.rados.0.smithi059.stdout:2724: seq_num 859 ranges {4538368=794624,5332992=614400,5947392=638976,6586368=352256} 2022-04-28T19:53:01.177 INFO:tasks.rados.rados.0.smithi059.stdout:2724: writing smithi05955964-29 from 4538368 to 5332992 tid 1 2022-04-28T19:53:01.180 INFO:tasks.rados.rados.0.smithi059.stdout:2724: writing smithi05955964-29 from 5332992 to 5947392 tid 2 2022-04-28T19:53:01.184 INFO:tasks.rados.rados.0.smithi059.stdout:2724: writing smithi05955964-29 from 5947392 to 6586368 tid 3 2022-04-28T19:53:01.187 INFO:tasks.rados.rados.0.smithi059.stdout:2724: writing smithi05955964-29 from 6586368 to 6938624 tid 4 2022-04-28T19:53:01.187 INFO:tasks.rados.rados.0.smithi059.stdout:2725: read oid 40 snap -1 2022-04-28T19:53:01.187 INFO:tasks.rados.rados.0.smithi059.stdout:2725: expect (ObjNum 839 snap 246 seq_num 839) 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2726: delete oid 28 current snap is 253 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2718: finishing write tid 4 to smithi05955964-11 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2718: finishing write tid 6 to smithi05955964-11 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2720: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2720: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 973 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2722: finishing write tid 1 to smithi05955964-5 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2718: finishing write tid 7 to smithi05955964-11 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1152 (ObjNum 857 snap 253 seq_num 857) dirty exists 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2718: left oid 11 (ObjNum 857 snap 253 seq_num 857) 2022-04-28T19:53:01.188 INFO:tasks.rados.rados.0.smithi059.stdout:2721: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:53:01.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:00 smithi059 bash[59745]: cluster 2022-04-28T19:52:59.934517+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2022-04-28T19:53:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:00 smithi059 bash[61152]: cluster 2022-04-28T19:52:59.934517+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2022-04-28T19:53:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:00 smithi139 bash[43301]: cluster 2022-04-28T19:52:59.934517+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e561: 8 total, 7 up, 8 in 2022-04-28T19:53:01.368 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:00 smithi139 systemd[1]: Stopped Ceph osd.5 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:53:01.368 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:01 smithi139 systemd[1]: Started Ceph osd.5 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:53:01.590 INFO:tasks.rados.rados.0.smithi059.stdout:2722: finishing write tid 2 to smithi05955964-5 2022-04-28T19:53:01.590 INFO:tasks.rados.rados.0.smithi059.stdout:2722: finishing write tid 3 to smithi05955964-5 2022-04-28T19:53:01.590 INFO:tasks.rados.rados.0.smithi059.stdout:2719: finishing copy_from to smithi05955964-4 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 982 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:2719: finishing copy_from racing read to smithi05955964-4 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:2722: finishing write tid 4 to smithi05955964-5 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:2722: finishing write tid 6 to smithi05955964-5 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:2722: finishing write tid 7 to smithi05955964-5 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1303 (ObjNum 858 snap 253 seq_num 858) dirty exists 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:2722: left oid 5 (ObjNum 858 snap 253 seq_num 858) 2022-04-28T19:53:01.591 INFO:tasks.rados.rados.0.smithi059.stdout:2723: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: finishing write tid 1 to smithi05955964-29 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: finishing write tid 2 to smithi05955964-29 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: finishing write tid 3 to smithi05955964-29 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: finishing write tid 4 to smithi05955964-29 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: finishing write tid 6 to smithi05955964-29 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: finishing write tid 7 to smithi05955964-29 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1309 (ObjNum 859 snap 253 seq_num 859) dirty exists 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2724: left oid 29 (ObjNum 859 snap 253 seq_num 859) 2022-04-28T19:53:01.704 INFO:tasks.rados.rados.0.smithi059.stdout:2725: expect (ObjNum 839 snap 246 seq_num 839) 2022-04-28T19:53:01.750 INFO:tasks.rados.rados.0.smithi059.stdout:2718: done (8 left) 2022-04-28T19:53:01.750 INFO:tasks.rados.rados.0.smithi059.stdout:2719: done (7 left) 2022-04-28T19:53:01.750 INFO:tasks.rados.rados.0.smithi059.stdout:2720: done (6 left) 2022-04-28T19:53:01.750 INFO:tasks.rados.rados.0.smithi059.stdout:2721: done (5 left) 2022-04-28T19:53:01.750 INFO:tasks.rados.rados.0.smithi059.stdout:2722: done (4 left) 2022-04-28T19:53:01.750 INFO:tasks.rados.rados.0.smithi059.stdout:2723: done (3 left) 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2724: done (2 left) 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2725: done (1 left) 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2726: done (0 left) 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2727: rollback oid 36 current snap is 253 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 219 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 253 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2728: seq_num 861 ranges {0=24576} 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2728: writing smithi05955964-39 from 0 to 24576 tid 1 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 253 2022-04-28T19:53:01.751 INFO:tasks.rados.rados.0.smithi059.stdout:2729: seq_num 862 ranges {3751936=40960} 2022-04-28T19:53:01.780 INFO:tasks.rados.rados.0.smithi059.stdout:2729: writing smithi05955964-20 from 3751936 to 3792896 tid 1 2022-04-28T19:53:01.780 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 253 2022-04-28T19:53:01.780 INFO:tasks.rados.rados.0.smithi059.stdout:2730: seq_num 863 ranges {3506176=57344} 2022-04-28T19:53:01.802 INFO:tasks.rados.rados.0.smithi059.stdout:2730: writing smithi05955964-19 from 3506176 to 3563520 tid 1 2022-04-28T19:53:01.802 INFO:tasks.rados.rados.0.smithi059.stdout:2731: read oid 43 snap -1 2022-04-28T19:53:01.802 INFO:tasks.rados.rados.0.smithi059.stdout:2731: expect (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2732: read oid 37 snap -1 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2732: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2733: rollback oid 17 current snap is 253 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 219 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2727: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2727: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2727: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1076 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2728: finishing write tid 1 to smithi05955964-39 2022-04-28T19:53:01.803 INFO:tasks.rados.rados.0.smithi059.stdout:2728: finishing write tid 3 to smithi05955964-39 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2728: finishing write tid 4 to smithi05955964-39 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1272 (ObjNum 861 snap 253 seq_num 861) dirty exists 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2728: left oid 39 (ObjNum 861 snap 253 seq_num 861) 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2729: finishing write tid 1 to smithi05955964-20 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2729: finishing write tid 3 to smithi05955964-20 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2729: finishing write tid 4 to smithi05955964-20 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 976 (ObjNum 862 snap 253 seq_num 862) dirty exists 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2729: left oid 20 (ObjNum 862 snap 253 seq_num 862) 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2727: done (6 left) 2022-04-28T19:53:01.804 INFO:tasks.rados.rados.0.smithi059.stdout:2728: done (5 left) 2022-04-28T19:53:01.805 INFO:tasks.rados.rados.0.smithi059.stdout:2729: done (4 left) 2022-04-28T19:53:01.805 INFO:tasks.rados.rados.0.smithi059.stdout:2734: read oid 16 snap -1 2022-04-28T19:53:01.805 INFO:tasks.rados.rados.0.smithi059.stdout:2734: expect (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:53:01.805 INFO:tasks.rados.rados.0.smithi059.stdout:2735: rmattr oid 36 current snap is 253 2022-04-28T19:53:01.805 INFO:tasks.rados.rados.0.smithi059.stdout:2735: done (5 left) 2022-04-28T19:53:01.805 INFO:tasks.rados.rados.0.smithi059.stdout:2736: snap_create 2022-04-28T19:53:01.807 INFO:tasks.rados.rados.0.smithi059.stdout:2730: finishing write tid 1 to smithi05955964-19 2022-04-28T19:53:01.807 INFO:tasks.rados.rados.0.smithi059.stdout:2730: finishing write tid 3 to smithi05955964-19 2022-04-28T19:53:01.811 INFO:tasks.rados.rados.0.smithi059.stdout:2730: finishing write tid 4 to smithi05955964-19 2022-04-28T19:53:01.812 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1142 (ObjNum 863 snap 253 seq_num 863) dirty exists 2022-04-28T19:53:01.812 INFO:tasks.rados.rados.0.smithi059.stdout:2730: left oid 19 (ObjNum 863 snap 253 seq_num 863) 2022-04-28T19:53:01.812 INFO:tasks.rados.rados.0.smithi059.stdout:2732: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:01.855 INFO:tasks.rados.rados.0.smithi059.stdout:2733: finishing rollback tid 0 to smithi05955964-17 2022-04-28T19:53:01.855 INFO:tasks.rados.rados.0.smithi059.stdout:2731: expect (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:53:01.874 INFO:tasks.rados.rados.0.smithi059.stdout:2733: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:53:01.874 INFO:tasks.rados.rados.0.smithi059.stdout:2733: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:53:01.874 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 1048 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:53:01.875 INFO:tasks.rados.rados.0.smithi059.stdout:2734: expect (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:53:01.911 INFO:tasks.rados.rados.0.smithi059.stdout:2730: done (5 left) 2022-04-28T19:53:01.911 INFO:tasks.rados.rados.0.smithi059.stdout:2731: done (4 left) 2022-04-28T19:53:01.911 INFO:tasks.rados.rados.0.smithi059.stdout:2732: done (3 left) 2022-04-28T19:53:01.911 INFO:tasks.rados.rados.0.smithi059.stdout:2733: done (2 left) 2022-04-28T19:53:01.911 INFO:tasks.rados.rados.0.smithi059.stdout:2734: done (1 left) 2022-04-28T19:53:01.912 INFO:tasks.rados.rados.0.smithi059.stdout:2736: done (0 left) 2022-04-28T19:53:01.912 INFO:tasks.rados.rados.0.smithi059.stdout:2737: delete oid 8 current snap is 254 2022-04-28T19:53:01.918 INFO:tasks.rados.rados.0.smithi059.stdout:2737: done (0 left) 2022-04-28T19:53:01.919 INFO:tasks.rados.rados.0.smithi059.stdout:2738: snap_create 2022-04-28T19:53:01.948 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: cluster 2022-04-28T19:53:00.934251+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2022-04-28T19:53:01.948 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: cluster 2022-04-28T19:53:00.940594+0000 mgr.y (mgr.24961) 209 : cluster [DBG] pgmap v328: 89 pgs: 1 active+undersized+degraded, 9 peering, 1 active+clean+snaptrim, 3 stale+active+clean, 7 active+undersized, 68 active+clean; 484 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.8 MiB/s wr, 17 op/s; 13/807 objects degraded (1.611%) 2022-04-28T19:53:01.948 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:01.632Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:01.949 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:01.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:01.949 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:01.633Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:02.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: cluster 2022-04-28T19:53:00.934251+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2022-04-28T19:53:02.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: cluster 2022-04-28T19:53:00.940594+0000 mgr.y (mgr.24961) 209 : cluster [DBG] pgmap v328: 89 pgs: 1 active+undersized+degraded, 9 peering, 1 active+clean+snaptrim, 3 stale+active+clean, 7 active+undersized, 68 active+clean; 484 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.8 MiB/s wr, 17 op/s; 13/807 objects degraded (1.611%) 2022-04-28T19:53:02.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: audit 2022-04-28T19:53:01.178138+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:02.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: audit 2022-04-28T19:53:01.179277+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:02.256 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: audit 2022-04-28T19:53:01.180985+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:53:02.256 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: audit 2022-04-28T19:53:01.182241+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:53:02.256 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: cephadm 2022-04-28T19:53:01.183307+0000 mgr.y (mgr.24961) 210 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:53:02.256 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: audit 2022-04-28T19:53:01.184111+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:53:02.256 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:01 smithi139 bash[43301]: cluster 2022-04-28T19:53:01.901467+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2022-04-28T19:53:02.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: audit 2022-04-28T19:53:01.178138+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: audit 2022-04-28T19:53:01.179277+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: audit 2022-04-28T19:53:01.180985+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:53:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: audit 2022-04-28T19:53:01.182241+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:53:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: cephadm 2022-04-28T19:53:01.183307+0000 mgr.y (mgr.24961) 210 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:53:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: audit 2022-04-28T19:53:01.184111+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:53:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[59745]: cluster 2022-04-28T19:53:01.901467+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2022-04-28T19:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: cluster 2022-04-28T19:53:00.934251+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e562: 8 total, 7 up, 8 in 2022-04-28T19:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: cluster 2022-04-28T19:53:00.940594+0000 mgr.y (mgr.24961) 209 : cluster [DBG] pgmap v328: 89 pgs: 1 active+undersized+degraded, 9 peering, 1 active+clean+snaptrim, 3 stale+active+clean, 7 active+undersized, 68 active+clean; 484 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 44 KiB/s rd, 1.8 MiB/s wr, 17 op/s; 13/807 objects degraded (1.611%) 2022-04-28T19:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: audit 2022-04-28T19:53:01.178138+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: audit 2022-04-28T19:53:01.179277+0000 mon.a (mon.0) 756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: audit 2022-04-28T19:53:01.180985+0000 mon.a (mon.0) 757 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:53:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: audit 2022-04-28T19:53:01.182241+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:53:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: cephadm 2022-04-28T19:53:01.183307+0000 mgr.y (mgr.24961) 210 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:53:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: audit 2022-04-28T19:53:01.184111+0000 mon.a (mon.0) 759 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:53:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:01 smithi059 bash[61152]: cluster 2022-04-28T19:53:01.901467+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e563: 8 total, 7 up, 8 in 2022-04-28T19:53:02.915 INFO:tasks.rados.rados.0.smithi059.stdout:2738: done (0 left) 2022-04-28T19:53:02.915 INFO:tasks.rados.rados.0.smithi059.stdout:2739: setattr oid 28 current snap is 255 2022-04-28T19:53:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2740: read oid 7 snap 238 2022-04-28T19:53:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2740: expect deleted 2022-04-28T19:53:02.917 INFO:tasks.rados.rados.0.smithi059.stdout:2741: snap_remove snap 254 2022-04-28T19:53:02.948 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2022-04-28T19:53:02.948 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2022-04-28T19:53:02.949 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2022-04-28T19:53:02.949 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2022-04-28T19:53:02.949 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2022-04-28T19:53:02.949 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2022-04-28T19:53:02.949 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[47164]: --> ceph-volume lvm activate successful for osd ID: 5 2022-04-28T19:53:03.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:02 smithi059 bash[61152]: cluster 2022-04-28T19:53:01.936665+0000 mon.a (mon.0) 761 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2022-04-28T19:53:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:02 smithi059 bash[61152]: cluster 2022-04-28T19:53:01.936698+0000 mon.a (mon.0) 762 : cluster [WRN] Health check failed: Degraded data redundancy: 13/807 objects degraded (1.611%), 1 pg degraded (PG_DEGRADED) 2022-04-28T19:53:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:02 smithi059 bash[61152]: cluster 2022-04-28T19:53:02.912880+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2022-04-28T19:53:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:02 smithi059 bash[59745]: cluster 2022-04-28T19:53:01.936665+0000 mon.a (mon.0) 761 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2022-04-28T19:53:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:02 smithi059 bash[59745]: cluster 2022-04-28T19:53:01.936698+0000 mon.a (mon.0) 762 : cluster [WRN] Health check failed: Degraded data redundancy: 13/807 objects degraded (1.611%), 1 pg degraded (PG_DEGRADED) 2022-04-28T19:53:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:02 smithi059 bash[59745]: cluster 2022-04-28T19:53:02.912880+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2022-04-28T19:53:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[43301]: cluster 2022-04-28T19:53:01.936665+0000 mon.a (mon.0) 761 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2022-04-28T19:53:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[43301]: cluster 2022-04-28T19:53:01.936698+0000 mon.a (mon.0) 762 : cluster [WRN] Health check failed: Degraded data redundancy: 13/807 objects degraded (1.611%), 1 pg degraded (PG_DEGRADED) 2022-04-28T19:53:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:02 smithi139 bash[43301]: cluster 2022-04-28T19:53:02.912880+0000 mon.a (mon.0) 763 : cluster [DBG] osdmap e564: 8 total, 7 up, 8 in 2022-04-28T19:53:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:03.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:03.526Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:03.526Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:53:03.923 INFO:tasks.rados.rados.0.smithi059.stdout:2741: done (2 left) 2022-04-28T19:53:03.923 INFO:tasks.rados.rados.0.smithi059.stdout:2742: snap_create 2022-04-28T19:53:03.923 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 984 (ObjNum -132850432 snap 21893 seq_num 4161793632) dirty exists 2022-04-28T19:53:04.329 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:03 smithi139 bash[43301]: cluster 2022-04-28T19:53:02.941308+0000 mgr.y (mgr.24961) 211 : cluster [DBG] pgmap v331: 89 pgs: 6 active+undersized+degraded, 9 peering, 1 active+clean+snaptrim, 13 active+undersized, 60 active+clean; 499 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.2 MiB/s wr, 29 op/s; 45/840 objects degraded (5.357%) 2022-04-28T19:53:04.329 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:03 smithi139 bash[43301]: cluster 2022-04-28T19:53:03.922506+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2022-04-28T19:53:04.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[59745]: cluster 2022-04-28T19:53:02.941308+0000 mgr.y (mgr.24961) 211 : cluster [DBG] pgmap v331: 89 pgs: 6 active+undersized+degraded, 9 peering, 1 active+clean+snaptrim, 13 active+undersized, 60 active+clean; 499 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.2 MiB/s wr, 29 op/s; 45/840 objects degraded (5.357%) 2022-04-28T19:53:04.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[59745]: cluster 2022-04-28T19:53:03.922506+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2022-04-28T19:53:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[61152]: cluster 2022-04-28T19:53:02.941308+0000 mgr.y (mgr.24961) 211 : cluster [DBG] pgmap v331: 89 pgs: 6 active+undersized+degraded, 9 peering, 1 active+clean+snaptrim, 13 active+undersized, 60 active+clean; 499 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.2 MiB/s wr, 29 op/s; 45/840 objects degraded (5.357%) 2022-04-28T19:53:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:03 smithi059 bash[61152]: cluster 2022-04-28T19:53:03.922506+0000 mon.a (mon.0) 764 : cluster [DBG] osdmap e565: 8 total, 7 up, 8 in 2022-04-28T19:53:04.618 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:04 smithi139 bash[47164]: debug 2022-04-28T19:53:04.457+0000 7f6c4ce6d700 1 -- 172.21.15.139:0/2955679572 <== mon.2 v2:172.21.15.139:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x556eadf9c840 con 0x556eae01ec00 2022-04-28T19:53:04.618 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:04 smithi139 bash[47164]: debug 2022-04-28T19:53:04.457+0000 7f6c4ce6d700 1 -- 172.21.15.139:0/2955679572 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556eadfa64e0 con 0x556eae01ec00 2022-04-28T19:53:04.928 INFO:tasks.rados.rados.0.smithi059.stdout:2739: done (2 left) 2022-04-28T19:53:04.928 INFO:tasks.rados.rados.0.smithi059.stdout:2740: done (1 left) 2022-04-28T19:53:04.929 INFO:tasks.rados.rados.0.smithi059.stdout:2742: done (0 left) 2022-04-28T19:53:04.929 INFO:tasks.rados.rados.0.smithi059.stdout:2743: setattr oid 44 current snap is 256 2022-04-28T19:53:04.930 INFO:tasks.rados.rados.0.smithi059.stdout:2744: snap_create 2022-04-28T19:53:04.933 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 964 (ObjNum 0 snap 32762 seq_num 4161716256) dirty exists 2022-04-28T19:53:05.368 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:05 smithi139 bash[47164]: debug 2022-04-28T19:53:05.017+0000 7f6c546c8200 -1 Falling back to public interface 2022-04-28T19:53:05.936 INFO:tasks.rados.rados.0.smithi059.stdout:2743: done (1 left) 2022-04-28T19:53:05.937 INFO:tasks.rados.rados.0.smithi059.stdout:2744: done (0 left) 2022-04-28T19:53:05.937 INFO:tasks.rados.rados.0.smithi059.stdout:2745: read oid 37 snap -1 2022-04-28T19:53:05.937 INFO:tasks.rados.rados.0.smithi059.stdout:2745: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:05.937 INFO:tasks.rados.rados.0.smithi059.stdout:2746: rollback oid 39 current snap is 257 2022-04-28T19:53:05.938 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 39 to 241 2022-04-28T19:53:05.938 INFO:tasks.rados.rados.0.smithi059.stdout:2747: read oid 18 snap -1 2022-04-28T19:53:05.938 INFO:tasks.rados.rados.0.smithi059.stdout:2747: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:53:05.938 INFO:tasks.rados.rados.0.smithi059.stdout:2748: read oid 41 snap -1 2022-04-28T19:53:05.939 INFO:tasks.rados.rados.0.smithi059.stdout:2748: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:05.939 INFO:tasks.rados.rados.0.smithi059.stdout:2749: snap_create 2022-04-28T19:53:05.945 INFO:tasks.rados.rados.0.smithi059.stdout:2746: finishing rollback tid 0 to smithi05955964-39 2022-04-28T19:53:05.946 INFO:tasks.rados.rados.0.smithi059.stdout:2746: finishing rollback tid 1 to smithi05955964-39 2022-04-28T19:53:05.947 INFO:tasks.rados.rados.0.smithi059.stdout:2748: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:05.978 INFO:tasks.rados.rados.0.smithi059.stdout:2746: finishing rollback tid 2 to smithi05955964-39 2022-04-28T19:53:05.979 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1276 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:53:05.979 INFO:tasks.rados.rados.0.smithi059.stdout:2745: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:06.021 INFO:tasks.rados.rados.0.smithi059.stdout:2747: expect (ObjNum 715 snap 207 seq_num 715) 2022-04-28T19:53:06.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:05 smithi059 bash[59745]: cluster 2022-04-28T19:53:04.923002+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2022-04-28T19:53:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:05 smithi059 bash[59745]: cluster 2022-04-28T19:53:04.941972+0000 mgr.y (mgr.24961) 212 : cluster [DBG] pgmap v334: 89 pgs: 13 active+undersized+degraded, 1 active+clean+snaptrim, 15 active+undersized, 60 active+clean; 512 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s; 112/858 objects degraded (13.054%) 2022-04-28T19:53:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:05 smithi059 bash[61152]: cluster 2022-04-28T19:53:04.923002+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2022-04-28T19:53:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:05 smithi059 bash[61152]: cluster 2022-04-28T19:53:04.941972+0000 mgr.y (mgr.24961) 212 : cluster [DBG] pgmap v334: 89 pgs: 13 active+undersized+degraded, 1 active+clean+snaptrim, 15 active+undersized, 60 active+clean; 512 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s; 112/858 objects degraded (13.054%) 2022-04-28T19:53:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:05 smithi139 bash[43301]: cluster 2022-04-28T19:53:04.923002+0000 mon.a (mon.0) 765 : cluster [DBG] osdmap e566: 8 total, 7 up, 8 in 2022-04-28T19:53:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:05 smithi139 bash[43301]: cluster 2022-04-28T19:53:04.941972+0000 mgr.y (mgr.24961) 212 : cluster [DBG] pgmap v334: 89 pgs: 13 active+undersized+degraded, 1 active+clean+snaptrim, 15 active+undersized, 60 active+clean; 512 MiB data, 1.0 GiB used, 711 GiB / 715 GiB avail; 5.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s; 112/858 objects degraded (13.054%) 2022-04-28T19:53:06.840 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[57771]: debug 2022-04-28T19:53:06.359+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:53:06.904 INFO:tasks.rados.rados.0.smithi059.stdout:2745: done (4 left) 2022-04-28T19:53:06.905 INFO:tasks.rados.rados.0.smithi059.stdout:2746: done (3 left) 2022-04-28T19:53:06.905 INFO:tasks.rados.rados.0.smithi059.stdout:2747: done (2 left) 2022-04-28T19:53:06.905 INFO:tasks.rados.rados.0.smithi059.stdout:2748: done (1 left) 2022-04-28T19:53:06.905 INFO:tasks.rados.rados.0.smithi059.stdout:2749: done (0 left) 2022-04-28T19:53:06.905 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 258 2022-04-28T19:53:06.905 INFO:tasks.rados.rados.0.smithi059.stdout:2750: seq_num 865 ranges {0=90112} 2022-04-28T19:53:06.906 INFO:tasks.rados.rados.0.smithi059.stdout:2750: writing smithi05955964-50 from 0 to 90112 tid 1 2022-04-28T19:53:06.906 INFO:tasks.rados.rados.0.smithi059.stdout:2751: read oid 34 snap -1 2022-04-28T19:53:06.906 INFO:tasks.rados.rados.0.smithi059.stdout:2751: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:53:06.906 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 258 2022-04-28T19:53:06.906 INFO:tasks.rados.rados.0.smithi059.stdout:2752: seq_num 866 ranges {3792896=106496} 2022-04-28T19:53:06.930 INFO:tasks.rados.rados.0.smithi059.stdout:2752: writing smithi05955964-18 from 3792896 to 3899392 tid 1 2022-04-28T19:53:06.930 INFO:tasks.rados.rados.0.smithi059.stdout:2753: read oid 9 snap -1 2022-04-28T19:53:06.930 INFO:tasks.rados.rados.0.smithi059.stdout:2753: expect (ObjNum 775 snap 233 seq_num 775) 2022-04-28T19:53:06.930 INFO:tasks.rados.rados.0.smithi059.stdout:2754: read oid 33 snap -1 2022-04-28T19:53:06.930 INFO:tasks.rados.rados.0.smithi059.stdout:2754: expect (ObjNum 845 snap 251 seq_num 845) 2022-04-28T19:53:06.931 INFO:tasks.rados.rados.0.smithi059.stdout:2755: snap_remove snap 250 2022-04-28T19:53:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: cluster 2022-04-28T19:53:05.917567+0000 mon.a (mon.0) 766 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2022-04-28T19:53:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: cluster 2022-04-28T19:53:05.933832+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2022-04-28T19:53:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.052064+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.340155+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.349527+0000 mon.a (mon.0) 770 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.353466+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.355509+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.357127+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: audit 2022-04-28T19:53:06.359566+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[59745]: cluster 2022-04-28T19:53:06.902073+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: cluster 2022-04-28T19:53:05.917567+0000 mon.a (mon.0) 766 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: cluster 2022-04-28T19:53:05.933832+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2022-04-28T19:53:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.052064+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.340155+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.349527+0000 mon.a (mon.0) 770 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.353466+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.355509+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.357127+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: audit 2022-04-28T19:53:06.359566+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:06 smithi059 bash[61152]: cluster 2022-04-28T19:53:06.902073+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2022-04-28T19:53:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: cluster 2022-04-28T19:53:05.917567+0000 mon.a (mon.0) 766 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2022-04-28T19:53:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: cluster 2022-04-28T19:53:05.933832+0000 mon.a (mon.0) 767 : cluster [DBG] osdmap e567: 8 total, 7 up, 8 in 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.052064+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.340155+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.349527+0000 mon.a (mon.0) 770 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.353466+0000 mon.a (mon.0) 771 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.355509+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.357127+0000 mon.a (mon.0) 773 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: audit 2022-04-28T19:53:06.359566+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:06 smithi139 bash[43301]: cluster 2022-04-28T19:53:06.902073+0000 mon.a (mon.0) 775 : cluster [DBG] osdmap e568: 8 total, 7 up, 8 in 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2755: done (5 left) 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2756: rollback oid 10 current snap is 258 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 10 to 257 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2750: finishing write tid 1 to smithi05955964-50 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2750: finishing write tid 3 to smithi05955964-50 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2750: finishing write tid 4 to smithi05955964-50 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1279 (ObjNum 865 snap 258 seq_num 865) dirty exists 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2750: left oid 50 (ObjNum 865 snap 258 seq_num 865) 2022-04-28T19:53:07.947 INFO:tasks.rados.rados.0.smithi059.stdout:2754: expect (ObjNum 845 snap 251 seq_num 845) 2022-04-28T19:53:08.136 INFO:tasks.rados.rados.0.smithi059.stdout:2751: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:53:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:07 smithi059 bash[61152]: audit 2022-04-28T19:53:06.359935+0000 mgr.y (mgr.24961) 213 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:07 smithi059 bash[61152]: cephadm 2022-04-28T19:53:06.361402+0000 mgr.y (mgr.24961) 214 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:53:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:07 smithi059 bash[61152]: cluster 2022-04-28T19:53:06.942639+0000 mgr.y (mgr.24961) 215 : cluster [DBG] pgmap v337: 89 pgs: 13 active+undersized+degraded, 15 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 2.5 MiB/s wr, 15 op/s; 114/870 objects degraded (13.103%) 2022-04-28T19:53:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:07 smithi059 bash[59745]: audit 2022-04-28T19:53:06.359935+0000 mgr.y (mgr.24961) 213 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:07 smithi059 bash[59745]: cephadm 2022-04-28T19:53:06.361402+0000 mgr.y (mgr.24961) 214 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:53:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:07 smithi059 bash[59745]: cluster 2022-04-28T19:53:06.942639+0000 mgr.y (mgr.24961) 215 : cluster [DBG] pgmap v337: 89 pgs: 13 active+undersized+degraded, 15 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 2.5 MiB/s wr, 15 op/s; 114/870 objects degraded (13.103%) 2022-04-28T19:53:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:07 smithi139 bash[43301]: audit 2022-04-28T19:53:06.359935+0000 mgr.y (mgr.24961) 213 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:07 smithi139 bash[43301]: cephadm 2022-04-28T19:53:06.361402+0000 mgr.y (mgr.24961) 214 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2022-04-28T19:53:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:07 smithi139 bash[43301]: cluster 2022-04-28T19:53:06.942639+0000 mgr.y (mgr.24961) 215 : cluster [DBG] pgmap v337: 89 pgs: 13 active+undersized+degraded, 15 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 2.5 MiB/s wr, 15 op/s; 114/870 objects degraded (13.103%) 2022-04-28T19:53:08.868 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:08 smithi139 bash[47164]: debug 2022-04-28T19:53:08.481+0000 7f6c546c8200 -1 osd.5 559 log_to_monitors {default=true} 2022-04-28T19:53:08.972 INFO:tasks.rados.rados.0.smithi059.stdout:2752: finishing write tid 1 to smithi05955964-18 2022-04-28T19:53:08.972 INFO:tasks.rados.rados.0.smithi059.stdout:2752: finishing write tid 3 to smithi05955964-18 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:2750: done (5 left) 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:2751: done (4 left) 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:2754: done (3 left) 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:2757: setattr oid 46 current snap is 258 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:2752: finishing write tid 4 to smithi05955964-18 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1218 (ObjNum 866 snap 258 seq_num 866) dirty exists 2022-04-28T19:53:08.973 INFO:tasks.rados.rados.0.smithi059.stdout:2752: left oid 18 (ObjNum 866 snap 258 seq_num 866) 2022-04-28T19:53:08.974 INFO:tasks.rados.rados.0.smithi059.stdout:2753: expect (ObjNum 775 snap 233 seq_num 775) 2022-04-28T19:53:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[61152]: cluster 2022-04-28T19:53:07.929555+0000 mon.a (mon.0) 776 : cluster [WRN] Health check update: Degraded data redundancy: 114/870 objects degraded (13.103%), 13 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[61152]: cluster 2022-04-28T19:53:07.946205+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2022-04-28T19:53:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[61152]: audit 2022-04-28T19:53:08.488130+0000 mon.b (mon.2) 5 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:53:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[61152]: audit 2022-04-28T19:53:08.490445+0000 mon.a (mon.0) 778 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:53:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[59745]: cluster 2022-04-28T19:53:07.929555+0000 mon.a (mon.0) 776 : cluster [WRN] Health check update: Degraded data redundancy: 114/870 objects degraded (13.103%), 13 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[59745]: cluster 2022-04-28T19:53:07.946205+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2022-04-28T19:53:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[59745]: audit 2022-04-28T19:53:08.488130+0000 mon.b (mon.2) 5 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:53:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:08 smithi059 bash[59745]: audit 2022-04-28T19:53:08.490445+0000 mon.a (mon.0) 778 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:53:09.368 INFO:journalctl@ceph.osd.5.smithi139.stdout:Apr 28 19:53:08 smithi139 bash[47164]: debug 2022-04-28T19:53:08.965+0000 7f6c45de2700 -1 osd.5 559 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:53:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:08 smithi139 bash[43301]: cluster 2022-04-28T19:53:07.929555+0000 mon.a (mon.0) 776 : cluster [WRN] Health check update: Degraded data redundancy: 114/870 objects degraded (13.103%), 13 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:08 smithi139 bash[43301]: cluster 2022-04-28T19:53:07.946205+0000 mon.a (mon.0) 777 : cluster [DBG] osdmap e569: 8 total, 7 up, 8 in 2022-04-28T19:53:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:08 smithi139 bash[43301]: audit 2022-04-28T19:53:08.488130+0000 mon.b (mon.2) 5 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:53:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:08 smithi139 bash[43301]: audit 2022-04-28T19:53:08.490445+0000 mon.a (mon.0) 778 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2022-04-28T19:53:09.808 INFO:tasks.rados.rados.0.smithi059.stdout:2756: finishing rollback tid 0 to smithi05955964-10 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2756: finishing rollback tid 1 to smithi05955964-10 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2756: finishing rollback tid 2 to smithi05955964-10 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1126 (ObjNum 817 snap 240 seq_num 817) dirty exists 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2752: done (3 left) 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2753: done (2 left) 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2756: done (1 left) 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2758: copy_from oid 31 from oid 29 current snap is 258 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 258 2022-04-28T19:53:09.809 INFO:tasks.rados.rados.0.smithi059.stdout:2759: seq_num 867 ranges {0=122880} 2022-04-28T19:53:09.810 INFO:tasks.rados.rados.0.smithi059.stdout:2759: writing smithi05955964-30 from 0 to 122880 tid 1 2022-04-28T19:53:09.810 INFO:tasks.rados.rados.0.smithi059.stdout:2760: copy_from oid 22 from oid 7 current snap is 258 2022-04-28T19:53:09.810 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 258 2022-04-28T19:53:09.810 INFO:tasks.rados.rados.0.smithi059.stdout:2761: seq_num 868 ranges {7340032=139264} 2022-04-28T19:53:09.857 INFO:tasks.rados.rados.0.smithi059.stdout:2761: writing smithi05955964-10 from 7340032 to 7479296 tid 1 2022-04-28T19:53:09.857 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 258 2022-04-28T19:53:09.857 INFO:tasks.rados.rados.0.smithi059.stdout:2762: seq_num 869 ranges {4661248=155648} 2022-04-28T19:53:09.884 INFO:tasks.rados.rados.0.smithi059.stdout:2762: writing smithi05955964-27 from 4661248 to 4816896 tid 1 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2763: read oid 24 snap -1 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2763: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2764: delete oid 9 current snap is 258 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2760: finishing copy_from to smithi05955964-22 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2760: got expected ENOENT (src dne) 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2760: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2759: finishing write tid 1 to smithi05955964-30 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2759: finishing write tid 3 to smithi05955964-30 2022-04-28T19:53:09.885 INFO:tasks.rados.rados.0.smithi059.stdout:2759: finishing write tid 4 to smithi05955964-30 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1220 (ObjNum 867 snap 258 seq_num 867) dirty exists 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:2759: left oid 30 (ObjNum 867 snap 258 seq_num 867) 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1316 (ObjNum 0 snap 32762 seq_num 4161990800) dirty exists 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:2758: finishing copy_from to smithi05955964-31 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1309 (ObjNum 859 snap 253 seq_num 859) dirty exists 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:2758: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:2761: finishing write tid 1 to smithi05955964-10 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:2761: finishing write tid 3 to smithi05955964-10 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:2761: finishing write tid 4 to smithi05955964-10 2022-04-28T19:53:09.886 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1128 (ObjNum 868 snap 258 seq_num 868) dirty exists 2022-04-28T19:53:09.887 INFO:tasks.rados.rados.0.smithi059.stdout:2761: left oid 10 (ObjNum 868 snap 258 seq_num 868) 2022-04-28T19:53:09.889 INFO:tasks.rados.rados.0.smithi059.stdout:2762: finishing write tid 1 to smithi05955964-27 2022-04-28T19:53:09.890 INFO:tasks.rados.rados.0.smithi059.stdout:2762: finishing write tid 3 to smithi05955964-27 2022-04-28T19:53:09.890 INFO:tasks.rados.rados.0.smithi059.stdout:2757: done (7 left) 2022-04-28T19:53:09.890 INFO:tasks.rados.rados.0.smithi059.stdout:2758: done (6 left) 2022-04-28T19:53:09.890 INFO:tasks.rados.rados.0.smithi059.stdout:2759: done (5 left) 2022-04-28T19:53:09.890 INFO:tasks.rados.rados.0.smithi059.stdout:2760: done (4 left) 2022-04-28T19:53:09.891 INFO:tasks.rados.rados.0.smithi059.stdout:2761: done (3 left) 2022-04-28T19:53:09.891 INFO:tasks.rados.rados.0.smithi059.stdout:2764: done (2 left) 2022-04-28T19:53:09.891 INFO:tasks.rados.rados.0.smithi059.stdout:2765: read oid 37 snap -1 2022-04-28T19:53:09.891 INFO:tasks.rados.rados.0.smithi059.stdout:2765: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:09.892 INFO:tasks.rados.rados.0.smithi059.stdout:2766: rmattr oid 32 current snap is 258 2022-04-28T19:53:09.892 INFO:tasks.rados.rados.0.smithi059.stdout:2767: snap_create 2022-04-28T19:53:09.893 INFO:tasks.rados.rados.0.smithi059.stdout:2762: finishing write tid 4 to smithi05955964-27 2022-04-28T19:53:09.893 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1131 (ObjNum 869 snap 258 seq_num 869) dirty exists 2022-04-28T19:53:09.893 INFO:tasks.rados.rados.0.smithi059.stdout:2762: left oid 27 (ObjNum 869 snap 258 seq_num 869) 2022-04-28T19:53:09.893 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1281 (ObjNum 819 snap 240 seq_num 819) dirty exists 2022-04-28T19:53:09.898 INFO:tasks.rados.rados.0.smithi059.stdout:2765: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:09.942 INFO:tasks.rados.rados.0.smithi059.stdout:2763: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:10.210 INFO:tasks.rados.rados.0.smithi059.stdout:2762: done (4 left) 2022-04-28T19:53:10.211 INFO:tasks.rados.rados.0.smithi059.stdout:2763: done (3 left) 2022-04-28T19:53:10.211 INFO:tasks.rados.rados.0.smithi059.stdout:2765: done (2 left) 2022-04-28T19:53:10.214 INFO:tasks.rados.rados.0.smithi059.stdout:2766: done (1 left) 2022-04-28T19:53:10.214 INFO:tasks.rados.rados.0.smithi059.stdout:2767: done (0 left) 2022-04-28T19:53:10.215 INFO:tasks.rados.rados.0.smithi059.stdout:2768: snap_create 2022-04-28T19:53:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[59745]: cluster 2022-04-28T19:53:08.943161+0000 mgr.y (mgr.24961) 216 : cluster [DBG] pgmap v340: 89 pgs: 13 active+undersized+degraded, 15 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 116/876 objects degraded (13.242%) 2022-04-28T19:53:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[59745]: audit 2022-04-28T19:53:08.946721+0000 mon.a (mon.0) 779 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-04-28T19:53:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[59745]: cluster 2022-04-28T19:53:08.946862+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2022-04-28T19:53:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[59745]: audit 2022-04-28T19:53:08.947779+0000 mon.b (mon.2) 6 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[59745]: audit 2022-04-28T19:53:08.954442+0000 mon.a (mon.0) 781 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[61152]: cluster 2022-04-28T19:53:08.943161+0000 mgr.y (mgr.24961) 216 : cluster [DBG] pgmap v340: 89 pgs: 13 active+undersized+degraded, 15 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 116/876 objects degraded (13.242%) 2022-04-28T19:53:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[61152]: audit 2022-04-28T19:53:08.946721+0000 mon.a (mon.0) 779 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-04-28T19:53:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[61152]: cluster 2022-04-28T19:53:08.946862+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2022-04-28T19:53:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[61152]: audit 2022-04-28T19:53:08.947779+0000 mon.b (mon.2) 6 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:09 smithi059 bash[61152]: audit 2022-04-28T19:53:08.954442+0000 mon.a (mon.0) 781 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:09 smithi139 bash[43301]: cluster 2022-04-28T19:53:08.943161+0000 mgr.y (mgr.24961) 216 : cluster [DBG] pgmap v340: 89 pgs: 13 active+undersized+degraded, 15 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 1.4 MiB/s wr, 13 op/s; 116/876 objects degraded (13.242%) 2022-04-28T19:53:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:09 smithi139 bash[43301]: audit 2022-04-28T19:53:08.946721+0000 mon.a (mon.0) 779 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2022-04-28T19:53:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:09 smithi139 bash[43301]: cluster 2022-04-28T19:53:08.946862+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e570: 8 total, 7 up, 8 in 2022-04-28T19:53:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:09 smithi139 bash[43301]: audit 2022-04-28T19:53:08.947779+0000 mon.b (mon.2) 6 : audit [INF] from='osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:09 smithi139 bash[43301]: audit 2022-04-28T19:53:08.954442+0000 mon.a (mon.0) 781 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:10.966 INFO:tasks.rados.rados.0.smithi059.stdout:2768: done (0 left) 2022-04-28T19:53:10.966 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 260 2022-04-28T19:53:10.966 INFO:tasks.rados.rados.0.smithi059.stdout:2769: seq_num 871 ranges {1777664=188416} 2022-04-28T19:53:10.979 INFO:tasks.rados.rados.0.smithi059.stdout:2769: writing smithi05955964-15 from 1777664 to 1966080 tid 1 2022-04-28T19:53:10.979 INFO:tasks.rados.rados.0.smithi059.stdout:2770: rollback oid 3 current snap is 260 2022-04-28T19:53:10.979 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 236 2022-04-28T19:53:10.979 INFO:tasks.rados.rados.0.smithi059.stdout:2771: read oid 50 snap 242 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2771: expect (ObjNum -132911664 snap 21893 seq_num 4161349616) 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2772: read oid 24 snap 258 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2772: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2773: read oid 43 snap -1 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2773: expect (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2774: read oid 33 snap 259 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2774: expect (ObjNum 845 snap 251 seq_num 845) 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2775: read oid 39 snap -1 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:2775: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:53:10.980 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 260 2022-04-28T19:53:10.981 INFO:tasks.rados.rados.0.smithi059.stdout:2776: seq_num 872 ranges {5554176=212992} 2022-04-28T19:53:11.015 INFO:tasks.rados.rados.0.smithi059.stdout:2776: writing smithi05955964-48 from 5554176 to 5767168 tid 1 2022-04-28T19:53:11.015 INFO:tasks.rados.rados.0.smithi059.stdout:2777: setattr oid 46 current snap is 260 2022-04-28T19:53:11.015 INFO:tasks.rados.rados.0.smithi059.stdout:2771: expect (ObjNum -132911664 snap 21893 seq_num 4161349616) 2022-04-28T19:53:11.017 INFO:tasks.rados.rados.0.smithi059.stdout:2775: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:53:11.023 INFO:tasks.rados.rados.0.smithi059.stdout:2770: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:53:11.023 INFO:tasks.rados.rados.0.smithi059.stdout:2769: finishing write tid 1 to smithi05955964-15 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:2770: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:2770: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 980 (ObjNum 709 snap 203 seq_num 709) dirty exists 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:2769: finishing write tid 3 to smithi05955964-15 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:2769: finishing write tid 4 to smithi05955964-15 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1311 (ObjNum 871 snap 260 seq_num 871) dirty exists 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:2769: left oid 15 (ObjNum 871 snap 260 seq_num 871) 2022-04-28T19:53:11.024 INFO:tasks.rados.rados.0.smithi059.stdout:2773: expect (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:53:11.046 INFO:tasks.rados.rados.0.smithi059.stdout:2772: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:11.305 INFO:tasks.rados.rados.0.smithi059.stdout:2774: expect (ObjNum 845 snap 251 seq_num 845) 2022-04-28T19:53:11.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[61152]: cluster 2022-04-28T19:53:09.951643+0000 mon.a (mon.0) 782 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:53:11.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[61152]: cluster 2022-04-28T19:53:09.957751+0000 mon.a (mon.0) 783 : cluster [INF] osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567] boot 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[61152]: cluster 2022-04-28T19:53:09.957890+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[61152]: audit 2022-04-28T19:53:09.962284+0000 mon.a (mon.0) 785 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[61152]: cluster 2022-04-28T19:53:10.961256+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[59745]: cluster 2022-04-28T19:53:09.951643+0000 mon.a (mon.0) 782 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[59745]: cluster 2022-04-28T19:53:09.957751+0000 mon.a (mon.0) 783 : cluster [INF] osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567] boot 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[59745]: cluster 2022-04-28T19:53:09.957890+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2022-04-28T19:53:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[59745]: audit 2022-04-28T19:53:09.962284+0000 mon.a (mon.0) 785 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:53:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:10 smithi059 bash[59745]: cluster 2022-04-28T19:53:10.961256+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2022-04-28T19:53:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:10 smithi139 bash[43301]: cluster 2022-04-28T19:53:09.951643+0000 mon.a (mon.0) 782 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:53:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:10 smithi139 bash[43301]: cluster 2022-04-28T19:53:09.957751+0000 mon.a (mon.0) 783 : cluster [INF] osd.5 [v2:172.21.15.139:6808/2702638567,v1:172.21.15.139:6809/2702638567] boot 2022-04-28T19:53:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:10 smithi139 bash[43301]: cluster 2022-04-28T19:53:09.957890+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2022-04-28T19:53:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:10 smithi139 bash[43301]: audit 2022-04-28T19:53:09.962284+0000 mon.a (mon.0) 785 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2022-04-28T19:53:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:10 smithi139 bash[43301]: cluster 2022-04-28T19:53:10.961256+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2022-04-28T19:53:11.486 INFO:tasks.rados.rados.0.smithi059.stdout:2776: finishing write tid 1 to smithi05955964-48 2022-04-28T19:53:11.486 INFO:tasks.rados.rados.0.smithi059.stdout:2776: finishing write tid 3 to smithi05955964-48 2022-04-28T19:53:11.486 INFO:tasks.rados.rados.0.smithi059.stdout:2776: finishing write tid 4 to smithi05955964-48 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1145 (ObjNum 872 snap 260 seq_num 872) dirty exists 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2776: left oid 48 (ObjNum 872 snap 260 seq_num 872) 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1318 (ObjNum 188341744 snap 32762 seq_num 188341744) dirty exists 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2769: done (8 left) 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2770: done (7 left) 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2771: done (6 left) 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2772: done (5 left) 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2773: done (4 left) 2022-04-28T19:53:11.487 INFO:tasks.rados.rados.0.smithi059.stdout:2774: done (3 left) 2022-04-28T19:53:11.488 INFO:tasks.rados.rados.0.smithi059.stdout:2775: done (2 left) 2022-04-28T19:53:11.488 INFO:tasks.rados.rados.0.smithi059.stdout:2776: done (1 left) 2022-04-28T19:53:11.488 INFO:tasks.rados.rados.0.smithi059.stdout:2777: done (0 left) 2022-04-28T19:53:11.488 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 260 2022-04-28T19:53:11.488 INFO:tasks.rados.rados.0.smithi059.stdout:2778: seq_num 873 ranges {6938624=229376} 2022-04-28T19:53:11.531 INFO:tasks.rados.rados.0.smithi059.stdout:2778: writing smithi05955964-45 from 6938624 to 7168000 tid 1 2022-04-28T19:53:11.532 INFO:tasks.rados.rados.0.smithi059.stdout:2779: read oid 31 snap 255 2022-04-28T19:53:11.532 INFO:tasks.rados.rados.0.smithi059.stdout:2779: expect (ObjNum -132988912 snap 21893 seq_num 4161525408) 2022-04-28T19:53:11.532 INFO:tasks.rados.rados.0.smithi059.stdout:2780: snap_create 2022-04-28T19:53:11.532 INFO:tasks.rados.rados.0.smithi059.stdout:2779: expect (ObjNum -132988912 snap 21893 seq_num 4161525408) 2022-04-28T19:53:11.538 INFO:tasks.rados.rados.0.smithi059.stdout:2778: finishing write tid 1 to smithi05955964-45 2022-04-28T19:53:11.539 INFO:tasks.rados.rados.0.smithi059.stdout:2778: finishing write tid 3 to smithi05955964-45 2022-04-28T19:53:11.539 INFO:tasks.rados.rados.0.smithi059.stdout:2778: finishing write tid 4 to smithi05955964-45 2022-04-28T19:53:11.539 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 983 (ObjNum 873 snap 260 seq_num 873) dirty exists 2022-04-28T19:53:11.539 INFO:tasks.rados.rados.0.smithi059.stdout:2778: left oid 45 (ObjNum 873 snap 260 seq_num 873) 2022-04-28T19:53:11.989 INFO:tasks.rados.rados.0.smithi059.stdout:2778: done (2 left) 2022-04-28T19:53:12.049 INFO:tasks.rados.rados.0.smithi059.stdout:2779: done (1 left) 2022-04-28T19:53:12.049 INFO:tasks.rados.rados.0.smithi059.stdout:2780: done (0 left) 2022-04-28T19:53:12.049 INFO:tasks.rados.rados.0.smithi059.stdout:2781: snap_remove snap 251 2022-04-28T19:53:12.050 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:11.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:12.052 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:11.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:12.052 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:11.633Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:12.052 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:11 smithi059 bash[59745]: cluster 2022-04-28T19:53:10.944321+0000 mgr.y (mgr.24961) 217 : cluster [DBG] pgmap v342: 89 pgs: 4 peering, 13 active+undersized+degraded, 11 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 93 KiB/s wr, 16 op/s; 117/885 objects degraded (13.220%) 2022-04-28T19:53:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:11 smithi059 bash[61152]: cluster 2022-04-28T19:53:10.944321+0000 mgr.y (mgr.24961) 217 : cluster [DBG] pgmap v342: 89 pgs: 4 peering, 13 active+undersized+degraded, 11 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 93 KiB/s wr, 16 op/s; 117/885 objects degraded (13.220%) 2022-04-28T19:53:12.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:11 smithi139 bash[43301]: cluster 2022-04-28T19:53:10.944321+0000 mgr.y (mgr.24961) 217 : cluster [DBG] pgmap v342: 89 pgs: 4 peering, 13 active+undersized+degraded, 11 active+undersized, 61 active+clean; 518 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 93 KiB/s wr, 16 op/s; 117/885 objects degraded (13.220%) 2022-04-28T19:53:13.006 INFO:tasks.rados.rados.0.smithi059.stdout:2781: done (0 left) 2022-04-28T19:53:13.007 INFO:tasks.rados.rados.0.smithi059.stdout:2782: read oid 1 snap -1 2022-04-28T19:53:13.007 INFO:tasks.rados.rados.0.smithi059.stdout:2782: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:53:13.007 INFO:tasks.rados.rados.0.smithi059.stdout:2783: copy_from oid 12 from oid 10 current snap is 261 2022-04-28T19:53:13.007 INFO:tasks.rados.rados.0.smithi059.stdout:2784: read oid 45 snap -1 2022-04-28T19:53:13.008 INFO:tasks.rados.rados.0.smithi059.stdout:2784: expect (ObjNum 873 snap 260 seq_num 873) 2022-04-28T19:53:13.008 INFO:tasks.rados.rados.0.smithi059.stdout:2785: copy_from oid 28 from oid 16 current snap is 261 2022-04-28T19:53:13.008 INFO:tasks.rados.rados.0.smithi059.stdout:2786: read oid 2 snap 260 2022-04-28T19:53:13.008 INFO:tasks.rados.rados.0.smithi059.stdout:2786: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:53:13.009 INFO:tasks.rados.rados.0.smithi059.stdout:2787: snap_create 2022-04-28T19:53:13.030 INFO:tasks.rados.rados.0.smithi059.stdout:2782: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:53:13.093 INFO:tasks.rados.rados.0.smithi059.stdout:2786: expect (ObjNum 719 snap 209 seq_num 719) 2022-04-28T19:53:13.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:13 smithi059 bash[61152]: cluster 2022-04-28T19:53:11.986572+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2022-04-28T19:53:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:12 smithi059 bash[59745]: cluster 2022-04-28T19:53:11.986572+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2022-04-28T19:53:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:12 smithi139 bash[43301]: cluster 2022-04-28T19:53:11.986572+0000 mon.a (mon.0) 787 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2022-04-28T19:53:13.508 INFO:tasks.rados.rados.0.smithi059.stdout:2785: finishing copy_from to smithi05955964-28 2022-04-28T19:53:13.509 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 982 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:53:13.509 INFO:tasks.rados.rados.0.smithi059.stdout:2785: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:53:13.509 INFO:tasks.rados.rados.0.smithi059.stdout:2784: expect (ObjNum 873 snap 260 seq_num 873) 2022-04-28T19:53:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:13.527Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:13.527Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:13.527Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:53:14.101 INFO:tasks.rados.rados.0.smithi059.stdout:2783: finishing copy_from to smithi05955964-12 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1128 (ObjNum 868 snap 258 seq_num 868) dirty exists 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2783: finishing copy_from racing read to smithi05955964-12 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2782: done (5 left) 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2783: done (4 left) 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2784: done (3 left) 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2785: done (2 left) 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2786: done (1 left) 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2787: done (0 left) 2022-04-28T19:53:14.102 INFO:tasks.rados.rados.0.smithi059.stdout:2788: setattr oid 18 current snap is 262 2022-04-28T19:53:14.104 INFO:tasks.rados.rados.0.smithi059.stdout:2789: copy_from oid 28 from oid 41 current snap is 262 2022-04-28T19:53:14.104 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 262 2022-04-28T19:53:14.104 INFO:tasks.rados.rados.0.smithi059.stdout:2790: seq_num 874 ranges {7569408=245760} 2022-04-28T19:53:14.155 INFO:tasks.rados.rados.0.smithi059.stdout:2790: writing smithi05955964-3 from 7569408 to 7815168 tid 1 2022-04-28T19:53:14.156 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 262 2022-04-28T19:53:14.156 INFO:tasks.rados.rados.0.smithi059.stdout:2791: seq_num 875 ranges {2383872=262144} 2022-04-28T19:53:14.170 INFO:tasks.rados.rados.0.smithi059.stdout:2791: writing smithi05955964-5 from 2383872 to 2646016 tid 1 2022-04-28T19:53:14.170 INFO:tasks.rados.rados.0.smithi059.stdout:2792: snap_remove snap 256 2022-04-28T19:53:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:14 smithi059 bash[61152]: cluster 2022-04-28T19:53:12.945588+0000 mgr.y (mgr.24961) 218 : cluster [DBG] pgmap v345: 89 pgs: 7 peering, 8 active+undersized+degraded, 8 active+undersized, 66 active+clean; 518 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 72 KiB/s wr, 27 op/s; 83/891 objects degraded (9.315%); 9.7 MiB/s, 3 objects/s recovering 2022-04-28T19:53:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:14 smithi059 bash[61152]: cluster 2022-04-28T19:53:12.989560+0000 mon.a (mon.0) 788 : cluster [WRN] Health check update: Degraded data redundancy: 83/891 objects degraded (9.315%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:14 smithi059 bash[61152]: cluster 2022-04-28T19:53:13.002600+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2022-04-28T19:53:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:14 smithi059 bash[59745]: cluster 2022-04-28T19:53:12.945588+0000 mgr.y (mgr.24961) 218 : cluster [DBG] pgmap v345: 89 pgs: 7 peering, 8 active+undersized+degraded, 8 active+undersized, 66 active+clean; 518 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 72 KiB/s wr, 27 op/s; 83/891 objects degraded (9.315%); 9.7 MiB/s, 3 objects/s recovering 2022-04-28T19:53:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:14 smithi059 bash[59745]: cluster 2022-04-28T19:53:12.989560+0000 mon.a (mon.0) 788 : cluster [WRN] Health check update: Degraded data redundancy: 83/891 objects degraded (9.315%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:14 smithi059 bash[59745]: cluster 2022-04-28T19:53:13.002600+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2022-04-28T19:53:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:14 smithi139 bash[43301]: cluster 2022-04-28T19:53:12.945588+0000 mgr.y (mgr.24961) 218 : cluster [DBG] pgmap v345: 89 pgs: 7 peering, 8 active+undersized+degraded, 8 active+undersized, 66 active+clean; 518 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 72 KiB/s wr, 27 op/s; 83/891 objects degraded (9.315%); 9.7 MiB/s, 3 objects/s recovering 2022-04-28T19:53:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:14 smithi139 bash[43301]: cluster 2022-04-28T19:53:12.989560+0000 mon.a (mon.0) 788 : cluster [WRN] Health check update: Degraded data redundancy: 83/891 objects degraded (9.315%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:14 smithi139 bash[43301]: cluster 2022-04-28T19:53:13.002600+0000 mon.a (mon.0) 789 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2022-04-28T19:53:15.030 INFO:tasks.rados.rados.0.smithi059.stdout:2792: done (4 left) 2022-04-28T19:53:15.030 INFO:tasks.rados.rados.0.smithi059.stdout:2793: rollback oid 20 current snap is 262 2022-04-28T19:53:15.030 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 246 2022-04-28T19:53:15.030 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1221 (ObjNum 866 snap 258 seq_num 866) dirty exists 2022-04-28T19:53:15.030 INFO:tasks.rados.rados.0.smithi059.stdout:2789: finishing copy_from to smithi05955964-28 2022-04-28T19:53:15.030 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1015 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2789: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2790: finishing write tid 1 to smithi05955964-3 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2790: finishing write tid 3 to smithi05955964-3 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2790: finishing write tid 4 to smithi05955964-3 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 986 (ObjNum 874 snap 262 seq_num 874) dirty exists 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2790: left oid 3 (ObjNum 874 snap 262 seq_num 874) 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2791: finishing write tid 1 to smithi05955964-5 2022-04-28T19:53:15.031 INFO:tasks.rados.rados.0.smithi059.stdout:2788: done (4 left) 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:2789: done (3 left) 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:2790: done (2 left) 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:2794: setattr oid 43 current snap is 262 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:2791: finishing write tid 3 to smithi05955964-5 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:2791: finishing write tid 4 to smithi05955964-5 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1325 (ObjNum 875 snap 262 seq_num 875) dirty exists 2022-04-28T19:53:15.032 INFO:tasks.rados.rados.0.smithi059.stdout:2791: left oid 5 (ObjNum 875 snap 262 seq_num 875) 2022-04-28T19:53:15.033 INFO:tasks.rados.rados.0.smithi059.stdout:2791: done (2 left) 2022-04-28T19:53:15.033 INFO:tasks.rados.rados.0.smithi059.stdout:2795: copy_from oid 45 from oid 34 current snap is 262 2022-04-28T19:53:15.033 INFO:tasks.rados.rados.0.smithi059.stdout:2796: read oid 15 snap -1 2022-04-28T19:53:15.034 INFO:tasks.rados.rados.0.smithi059.stdout:2796: expect (ObjNum 871 snap 260 seq_num 871) 2022-04-28T19:53:15.034 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 262 2022-04-28T19:53:15.034 INFO:tasks.rados.rados.0.smithi059.stdout:2797: seq_num 876 ranges {2056192=278528} 2022-04-28T19:53:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:2797: writing smithi05955964-13 from 2056192 to 2334720 tid 1 2022-04-28T19:53:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:2798: rollback oid 18 current snap is 262 2022-04-28T19:53:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 259 2022-04-28T19:53:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:2793: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:53:15.045 INFO:tasks.rados.rados.0.smithi059.stdout:2793: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:53:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:2793: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:53:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 990 (ObjNum 821 snap 240 seq_num 821) dirty exists 2022-04-28T19:53:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:2793: done (5 left) 2022-04-28T19:53:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:2799: read oid 35 snap -1 2022-04-28T19:53:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:2799: expect (ObjNum 854 snap 253 seq_num 854) 2022-04-28T19:53:15.046 INFO:tasks.rados.rados.0.smithi059.stdout:2800: delete oid 39 current snap is 262 2022-04-28T19:53:15.048 INFO:tasks.rados.rados.0.smithi059.stdout:2796: expect (ObjNum 871 snap 260 seq_num 871) 2022-04-28T19:53:15.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1299 (ObjNum 768 snap 230 seq_num 768) dirty exists 2022-04-28T19:53:15.135 INFO:tasks.rados.rados.0.smithi059.stdout:2797: finishing write tid 1 to smithi05955964-13 2022-04-28T19:53:15.135 INFO:tasks.rados.rados.0.smithi059.stdout:2797: finishing write tid 3 to smithi05955964-13 2022-04-28T19:53:15.135 INFO:tasks.rados.rados.0.smithi059.stdout:2797: finishing write tid 4 to smithi05955964-13 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1050 (ObjNum 876 snap 262 seq_num 876) dirty exists 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2797: left oid 13 (ObjNum 876 snap 262 seq_num 876) 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2798: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2798: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2798: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1224 (ObjNum 866 snap 258 seq_num 866) dirty exists 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2794: done (6 left) 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2796: done (5 left) 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2797: done (4 left) 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2798: done (3 left) 2022-04-28T19:53:15.136 INFO:tasks.rados.rados.0.smithi059.stdout:2800: done (2 left) 2022-04-28T19:53:15.137 INFO:tasks.rados.rados.0.smithi059.stdout:2801: read oid 44 snap -1 2022-04-28T19:53:15.137 INFO:tasks.rados.rados.0.smithi059.stdout:2801: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:53:15.137 INFO:tasks.rados.rados.0.smithi059.stdout:2802: rollback oid 13 current snap is 262 2022-04-28T19:53:15.137 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 13 to 258 2022-04-28T19:53:15.137 INFO:tasks.rados.rados.0.smithi059.stdout:2799: expect (ObjNum 854 snap 253 seq_num 854) 2022-04-28T19:53:15.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:15 smithi059 bash[61152]: cluster 2022-04-28T19:53:14.009429+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2022-04-28T19:53:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:15 smithi059 bash[59745]: cluster 2022-04-28T19:53:14.009429+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2022-04-28T19:53:15.351 INFO:tasks.rados.rados.0.smithi059.stdout:2795: finishing copy_from to smithi05955964-45 2022-04-28T19:53:15.351 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1215 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:53:15.351 INFO:tasks.rados.rados.0.smithi059.stdout:2795: finishing copy_from racing read to smithi05955964-45 2022-04-28T19:53:15.351 INFO:tasks.rados.rados.0.smithi059.stdout:2795: done (3 left) 2022-04-28T19:53:15.351 INFO:tasks.rados.rados.0.smithi059.stdout:2799: done (2 left) 2022-04-28T19:53:15.352 INFO:tasks.rados.rados.0.smithi059.stdout:2803: read oid 1 snap -1 2022-04-28T19:53:15.352 INFO:tasks.rados.rados.0.smithi059.stdout:2803: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:53:15.352 INFO:tasks.rados.rados.0.smithi059.stdout:2804: rmattr oid 23 current snap is 262 2022-04-28T19:53:15.352 INFO:tasks.rados.rados.0.smithi059.stdout:2801: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:53:15.355 INFO:tasks.rados.rados.0.smithi059.stdout:2802: finishing rollback tid 0 to smithi05955964-13 2022-04-28T19:53:15.355 INFO:tasks.rados.rados.0.smithi059.stdout:2802: finishing rollback tid 1 to smithi05955964-13 2022-04-28T19:53:15.355 INFO:tasks.rados.rados.0.smithi059.stdout:2802: finishing rollback tid 2 to smithi05955964-13 2022-04-28T19:53:15.355 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1053 (ObjNum 792 snap 235 seq_num 792) dirty exists 2022-04-28T19:53:15.355 INFO:tasks.rados.rados.0.smithi059.stdout:2801: done (3 left) 2022-04-28T19:53:15.355 INFO:tasks.rados.rados.0.smithi059.stdout:2802: done (2 left) 2022-04-28T19:53:15.356 INFO:tasks.rados.rados.0.smithi059.stdout:2805: rmattr oid 30 current snap is 262 2022-04-28T19:53:15.356 INFO:tasks.rados.rados.0.smithi059.stdout:2806: copy_from oid 13 from oid 42 current snap is 262 2022-04-28T19:53:15.356 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 262 2022-04-28T19:53:15.356 INFO:tasks.rados.rados.0.smithi059.stdout:2807: seq_num 878 ranges {5554176=311296} 2022-04-28T19:53:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:15 smithi139 bash[43301]: cluster 2022-04-28T19:53:14.009429+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2022-04-28T19:53:15.387 INFO:tasks.rados.rados.0.smithi059.stdout:2807: writing smithi05955964-21 from 5554176 to 5865472 tid 1 2022-04-28T19:53:15.387 INFO:tasks.rados.rados.0.smithi059.stdout:2808: read oid 36 snap -1 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:2808: expect (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:2809: setattr oid 33 current snap is 262 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1133 (ObjNum 600 snap 169 seq_num 600) dirty exists 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:2806: finishing copy_from to smithi05955964-13 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:2806: got expected ENOENT (src dne) 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:2806: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1225 (ObjNum 867 snap 258 seq_num 867) dirty exists 2022-04-28T19:53:15.388 INFO:tasks.rados.rados.0.smithi059.stdout:2803: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:53:15.450 INFO:tasks.rados.rados.0.smithi059.stdout:2807: finishing write tid 1 to smithi05955964-21 2022-04-28T19:53:15.450 INFO:tasks.rados.rados.0.smithi059.stdout:2807: finishing write tid 3 to smithi05955964-21 2022-04-28T19:53:15.450 INFO:tasks.rados.rados.0.smithi059.stdout:2808: expect (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:2807: finishing write tid 4 to smithi05955964-21 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1087 (ObjNum 878 snap 262 seq_num 878) dirty exists 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:2807: left oid 21 (ObjNum 878 snap 262 seq_num 878) 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:2803: done (6 left) 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:2804: done (5 left) 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:2805: done (4 left) 2022-04-28T19:53:15.491 INFO:tasks.rados.rados.0.smithi059.stdout:2806: done (3 left) 2022-04-28T19:53:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:2807: done (2 left) 2022-04-28T19:53:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:2808: done (1 left) 2022-04-28T19:53:15.492 INFO:tasks.rados.rados.0.smithi059.stdout:2810: snap_create 2022-04-28T19:53:15.496 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1309 (ObjNum 845 snap 251 seq_num 845) dirty exists 2022-04-28T19:53:16.033 INFO:tasks.rados.rados.0.smithi059.stdout:2809: done (1 left) 2022-04-28T19:53:16.033 INFO:tasks.rados.rados.0.smithi059.stdout:2810: done (0 left) 2022-04-28T19:53:16.033 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 263 2022-04-28T19:53:16.033 INFO:tasks.rados.rados.0.smithi059.stdout:2811: seq_num 879 ranges {5996544=327680} 2022-04-28T19:53:16.067 INFO:tasks.rados.rados.0.smithi059.stdout:2811: writing smithi05955964-14 from 5996544 to 6324224 tid 1 2022-04-28T19:53:16.067 INFO:tasks.rados.rados.0.smithi059.stdout:2812: rollback oid 15 current snap is 263 2022-04-28T19:53:16.067 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 242 2022-04-28T19:53:16.067 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 263 2022-04-28T19:53:16.067 INFO:tasks.rados.rados.0.smithi059.stdout:2813: seq_num 880 ranges {0=344064} 2022-04-28T19:53:16.069 INFO:tasks.rados.rados.0.smithi059.stdout:2813: writing smithi05955964-46 from 0 to 344064 tid 1 2022-04-28T19:53:16.069 INFO:tasks.rados.rados.0.smithi059.stdout:2814: read oid 49 snap 253 2022-04-28T19:53:16.069 INFO:tasks.rados.rados.0.smithi059.stdout:2814: expect (ObjNum 850 snap 251 seq_num 850) 2022-04-28T19:53:16.069 INFO:tasks.rados.rados.0.smithi059.stdout:2815: snap_remove snap 262 2022-04-28T19:53:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[59745]: cluster 2022-04-28T19:53:14.946323+0000 mgr.y (mgr.24961) 219 : cluster [DBG] pgmap v348: 89 pgs: 7 peering, 82 active+clean; 540 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 28 op/s; 24 MiB/s, 8 objects/s recovering 2022-04-28T19:53:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[59745]: cluster 2022-04-28T19:53:15.011241+0000 mon.a (mon.0) 791 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/891 objects degraded (9.315%), 8 pgs degraded) 2022-04-28T19:53:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[59745]: cluster 2022-04-28T19:53:15.011315+0000 mon.a (mon.0) 792 : cluster [INF] Cluster is now healthy 2022-04-28T19:53:16.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[59745]: cluster 2022-04-28T19:53:15.028535+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2022-04-28T19:53:16.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[61152]: cluster 2022-04-28T19:53:14.946323+0000 mgr.y (mgr.24961) 219 : cluster [DBG] pgmap v348: 89 pgs: 7 peering, 82 active+clean; 540 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 28 op/s; 24 MiB/s, 8 objects/s recovering 2022-04-28T19:53:16.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[61152]: cluster 2022-04-28T19:53:15.011241+0000 mon.a (mon.0) 791 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/891 objects degraded (9.315%), 8 pgs degraded) 2022-04-28T19:53:16.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[61152]: cluster 2022-04-28T19:53:15.011315+0000 mon.a (mon.0) 792 : cluster [INF] Cluster is now healthy 2022-04-28T19:53:16.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:16 smithi059 bash[61152]: cluster 2022-04-28T19:53:15.028535+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2022-04-28T19:53:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:16 smithi139 bash[43301]: cluster 2022-04-28T19:53:14.946323+0000 mgr.y (mgr.24961) 219 : cluster [DBG] pgmap v348: 89 pgs: 7 peering, 82 active+clean; 540 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 28 op/s; 24 MiB/s, 8 objects/s recovering 2022-04-28T19:53:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:16 smithi139 bash[43301]: cluster 2022-04-28T19:53:15.011241+0000 mon.a (mon.0) 791 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/891 objects degraded (9.315%), 8 pgs degraded) 2022-04-28T19:53:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:16 smithi139 bash[43301]: cluster 2022-04-28T19:53:15.011315+0000 mon.a (mon.0) 792 : cluster [INF] Cluster is now healthy 2022-04-28T19:53:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:16 smithi139 bash[43301]: cluster 2022-04-28T19:53:15.028535+0000 mon.a (mon.0) 793 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2022-04-28T19:53:16.909 INFO:tasks.rados.rados.0.smithi059.stdout:2815: done (4 left) 2022-04-28T19:53:16.909 INFO:tasks.rados.rados.0.smithi059.stdout:2816: copy_from oid 21 from oid 38 current snap is 263 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2811: finishing write tid 1 to smithi05955964-14 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2811: finishing write tid 3 to smithi05955964-14 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2812: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2813: finishing write tid 1 to smithi05955964-46 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2813: finishing write tid 3 to smithi05955964-46 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2812: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2812: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1314 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:2813: finishing write tid 4 to smithi05955964-46 2022-04-28T19:53:16.910 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1330 (ObjNum 880 snap 263 seq_num 880) dirty exists 2022-04-28T19:53:16.911 INFO:tasks.rados.rados.0.smithi059.stdout:2813: left oid 46 (ObjNum 880 snap 263 seq_num 880) 2022-04-28T19:53:16.911 INFO:tasks.rados.rados.0.smithi059.stdout:2811: finishing write tid 4 to smithi05955964-14 2022-04-28T19:53:16.911 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1148 (ObjNum 879 snap 263 seq_num 879) dirty exists 2022-04-28T19:53:16.911 INFO:tasks.rados.rados.0.smithi059.stdout:2811: left oid 14 (ObjNum 879 snap 263 seq_num 879) 2022-04-28T19:53:16.911 INFO:tasks.rados.rados.0.smithi059.stdout:2814: expect (ObjNum 850 snap 251 seq_num 850) 2022-04-28T19:53:17.107 INFO:tasks.rados.rados.0.smithi059.stdout:2816: finishing copy_from to smithi05955964-21 2022-04-28T19:53:17.108 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1147 (ObjNum 849 snap 251 seq_num 849) dirty exists 2022-04-28T19:53:17.108 INFO:tasks.rados.rados.0.smithi059.stdout:2816: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:53:17.108 INFO:tasks.rados.rados.0.smithi059.stdout:2811: done (4 left) 2022-04-28T19:53:17.108 INFO:tasks.rados.rados.0.smithi059.stdout:2812: done (3 left) 2022-04-28T19:53:17.109 INFO:tasks.rados.rados.0.smithi059.stdout:2813: done (2 left) 2022-04-28T19:53:17.109 INFO:tasks.rados.rados.0.smithi059.stdout:2814: done (1 left) 2022-04-28T19:53:17.109 INFO:tasks.rados.rados.0.smithi059.stdout:2816: done (0 left) 2022-04-28T19:53:17.109 INFO:tasks.rados.rados.0.smithi059.stdout:2817: delete oid 10 current snap is 263 2022-04-28T19:53:17.112 INFO:tasks.rados.rados.0.smithi059.stdout:2817: done (0 left) 2022-04-28T19:53:17.113 INFO:tasks.rados.rados.0.smithi059.stdout:2818: snap_remove snap 238 2022-04-28T19:53:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[59745]: cluster 2022-04-28T19:53:16.028459+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2022-04-28T19:53:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[59745]: audit 2022-04-28T19:53:16.899710+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T19:53:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[59745]: audit 2022-04-28T19:53:16.900244+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T19:53:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[59745]: cluster 2022-04-28T19:53:16.905563+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2022-04-28T19:53:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[59745]: audit 2022-04-28T19:53:16.907170+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:53:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[61152]: cluster 2022-04-28T19:53:16.028459+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2022-04-28T19:53:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[61152]: audit 2022-04-28T19:53:16.899710+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T19:53:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[61152]: audit 2022-04-28T19:53:16.900244+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T19:53:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[61152]: cluster 2022-04-28T19:53:16.905563+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2022-04-28T19:53:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:17 smithi059 bash[61152]: audit 2022-04-28T19:53:16.907170+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:53:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:17 smithi139 bash[43301]: cluster 2022-04-28T19:53:16.028459+0000 mon.a (mon.0) 794 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2022-04-28T19:53:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:17 smithi139 bash[43301]: audit 2022-04-28T19:53:16.899710+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T19:53:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:17 smithi139 bash[43301]: audit 2022-04-28T19:53:16.900244+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T19:53:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:17 smithi139 bash[43301]: cluster 2022-04-28T19:53:16.905563+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2022-04-28T19:53:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:17 smithi139 bash[43301]: audit 2022-04-28T19:53:16.907170+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:53:18.050 INFO:tasks.rados.rados.0.smithi059.stdout:2818: done (0 left) 2022-04-28T19:53:18.050 INFO:tasks.rados.rados.0.smithi059.stdout:2819: read oid 39 snap -1 2022-04-28T19:53:18.051 INFO:tasks.rados.rados.0.smithi059.stdout:2819: expect deleted 2022-04-28T19:53:18.051 INFO:tasks.rados.rados.0.smithi059.stdout:2820: setattr oid 34 current snap is 263 2022-04-28T19:53:18.051 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 263 2022-04-28T19:53:18.051 INFO:tasks.rados.rados.0.smithi059.stdout:2821: seq_num 882 ranges {2228224=376832} 2022-04-28T19:53:18.067 INFO:tasks.rados.rados.0.smithi059.stdout:2821: writing smithi05955964-38 from 2228224 to 2605056 tid 1 2022-04-28T19:53:18.067 INFO:tasks.rados.rados.0.smithi059.stdout:2822: read oid 31 snap -1 2022-04-28T19:53:18.067 INFO:tasks.rados.rados.0.smithi059.stdout:2822: expect (ObjNum 859 snap 253 seq_num 859) 2022-04-28T19:53:18.068 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 263 2022-04-28T19:53:18.068 INFO:tasks.rados.rados.0.smithi059.stdout:2823: seq_num 883 ranges {2228224=393216} 2022-04-28T19:53:18.082 INFO:tasks.rados.rados.0.smithi059.stdout:2823: writing smithi05955964-21 from 2228224 to 2621440 tid 1 2022-04-28T19:53:18.082 INFO:tasks.rados.rados.0.smithi059.stdout:2824: delete oid 10 current snap is 263 2022-04-28T19:53:18.083 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1226 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:53:18.083 INFO:tasks.rados.rados.0.smithi059.stdout:2821: finishing write tid 1 to smithi05955964-38 2022-04-28T19:53:18.083 INFO:tasks.rados.rados.0.smithi059.stdout:2821: finishing write tid 3 to smithi05955964-38 2022-04-28T19:53:18.083 INFO:tasks.rados.rados.0.smithi059.stdout:2821: finishing write tid 4 to smithi05955964-38 2022-04-28T19:53:18.083 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1154 (ObjNum 882 snap 263 seq_num 882) dirty exists 2022-04-28T19:53:18.084 INFO:tasks.rados.rados.0.smithi059.stdout:2821: left oid 38 (ObjNum 882 snap 263 seq_num 882) 2022-04-28T19:53:18.085 INFO:tasks.rados.rados.0.smithi059.stdout:2819: done (5 left) 2022-04-28T19:53:18.085 INFO:tasks.rados.rados.0.smithi059.stdout:2820: done (4 left) 2022-04-28T19:53:18.085 INFO:tasks.rados.rados.0.smithi059.stdout:2821: done (3 left) 2022-04-28T19:53:18.085 INFO:tasks.rados.rados.0.smithi059.stdout:2824: done (2 left) 2022-04-28T19:53:18.086 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 263 2022-04-28T19:53:18.086 INFO:tasks.rados.rados.0.smithi059.stdout:2825: seq_num 885 ranges {0=425984} 2022-04-28T19:53:18.088 INFO:tasks.rados.rados.0.smithi059.stdout:2825: writing smithi05955964-47 from 0 to 425984 tid 1 2022-04-28T19:53:18.088 INFO:tasks.rados.rados.0.smithi059.stdout:2826: read oid 41 snap 241 2022-04-28T19:53:18.089 INFO:tasks.rados.rados.0.smithi059.stdout:2826: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:18.089 INFO:tasks.rados.rados.0.smithi059.stdout:2827: copy_from oid 37 from oid 33 current snap is 263 2022-04-28T19:53:18.089 INFO:tasks.rados.rados.0.smithi059.stdout:2823: finishing write tid 1 to smithi05955964-21 2022-04-28T19:53:18.089 INFO:tasks.rados.rados.0.smithi059.stdout:2823: finishing write tid 3 to smithi05955964-21 2022-04-28T19:53:18.089 INFO:tasks.rados.rados.0.smithi059.stdout:2828: delete oid 24 current snap is 263 2022-04-28T19:53:18.090 INFO:tasks.rados.rados.0.smithi059.stdout:2823: finishing write tid 4 to smithi05955964-21 2022-04-28T19:53:18.090 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1149 (ObjNum 883 snap 263 seq_num 883) dirty exists 2022-04-28T19:53:18.090 INFO:tasks.rados.rados.0.smithi059.stdout:2823: left oid 21 (ObjNum 883 snap 263 seq_num 883) 2022-04-28T19:53:18.091 INFO:tasks.rados.rados.0.smithi059.stdout:2823: done (5 left) 2022-04-28T19:53:18.092 INFO:tasks.rados.rados.0.smithi059.stdout:2828: done (4 left) 2022-04-28T19:53:18.092 INFO:tasks.rados.rados.0.smithi059.stdout:2829: snap_remove snap 259 2022-04-28T19:53:18.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:18 smithi059 bash[59745]: cluster 2022-04-28T19:53:16.946750+0000 mgr.y (mgr.24961) 220 : cluster [DBG] pgmap v352: 89 pgs: 89 active+clean; 540 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.8 MiB/s wr, 14 op/s; 14 MiB/s, 4 objects/s recovering 2022-04-28T19:53:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:18 smithi059 bash[59745]: audit 2022-04-28T19:53:17.123178+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:53:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:18 smithi059 bash[59745]: audit 2022-04-28T19:53:17.140868+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:53:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:18 smithi059 bash[61152]: cluster 2022-04-28T19:53:16.946750+0000 mgr.y (mgr.24961) 220 : cluster [DBG] pgmap v352: 89 pgs: 89 active+clean; 540 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.8 MiB/s wr, 14 op/s; 14 MiB/s, 4 objects/s recovering 2022-04-28T19:53:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:18 smithi059 bash[61152]: audit 2022-04-28T19:53:17.123178+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:53:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:18 smithi059 bash[61152]: audit 2022-04-28T19:53:17.140868+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:53:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:18 smithi139 bash[43301]: cluster 2022-04-28T19:53:16.946750+0000 mgr.y (mgr.24961) 220 : cluster [DBG] pgmap v352: 89 pgs: 89 active+clean; 540 MiB data, 1.1 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 3.8 MiB/s wr, 14 op/s; 14 MiB/s, 4 objects/s recovering 2022-04-28T19:53:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:18 smithi139 bash[43301]: audit 2022-04-28T19:53:17.123178+0000 mon.a (mon.0) 799 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:53:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:18 smithi139 bash[43301]: audit 2022-04-28T19:53:17.140868+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:53:19.064 INFO:tasks.rados.rados.0.smithi059.stdout:2829: done (4 left) 2022-04-28T19:53:19.064 INFO:tasks.rados.rados.0.smithi059.stdout:2830: read oid 40 snap 229 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2830: expect (ObjNum 755 snap 226 seq_num 755) 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2831: rollback oid 1 current snap is 263 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 244 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2832: snap_create 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2825: finishing write tid 1 to smithi05955964-47 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2825: finishing write tid 3 to smithi05955964-47 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2825: finishing write tid 4 to smithi05955964-47 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1138 (ObjNum 885 snap 263 seq_num 885) dirty exists 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2825: left oid 47 (ObjNum 885 snap 263 seq_num 885) 2022-04-28T19:53:19.065 INFO:tasks.rados.rados.0.smithi059.stdout:2826: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:19.093 INFO:tasks.rados.rados.0.smithi059.stdout:2822: expect (ObjNum 859 snap 253 seq_num 859) 2022-04-28T19:53:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:19 smithi059 bash[61152]: cluster 2022-04-28T19:53:18.049304+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2022-04-28T19:53:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:19 smithi059 bash[59745]: cluster 2022-04-28T19:53:18.049304+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2022-04-28T19:53:19.365 INFO:tasks.rados.rados.0.smithi059.stdout:2827: finishing copy_from to smithi05955964-37 2022-04-28T19:53:19.461 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1309 (ObjNum 845 snap 251 seq_num 845) dirty exists 2022-04-28T19:53:19.461 INFO:tasks.rados.rados.0.smithi059.stdout:2827: finishing copy_from racing read to smithi05955964-37 2022-04-28T19:53:19.461 INFO:tasks.rados.rados.0.smithi059.stdout:2830: expect (ObjNum 755 snap 226 seq_num 755) 2022-04-28T19:53:19.462 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:19 smithi139 bash[43301]: cluster 2022-04-28T19:53:18.049304+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2022-04-28T19:53:19.462 INFO:tasks.rados.rados.0.smithi059.stdout:2831: finishing rollback tid 0 to smithi05955964-1 2022-04-28T19:53:19.463 INFO:tasks.rados.rados.0.smithi059.stdout:2831: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:53:19.463 INFO:tasks.rados.rados.0.smithi059.stdout:2831: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:53:19.463 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1079 (ObjNum 792 snap 235 seq_num 792) dirty exists 2022-04-28T19:53:20.064 INFO:tasks.rados.rados.0.smithi059.stdout:2822: done (6 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:2825: done (5 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:2826: done (4 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:2827: done (3 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:2830: done (2 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:2831: done (1 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:2832: done (0 left) 2022-04-28T19:53:20.065 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 264 2022-04-28T19:53:20.066 INFO:tasks.rados.rados.0.smithi059.stdout:2833: seq_num 887 ranges {122880=458752} 2022-04-28T19:53:20.068 INFO:tasks.rados.rados.0.smithi059.stdout:2833: writing smithi05955964-30 from 122880 to 581632 tid 1 2022-04-28T19:53:20.068 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 264 2022-04-28T19:53:20.068 INFO:tasks.rados.rados.0.smithi059.stdout:2834: seq_num 888 ranges {2605056=475136} 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:2834: writing smithi05955964-38 from 2605056 to 3080192 tid 1 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:2835: snap_create 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:2833: finishing write tid 1 to smithi05955964-30 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:2833: finishing write tid 3 to smithi05955964-30 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:2833: finishing write tid 4 to smithi05955964-30 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1311 (ObjNum 887 snap 264 seq_num 887) dirty exists 2022-04-28T19:53:20.086 INFO:tasks.rados.rados.0.smithi059.stdout:2833: left oid 30 (ObjNum 887 snap 264 seq_num 887) 2022-04-28T19:53:20.091 INFO:tasks.rados.rados.0.smithi059.stdout:2834: finishing write tid 1 to smithi05955964-38 2022-04-28T19:53:20.091 INFO:tasks.rados.rados.0.smithi059.stdout:2834: finishing write tid 3 to smithi05955964-38 2022-04-28T19:53:20.092 INFO:tasks.rados.rados.0.smithi059.stdout:2834: finishing write tid 4 to smithi05955964-38 2022-04-28T19:53:20.092 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1157 (ObjNum 888 snap 264 seq_num 888) dirty exists 2022-04-28T19:53:20.092 INFO:tasks.rados.rados.0.smithi059.stdout:2834: left oid 38 (ObjNum 888 snap 264 seq_num 888) 2022-04-28T19:53:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:20 smithi059 bash[59745]: cluster 2022-04-28T19:53:18.947629+0000 mgr.y (mgr.24961) 221 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 544 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1006 KiB/s wr, 18 op/s 2022-04-28T19:53:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:20 smithi059 bash[59745]: cluster 2022-04-28T19:53:19.058924+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2022-04-28T19:53:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:20 smithi059 bash[61152]: cluster 2022-04-28T19:53:18.947629+0000 mgr.y (mgr.24961) 221 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 544 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1006 KiB/s wr, 18 op/s 2022-04-28T19:53:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:20 smithi059 bash[61152]: cluster 2022-04-28T19:53:19.058924+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2022-04-28T19:53:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:20 smithi139 bash[43301]: cluster 2022-04-28T19:53:18.947629+0000 mgr.y (mgr.24961) 221 : cluster [DBG] pgmap v354: 89 pgs: 89 active+clean; 544 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 1006 KiB/s wr, 18 op/s 2022-04-28T19:53:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:20 smithi139 bash[43301]: cluster 2022-04-28T19:53:19.058924+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2022-04-28T19:53:21.081 INFO:tasks.rados.rados.0.smithi059.stdout:2833: done (2 left) 2022-04-28T19:53:21.081 INFO:tasks.rados.rados.0.smithi059.stdout:2834: done (1 left) 2022-04-28T19:53:21.082 INFO:tasks.rados.rados.0.smithi059.stdout:2835: done (0 left) 2022-04-28T19:53:21.082 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 265 2022-04-28T19:53:21.082 INFO:tasks.rados.rados.0.smithi059.stdout:2836: seq_num 889 ranges {6938624=491520} 2022-04-28T19:53:21.126 INFO:tasks.rados.rados.0.smithi059.stdout:2836: writing smithi05955964-31 from 6938624 to 7430144 tid 1 2022-04-28T19:53:21.126 INFO:tasks.rados.rados.0.smithi059.stdout:2837: rmattr oid 35 current snap is 265 2022-04-28T19:53:21.126 INFO:tasks.rados.rados.0.smithi059.stdout:2838: read oid 46 snap 242 2022-04-28T19:53:21.126 INFO:tasks.rados.rados.0.smithi059.stdout:2838: expect (ObjNum 817 snap 240 seq_num 817) 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2839: rmattr oid 42 current snap is 265 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2839: done (3 left) 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2840: copy_from oid 13 from oid 17 current snap is 265 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2841: read oid 3 snap -1 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2841: expect (ObjNum 874 snap 262 seq_num 874) 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2842: rmattr oid 15 current snap is 265 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:2843: rollback oid 14 current snap is 265 2022-04-28T19:53:21.127 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 229 2022-04-28T19:53:21.128 INFO:tasks.rados.rados.0.smithi059.stdout:2844: copy_from oid 23 from oid 22 current snap is 265 2022-04-28T19:53:21.128 INFO:tasks.rados.rados.0.smithi059.stdout:2845: read oid 11 snap 246 2022-04-28T19:53:21.128 INFO:tasks.rados.rados.0.smithi059.stdout:2845: expect deleted 2022-04-28T19:53:21.128 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 265 2022-04-28T19:53:21.128 INFO:tasks.rados.rados.0.smithi059.stdout:2846: seq_num 890 ranges {8044544=507904} 2022-04-28T19:53:21.176 INFO:tasks.rados.rados.0.smithi059.stdout:2846: writing smithi05955964-25 from 8044544 to 8552448 tid 1 2022-04-28T19:53:21.176 INFO:tasks.rados.rados.0.smithi059.stdout:2847: delete oid 26 current snap is 265 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:2836: finishing write tid 1 to smithi05955964-31 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:2836: finishing write tid 3 to smithi05955964-31 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1317 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:2836: finishing write tid 4 to smithi05955964-31 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1316 (ObjNum 889 snap 265 seq_num 889) dirty exists 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:2836: left oid 31 (ObjNum 889 snap 265 seq_num 889) 2022-04-28T19:53:21.177 INFO:tasks.rados.rados.0.smithi059.stdout:2843: finishing rollback tid 0 to smithi05955964-14 2022-04-28T19:53:21.178 INFO:tasks.rados.rados.0.smithi059.stdout:2843: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:53:21.178 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1151 (ObjNum -132379008 snap 21893 seq_num 188341504) dirty dne 2022-04-28T19:53:21.178 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 952 (ObjNum 854 snap 253 seq_num 854) dirty exists 2022-04-28T19:53:21.178 INFO:tasks.rados.rados.0.smithi059.stdout:2841: expect (ObjNum 874 snap 262 seq_num 874) 2022-04-28T19:53:21.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:21 smithi059 bash[61152]: cluster 2022-04-28T19:53:20.061806+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2022-04-28T19:53:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:21 smithi059 bash[59745]: cluster 2022-04-28T19:53:20.061806+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2022-04-28T19:53:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:21 smithi139 bash[43301]: cluster 2022-04-28T19:53:20.061806+0000 mon.a (mon.0) 803 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2022-04-28T19:53:21.759 INFO:tasks.rados.rados.0.smithi059.stdout:2838: expect (ObjNum 817 snap 240 seq_num 817) 2022-04-28T19:53:22.077 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:21.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:22.077 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:21.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:22.077 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:21.633Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:22.078 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[59745]: cluster 2022-04-28T19:53:20.948525+0000 mgr.y (mgr.24961) 222 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 562 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 5.5 MiB/s wr, 43 op/s 2022-04-28T19:53:22.078 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[59745]: cluster 2022-04-28T19:53:21.080036+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2022-04-28T19:53:22.078 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[59745]: audit 2022-04-28T19:53:21.365964+0000 mon.a (mon.0) 805 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:22.078 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[59745]: cluster 2022-04-28T19:53:21.910876+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:2840: finishing copy_from to smithi05955964-13 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1048 (ObjNum 740 snap 219 seq_num 740) dirty exists 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:2844: finishing copy_from to smithi05955964-23 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 949 (ObjNum 812 snap 240 seq_num 812) dirty exists 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:2840: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:2844: finishing copy_from racing read to smithi05955964-23 2022-04-28T19:53:22.162 INFO:tasks.rados.rados.0.smithi059.stdout:2846: finishing write tid 1 to smithi05955964-25 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2846: finishing write tid 3 to smithi05955964-25 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2846: finishing write tid 4 to smithi05955964-25 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1343 (ObjNum 890 snap 265 seq_num 890) dirty exists 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2846: left oid 25 (ObjNum 890 snap 265 seq_num 890) 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2836: done (10 left) 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2837: done (9 left) 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2838: done (8 left) 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2840: done (7 left) 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2841: done (6 left) 2022-04-28T19:53:22.163 INFO:tasks.rados.rados.0.smithi059.stdout:2842: done (5 left) 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:2843: done (4 left) 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:2844: done (3 left) 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:2845: done (2 left) 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:2846: done (1 left) 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:2847: done (0 left) 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 265 2022-04-28T19:53:22.164 INFO:tasks.rados.rados.0.smithi059.stdout:2848: seq_num 892 ranges {3080192=548864} 2022-04-28T19:53:22.191 INFO:tasks.rados.rados.0.smithi059.stdout:2848: writing smithi05955964-38 from 3080192 to 3629056 tid 1 2022-04-28T19:53:22.191 INFO:tasks.rados.rados.0.smithi059.stdout:2849: snap_create 2022-04-28T19:53:22.197 INFO:tasks.rados.rados.0.smithi059.stdout:2848: finishing write tid 1 to smithi05955964-38 2022-04-28T19:53:22.197 INFO:tasks.rados.rados.0.smithi059.stdout:2848: finishing write tid 3 to smithi05955964-38 2022-04-28T19:53:22.198 INFO:tasks.rados.rados.0.smithi059.stdout:2848: finishing write tid 4 to smithi05955964-38 2022-04-28T19:53:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1159 (ObjNum 892 snap 265 seq_num 892) dirty exists 2022-04-28T19:53:22.199 INFO:tasks.rados.rados.0.smithi059.stdout:2848: left oid 38 (ObjNum 892 snap 265 seq_num 892) 2022-04-28T19:53:22.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[61152]: cluster 2022-04-28T19:53:20.948525+0000 mgr.y (mgr.24961) 222 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 562 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 5.5 MiB/s wr, 43 op/s 2022-04-28T19:53:22.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[61152]: cluster 2022-04-28T19:53:21.080036+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2022-04-28T19:53:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[61152]: audit 2022-04-28T19:53:21.365964+0000 mon.a (mon.0) 805 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:22 smithi059 bash[61152]: cluster 2022-04-28T19:53:21.910876+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2022-04-28T19:53:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:22 smithi139 bash[43301]: cluster 2022-04-28T19:53:20.948525+0000 mgr.y (mgr.24961) 222 : cluster [DBG] pgmap v357: 89 pgs: 89 active+clean; 562 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 18 MiB/s rd, 5.5 MiB/s wr, 43 op/s 2022-04-28T19:53:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:22 smithi139 bash[43301]: cluster 2022-04-28T19:53:21.080036+0000 mon.a (mon.0) 804 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2022-04-28T19:53:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:22 smithi139 bash[43301]: audit 2022-04-28T19:53:21.365964+0000 mon.a (mon.0) 805 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:22 smithi139 bash[43301]: cluster 2022-04-28T19:53:21.910876+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2022-04-28T19:53:23.086 INFO:tasks.rados.rados.0.smithi059.stdout:2848: done (1 left) 2022-04-28T19:53:23.086 INFO:tasks.rados.rados.0.smithi059.stdout:2849: done (0 left) 2022-04-28T19:53:23.086 INFO:tasks.rados.rados.0.smithi059.stdout:2850: copy_from oid 40 from oid 20 current snap is 266 2022-04-28T19:53:23.086 INFO:tasks.rados.rados.0.smithi059.stdout:2851: rmattr oid 46 current snap is 266 2022-04-28T19:53:23.086 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 266 2022-04-28T19:53:23.086 INFO:tasks.rados.rados.0.smithi059.stdout:2852: seq_num 893 ranges {2367488=565248} 2022-04-28T19:53:23.104 INFO:tasks.rados.rados.0.smithi059.stdout:2852: writing smithi05955964-11 from 2367488 to 2932736 tid 1 2022-04-28T19:53:23.104 INFO:tasks.rados.rados.0.smithi059.stdout:2853: snap_create 2022-04-28T19:53:23.105 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1345 (ObjNum 880 snap 263 seq_num 880) dirty exists 2022-04-28T19:53:23.106 INFO:tasks.rados.rados.0.smithi059.stdout:2850: finishing copy_from to smithi05955964-40 2022-04-28T19:53:23.106 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 990 (ObjNum 821 snap 240 seq_num 821) dirty exists 2022-04-28T19:53:23.107 INFO:tasks.rados.rados.0.smithi059.stdout:2850: finishing copy_from racing read to smithi05955964-40 2022-04-28T19:53:23.109 INFO:tasks.rados.rados.0.smithi059.stdout:2852: finishing write tid 1 to smithi05955964-11 2022-04-28T19:53:23.110 INFO:tasks.rados.rados.0.smithi059.stdout:2852: finishing write tid 3 to smithi05955964-11 2022-04-28T19:53:23.111 INFO:tasks.rados.rados.0.smithi059.stdout:2852: finishing write tid 4 to smithi05955964-11 2022-04-28T19:53:23.111 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1161 (ObjNum 893 snap 266 seq_num 893) dirty exists 2022-04-28T19:53:23.111 INFO:tasks.rados.rados.0.smithi059.stdout:2852: left oid 11 (ObjNum 893 snap 266 seq_num 893) 2022-04-28T19:53:23.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[61152]: audit 2022-04-28T19:53:21.366363+0000 mgr.y (mgr.24961) 223 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[61152]: cephadm 2022-04-28T19:53:21.368579+0000 mgr.y (mgr.24961) 224 : cephadm [INF] Upgrade: osd.6 is safe to restart 2022-04-28T19:53:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[61152]: audit 2022-04-28T19:53:22.363939+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[59745]: audit 2022-04-28T19:53:21.366363+0000 mgr.y (mgr.24961) 223 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[59745]: cephadm 2022-04-28T19:53:21.368579+0000 mgr.y (mgr.24961) 224 : cephadm [INF] Upgrade: osd.6 is safe to restart 2022-04-28T19:53:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[59745]: audit 2022-04-28T19:53:22.363939+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:23.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:23 smithi139 bash[43301]: audit 2022-04-28T19:53:21.366363+0000 mgr.y (mgr.24961) 223 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2022-04-28T19:53:23.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:23 smithi139 bash[43301]: cephadm 2022-04-28T19:53:21.368579+0000 mgr.y (mgr.24961) 224 : cephadm [INF] Upgrade: osd.6 is safe to restart 2022-04-28T19:53:23.361 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:23 smithi139 bash[43301]: audit 2022-04-28T19:53:22.363939+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:23.527Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:23.527Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:23.527Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:24.108 INFO:tasks.rados.rados.0.smithi059.stdout:2850: done (3 left) 2022-04-28T19:53:24.108 INFO:tasks.rados.rados.0.smithi059.stdout:2851: done (2 left) 2022-04-28T19:53:24.108 INFO:tasks.rados.rados.0.smithi059.stdout:2852: done (1 left) 2022-04-28T19:53:24.109 INFO:tasks.rados.rados.0.smithi059.stdout:2853: done (0 left) 2022-04-28T19:53:24.109 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 267 2022-04-28T19:53:24.109 INFO:tasks.rados.rados.0.smithi059.stdout:2854: seq_num 894 ranges {0=581632} 2022-04-28T19:53:24.113 INFO:tasks.rados.rados.0.smithi059.stdout:2854: writing smithi05955964-39 from 0 to 581632 tid 1 2022-04-28T19:53:24.113 INFO:tasks.rados.rados.0.smithi059.stdout:2855: read oid 17 snap -1 2022-04-28T19:53:24.113 INFO:tasks.rados.rados.0.smithi059.stdout:2855: expect (ObjNum 740 snap 219 seq_num 740) 2022-04-28T19:53:24.113 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 267 2022-04-28T19:53:24.113 INFO:tasks.rados.rados.0.smithi059.stdout:2856: seq_num 895 ranges {0=598016} 2022-04-28T19:53:24.118 INFO:tasks.rados.rados.0.smithi059.stdout:2856: writing smithi05955964-42 from 0 to 598016 tid 1 2022-04-28T19:53:24.118 INFO:tasks.rados.rados.0.smithi059.stdout:2857: read oid 11 snap -1 2022-04-28T19:53:24.118 INFO:tasks.rados.rados.0.smithi059.stdout:2857: expect (ObjNum 893 snap 266 seq_num 893) 2022-04-28T19:53:24.118 INFO:tasks.rados.rados.0.smithi059.stdout:2858: rollback oid 7 current snap is 267 2022-04-28T19:53:24.119 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 7 to 244 2022-04-28T19:53:24.119 INFO:tasks.rados.rados.0.smithi059.stdout:2859: snap_create 2022-04-28T19:53:24.126 INFO:tasks.rados.rados.0.smithi059.stdout:2856: finishing write tid 1 to smithi05955964-42 2022-04-28T19:53:24.126 INFO:tasks.rados.rados.0.smithi059.stdout:2856: finishing write tid 3 to smithi05955964-42 2022-04-28T19:53:24.127 INFO:tasks.rados.rados.0.smithi059.stdout:2854: finishing write tid 1 to smithi05955964-39 2022-04-28T19:53:24.128 INFO:tasks.rados.rados.0.smithi059.stdout:2856: finishing write tid 4 to smithi05955964-42 2022-04-28T19:53:24.128 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1313 (ObjNum 895 snap 267 seq_num 895) dirty exists 2022-04-28T19:53:24.128 INFO:tasks.rados.rados.0.smithi059.stdout:2856: left oid 42 (ObjNum 895 snap 267 seq_num 895) 2022-04-28T19:53:24.128 INFO:tasks.rados.rados.0.smithi059.stdout:2854: finishing write tid 3 to smithi05955964-39 2022-04-28T19:53:24.129 INFO:tasks.rados.rados.0.smithi059.stdout:2858: finishing rollback tid 1 to smithi05955964-7 2022-04-28T19:53:24.129 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 964 (ObjNum 0 snap 0 seq_num 188341408) dirty dne 2022-04-28T19:53:24.131 INFO:tasks.rados.rados.0.smithi059.stdout:2854: finishing write tid 4 to smithi05955964-39 2022-04-28T19:53:24.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1325 (ObjNum 894 snap 267 seq_num 894) dirty exists 2022-04-28T19:53:24.131 INFO:tasks.rados.rados.0.smithi059.stdout:2854: left oid 39 (ObjNum 894 snap 267 seq_num 894) 2022-04-28T19:53:24.134 INFO:tasks.rados.rados.0.smithi059.stdout:2857: expect (ObjNum 893 snap 266 seq_num 893) 2022-04-28T19:53:24.249 INFO:tasks.rados.rados.0.smithi059.stdout:2855: expect (ObjNum 740 snap 219 seq_num 740) 2022-04-28T19:53:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:24 smithi139 bash[43301]: cluster 2022-04-28T19:53:22.949407+0000 mgr.y (mgr.24961) 225 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 564 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 5.0 MiB/s wr, 32 op/s 2022-04-28T19:53:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:24 smithi139 bash[43301]: cluster 2022-04-28T19:53:23.085232+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2022-04-28T19:53:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:24 smithi139 bash[43301]: audit 2022-04-28T19:53:24.003623+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:24 smithi139 bash[43301]: audit 2022-04-28T19:53:24.008345+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "container_image"}]: dispatch 2022-04-28T19:53:24.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[59745]: cluster 2022-04-28T19:53:22.949407+0000 mgr.y (mgr.24961) 225 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 564 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 5.0 MiB/s wr, 32 op/s 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[59745]: cluster 2022-04-28T19:53:23.085232+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[59745]: audit 2022-04-28T19:53:24.003623+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[59745]: audit 2022-04-28T19:53:24.008345+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "container_image"}]: dispatch 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[61152]: cluster 2022-04-28T19:53:22.949407+0000 mgr.y (mgr.24961) 225 : cluster [DBG] pgmap v360: 89 pgs: 89 active+clean; 564 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 20 MiB/s rd, 5.0 MiB/s wr, 32 op/s 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[61152]: cluster 2022-04-28T19:53:23.085232+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[61152]: audit 2022-04-28T19:53:24.003623+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:24 smithi059 bash[61152]: audit 2022-04-28T19:53:24.008345+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "container_image"}]: dispatch 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2854: done (5 left) 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2855: done (4 left) 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2856: done (3 left) 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2857: done (2 left) 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2858: done (1 left) 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2859: done (0 left) 2022-04-28T19:53:25.108 INFO:tasks.rados.rados.0.smithi059.stdout:2860: copy_from oid 35 from oid 31 current snap is 268 2022-04-28T19:53:25.109 INFO:tasks.rados.rados.0.smithi059.stdout:2861: snap_remove snap 241 2022-04-28T19:53:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:25 smithi139 bash[43301]: cephadm 2022-04-28T19:53:23.994791+0000 mgr.y (mgr.24961) 226 : cephadm [INF] Upgrade: Updating osd.6 2022-04-28T19:53:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:25 smithi139 bash[43301]: cephadm 2022-04-28T19:53:24.007029+0000 mgr.y (mgr.24961) 227 : cephadm [INF] Deploying daemon osd.6 on smithi139 2022-04-28T19:53:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:25 smithi139 bash[43301]: cluster 2022-04-28T19:53:24.107855+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2022-04-28T19:53:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:25 smithi059 bash[59745]: cephadm 2022-04-28T19:53:23.994791+0000 mgr.y (mgr.24961) 226 : cephadm [INF] Upgrade: Updating osd.6 2022-04-28T19:53:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:25 smithi059 bash[59745]: cephadm 2022-04-28T19:53:24.007029+0000 mgr.y (mgr.24961) 227 : cephadm [INF] Deploying daemon osd.6 on smithi139 2022-04-28T19:53:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:25 smithi059 bash[59745]: cluster 2022-04-28T19:53:24.107855+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2022-04-28T19:53:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:25 smithi059 bash[61152]: cephadm 2022-04-28T19:53:23.994791+0000 mgr.y (mgr.24961) 226 : cephadm [INF] Upgrade: Updating osd.6 2022-04-28T19:53:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:25 smithi059 bash[61152]: cephadm 2022-04-28T19:53:24.007029+0000 mgr.y (mgr.24961) 227 : cephadm [INF] Deploying daemon osd.6 on smithi139 2022-04-28T19:53:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:25 smithi059 bash[61152]: cluster 2022-04-28T19:53:24.107855+0000 mon.a (mon.0) 811 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2022-04-28T19:53:26.128 INFO:tasks.rados.rados.0.smithi059.stdout:2861: done (1 left) 2022-04-28T19:53:26.129 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 268 2022-04-28T19:53:26.129 INFO:tasks.rados.rados.0.smithi059.stdout:2862: seq_num 896 ranges {0=614400} 2022-04-28T19:53:26.132 INFO:tasks.rados.rados.0.smithi059.stdout:2862: writing smithi05955964-9 from 0 to 614400 tid 1 2022-04-28T19:53:26.133 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 268 2022-04-28T19:53:26.133 INFO:tasks.rados.rados.0.smithi059.stdout:2863: seq_num 897 ranges {425984=630784} 2022-04-28T19:53:26.139 INFO:tasks.rados.rados.0.smithi059.stdout:2863: writing smithi05955964-47 from 425984 to 1056768 tid 1 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2864: setattr oid 21 current snap is 268 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2860: finishing copy_from to smithi05955964-35 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1316 (ObjNum 889 snap 265 seq_num 889) dirty exists 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2860: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2862: finishing write tid 1 to smithi05955964-9 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2862: finishing write tid 3 to smithi05955964-9 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2862: finishing write tid 4 to smithi05955964-9 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 967 (ObjNum 896 snap 268 seq_num 896) dirty exists 2022-04-28T19:53:26.140 INFO:tasks.rados.rados.0.smithi059.stdout:2862: left oid 9 (ObjNum 896 snap 268 seq_num 896) 2022-04-28T19:53:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2860: done (3 left) 2022-04-28T19:53:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2862: done (2 left) 2022-04-28T19:53:26.141 INFO:tasks.rados.rados.0.smithi059.stdout:2865: snap_remove snap 267 2022-04-28T19:53:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:26 smithi139 bash[43301]: cluster 2022-04-28T19:53:24.950150+0000 mgr.y (mgr.24961) 228 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 582 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 5.0 MiB/s wr, 9 op/s 2022-04-28T19:53:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:26 smithi139 bash[43301]: cluster 2022-04-28T19:53:25.107395+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2022-04-28T19:53:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:26 smithi059 bash[61152]: cluster 2022-04-28T19:53:24.950150+0000 mgr.y (mgr.24961) 228 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 582 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 5.0 MiB/s wr, 9 op/s 2022-04-28T19:53:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:26 smithi059 bash[61152]: cluster 2022-04-28T19:53:25.107395+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2022-04-28T19:53:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:26 smithi059 bash[59745]: cluster 2022-04-28T19:53:24.950150+0000 mgr.y (mgr.24961) 228 : cluster [DBG] pgmap v363: 89 pgs: 89 active+clean; 582 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 11 MiB/s rd, 5.0 MiB/s wr, 9 op/s 2022-04-28T19:53:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:26 smithi059 bash[59745]: cluster 2022-04-28T19:53:25.107395+0000 mon.a (mon.0) 812 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2022-04-28T19:53:26.916 INFO:tasks.rados.rados.0.smithi059.stdout:2865: done (2 left) 2022-04-28T19:53:26.917 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 268 2022-04-28T19:53:26.917 INFO:tasks.rados.rados.0.smithi059.stdout:2866: seq_num 898 ranges {4636672=647168} 2022-04-28T19:53:26.951 INFO:tasks.rados.rados.0.smithi059.stdout:2866: writing smithi05955964-37 from 4636672 to 5283840 tid 1 2022-04-28T19:53:26.951 INFO:tasks.rados.rados.0.smithi059.stdout:2867: read oid 10 snap -1 2022-04-28T19:53:26.951 INFO:tasks.rados.rados.0.smithi059.stdout:2867: expect deleted 2022-04-28T19:53:26.951 INFO:tasks.rados.rados.0.smithi059.stdout:2868: setattr oid 41 current snap is 268 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:2863: finishing write tid 1 to smithi05955964-47 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:2863: finishing write tid 3 to smithi05955964-47 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:2863: finishing write tid 4 to smithi05955964-47 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1143 (ObjNum 897 snap 268 seq_num 897) dirty exists 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:2863: left oid 47 (ObjNum 897 snap 268 seq_num 897) 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1150 (ObjNum 883 snap 263 seq_num 883) dirty exists 2022-04-28T19:53:26.952 INFO:tasks.rados.rados.0.smithi059.stdout:2863: done (4 left) 2022-04-28T19:53:26.953 INFO:tasks.rados.rados.0.smithi059.stdout:2864: done (3 left) 2022-04-28T19:53:26.953 INFO:tasks.rados.rados.0.smithi059.stdout:2869: setattr oid 49 current snap is 268 2022-04-28T19:53:26.954 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 268 2022-04-28T19:53:26.954 INFO:tasks.rados.rados.0.smithi059.stdout:2870: seq_num 899 ranges {598016=663552} 2022-04-28T19:53:26.962 INFO:tasks.rados.rados.0.smithi059.stdout:2870: writing smithi05955964-42 from 598016 to 1261568 tid 1 2022-04-28T19:53:26.962 INFO:tasks.rados.rados.0.smithi059.stdout:2871: delete oid 46 current snap is 268 2022-04-28T19:53:26.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1016 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:53:26.967 INFO:tasks.rados.rados.0.smithi059.stdout:2866: finishing write tid 1 to smithi05955964-37 2022-04-28T19:53:26.967 INFO:tasks.rados.rados.0.smithi059.stdout:2866: finishing write tid 3 to smithi05955964-37 2022-04-28T19:53:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1080 (ObjNum 850 snap 251 seq_num 850) dirty exists 2022-04-28T19:53:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:2867: done (5 left) 2022-04-28T19:53:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:2868: done (4 left) 2022-04-28T19:53:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:2869: done (3 left) 2022-04-28T19:53:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:2871: done (2 left) 2022-04-28T19:53:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:2872: rollback oid 46 current snap is 268 2022-04-28T19:53:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 257 2022-04-28T19:53:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 268 2022-04-28T19:53:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:2873: seq_num 901 ranges {6938624=696320} 2022-04-28T19:53:27.012 INFO:tasks.rados.rados.0.smithi059.stdout:2873: writing smithi05955964-23 from 6938624 to 7634944 tid 1 2022-04-28T19:53:27.012 INFO:tasks.rados.rados.0.smithi059.stdout:2874: read oid 20 snap -1 2022-04-28T19:53:27.012 INFO:tasks.rados.rados.0.smithi059.stdout:2874: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:53:27.012 INFO:tasks.rados.rados.0.smithi059.stdout:2875: delete oid 22 current snap is 268 2022-04-28T19:53:27.012 INFO:tasks.rados.rados.0.smithi059.stdout:2870: finishing write tid 1 to smithi05955964-42 2022-04-28T19:53:27.012 INFO:tasks.rados.rados.0.smithi059.stdout:2870: finishing write tid 3 to smithi05955964-42 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:2866: finishing write tid 4 to smithi05955964-37 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1315 (ObjNum 898 snap 268 seq_num 898) dirty exists 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:2866: left oid 37 (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:2870: finishing write tid 4 to smithi05955964-42 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1317 (ObjNum 899 snap 268 seq_num 899) dirty exists 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:2870: left oid 42 (ObjNum 899 snap 268 seq_num 899) 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:2872: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:53:27.013 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1353 (ObjNum 0 snap 0 seq_num 0) dirty dne 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2866: done (5 left) 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2870: done (4 left) 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2872: done (3 left) 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2875: done (2 left) 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2876: read oid 28 snap -1 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2876: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2877: read oid 46 snap 261 2022-04-28T19:53:27.015 INFO:tasks.rados.rados.0.smithi059.stdout:2877: expect (ObjNum -133343456 snap 21893 seq_num 4162579040) 2022-04-28T19:53:27.016 INFO:tasks.rados.rados.0.smithi059.stdout:2878: snap_remove snap 257 2022-04-28T19:53:27.277 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:53:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:27 smithi139 bash[43301]: cluster 2022-04-28T19:53:26.126835+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2022-04-28T19:53:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:27 smithi139 bash[43301]: cluster 2022-04-28T19:53:26.915471+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2022-04-28T19:53:27.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:27 smithi059 bash[61152]: cluster 2022-04-28T19:53:26.126835+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2022-04-28T19:53:27.632 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:27 smithi059 bash[61152]: cluster 2022-04-28T19:53:26.915471+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2022-04-28T19:53:27.632 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:27 smithi059 bash[59745]: cluster 2022-04-28T19:53:26.126835+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2022-04-28T19:53:27.632 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:27 smithi059 bash[59745]: cluster 2022-04-28T19:53:26.915471+0000 mon.a (mon.0) 814 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (12m) 81s ago 12m 14.7M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (12m) 21s ago 12m 26.3M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (10m) 81s ago 10m 14.9M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (10m) 21s ago 10m 19.0M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (4m) 21s ago 16m 376M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (5m) 81s ago 18m 415M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:53:27.633 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (4m) 81s ago 18m 66.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (3m) 21s ago 16m 48.7M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (3m) 81s ago 17m 52.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (12m) 81s ago 13m 11.2M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (12m) 21s ago 13m 11.2M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (2m) 81s ago 16m 294M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (2m) 81s ago 15m 215M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (112s) 81s ago 15m 256M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (83s) 81s ago 15m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (54s) 21s ago 14m 249M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:53:27.634 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (24s) 21s ago 14m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 4d55ca87a6a9 2022-04-28T19:53:27.635 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (13m) 21s ago 13m 247M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 c36b8738a469 2022-04-28T19:53:27.635 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (13m) 21s ago 13m 207M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:53:27.635 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (5m) 21s ago 13m 39.9M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:53:27.934 INFO:tasks.rados.rados.0.smithi059.stdout:2878: done (4 left) 2022-04-28T19:53:27.934 INFO:tasks.rados.rados.0.smithi059.stdout:2879: delete oid 22 current snap is 268 2022-04-28T19:53:27.934 INFO:tasks.rados.rados.0.smithi059.stdout:2877: expect (ObjNum -133343456 snap 21893 seq_num 4162579040) 2022-04-28T19:53:27.938 INFO:tasks.rados.rados.0.smithi059.stdout:2876: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:27.964 INFO:tasks.rados.rados.0.smithi059.stdout:2874: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:53:27.990 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:53:27.990 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:53:27.990 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:53:27.990 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:27.990 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:53:27.990 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2, 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 6 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:53:27.991 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 4, 2022-04-28T19:53:27.992 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 11 2022-04-28T19:53:27.992 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:53:27.992 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:53:28.113 INFO:tasks.rados.rados.0.smithi059.stdout:2873: finishing write tid 1 to smithi05955964-23 2022-04-28T19:53:28.113 INFO:tasks.rados.rados.0.smithi059.stdout:2873: finishing write tid 3 to smithi05955964-23 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2873: finishing write tid 4 to smithi05955964-23 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1146 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2873: left oid 23 (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2873: done (4 left) 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2874: done (3 left) 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2876: done (2 left) 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2877: done (1 left) 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2879: done (0 left) 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:2880: rollback oid 32 current snap is 268 2022-04-28T19:53:28.114 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 253 2022-04-28T19:53:28.115 INFO:tasks.rados.rados.0.smithi059.stdout:2881: read oid 10 snap -1 2022-04-28T19:53:28.115 INFO:tasks.rados.rados.0.smithi059.stdout:2881: expect deleted 2022-04-28T19:53:28.115 INFO:tasks.rados.rados.0.smithi059.stdout:2882: copy_from oid 48 from oid 24 current snap is 268 2022-04-28T19:53:28.115 INFO:tasks.rados.rados.0.smithi059.stdout:2883: snap_create 2022-04-28T19:53:28.119 INFO:tasks.rados.rados.0.smithi059.stdout:2882: finishing copy_from to smithi05955964-48 2022-04-28T19:53:28.119 INFO:tasks.rados.rados.0.smithi059.stdout:2882: got expected ENOENT (src dne) 2022-04-28T19:53:28.119 INFO:tasks.rados.rados.0.smithi059.stdout:2882: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:53:28.123 INFO:tasks.rados.rados.0.smithi059.stdout:2880: finishing rollback tid 0 to smithi05955964-32 2022-04-28T19:53:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:2880: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:53:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:2880: finishing rollback tid 2 to smithi05955964-32 2022-04-28T19:53:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1339 (ObjNum 819 snap 240 seq_num 819) dirty exists 2022-04-28T19:53:28.368 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:28 smithi139 systemd[1]: Stopping Ceph osd.6 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: cluster 2022-04-28T19:53:26.950923+0000 mgr.y (mgr.24961) 229 : cluster [DBG] pgmap v367: 89 pgs: 89 active+clean; 583 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.1 MiB/s wr, 9 op/s 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.263337+0000 mgr.y (mgr.24961) 230 : audit [DBG] from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.590867+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.591701+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.592314+0000 mon.a (mon.0) 817 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.592969+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.593610+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.594233+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.596811+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.597501+0000 mon.a (mon.0) 822 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.598241+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.598945+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.599642+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: cluster 2022-04-28T19:53:27.933260+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2022-04-28T19:53:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[43301]: audit 2022-04-28T19:53:27.990295+0000 mon.c (mon.1) 13 : audit [DBG] from='client.? 172.21.15.59:0/186798868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: cluster 2022-04-28T19:53:26.950923+0000 mgr.y (mgr.24961) 229 : cluster [DBG] pgmap v367: 89 pgs: 89 active+clean; 583 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.1 MiB/s wr, 9 op/s 2022-04-28T19:53:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.263337+0000 mgr.y (mgr.24961) 230 : audit [DBG] from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.590867+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.591701+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.592314+0000 mon.a (mon.0) 817 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.592969+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.593610+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.594233+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.596811+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.597501+0000 mon.a (mon.0) 822 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.598241+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.598945+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.599642+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: cluster 2022-04-28T19:53:27.933260+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2022-04-28T19:53:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[59745]: audit 2022-04-28T19:53:27.990295+0000 mon.c (mon.1) 13 : audit [DBG] from='client.? 172.21.15.59:0/186798868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: cluster 2022-04-28T19:53:26.950923+0000 mgr.y (mgr.24961) 229 : cluster [DBG] pgmap v367: 89 pgs: 89 active+clean; 583 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.1 MiB/s wr, 9 op/s 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.263337+0000 mgr.y (mgr.24961) 230 : audit [DBG] from='client.44313 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.590867+0000 mon.a (mon.0) 815 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.591701+0000 mon.a (mon.0) 816 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.592314+0000 mon.a (mon.0) 817 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.592969+0000 mon.a (mon.0) 818 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.593610+0000 mon.a (mon.0) 819 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.594233+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.596811+0000 mon.a (mon.0) 821 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:28.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.597501+0000 mon.a (mon.0) 822 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.598241+0000 mon.a (mon.0) 823 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.598945+0000 mon.a (mon.0) 824 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.599642+0000 mon.a (mon.0) 825 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:28.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: cluster 2022-04-28T19:53:27.933260+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2022-04-28T19:53:28.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:28 smithi059 bash[61152]: audit 2022-04-28T19:53:27.990295+0000 mon.c (mon.1) 13 : audit [DBG] from='client.? 172.21.15.59:0/186798868' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:28.868 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[29993]: debug 2022-04-28T19:53:28.361+0000 7fa53f155700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:53:28.868 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[29993]: debug 2022-04-28T19:53:28.361+0000 7fa53f155700 -1 osd.6 589 *** Got signal Terminated *** 2022-04-28T19:53:28.868 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:28 smithi139 bash[29993]: debug 2022-04-28T19:53:28.361+0000 7fa53f155700 -1 osd.6 589 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:53:28.941 INFO:tasks.rados.rados.0.smithi059.stdout:2880: done (3 left) 2022-04-28T19:53:28.941 INFO:tasks.rados.rados.0.smithi059.stdout:2881: done (2 left) 2022-04-28T19:53:28.942 INFO:tasks.rados.rados.0.smithi059.stdout:2882: done (1 left) 2022-04-28T19:53:28.942 INFO:tasks.rados.rados.0.smithi059.stdout:2883: done (0 left) 2022-04-28T19:53:28.942 INFO:tasks.rados.rados.0.smithi059.stdout:2884: snap_create 2022-04-28T19:53:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: audit 2022-04-28T19:53:27.585089+0000 mgr.y (mgr.24961) 231 : audit [DBG] from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.688116+0000 mon.a (mon.0) 827 : cluster [DBG] osd.6 reported immediately failed by osd.7 2022-04-28T19:53:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.688378+0000 mon.a (mon.0) 828 : cluster [INF] osd.6 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:53:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.689009+0000 mon.a (mon.0) 829 : cluster [DBG] osd.6 reported immediately failed by osd.7 2022-04-28T19:53:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.689500+0000 mon.a (mon.0) 830 : cluster [DBG] osd.6 reported immediately failed by osd.2 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.689960+0000 mon.a (mon.0) 831 : cluster [DBG] osd.6 reported immediately failed by osd.1 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.690319+0000 mon.a (mon.0) 832 : cluster [DBG] osd.6 reported immediately failed by osd.5 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.690539+0000 mon.a (mon.0) 833 : cluster [DBG] osd.6 reported immediately failed by osd.2 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.690824+0000 mon.a (mon.0) 834 : cluster [DBG] osd.6 reported immediately failed by osd.3 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.691208+0000 mon.a (mon.0) 835 : cluster [DBG] osd.6 reported immediately failed by osd.5 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.691413+0000 mon.a (mon.0) 836 : cluster [DBG] osd.6 reported immediately failed by osd.0 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.691910+0000 mon.a (mon.0) 837 : cluster [DBG] osd.6 reported immediately failed by osd.4 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.692121+0000 mon.a (mon.0) 838 : cluster [DBG] osd.6 reported immediately failed by osd.1 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.692643+0000 mon.a (mon.0) 839 : cluster [DBG] osd.6 reported immediately failed by osd.4 2022-04-28T19:53:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.692903+0000 mon.a (mon.0) 840 : cluster [DBG] osd.6 reported immediately failed by osd.3 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.693325+0000 mon.a (mon.0) 841 : cluster [DBG] osd.6 reported immediately failed by osd.0 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.928750+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.941062+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e590: 8 total, 7 up, 8 in 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: audit 2022-04-28T19:53:27.585089+0000 mgr.y (mgr.24961) 231 : audit [DBG] from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.688116+0000 mon.a (mon.0) 827 : cluster [DBG] osd.6 reported immediately failed by osd.7 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.688378+0000 mon.a (mon.0) 828 : cluster [INF] osd.6 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.689009+0000 mon.a (mon.0) 829 : cluster [DBG] osd.6 reported immediately failed by osd.7 2022-04-28T19:53:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.689500+0000 mon.a (mon.0) 830 : cluster [DBG] osd.6 reported immediately failed by osd.2 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.689960+0000 mon.a (mon.0) 831 : cluster [DBG] osd.6 reported immediately failed by osd.1 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.690319+0000 mon.a (mon.0) 832 : cluster [DBG] osd.6 reported immediately failed by osd.5 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.690539+0000 mon.a (mon.0) 833 : cluster [DBG] osd.6 reported immediately failed by osd.2 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.690824+0000 mon.a (mon.0) 834 : cluster [DBG] osd.6 reported immediately failed by osd.3 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.691208+0000 mon.a (mon.0) 835 : cluster [DBG] osd.6 reported immediately failed by osd.5 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.691413+0000 mon.a (mon.0) 836 : cluster [DBG] osd.6 reported immediately failed by osd.0 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.691910+0000 mon.a (mon.0) 837 : cluster [DBG] osd.6 reported immediately failed by osd.4 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.692121+0000 mon.a (mon.0) 838 : cluster [DBG] osd.6 reported immediately failed by osd.1 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.692643+0000 mon.a (mon.0) 839 : cluster [DBG] osd.6 reported immediately failed by osd.4 2022-04-28T19:53:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.692903+0000 mon.a (mon.0) 840 : cluster [DBG] osd.6 reported immediately failed by osd.3 2022-04-28T19:53:29.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.693325+0000 mon.a (mon.0) 841 : cluster [DBG] osd.6 reported immediately failed by osd.0 2022-04-28T19:53:29.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.928750+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:53:29.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:29 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.941062+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e590: 8 total, 7 up, 8 in 2022-04-28T19:53:29.618 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[48531]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.6 2022-04-28T19:53:29.618 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[48531]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-6 2022-04-28T19:53:29.618 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[48531]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.6 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: audit 2022-04-28T19:53:27.585089+0000 mgr.y (mgr.24961) 231 : audit [DBG] from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.688116+0000 mon.a (mon.0) 827 : cluster [DBG] osd.6 reported immediately failed by osd.7 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.688378+0000 mon.a (mon.0) 828 : cluster [INF] osd.6 failed (root=default,host=smithi139) (connection refused reported by osd.7) 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.689009+0000 mon.a (mon.0) 829 : cluster [DBG] osd.6 reported immediately failed by osd.7 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.689500+0000 mon.a (mon.0) 830 : cluster [DBG] osd.6 reported immediately failed by osd.2 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.689960+0000 mon.a (mon.0) 831 : cluster [DBG] osd.6 reported immediately failed by osd.1 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.690319+0000 mon.a (mon.0) 832 : cluster [DBG] osd.6 reported immediately failed by osd.5 2022-04-28T19:53:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.690539+0000 mon.a (mon.0) 833 : cluster [DBG] osd.6 reported immediately failed by osd.2 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.690824+0000 mon.a (mon.0) 834 : cluster [DBG] osd.6 reported immediately failed by osd.3 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.691208+0000 mon.a (mon.0) 835 : cluster [DBG] osd.6 reported immediately failed by osd.5 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.691413+0000 mon.a (mon.0) 836 : cluster [DBG] osd.6 reported immediately failed by osd.0 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.691910+0000 mon.a (mon.0) 837 : cluster [DBG] osd.6 reported immediately failed by osd.4 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.692121+0000 mon.a (mon.0) 838 : cluster [DBG] osd.6 reported immediately failed by osd.1 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.692643+0000 mon.a (mon.0) 839 : cluster [DBG] osd.6 reported immediately failed by osd.4 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.692903+0000 mon.a (mon.0) 840 : cluster [DBG] osd.6 reported immediately failed by osd.3 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.693325+0000 mon.a (mon.0) 841 : cluster [DBG] osd.6 reported immediately failed by osd.0 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.928750+0000 mon.a (mon.0) 842 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:53:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:29 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.941062+0000 mon.a (mon.0) 843 : cluster [DBG] osdmap e590: 8 total, 7 up, 8 in 2022-04-28T19:53:29.948 INFO:tasks.rados.rados.0.smithi059.stdout:2884: done (0 left) 2022-04-28T19:53:29.948 INFO:tasks.rados.rados.0.smithi059.stdout:2885: snap_create 2022-04-28T19:53:30.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:30 smithi139 bash[43301]: cluster 2022-04-28T19:53:28.951458+0000 mgr.y (mgr.24961) 232 : cluster [DBG] pgmap v370: 89 pgs: 9 peering, 5 stale+active+clean, 75 active+clean; 591 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2022-04-28T19:53:30.390 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:30 smithi139 bash[43301]: cluster 2022-04-28T19:53:29.943080+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e591: 8 total, 7 up, 8 in 2022-04-28T19:53:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:30 smithi059 bash[61152]: cluster 2022-04-28T19:53:28.951458+0000 mgr.y (mgr.24961) 232 : cluster [DBG] pgmap v370: 89 pgs: 9 peering, 5 stale+active+clean, 75 active+clean; 591 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2022-04-28T19:53:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:30 smithi059 bash[61152]: cluster 2022-04-28T19:53:29.943080+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e591: 8 total, 7 up, 8 in 2022-04-28T19:53:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:30 smithi059 bash[59745]: cluster 2022-04-28T19:53:28.951458+0000 mgr.y (mgr.24961) 232 : cluster [DBG] pgmap v370: 89 pgs: 9 peering, 5 stale+active+clean, 75 active+clean; 591 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 9.0 MiB/s rd, 2.3 MiB/s wr, 11 op/s 2022-04-28T19:53:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:30 smithi059 bash[59745]: cluster 2022-04-28T19:53:29.943080+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e591: 8 total, 7 up, 8 in 2022-04-28T19:53:30.951 INFO:tasks.rados.rados.0.smithi059.stdout:2885: done (0 left) 2022-04-28T19:53:30.951 INFO:tasks.rados.rados.0.smithi059.stdout:2886: rmattr oid 31 current snap is 271 2022-04-28T19:53:30.951 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 271 2022-04-28T19:53:30.951 INFO:tasks.rados.rados.0.smithi059.stdout:2887: seq_num 904 ranges {5283840=745472} 2022-04-28T19:53:30.989 INFO:tasks.rados.rados.0.smithi059.stdout:2887: writing smithi05955964-37 from 5283840 to 6029312 tid 1 2022-04-28T19:53:30.989 INFO:tasks.rados.rados.0.smithi059.stdout:2888: copy_from oid 2 from oid 39 current snap is 271 2022-04-28T19:53:30.989 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1318 (ObjNum 889 snap 265 seq_num 889) dirty exists 2022-04-28T19:53:30.989 INFO:tasks.rados.rados.0.smithi059.stdout:2886: done (2 left) 2022-04-28T19:53:30.989 INFO:tasks.rados.rados.0.smithi059.stdout:2889: rollback oid 14 current snap is 271 2022-04-28T19:53:30.990 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 236 2022-04-28T19:53:30.990 INFO:tasks.rados.rados.0.smithi059.stdout:2890: read oid 28 snap -1 2022-04-28T19:53:30.990 INFO:tasks.rados.rados.0.smithi059.stdout:2890: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:30.990 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 271 2022-04-28T19:53:30.990 INFO:tasks.rados.rados.0.smithi059.stdout:2891: seq_num 905 ranges {90112=761856} 2022-04-28T19:53:30.995 INFO:tasks.rados.rados.0.smithi059.stdout:2891: writing smithi05955964-50 from 90112 to 851968 tid 1 2022-04-28T19:53:30.995 INFO:tasks.rados.rados.0.smithi059.stdout:2892: read oid 20 snap -1 2022-04-28T19:53:30.995 INFO:tasks.rados.rados.0.smithi059.stdout:2892: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:53:30.995 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 271 2022-04-28T19:53:30.995 INFO:tasks.rados.rados.0.smithi059.stdout:2893: seq_num 906 ranges {7479296=778240} 2022-04-28T19:53:31.042 INFO:tasks.rados.rados.0.smithi059.stdout:2893: writing smithi05955964-12 from 7479296 to 8257536 tid 1 2022-04-28T19:53:31.042 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 271 2022-04-28T19:53:31.043 INFO:tasks.rados.rados.0.smithi059.stdout:2894: seq_num 907 ranges {3629056=794624} 2022-04-28T19:53:31.067 INFO:tasks.rados.rados.0.smithi059.stdout:2894: writing smithi05955964-38 from 3629056 to 4423680 tid 1 2022-04-28T19:53:31.067 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 271 2022-04-28T19:53:31.067 INFO:tasks.rados.rados.0.smithi059.stdout:2895: seq_num 908 ranges {5382144=409600,5791744=401408} 2022-04-28T19:53:31.100 INFO:tasks.rados.rados.0.smithi059.stdout:2895: writing smithi05955964-49 from 5382144 to 5791744 tid 1 2022-04-28T19:53:31.102 INFO:tasks.rados.rados.0.smithi059.stdout:2895: writing smithi05955964-49 from 5791744 to 6193152 tid 2 2022-04-28T19:53:31.102 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 271 2022-04-28T19:53:31.103 INFO:tasks.rados.rados.0.smithi059.stdout:2896: seq_num 909 ranges {8552448=425984,8978432=401408} 2022-04-28T19:53:31.153 INFO:tasks.rados.rados.0.smithi059.stdout:2896: writing smithi05955964-25 from 8552448 to 8978432 tid 1 2022-04-28T19:53:31.155 INFO:tasks.rados.rados.0.smithi059.stdout:2896: writing smithi05955964-25 from 8978432 to 9379840 tid 2 2022-04-28T19:53:31.156 INFO:tasks.rados.rados.0.smithi059.stdout:2897: read oid 18 snap -1 2022-04-28T19:53:31.156 INFO:tasks.rados.rados.0.smithi059.stdout:2897: expect (ObjNum 866 snap 258 seq_num 866) 2022-04-28T19:53:31.156 INFO:tasks.rados.rados.0.smithi059.stdout:2898: snap_remove snap 258 2022-04-28T19:53:31.868 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:31 smithi139 systemd[1]: Stopped Ceph osd.6 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:53:31.868 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:31 smithi139 systemd[1]: Started Ceph osd.6 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:53:31.953 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:31.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:31.953 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:31.633Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:31.953 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:31.633Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:31.961 INFO:tasks.rados.rados.0.smithi059.stdout:2898: done (11 left) 2022-04-28T19:53:31.961 INFO:tasks.rados.rados.0.smithi059.stdout:2899: delete oid 21 current snap is 271 2022-04-28T19:53:31.961 INFO:tasks.rados.rados.0.smithi059.stdout:2889: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:53:31.961 INFO:tasks.rados.rados.0.smithi059.stdout:2889: finishing rollback tid 2 to smithi05955964-14 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1158 (ObjNum 800 snap 235 seq_num 800) dirty exists 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2887: finishing write tid 1 to smithi05955964-37 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2887: finishing write tid 3 to smithi05955964-37 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2887: finishing write tid 4 to smithi05955964-37 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1319 (ObjNum 904 snap 271 seq_num 904) dirty exists 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2887: left oid 37 (ObjNum 904 snap 271 seq_num 904) 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2891: finishing write tid 1 to smithi05955964-50 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2891: finishing write tid 3 to smithi05955964-50 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2891: finishing write tid 4 to smithi05955964-50 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1352 (ObjNum 905 snap 271 seq_num 905) dirty exists 2022-04-28T19:53:31.962 INFO:tasks.rados.rados.0.smithi059.stdout:2891: left oid 50 (ObjNum 905 snap 271 seq_num 905) 2022-04-28T19:53:31.963 INFO:tasks.rados.rados.0.smithi059.stdout:2890: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:31.990 INFO:tasks.rados.rados.0.smithi059.stdout:2892: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:53:32.145 INFO:tasks.rados.rados.0.smithi059.stdout:2888: finishing copy_from to smithi05955964-2 2022-04-28T19:53:32.145 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1325 (ObjNum 894 snap 267 seq_num 894) dirty exists 2022-04-28T19:53:32.145 INFO:tasks.rados.rados.0.smithi059.stdout:2888: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:53:32.145 INFO:tasks.rados.rados.0.smithi059.stdout:2893: finishing write tid 1 to smithi05955964-12 2022-04-28T19:53:32.145 INFO:tasks.rados.rados.0.smithi059.stdout:2893: finishing write tid 3 to smithi05955964-12 2022-04-28T19:53:32.145 INFO:tasks.rados.rados.0.smithi059.stdout:2893: finishing write tid 4 to smithi05955964-12 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1320 (ObjNum 906 snap 271 seq_num 906) dirty exists 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2893: left oid 12 (ObjNum 906 snap 271 seq_num 906) 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2894: finishing write tid 1 to smithi05955964-38 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2894: finishing write tid 3 to smithi05955964-38 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2894: finishing write tid 4 to smithi05955964-38 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1163 (ObjNum 907 snap 271 seq_num 907) dirty exists 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2894: left oid 38 (ObjNum 907 snap 271 seq_num 907) 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2895: finishing write tid 1 to smithi05955964-49 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2895: finishing write tid 2 to smithi05955964-49 2022-04-28T19:53:32.146 INFO:tasks.rados.rados.0.smithi059.stdout:2887: done (11 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2888: done (10 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2889: done (9 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2890: done (8 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2891: done (7 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2892: done (6 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2893: done (5 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2894: done (4 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2899: done (3 left) 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:2900: rollback oid 11 current snap is 271 2022-04-28T19:53:32.147 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 260 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2895: finishing write tid 4 to smithi05955964-49 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2895: finishing write tid 5 to smithi05955964-49 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1083 (ObjNum 908 snap 271 seq_num 908) dirty exists 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2895: left oid 49 (ObjNum 908 snap 271 seq_num 908) 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2896: finishing write tid 1 to smithi05955964-25 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2896: finishing write tid 2 to smithi05955964-25 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2896: finishing write tid 4 to smithi05955964-25 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2896: finishing write tid 5 to smithi05955964-25 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1362 (ObjNum 909 snap 271 seq_num 909) dirty exists 2022-04-28T19:53:32.148 INFO:tasks.rados.rados.0.smithi059.stdout:2896: left oid 25 (ObjNum 909 snap 271 seq_num 909) 2022-04-28T19:53:32.149 INFO:tasks.rados.rados.0.smithi059.stdout:2897: expect (ObjNum 866 snap 258 seq_num 866) 2022-04-28T19:53:32.328 INFO:tasks.rados.rados.0.smithi059.stdout:2900: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2900: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2900: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1166 (ObjNum 857 snap 253 seq_num 857) dirty exists 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2895: done (3 left) 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2896: done (2 left) 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2897: done (1 left) 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2900: done (0 left) 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 271 2022-04-28T19:53:32.329 INFO:tasks.rados.rados.0.smithi059.stdout:2901: seq_num 911 ranges {0=466944,466944=401408} 2022-04-28T19:53:32.336 INFO:tasks.rados.rados.0.smithi059.stdout:2901: writing smithi05955964-46 from 0 to 466944 tid 1 2022-04-28T19:53:32.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: cluster 2022-04-28T19:53:30.950740+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e592: 8 total, 7 up, 8 in 2022-04-28T19:53:32.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: cluster 2022-04-28T19:53:30.952079+0000 mgr.y (mgr.24961) 233 : cluster [DBG] pgmap v373: 89 pgs: 15 peering, 3 stale+active+clean, 8 active+undersized, 63 active+clean; 595 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.8 MiB/s wr, 30 op/s 2022-04-28T19:53:32.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: audit 2022-04-28T19:53:31.666456+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:32.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: audit 2022-04-28T19:53:31.668388+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: audit 2022-04-28T19:53:31.671142+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: audit 2022-04-28T19:53:31.672954+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[59745]: audit 2022-04-28T19:53:31.677026+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: cluster 2022-04-28T19:53:30.950740+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e592: 8 total, 7 up, 8 in 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: cluster 2022-04-28T19:53:30.952079+0000 mgr.y (mgr.24961) 233 : cluster [DBG] pgmap v373: 89 pgs: 15 peering, 3 stale+active+clean, 8 active+undersized, 63 active+clean; 595 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.8 MiB/s wr, 30 op/s 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: audit 2022-04-28T19:53:31.666456+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: audit 2022-04-28T19:53:31.668388+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: audit 2022-04-28T19:53:31.671142+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:53:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: audit 2022-04-28T19:53:31.672954+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:53:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:31 smithi059 bash[61152]: audit 2022-04-28T19:53:31.677026+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:53:32.343 INFO:tasks.rados.rados.0.smithi059.stdout:2901: writing smithi05955964-46 from 466944 to 868352 tid 2 2022-04-28T19:53:32.343 INFO:tasks.rados.rados.0.smithi059.stdout:2902: read oid 45 snap 219 2022-04-28T19:53:32.343 INFO:tasks.rados.rados.0.smithi059.stdout:2902: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:32.343 INFO:tasks.rados.rados.0.smithi059.stdout:2903: delete oid 31 current snap is 271 2022-04-28T19:53:32.344 INFO:tasks.rados.rados.0.smithi059.stdout:2901: finishing write tid 1 to smithi05955964-46 2022-04-28T19:53:32.345 INFO:tasks.rados.rados.0.smithi059.stdout:2903: done (2 left) 2022-04-28T19:53:32.346 INFO:tasks.rados.rados.0.smithi059.stdout:2904: snap_create 2022-04-28T19:53:32.348 INFO:tasks.rados.rados.0.smithi059.stdout:2901: finishing write tid 2 to smithi05955964-46 2022-04-28T19:53:32.349 INFO:tasks.rados.rados.0.smithi059.stdout:2901: finishing write tid 4 to smithi05955964-46 2022-04-28T19:53:32.350 INFO:tasks.rados.rados.0.smithi059.stdout:2901: finishing write tid 5 to smithi05955964-46 2022-04-28T19:53:32.350 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1365 (ObjNum 911 snap 271 seq_num 911) dirty exists 2022-04-28T19:53:32.350 INFO:tasks.rados.rados.0.smithi059.stdout:2901: left oid 46 (ObjNum 911 snap 271 seq_num 911) 2022-04-28T19:53:32.355 INFO:tasks.rados.rados.0.smithi059.stdout:2902: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: cluster 2022-04-28T19:53:30.950740+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e592: 8 total, 7 up, 8 in 2022-04-28T19:53:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: cluster 2022-04-28T19:53:30.952079+0000 mgr.y (mgr.24961) 233 : cluster [DBG] pgmap v373: 89 pgs: 15 peering, 3 stale+active+clean, 8 active+undersized, 63 active+clean; 595 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.8 MiB/s wr, 30 op/s 2022-04-28T19:53:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: audit 2022-04-28T19:53:31.666456+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: audit 2022-04-28T19:53:31.668388+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: audit 2022-04-28T19:53:31.671142+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:53:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: audit 2022-04-28T19:53:31.672954+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:53:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:31 smithi139 bash[43301]: audit 2022-04-28T19:53:31.677026+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:53:32.970 INFO:tasks.rados.rados.0.smithi059.stdout:2901: done (2 left) 2022-04-28T19:53:32.970 INFO:tasks.rados.rados.0.smithi059.stdout:2902: done (1 left) 2022-04-28T19:53:32.971 INFO:tasks.rados.rados.0.smithi059.stdout:2904: done (0 left) 2022-04-28T19:53:32.971 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 272 2022-04-28T19:53:32.971 INFO:tasks.rados.rados.0.smithi059.stdout:2905: seq_num 913 ranges {11804672=499712,12304384=401408} 2022-04-28T19:53:33.039 INFO:tasks.rados.rados.0.smithi059.stdout:2905: writing smithi05955964-45 from 11804672 to 12304384 tid 1 2022-04-28T19:53:33.042 INFO:tasks.rados.rados.0.smithi059.stdout:2905: writing smithi05955964-45 from 12304384 to 12705792 tid 2 2022-04-28T19:53:33.042 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 272 2022-04-28T19:53:33.042 INFO:tasks.rados.rados.0.smithi059.stdout:2906: seq_num 914 ranges {2056192=516096,2572288=401408} 2022-04-28T19:53:33.057 INFO:tasks.rados.rados.0.smithi059.stdout:2906: writing smithi05955964-1 from 2056192 to 2572288 tid 1 2022-04-28T19:53:33.059 INFO:tasks.rados.rados.0.smithi059.stdout:2906: writing smithi05955964-1 from 2572288 to 2973696 tid 2 2022-04-28T19:53:33.059 INFO:tasks.rados.rados.0.smithi059.stdout:2907: snap_remove snap 265 2022-04-28T19:53:33.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:32 smithi139 bash[43301]: cephadm 2022-04-28T19:53:31.674502+0000 mgr.y (mgr.24961) 234 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:53:33.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:32 smithi139 bash[43301]: cluster 2022-04-28T19:53:31.959619+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e593: 8 total, 7 up, 8 in 2022-04-28T19:53:33.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:32 smithi059 bash[61152]: cephadm 2022-04-28T19:53:31.674502+0000 mgr.y (mgr.24961) 234 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:53:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:32 smithi059 bash[61152]: cluster 2022-04-28T19:53:31.959619+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e593: 8 total, 7 up, 8 in 2022-04-28T19:53:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:32 smithi059 bash[59745]: cephadm 2022-04-28T19:53:31.674502+0000 mgr.y (mgr.24961) 234 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:53:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:32 smithi059 bash[59745]: cluster 2022-04-28T19:53:31.959619+0000 mon.a (mon.0) 851 : cluster [DBG] osdmap e593: 8 total, 7 up, 8 in 2022-04-28T19:53:33.618 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2022-04-28T19:53:33.618 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2022-04-28T19:53:33.619 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2022-04-28T19:53:33.619 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2022-04-28T19:53:33.619 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2022-04-28T19:53:33.619 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2022-04-28T19:53:33.619 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[48787]: --> ceph-volume lvm activate successful for osd ID: 6 2022-04-28T19:53:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:33.527Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:33.527Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:33.527Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2907: done (2 left) 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2908: read oid 9 snap 219 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2908: expect (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2909: rmattr oid 31 current snap is 272 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2909: done (3 left) 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2910: read oid 22 snap 268 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2910: expect deleted 2022-04-28T19:53:33.983 INFO:tasks.rados.rados.0.smithi059.stdout:2911: snap_create 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2905: finishing write tid 1 to smithi05955964-45 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2905: finishing write tid 2 to smithi05955964-45 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2905: finishing write tid 4 to smithi05955964-45 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2905: finishing write tid 5 to smithi05955964-45 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1218 (ObjNum 913 snap 272 seq_num 913) dirty exists 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2905: left oid 45 (ObjNum 913 snap 272 seq_num 913) 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2906: finishing write tid 1 to smithi05955964-1 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2906: finishing write tid 2 to smithi05955964-1 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2906: finishing write tid 4 to smithi05955964-1 2022-04-28T19:53:33.984 INFO:tasks.rados.rados.0.smithi059.stdout:2906: finishing write tid 5 to smithi05955964-1 2022-04-28T19:53:33.985 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1086 (ObjNum 914 snap 272 seq_num 914) dirty exists 2022-04-28T19:53:33.985 INFO:tasks.rados.rados.0.smithi059.stdout:2906: left oid 1 (ObjNum 914 snap 272 seq_num 914) 2022-04-28T19:53:34.014 INFO:tasks.rados.rados.0.smithi059.stdout:2908: expect (ObjNum 716 snap 208 seq_num 716) 2022-04-28T19:53:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[59745]: cluster 2022-04-28T19:53:32.953026+0000 mgr.y (mgr.24961) 235 : cluster [DBG] pgmap v375: 89 pgs: 5 active+undersized+degraded, 15 peering, 11 active+undersized, 58 active+clean; 597 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.3 MiB/s wr, 25 op/s; 80/1041 objects degraded (7.685%) 2022-04-28T19:53:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[59745]: cluster 2022-04-28T19:53:32.968336+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e594: 8 total, 7 up, 8 in 2022-04-28T19:53:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[59745]: cluster 2022-04-28T19:53:33.952961+0000 mon.a (mon.0) 853 : cluster [WRN] Health check failed: Degraded data redundancy: 80/1041 objects degraded (7.685%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[61152]: cluster 2022-04-28T19:53:32.953026+0000 mgr.y (mgr.24961) 235 : cluster [DBG] pgmap v375: 89 pgs: 5 active+undersized+degraded, 15 peering, 11 active+undersized, 58 active+clean; 597 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.3 MiB/s wr, 25 op/s; 80/1041 objects degraded (7.685%) 2022-04-28T19:53:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[61152]: cluster 2022-04-28T19:53:32.968336+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e594: 8 total, 7 up, 8 in 2022-04-28T19:53:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:33 smithi059 bash[61152]: cluster 2022-04-28T19:53:33.952961+0000 mon.a (mon.0) 853 : cluster [WRN] Health check failed: Degraded data redundancy: 80/1041 objects degraded (7.685%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[43301]: cluster 2022-04-28T19:53:32.953026+0000 mgr.y (mgr.24961) 235 : cluster [DBG] pgmap v375: 89 pgs: 5 active+undersized+degraded, 15 peering, 11 active+undersized, 58 active+clean; 597 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.3 MiB/s wr, 25 op/s; 80/1041 objects degraded (7.685%) 2022-04-28T19:53:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[43301]: cluster 2022-04-28T19:53:32.968336+0000 mon.a (mon.0) 852 : cluster [DBG] osdmap e594: 8 total, 7 up, 8 in 2022-04-28T19:53:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:33 smithi139 bash[43301]: cluster 2022-04-28T19:53:33.952961+0000 mon.a (mon.0) 853 : cluster [WRN] Health check failed: Degraded data redundancy: 80/1041 objects degraded (7.685%), 5 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:34.986 INFO:tasks.rados.rados.0.smithi059.stdout:2905: done (4 left) 2022-04-28T19:53:34.986 INFO:tasks.rados.rados.0.smithi059.stdout:2906: done (3 left) 2022-04-28T19:53:34.986 INFO:tasks.rados.rados.0.smithi059.stdout:2908: done (2 left) 2022-04-28T19:53:34.986 INFO:tasks.rados.rados.0.smithi059.stdout:2910: done (1 left) 2022-04-28T19:53:34.986 INFO:tasks.rados.rados.0.smithi059.stdout:2911: done (0 left) 2022-04-28T19:53:34.987 INFO:tasks.rados.rados.0.smithi059.stdout:2912: snap_remove snap 263 2022-04-28T19:53:35.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:34 smithi059 bash[59745]: cluster 2022-04-28T19:53:33.979584+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e595: 8 total, 7 up, 8 in 2022-04-28T19:53:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:34 smithi059 bash[61152]: cluster 2022-04-28T19:53:33.979584+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e595: 8 total, 7 up, 8 in 2022-04-28T19:53:35.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:34 smithi139 bash[43301]: cluster 2022-04-28T19:53:33.979584+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e595: 8 total, 7 up, 8 in 2022-04-28T19:53:35.368 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:35 smithi139 bash[48787]: debug 2022-04-28T19:53:35.145+0000 7f1b660b3700 1 -- 172.21.15.139:0/3221260664 <== mon.2 v2:172.21.15.139:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x5571618ca840 con 0x55716194d400 2022-04-28T19:53:35.368 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:35 smithi139 bash[48787]: debug 2022-04-28T19:53:35.145+0000 7f1b660b3700 1 -- 172.21.15.139:0/3221260664 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5571618d44e0 con 0x55716194d400 2022-04-28T19:53:35.993 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:35 smithi139 bash[48787]: debug 2022-04-28T19:53:35.705+0000 7f1b6d90e200 -1 Falling back to public interface 2022-04-28T19:53:36.006 INFO:tasks.rados.rados.0.smithi059.stdout:2912: done (0 left) 2022-04-28T19:53:36.006 INFO:tasks.rados.rados.0.smithi059.stdout:2913: rollback oid 24 current snap is 273 2022-04-28T19:53:36.006 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 242 2022-04-28T19:53:36.007 INFO:tasks.rados.rados.0.smithi059.stdout:2914: copy_from oid 33 from oid 15 current snap is 273 2022-04-28T19:53:36.007 INFO:tasks.rados.rados.0.smithi059.stdout:2915: snap_remove snap 244 2022-04-28T19:53:36.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:35 smithi059 bash[61152]: cluster 2022-04-28T19:53:34.953961+0000 mgr.y (mgr.24961) 236 : cluster [DBG] pgmap v378: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 601 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 13 op/s; 103/1059 objects degraded (9.726%) 2022-04-28T19:53:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:35 smithi059 bash[61152]: cluster 2022-04-28T19:53:34.983312+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e596: 8 total, 7 up, 8 in 2022-04-28T19:53:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:35 smithi059 bash[59745]: cluster 2022-04-28T19:53:34.953961+0000 mgr.y (mgr.24961) 236 : cluster [DBG] pgmap v378: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 601 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 13 op/s; 103/1059 objects degraded (9.726%) 2022-04-28T19:53:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:35 smithi059 bash[59745]: cluster 2022-04-28T19:53:34.983312+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e596: 8 total, 7 up, 8 in 2022-04-28T19:53:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:35 smithi139 bash[43301]: cluster 2022-04-28T19:53:34.953961+0000 mgr.y (mgr.24961) 236 : cluster [DBG] pgmap v378: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 601 MiB data, 1.2 GiB used, 712 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 13 op/s; 103/1059 objects degraded (9.726%) 2022-04-28T19:53:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:35 smithi139 bash[43301]: cluster 2022-04-28T19:53:34.983312+0000 mon.a (mon.0) 855 : cluster [DBG] osdmap e596: 8 total, 7 up, 8 in 2022-04-28T19:53:36.920 INFO:tasks.rados.rados.0.smithi059.stdout:2915: done (2 left) 2022-04-28T19:53:36.920 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 273 2022-04-28T19:53:36.920 INFO:tasks.rados.rados.0.smithi059.stdout:2916: seq_num 915 ranges {3751936=532480,4284416=401408} 2022-04-28T19:53:36.949 INFO:tasks.rados.rados.0.smithi059.stdout:2916: writing smithi05955964-40 from 3751936 to 4284416 tid 1 2022-04-28T19:53:36.953 INFO:tasks.rados.rados.0.smithi059.stdout:2916: writing smithi05955964-40 from 4284416 to 4685824 tid 2 2022-04-28T19:53:36.954 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 273 2022-04-28T19:53:36.954 INFO:tasks.rados.rados.0.smithi059.stdout:2917: seq_num 916 ranges {0=548864,548864=401408} 2022-04-28T19:53:36.955 INFO:tasks.rados.rados.0.smithi059.stdout:2917: writing smithi05955964-21 from 0 to 548864 tid 1 2022-04-28T19:53:36.958 INFO:tasks.rados.rados.0.smithi059.stdout:2917: writing smithi05955964-21 from 548864 to 950272 tid 2 2022-04-28T19:53:36.959 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 273 2022-04-28T19:53:36.959 INFO:tasks.rados.rados.0.smithi059.stdout:2918: seq_num 917 ranges {3563520=565248,4128768=401408} 2022-04-28T19:53:36.982 INFO:tasks.rados.rados.0.smithi059.stdout:2918: writing smithi05955964-19 from 3563520 to 4128768 tid 1 2022-04-28T19:53:36.985 INFO:tasks.rados.rados.0.smithi059.stdout:2918: writing smithi05955964-19 from 4128768 to 4530176 tid 2 2022-04-28T19:53:36.985 INFO:tasks.rados.rados.0.smithi059.stdout:2919: delete oid 46 current snap is 273 2022-04-28T19:53:36.985 INFO:tasks.rados.rados.0.smithi059.stdout:2913: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:53:36.985 INFO:tasks.rados.rados.0.smithi059.stdout:2913: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1168 (ObjNum 806 snap 239 seq_num 806) dirty exists 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2914: finishing copy_from to smithi05955964-33 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1317 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2914: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2916: finishing write tid 1 to smithi05955964-40 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2916: finishing write tid 2 to smithi05955964-40 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2916: finishing write tid 4 to smithi05955964-40 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2916: finishing write tid 5 to smithi05955964-40 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1089 (ObjNum 915 snap 273 seq_num 915) dirty exists 2022-04-28T19:53:36.986 INFO:tasks.rados.rados.0.smithi059.stdout:2916: left oid 40 (ObjNum 915 snap 273 seq_num 915) 2022-04-28T19:53:36.987 INFO:tasks.rados.rados.0.smithi059.stdout:2917: finishing write tid 1 to smithi05955964-21 2022-04-28T19:53:36.987 INFO:tasks.rados.rados.0.smithi059.stdout:2917: finishing write tid 2 to smithi05955964-21 2022-04-28T19:53:36.987 INFO:tasks.rados.rados.0.smithi059.stdout:2917: finishing write tid 4 to smithi05955964-21 2022-04-28T19:53:36.987 INFO:tasks.rados.rados.0.smithi059.stdout:2917: finishing write tid 5 to smithi05955964-21 2022-04-28T19:53:36.987 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1154 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:53:36.987 INFO:tasks.rados.rados.0.smithi059.stdout:2917: left oid 21 (ObjNum 916 snap 273 seq_num 916) 2022-04-28T19:53:36.988 INFO:tasks.rados.rados.0.smithi059.stdout:2918: finishing write tid 1 to smithi05955964-19 2022-04-28T19:53:36.989 INFO:tasks.rados.rados.0.smithi059.stdout:2913: done (5 left) 2022-04-28T19:53:36.989 INFO:tasks.rados.rados.0.smithi059.stdout:2914: done (4 left) 2022-04-28T19:53:36.989 INFO:tasks.rados.rados.0.smithi059.stdout:2916: done (3 left) 2022-04-28T19:53:36.989 INFO:tasks.rados.rados.0.smithi059.stdout:2917: done (2 left) 2022-04-28T19:53:36.989 INFO:tasks.rados.rados.0.smithi059.stdout:2919: done (1 left) 2022-04-28T19:53:36.990 INFO:tasks.rados.rados.0.smithi059.stdout:2920: snap_create 2022-04-28T19:53:36.990 INFO:tasks.rados.rados.0.smithi059.stdout:2918: finishing write tid 2 to smithi05955964-19 2022-04-28T19:53:36.990 INFO:tasks.rados.rados.0.smithi059.stdout:2918: finishing write tid 4 to smithi05955964-19 2022-04-28T19:53:36.993 INFO:tasks.rados.rados.0.smithi059.stdout:2918: finishing write tid 5 to smithi05955964-19 2022-04-28T19:53:36.994 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1162 (ObjNum 917 snap 273 seq_num 917) dirty exists 2022-04-28T19:53:36.994 INFO:tasks.rados.rados.0.smithi059.stdout:2918: left oid 19 (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:53:37.000 INFO:journalctl@ceph.mgr.y.smithi059.stdout:Apr 28 19:53:36 smithi059 bash[57771]: debug 2022-04-28T19:53:36.635+0000 7f2d0ce78700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:53:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: cluster 2022-04-28T19:53:36.002053+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e597: 8 total, 7 up, 8 in 2022-04-28T19:53:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.610889+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.619472+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.628475+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.632336+0000 mon.a (mon.0) 860 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.634265+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.635821+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: audit 2022-04-28T19:53:36.638285+0000 mon.a (mon.0) 863 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[59745]: cluster 2022-04-28T19:53:36.917577+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e598: 8 total, 7 up, 8 in 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: cluster 2022-04-28T19:53:36.002053+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e597: 8 total, 7 up, 8 in 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.610889+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.619472+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.628475+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.632336+0000 mon.a (mon.0) 860 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.634265+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.635821+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: audit 2022-04-28T19:53:36.638285+0000 mon.a (mon.0) 863 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:37 smithi059 bash[61152]: cluster 2022-04-28T19:53:36.917577+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e598: 8 total, 7 up, 8 in 2022-04-28T19:53:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:36 smithi139 bash[43301]: cluster 2022-04-28T19:53:36.002053+0000 mon.a (mon.0) 856 : cluster [DBG] osdmap e597: 8 total, 7 up, 8 in 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:36 smithi139 bash[43301]: audit 2022-04-28T19:53:36.610889+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:36 smithi139 bash[43301]: audit 2022-04-28T19:53:36.619472+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:36 smithi139 bash[43301]: audit 2022-04-28T19:53:36.628475+0000 mon.a (mon.0) 859 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:37 smithi139 bash[43301]: audit 2022-04-28T19:53:36.632336+0000 mon.a (mon.0) 860 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:37 smithi139 bash[43301]: audit 2022-04-28T19:53:36.634265+0000 mon.a (mon.0) 861 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:37 smithi139 bash[43301]: audit 2022-04-28T19:53:36.635821+0000 mon.a (mon.0) 862 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:37 smithi139 bash[43301]: audit 2022-04-28T19:53:36.638285+0000 mon.a (mon.0) 863 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:37 smithi139 bash[43301]: cluster 2022-04-28T19:53:36.917577+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e598: 8 total, 7 up, 8 in 2022-04-28T19:53:37.939 INFO:tasks.rados.rados.0.smithi059.stdout:2918: done (1 left) 2022-04-28T19:53:37.939 INFO:tasks.rados.rados.0.smithi059.stdout:2920: done (0 left) 2022-04-28T19:53:37.939 INFO:tasks.rados.rados.0.smithi059.stdout:2921: read oid 21 snap 246 2022-04-28T19:53:37.940 INFO:tasks.rados.rados.0.smithi059.stdout:2921: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:53:37.940 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 274 2022-04-28T19:53:37.940 INFO:tasks.rados.rados.0.smithi059.stdout:2922: seq_num 919 ranges {7970816=598016,8568832=401408} 2022-04-28T19:53:37.991 INFO:tasks.rados.rados.0.smithi059.stdout:2922: writing smithi05955964-17 from 7970816 to 8568832 tid 1 2022-04-28T19:53:37.994 INFO:tasks.rados.rados.0.smithi059.stdout:2922: writing smithi05955964-17 from 8568832 to 8970240 tid 2 2022-04-28T19:53:37.994 INFO:tasks.rados.rados.0.smithi059.stdout:2923: snap_create 2022-04-28T19:53:37.994 INFO:tasks.rados.rados.0.smithi059.stdout:2921: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:53:38.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[59745]: audit 2022-04-28T19:53:36.638692+0000 mgr.y (mgr.24961) 237 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[59745]: cephadm 2022-04-28T19:53:36.640095+0000 mgr.y (mgr.24961) 238 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:53:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[59745]: cluster 2022-04-28T19:53:36.954514+0000 mgr.y (mgr.24961) 239 : cluster [DBG] pgmap v382: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 606 MiB data, 1.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 105/1071 objects degraded (9.804%) 2022-04-28T19:53:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[59745]: cluster 2022-04-28T19:53:37.938137+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e599: 8 total, 7 up, 8 in 2022-04-28T19:53:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[61152]: audit 2022-04-28T19:53:36.638692+0000 mgr.y (mgr.24961) 237 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[61152]: cephadm 2022-04-28T19:53:36.640095+0000 mgr.y (mgr.24961) 238 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:53:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[61152]: cluster 2022-04-28T19:53:36.954514+0000 mgr.y (mgr.24961) 239 : cluster [DBG] pgmap v382: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 606 MiB data, 1.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 105/1071 objects degraded (9.804%) 2022-04-28T19:53:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:38 smithi059 bash[61152]: cluster 2022-04-28T19:53:37.938137+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e599: 8 total, 7 up, 8 in 2022-04-28T19:53:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:38 smithi139 bash[43301]: audit 2022-04-28T19:53:36.638692+0000 mgr.y (mgr.24961) 237 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:38 smithi139 bash[43301]: cephadm 2022-04-28T19:53:36.640095+0000 mgr.y (mgr.24961) 238 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2022-04-28T19:53:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:38 smithi139 bash[43301]: cluster 2022-04-28T19:53:36.954514+0000 mgr.y (mgr.24961) 239 : cluster [DBG] pgmap v382: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 606 MiB data, 1.3 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 105/1071 objects degraded (9.804%) 2022-04-28T19:53:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:38 smithi139 bash[43301]: cluster 2022-04-28T19:53:37.938137+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e599: 8 total, 7 up, 8 in 2022-04-28T19:53:38.398 INFO:tasks.rados.rados.0.smithi059.stdout:2922: finishing write tid 1 to smithi05955964-17 2022-04-28T19:53:38.398 INFO:tasks.rados.rados.0.smithi059.stdout:2922: finishing write tid 2 to smithi05955964-17 2022-04-28T19:53:38.398 INFO:tasks.rados.rados.0.smithi059.stdout:2922: finishing write tid 4 to smithi05955964-17 2022-04-28T19:53:38.398 INFO:tasks.rados.rados.0.smithi059.stdout:2922: finishing write tid 5 to smithi05955964-17 2022-04-28T19:53:38.398 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 1056 (ObjNum 919 snap 274 seq_num 919) dirty exists 2022-04-28T19:53:38.398 INFO:tasks.rados.rados.0.smithi059.stdout:2922: left oid 17 (ObjNum 919 snap 274 seq_num 919) 2022-04-28T19:53:38.942 INFO:tasks.rados.rados.0.smithi059.stdout:2921: done (2 left) 2022-04-28T19:53:38.942 INFO:tasks.rados.rados.0.smithi059.stdout:2922: done (1 left) 2022-04-28T19:53:38.942 INFO:tasks.rados.rados.0.smithi059.stdout:2923: done (0 left) 2022-04-28T19:53:38.942 INFO:tasks.rados.rados.0.smithi059.stdout:2924: copy_from oid 14 from oid 19 current snap is 275 2022-04-28T19:53:38.942 INFO:tasks.rados.rados.0.smithi059.stdout:2925: rmattr oid 49 current snap is 275 2022-04-28T19:53:38.942 INFO:tasks.rados.rados.0.smithi059.stdout:2926: delete oid 31 current snap is 275 2022-04-28T19:53:38.948 INFO:tasks.rados.rados.0.smithi059.stdout:2926: done (2 left) 2022-04-28T19:53:38.948 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 275 2022-04-28T19:53:38.948 INFO:tasks.rados.rados.0.smithi059.stdout:2927: seq_num 921 ranges {1507328=630784,2138112=401408} 2022-04-28T19:53:38.961 INFO:tasks.rados.rados.0.smithi059.stdout:2927: writing smithi05955964-24 from 1507328 to 2138112 tid 1 2022-04-28T19:53:38.963 INFO:tasks.rados.rados.0.smithi059.stdout:2927: writing smithi05955964-24 from 2138112 to 2539520 tid 2 2022-04-28T19:53:38.964 INFO:tasks.rados.rados.0.smithi059.stdout:2928: delete oid 38 current snap is 275 2022-04-28T19:53:38.964 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1090 (ObjNum 908 snap 271 seq_num 908) dirty exists 2022-04-28T19:53:38.967 INFO:tasks.rados.rados.0.smithi059.stdout:2924: finishing copy_from to smithi05955964-14 2022-04-28T19:53:38.967 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1162 (ObjNum 917 snap 273 seq_num 917) dirty exists 2022-04-28T19:53:38.968 INFO:tasks.rados.rados.0.smithi059.stdout:2924: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:53:38.969 INFO:tasks.rados.rados.0.smithi059.stdout:2927: finishing write tid 1 to smithi05955964-24 2022-04-28T19:53:38.970 INFO:tasks.rados.rados.0.smithi059.stdout:2927: finishing write tid 2 to smithi05955964-24 2022-04-28T19:53:38.970 INFO:tasks.rados.rados.0.smithi059.stdout:2927: finishing write tid 4 to smithi05955964-24 2022-04-28T19:53:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:2924: done (3 left) 2022-04-28T19:53:38.972 INFO:tasks.rados.rados.0.smithi059.stdout:2925: done (2 left) 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2928: done (1 left) 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2929: rmattr oid 47 current snap is 275 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2929: done (1 left) 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2930: read oid 47 snap 255 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2930: expect deleted 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2931: read oid 15 snap -1 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2931: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2932: read oid 32 snap -1 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2932: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:53:38.973 INFO:tasks.rados.rados.0.smithi059.stdout:2933: read oid 1 snap -1 2022-04-28T19:53:38.974 INFO:tasks.rados.rados.0.smithi059.stdout:2933: expect (ObjNum 914 snap 272 seq_num 914) 2022-04-28T19:53:38.974 INFO:tasks.rados.rados.0.smithi059.stdout:2934: read oid 43 snap -1 2022-04-28T19:53:38.974 INFO:tasks.rados.rados.0.smithi059.stdout:2934: expect (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:53:38.974 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 275 2022-04-28T19:53:38.974 INFO:tasks.rados.rados.0.smithi059.stdout:2935: seq_num 923 ranges {15851520=663552,16515072=401408} 2022-04-28T19:53:39.069 INFO:tasks.rados.rados.0.smithi059.stdout:2935: writing smithi05955964-6 from 15851520 to 16515072 tid 1 2022-04-28T19:53:39.072 INFO:tasks.rados.rados.0.smithi059.stdout:2935: writing smithi05955964-6 from 16515072 to 16916480 tid 2 2022-04-28T19:53:39.072 INFO:tasks.rados.rados.0.smithi059.stdout:2936: snap_create 2022-04-28T19:53:39.072 INFO:tasks.rados.rados.0.smithi059.stdout:2927: finishing write tid 5 to smithi05955964-24 2022-04-28T19:53:39.072 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1171 (ObjNum 921 snap 275 seq_num 921) dirty exists 2022-04-28T19:53:39.073 INFO:tasks.rados.rados.0.smithi059.stdout:2927: left oid 24 (ObjNum 921 snap 275 seq_num 921) 2022-04-28T19:53:39.073 INFO:tasks.rados.rados.0.smithi059.stdout:2933: expect (ObjNum 914 snap 272 seq_num 914) 2022-04-28T19:53:39.193 INFO:tasks.rados.rados.0.smithi059.stdout:2931: expect (ObjNum 822 snap 241 seq_num 822) 2022-04-28T19:53:39.235 INFO:tasks.rados.rados.0.smithi059.stdout:2932: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:53:39.361 INFO:tasks.rados.rados.0.smithi059.stdout:2934: expect (ObjNum 768 snap 230 seq_num 768) 2022-04-28T19:53:39.381 INFO:tasks.rados.rados.0.smithi059.stdout:2935: finishing write tid 1 to smithi05955964-6 2022-04-28T19:53:39.382 INFO:tasks.rados.rados.0.smithi059.stdout:2935: finishing write tid 2 to smithi05955964-6 2022-04-28T19:53:39.382 INFO:tasks.rados.rados.0.smithi059.stdout:2935: finishing write tid 4 to smithi05955964-6 2022-04-28T19:53:39.382 INFO:tasks.rados.rados.0.smithi059.stdout:2935: finishing write tid 5 to smithi05955964-6 2022-04-28T19:53:39.382 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1221 (ObjNum 923 snap 275 seq_num 923) dirty exists 2022-04-28T19:53:39.382 INFO:tasks.rados.rados.0.smithi059.stdout:2935: left oid 6 (ObjNum 923 snap 275 seq_num 923) 2022-04-28T19:53:39.618 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:39 smithi139 bash[48787]: debug 2022-04-28T19:53:39.125+0000 7f1b6d90e200 -1 osd.6 589 log_to_monitors {default=true} 2022-04-28T19:53:39.957 INFO:tasks.rados.rados.0.smithi059.stdout:2927: done (7 left) 2022-04-28T19:53:39.957 INFO:tasks.rados.rados.0.smithi059.stdout:2930: done (6 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2931: done (5 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2932: done (4 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2933: done (3 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2934: done (2 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2935: done (1 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2936: done (0 left) 2022-04-28T19:53:39.958 INFO:tasks.rados.rados.0.smithi059.stdout:2937: snap_remove snap 219 2022-04-28T19:53:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[61152]: cluster 2022-04-28T19:53:38.941013+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e600: 8 total, 7 up, 8 in 2022-04-28T19:53:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[61152]: cluster 2022-04-28T19:53:38.955251+0000 mgr.y (mgr.24961) 240 : cluster [DBG] pgmap v385: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 609 MiB data, 1.3 GiB used, 712 GiB / 715 GiB avail; 852 KiB/s rd, 1020 KiB/s wr, 5 op/s; 106/1080 objects degraded (9.815%) 2022-04-28T19:53:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[61152]: audit 2022-04-28T19:53:39.129979+0000 mon.b (mon.2) 7 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:53:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[61152]: audit 2022-04-28T19:53:39.132257+0000 mon.a (mon.0) 867 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:53:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[59745]: cluster 2022-04-28T19:53:38.941013+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e600: 8 total, 7 up, 8 in 2022-04-28T19:53:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[59745]: cluster 2022-04-28T19:53:38.955251+0000 mgr.y (mgr.24961) 240 : cluster [DBG] pgmap v385: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 609 MiB data, 1.3 GiB used, 712 GiB / 715 GiB avail; 852 KiB/s rd, 1020 KiB/s wr, 5 op/s; 106/1080 objects degraded (9.815%) 2022-04-28T19:53:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[59745]: audit 2022-04-28T19:53:39.129979+0000 mon.b (mon.2) 7 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:53:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:39 smithi059 bash[59745]: audit 2022-04-28T19:53:39.132257+0000 mon.a (mon.0) 867 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:53:40.368 INFO:journalctl@ceph.osd.6.smithi139.stdout:Apr 28 19:53:39 smithi139 bash[48787]: debug 2022-04-28T19:53:39.965+0000 7f1b5c823700 -1 osd.6 589 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:53:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:39 smithi139 bash[43301]: cluster 2022-04-28T19:53:38.941013+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e600: 8 total, 7 up, 8 in 2022-04-28T19:53:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:39 smithi139 bash[43301]: cluster 2022-04-28T19:53:38.955251+0000 mgr.y (mgr.24961) 240 : cluster [DBG] pgmap v385: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 609 MiB data, 1.3 GiB used, 712 GiB / 715 GiB avail; 852 KiB/s rd, 1020 KiB/s wr, 5 op/s; 106/1080 objects degraded (9.815%) 2022-04-28T19:53:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:39 smithi139 bash[43301]: audit 2022-04-28T19:53:39.129979+0000 mon.b (mon.2) 7 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:53:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:39 smithi139 bash[43301]: audit 2022-04-28T19:53:39.132257+0000 mon.a (mon.0) 867 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2022-04-28T19:53:40.963 INFO:tasks.rados.rados.0.smithi059.stdout:2937: done (0 left) 2022-04-28T19:53:40.963 INFO:tasks.rados.rados.0.smithi059.stdout:2938: copy_from oid 49 from oid 17 current snap is 276 2022-04-28T19:53:40.963 INFO:tasks.rados.rados.0.smithi059.stdout:2939: snap_remove snap 260 2022-04-28T19:53:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[61152]: cluster 2022-04-28T19:53:39.935220+0000 mon.a (mon.0) 868 : cluster [WRN] Health check update: Degraded data redundancy: 106/1080 objects degraded (9.815%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[61152]: audit 2022-04-28T19:53:39.950421+0000 mon.a (mon.0) 869 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-04-28T19:53:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[61152]: cluster 2022-04-28T19:53:39.950546+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e601: 8 total, 7 up, 8 in 2022-04-28T19:53:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[61152]: audit 2022-04-28T19:53:39.952276+0000 mon.b (mon.2) 8 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[61152]: audit 2022-04-28T19:53:39.954715+0000 mon.a (mon.0) 871 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[59745]: cluster 2022-04-28T19:53:39.935220+0000 mon.a (mon.0) 868 : cluster [WRN] Health check update: Degraded data redundancy: 106/1080 objects degraded (9.815%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:41.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[59745]: audit 2022-04-28T19:53:39.950421+0000 mon.a (mon.0) 869 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-04-28T19:53:41.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[59745]: cluster 2022-04-28T19:53:39.950546+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e601: 8 total, 7 up, 8 in 2022-04-28T19:53:41.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[59745]: audit 2022-04-28T19:53:39.952276+0000 mon.b (mon.2) 8 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:41.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:40 smithi059 bash[59745]: audit 2022-04-28T19:53:39.954715+0000 mon.a (mon.0) 871 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:40 smithi139 bash[43301]: cluster 2022-04-28T19:53:39.935220+0000 mon.a (mon.0) 868 : cluster [WRN] Health check update: Degraded data redundancy: 106/1080 objects degraded (9.815%), 8 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:40 smithi139 bash[43301]: audit 2022-04-28T19:53:39.950421+0000 mon.a (mon.0) 869 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2022-04-28T19:53:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:40 smithi139 bash[43301]: cluster 2022-04-28T19:53:39.950546+0000 mon.a (mon.0) 870 : cluster [DBG] osdmap e601: 8 total, 7 up, 8 in 2022-04-28T19:53:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:40 smithi139 bash[43301]: audit 2022-04-28T19:53:39.952276+0000 mon.b (mon.2) 8 : audit [INF] from='osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:40 smithi139 bash[43301]: audit 2022-04-28T19:53:39.954715+0000 mon.a (mon.0) 871 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:53:41.962 INFO:tasks.rados.rados.0.smithi059.stdout:2939: done (1 left) 2022-04-28T19:53:41.962 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 276 2022-04-28T19:53:41.963 INFO:tasks.rados.rados.0.smithi059.stdout:2940: seq_num 924 ranges {1155072=679936,1835008=401408} 2022-04-28T19:53:41.974 INFO:tasks.rados.rados.0.smithi059.stdout:2940: writing smithi05955964-28 from 1155072 to 1835008 tid 1 2022-04-28T19:53:41.977 INFO:tasks.rados.rados.0.smithi059.stdout:2940: writing smithi05955964-28 from 1835008 to 2236416 tid 2 2022-04-28T19:53:41.977 INFO:tasks.rados.rados.0.smithi059.stdout:2941: snap_remove snap 273 2022-04-28T19:53:41.979 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:41.634Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:41.979 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:41.634Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:41.979 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:41.634Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:53:41.980 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[61152]: cluster 2022-04-28T19:53:40.945749+0000 mon.a (mon.0) 872 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:53:41.981 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[61152]: cluster 2022-04-28T19:53:40.955903+0000 mgr.y (mgr.24961) 241 : cluster [DBG] pgmap v388: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 619 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 108/1101 objects degraded (9.809%) 2022-04-28T19:53:41.981 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[61152]: cluster 2022-04-28T19:53:40.959417+0000 mon.a (mon.0) 873 : cluster [INF] osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303] boot 2022-04-28T19:53:41.981 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[61152]: cluster 2022-04-28T19:53:40.959656+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2022-04-28T19:53:41.981 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[61152]: audit 2022-04-28T19:53:40.961435+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:53:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[59745]: cluster 2022-04-28T19:53:40.945749+0000 mon.a (mon.0) 872 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:53:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[59745]: cluster 2022-04-28T19:53:40.955903+0000 mgr.y (mgr.24961) 241 : cluster [DBG] pgmap v388: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 619 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 108/1101 objects degraded (9.809%) 2022-04-28T19:53:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[59745]: cluster 2022-04-28T19:53:40.959417+0000 mon.a (mon.0) 873 : cluster [INF] osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303] boot 2022-04-28T19:53:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[59745]: cluster 2022-04-28T19:53:40.959656+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2022-04-28T19:53:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:41 smithi059 bash[59745]: audit 2022-04-28T19:53:40.961435+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:53:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:41 smithi139 bash[43301]: cluster 2022-04-28T19:53:40.945749+0000 mon.a (mon.0) 872 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:53:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:41 smithi139 bash[43301]: cluster 2022-04-28T19:53:40.955903+0000 mgr.y (mgr.24961) 241 : cluster [DBG] pgmap v388: 89 pgs: 8 active+undersized+degraded, 23 active+undersized, 58 active+clean; 619 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 108/1101 objects degraded (9.809%) 2022-04-28T19:53:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:41 smithi139 bash[43301]: cluster 2022-04-28T19:53:40.959417+0000 mon.a (mon.0) 873 : cluster [INF] osd.6 [v2:172.21.15.139:6816/3450296303,v1:172.21.15.139:6817/3450296303] boot 2022-04-28T19:53:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:41 smithi139 bash[43301]: cluster 2022-04-28T19:53:40.959656+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2022-04-28T19:53:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:41 smithi139 bash[43301]: audit 2022-04-28T19:53:40.961435+0000 mon.a (mon.0) 875 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:2941: done (2 left) 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:2942: delete oid 27 current snap is 276 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:2938: finishing copy_from to smithi05955964-49 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1056 (ObjNum 919 snap 274 seq_num 919) dirty exists 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:2938: finishing copy_from racing read to smithi05955964-49 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:2940: finishing write tid 1 to smithi05955964-28 2022-04-28T19:53:42.998 INFO:tasks.rados.rados.0.smithi059.stdout:2940: finishing write tid 2 to smithi05955964-28 2022-04-28T19:53:42.999 INFO:tasks.rados.rados.0.smithi059.stdout:2940: finishing write tid 4 to smithi05955964-28 2022-04-28T19:53:42.999 INFO:tasks.rados.rados.0.smithi059.stdout:2940: finishing write tid 5 to smithi05955964-28 2022-04-28T19:53:42.999 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1018 (ObjNum 924 snap 276 seq_num 924) dirty exists 2022-04-28T19:53:42.999 INFO:tasks.rados.rados.0.smithi059.stdout:2940: left oid 28 (ObjNum 924 snap 276 seq_num 924) 2022-04-28T19:53:43.002 INFO:tasks.rados.rados.0.smithi059.stdout:2938: done (2 left) 2022-04-28T19:53:43.002 INFO:tasks.rados.rados.0.smithi059.stdout:2940: done (1 left) 2022-04-28T19:53:43.002 INFO:tasks.rados.rados.0.smithi059.stdout:2942: done (0 left) 2022-04-28T19:53:43.002 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 276 2022-04-28T19:53:43.002 INFO:tasks.rados.rados.0.smithi059.stdout:2943: seq_num 926 ranges {1556480=712704,2269184=401408} 2022-04-28T19:53:43.015 INFO:tasks.rados.rados.0.smithi059.stdout:2943: writing smithi05955964-4 from 1556480 to 2269184 tid 1 2022-04-28T19:53:43.017 INFO:tasks.rados.rados.0.smithi059.stdout:2943: writing smithi05955964-4 from 2269184 to 2670592 tid 2 2022-04-28T19:53:43.017 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 276 2022-04-28T19:53:43.017 INFO:tasks.rados.rados.0.smithi059.stdout:2944: seq_num 927 ranges {4685824=729088,5414912=401408} 2022-04-28T19:53:43.048 INFO:tasks.rados.rados.0.smithi059.stdout:2944: writing smithi05955964-40 from 4685824 to 5414912 tid 1 2022-04-28T19:53:43.051 INFO:tasks.rados.rados.0.smithi059.stdout:2944: writing smithi05955964-40 from 5414912 to 5816320 tid 2 2022-04-28T19:53:43.051 INFO:tasks.rados.rados.0.smithi059.stdout:2945: copy_from oid 33 from oid 22 current snap is 276 2022-04-28T19:53:43.051 INFO:tasks.rados.rados.0.smithi059.stdout:2943: finishing write tid 1 to smithi05955964-4 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:2943: finishing write tid 2 to smithi05955964-4 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:2943: finishing write tid 4 to smithi05955964-4 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:2943: finishing write tid 5 to smithi05955964-4 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1175 (ObjNum 926 snap 276 seq_num 926) dirty exists 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:2943: left oid 4 (ObjNum 926 snap 276 seq_num 926) 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:2943: done (2 left) 2022-04-28T19:53:43.052 INFO:tasks.rados.rados.0.smithi059.stdout:2946: snap_remove snap 266 2022-04-28T19:53:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:42 smithi059 bash[61152]: cluster 2022-04-28T19:53:41.961667+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2022-04-28T19:53:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:42 smithi059 bash[59745]: cluster 2022-04-28T19:53:41.961667+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2022-04-28T19:53:43.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:42 smithi139 bash[43301]: cluster 2022-04-28T19:53:41.961667+0000 mon.a (mon.0) 876 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2022-04-28T19:53:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:43.528Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:43.528Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:43.528Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:53:44.010 INFO:tasks.rados.rados.0.smithi059.stdout:2946: done (2 left) 2022-04-28T19:53:44.010 INFO:tasks.rados.rados.0.smithi059.stdout:2947: rmattr oid 45 current snap is 276 2022-04-28T19:53:44.010 INFO:tasks.rados.rados.0.smithi059.stdout:2945: finishing copy_from to smithi05955964-33 2022-04-28T19:53:44.010 INFO:tasks.rados.rados.0.smithi059.stdout:2945: got expected ENOENT (src dne) 2022-04-28T19:53:44.010 INFO:tasks.rados.rados.0.smithi059.stdout:2945: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2944: finishing write tid 1 to smithi05955964-40 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2944: finishing write tid 2 to smithi05955964-40 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2944: finishing write tid 4 to smithi05955964-40 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2944: finishing write tid 5 to smithi05955964-40 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1093 (ObjNum 927 snap 276 seq_num 927) dirty exists 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2944: left oid 40 (ObjNum 927 snap 276 seq_num 927) 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2944: done (2 left) 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2945: done (1 left) 2022-04-28T19:53:44.011 INFO:tasks.rados.rados.0.smithi059.stdout:2948: setattr oid 27 current snap is 276 2022-04-28T19:53:44.012 INFO:tasks.rados.rados.0.smithi059.stdout:2949: read oid 35 snap -1 2022-04-28T19:53:44.012 INFO:tasks.rados.rados.0.smithi059.stdout:2949: expect (ObjNum 889 snap 265 seq_num 889) 2022-04-28T19:53:44.012 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 276 2022-04-28T19:53:44.012 INFO:tasks.rados.rados.0.smithi059.stdout:2950: seq_num 928 ranges {8970240=745472,9715712=401408} 2022-04-28T19:53:44.068 INFO:tasks.rados.rados.0.smithi059.stdout:2950: writing smithi05955964-49 from 8970240 to 9715712 tid 1 2022-04-28T19:53:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:2950: writing smithi05955964-49 from 9715712 to 10117120 tid 2 2022-04-28T19:53:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:2951: copy_from oid 38 from oid 28 current snap is 276 2022-04-28T19:53:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1222 (ObjNum 913 snap 272 seq_num 913) dirty exists 2022-04-28T19:53:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1149 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:53:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:2949: expect (ObjNum 889 snap 265 seq_num 889) 2022-04-28T19:53:44.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[59745]: cluster 2022-04-28T19:53:42.956779+0000 mgr.y (mgr.24961) 242 : cluster [DBG] pgmap v390: 89 pgs: 6 active+undersized+degraded, 21 active+undersized, 62 active+clean; 612 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.5 MiB/s wr, 13 op/s; 80/1080 objects degraded (7.407%); 4.5 MiB/s, 0 objects/s recovering 2022-04-28T19:53:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[59745]: cluster 2022-04-28T19:53:42.994496+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2022-04-28T19:53:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[61152]: cluster 2022-04-28T19:53:42.956779+0000 mgr.y (mgr.24961) 242 : cluster [DBG] pgmap v390: 89 pgs: 6 active+undersized+degraded, 21 active+undersized, 62 active+clean; 612 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.5 MiB/s wr, 13 op/s; 80/1080 objects degraded (7.407%); 4.5 MiB/s, 0 objects/s recovering 2022-04-28T19:53:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:43 smithi059 bash[61152]: cluster 2022-04-28T19:53:42.994496+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2022-04-28T19:53:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:43 smithi139 bash[43301]: cluster 2022-04-28T19:53:42.956779+0000 mgr.y (mgr.24961) 242 : cluster [DBG] pgmap v390: 89 pgs: 6 active+undersized+degraded, 21 active+undersized, 62 active+clean; 612 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 2.5 MiB/s wr, 13 op/s; 80/1080 objects degraded (7.407%); 4.5 MiB/s, 0 objects/s recovering 2022-04-28T19:53:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:43 smithi139 bash[43301]: cluster 2022-04-28T19:53:42.994496+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2022-04-28T19:53:44.487 INFO:tasks.rados.rados.0.smithi059.stdout:2950: finishing write tid 1 to smithi05955964-49 2022-04-28T19:53:44.487 INFO:tasks.rados.rados.0.smithi059.stdout:2950: finishing write tid 2 to smithi05955964-49 2022-04-28T19:53:44.487 INFO:tasks.rados.rados.0.smithi059.stdout:2950: finishing write tid 4 to smithi05955964-49 2022-04-28T19:53:44.487 INFO:tasks.rados.rados.0.smithi059.stdout:2950: finishing write tid 5 to smithi05955964-49 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1096 (ObjNum 928 snap 276 seq_num 928) dirty exists 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2950: left oid 49 (ObjNum 928 snap 276 seq_num 928) 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2951: finishing copy_from to smithi05955964-38 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1018 (ObjNum 924 snap 276 seq_num 924) dirty exists 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2951: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2947: done (4 left) 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2948: done (3 left) 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2949: done (2 left) 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2950: done (1 left) 2022-04-28T19:53:44.488 INFO:tasks.rados.rados.0.smithi059.stdout:2951: done (0 left) 2022-04-28T19:53:44.489 INFO:tasks.rados.rados.0.smithi059.stdout:2952: snap_remove snap 242 2022-04-28T19:53:45.016 INFO:tasks.rados.rados.0.smithi059.stdout:2952: done (0 left) 2022-04-28T19:53:45.016 INFO:tasks.rados.rados.0.smithi059.stdout:2953: copy_from oid 35 from oid 17 current snap is 276 2022-04-28T19:53:45.016 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 276 2022-04-28T19:53:45.016 INFO:tasks.rados.rados.0.smithi059.stdout:2954: seq_num 929 ranges {10117120=761856,10878976=401408} 2022-04-28T19:53:45.076 INFO:tasks.rados.rados.0.smithi059.stdout:2954: writing smithi05955964-49 from 10117120 to 10878976 tid 1 2022-04-28T19:53:45.078 INFO:tasks.rados.rados.0.smithi059.stdout:2954: writing smithi05955964-49 from 10878976 to 11280384 tid 2 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:2955: rollback oid 36 current snap is 276 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 270 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:2953: finishing copy_from to smithi05955964-35 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1056 (ObjNum 919 snap 274 seq_num 919) dirty exists 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:2953: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:2953: done (2 left) 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:2956: rollback oid 31 current snap is 276 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 270 2022-04-28T19:53:45.079 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 276 2022-04-28T19:53:45.080 INFO:tasks.rados.rados.0.smithi059.stdout:2957: seq_num 930 ranges {1056768=786432,1843200=401408} 2022-04-28T19:53:45.088 INFO:tasks.rados.rados.0.smithi059.stdout:2957: writing smithi05955964-47 from 1056768 to 1843200 tid 1 2022-04-28T19:53:45.091 INFO:tasks.rados.rados.0.smithi059.stdout:2957: writing smithi05955964-47 from 1843200 to 2244608 tid 2 2022-04-28T19:53:45.091 INFO:tasks.rados.rados.0.smithi059.stdout:2958: delete oid 25 current snap is 276 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2956: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2954: finishing write tid 1 to smithi05955964-49 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2956: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1323 (ObjNum 889 snap 265 seq_num 889) dirty exists 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2954: finishing write tid 2 to smithi05955964-49 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2954: finishing write tid 4 to smithi05955964-49 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2954: finishing write tid 5 to smithi05955964-49 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1099 (ObjNum 929 snap 276 seq_num 929) dirty exists 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2954: left oid 49 (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:53:45.092 INFO:tasks.rados.rados.0.smithi059.stdout:2955: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:53:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:2955: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:53:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:2955: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:53:45.093 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1102 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:53:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:2957: finishing write tid 1 to smithi05955964-47 2022-04-28T19:53:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:2957: finishing write tid 2 to smithi05955964-47 2022-04-28T19:53:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:2954: done (4 left) 2022-04-28T19:53:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:2955: done (3 left) 2022-04-28T19:53:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:2956: done (2 left) 2022-04-28T19:53:45.094 INFO:tasks.rados.rados.0.smithi059.stdout:2958: done (1 left) 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:2959: read oid 16 snap -1 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:2959: expect (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:2960: delete oid 30 current snap is 276 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:2957: finishing write tid 4 to smithi05955964-47 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:2957: finishing write tid 5 to smithi05955964-47 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1153 (ObjNum 930 snap 276 seq_num 930) dirty exists 2022-04-28T19:53:45.095 INFO:tasks.rados.rados.0.smithi059.stdout:2957: left oid 47 (ObjNum 930 snap 276 seq_num 930) 2022-04-28T19:53:45.098 INFO:tasks.rados.rados.0.smithi059.stdout:2957: done (2 left) 2022-04-28T19:53:45.098 INFO:tasks.rados.rados.0.smithi059.stdout:2960: done (1 left) 2022-04-28T19:53:45.098 INFO:tasks.rados.rados.0.smithi059.stdout:2961: copy_from oid 47 from oid 37 current snap is 276 2022-04-28T19:53:45.098 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 276 2022-04-28T19:53:45.098 INFO:tasks.rados.rados.0.smithi059.stdout:2962: seq_num 933 ranges {2973696=434176,3407872=548864,3956736=253952} 2022-04-28T19:53:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:2962: writing smithi05955964-1 from 2973696 to 3407872 tid 1 2022-04-28T19:53:45.121 INFO:tasks.rados.rados.0.smithi059.stdout:2962: writing smithi05955964-1 from 3407872 to 3956736 tid 2 2022-04-28T19:53:45.123 INFO:tasks.rados.rados.0.smithi059.stdout:2962: writing smithi05955964-1 from 3956736 to 4210688 tid 3 2022-04-28T19:53:45.124 INFO:tasks.rados.rados.0.smithi059.stdout:2963: read oid 50 snap -1 2022-04-28T19:53:45.124 INFO:tasks.rados.rados.0.smithi059.stdout:2963: expect (ObjNum 905 snap 271 seq_num 905) 2022-04-28T19:53:45.124 INFO:tasks.rados.rados.0.smithi059.stdout:2964: read oid 20 snap 275 2022-04-28T19:53:45.124 INFO:tasks.rados.rados.0.smithi059.stdout:2964: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:53:45.124 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 276 2022-04-28T19:53:45.124 INFO:tasks.rados.rados.0.smithi059.stdout:2965: seq_num 934 ranges {16916480=450560,17367040=434176,17801216=368640} 2022-04-28T19:53:45.219 INFO:tasks.rados.rados.0.smithi059.stdout:2965: writing smithi05955964-6 from 16916480 to 17367040 tid 1 2022-04-28T19:53:45.221 INFO:tasks.rados.rados.0.smithi059.stdout:2965: writing smithi05955964-6 from 17367040 to 17801216 tid 2 2022-04-28T19:53:45.224 INFO:tasks.rados.rados.0.smithi059.stdout:2965: writing smithi05955964-6 from 17801216 to 18169856 tid 3 2022-04-28T19:53:45.224 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 276 2022-04-28T19:53:45.224 INFO:tasks.rados.rados.0.smithi059.stdout:2966: seq_num 935 ranges {7430144=466944,7897088=720896,8617984=81920} 2022-04-28T19:53:45.270 INFO:tasks.rados.rados.0.smithi059.stdout:2966: writing smithi05955964-31 from 7430144 to 7897088 tid 1 2022-04-28T19:53:45.272 INFO:tasks.rados.rados.0.smithi059.stdout:2966: writing smithi05955964-31 from 7897088 to 8617984 tid 2 2022-04-28T19:53:45.272 INFO:tasks.rados.rados.0.smithi059.stdout:2966: writing smithi05955964-31 from 8617984 to 8699904 tid 3 2022-04-28T19:53:45.272 INFO:tasks.rados.rados.0.smithi059.stdout:2967: snap_remove snap 274 2022-04-28T19:53:45.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:45 smithi059 bash[59745]: cluster 2022-04-28T19:53:44.009463+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2022-04-28T19:53:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:45 smithi059 bash[61152]: cluster 2022-04-28T19:53:44.009463+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2022-04-28T19:53:45.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:45 smithi139 bash[43301]: cluster 2022-04-28T19:53:44.009463+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2022-04-28T19:53:46.025 INFO:tasks.rados.rados.0.smithi059.stdout:2967: done (7 left) 2022-04-28T19:53:46.025 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 276 2022-04-28T19:53:46.025 INFO:tasks.rados.rados.0.smithi059.stdout:2968: seq_num 936 ranges {8257536=483328,8740864=655360,9396224=147456} 2022-04-28T19:53:46.073 INFO:tasks.rados.rados.0.smithi059.stdout:2968: writing smithi05955964-12 from 8257536 to 8740864 tid 1 2022-04-28T19:53:46.077 INFO:tasks.rados.rados.0.smithi059.stdout:2968: writing smithi05955964-12 from 8740864 to 9396224 tid 2 2022-04-28T19:53:46.078 INFO:tasks.rados.rados.0.smithi059.stdout:2968: writing smithi05955964-12 from 9396224 to 9543680 tid 3 2022-04-28T19:53:46.078 INFO:tasks.rados.rados.0.smithi059.stdout:2969: rollback oid 4 current snap is 276 2022-04-28T19:53:46.078 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 246 2022-04-28T19:53:46.078 INFO:tasks.rados.rados.0.smithi059.stdout:2959: expect (ObjNum 809 snap 240 seq_num 809) 2022-04-28T19:53:46.124 INFO:tasks.rados.rados.0.smithi059.stdout:2962: finishing write tid 1 to smithi05955964-1 2022-04-28T19:53:46.125 INFO:tasks.rados.rados.0.smithi059.stdout:2962: finishing write tid 2 to smithi05955964-1 2022-04-28T19:53:46.125 INFO:tasks.rados.rados.0.smithi059.stdout:2962: finishing write tid 3 to smithi05955964-1 2022-04-28T19:53:46.125 INFO:tasks.rados.rados.0.smithi059.stdout:2959: done (8 left) 2022-04-28T19:53:46.125 INFO:tasks.rados.rados.0.smithi059.stdout:2970: snap_remove snap 269 2022-04-28T19:53:46.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:46 smithi059 bash[61152]: cluster 2022-04-28T19:53:44.957442+0000 mgr.y (mgr.24961) 243 : cluster [DBG] pgmap v393: 89 pgs: 3 active+undersized+degraded, 12 active+undersized, 74 active+clean; 622 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 57/1092 objects degraded (5.220%); 10 MiB/s, 1 objects/s recovering 2022-04-28T19:53:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:46 smithi059 bash[61152]: cluster 2022-04-28T19:53:44.998403+0000 mon.a (mon.0) 879 : cluster [WRN] Health check update: Degraded data redundancy: 57/1092 objects degraded (5.220%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:46 smithi059 bash[61152]: cluster 2022-04-28T19:53:45.015346+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2022-04-28T19:53:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:46 smithi059 bash[59745]: cluster 2022-04-28T19:53:44.957442+0000 mgr.y (mgr.24961) 243 : cluster [DBG] pgmap v393: 89 pgs: 3 active+undersized+degraded, 12 active+undersized, 74 active+clean; 622 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 57/1092 objects degraded (5.220%); 10 MiB/s, 1 objects/s recovering 2022-04-28T19:53:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:46 smithi059 bash[59745]: cluster 2022-04-28T19:53:44.998403+0000 mon.a (mon.0) 879 : cluster [WRN] Health check update: Degraded data redundancy: 57/1092 objects degraded (5.220%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:46 smithi059 bash[59745]: cluster 2022-04-28T19:53:45.015346+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2022-04-28T19:53:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:46 smithi139 bash[43301]: cluster 2022-04-28T19:53:44.957442+0000 mgr.y (mgr.24961) 243 : cluster [DBG] pgmap v393: 89 pgs: 3 active+undersized+degraded, 12 active+undersized, 74 active+clean; 622 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.9 MiB/s wr, 12 op/s; 57/1092 objects degraded (5.220%); 10 MiB/s, 1 objects/s recovering 2022-04-28T19:53:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:46 smithi139 bash[43301]: cluster 2022-04-28T19:53:44.998403+0000 mon.a (mon.0) 879 : cluster [WRN] Health check update: Degraded data redundancy: 57/1092 objects degraded (5.220%), 3 pgs degraded (PG_DEGRADED) 2022-04-28T19:53:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:46 smithi139 bash[43301]: cluster 2022-04-28T19:53:45.015346+0000 mon.a (mon.0) 880 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2022-04-28T19:53:47.036 INFO:tasks.rados.rados.0.smithi059.stdout:2970: done (8 left) 2022-04-28T19:53:47.036 INFO:tasks.rados.rados.0.smithi059.stdout:2971: rmattr oid 39 current snap is 276 2022-04-28T19:53:47.036 INFO:tasks.rados.rados.0.smithi059.stdout:2971: done (8 left) 2022-04-28T19:53:47.036 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 276 2022-04-28T19:53:47.036 INFO:tasks.rados.rados.0.smithi059.stdout:2972: seq_num 937 ranges {0=499712,499712=532480,1032192=270336} 2022-04-28T19:53:47.039 INFO:tasks.rados.rados.0.smithi059.stdout:2972: writing smithi05955964-25 from 0 to 499712 tid 1 2022-04-28T19:53:47.042 INFO:tasks.rados.rados.0.smithi059.stdout:2972: writing smithi05955964-25 from 499712 to 1032192 tid 2 2022-04-28T19:53:47.044 INFO:tasks.rados.rados.0.smithi059.stdout:2972: writing smithi05955964-25 from 1032192 to 1302528 tid 3 2022-04-28T19:53:47.044 INFO:tasks.rados.rados.0.smithi059.stdout:2973: snap_remove snap 245 2022-04-28T19:53:47.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:47 smithi059 bash[61152]: cluster 2022-04-28T19:53:46.024254+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2022-04-28T19:53:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:47 smithi059 bash[59745]: cluster 2022-04-28T19:53:46.024254+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2022-04-28T19:53:47.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:47 smithi139 bash[43301]: cluster 2022-04-28T19:53:46.024254+0000 mon.a (mon.0) 881 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2022-04-28T19:53:48.043 INFO:tasks.rados.rados.0.smithi059.stdout:2973: done (9 left) 2022-04-28T19:53:48.043 INFO:tasks.rados.rados.0.smithi059.stdout:2974: copy_from oid 18 from oid 34 current snap is 276 2022-04-28T19:53:48.043 INFO:tasks.rados.rados.0.smithi059.stdout:2962: finishing write tid 5 to smithi05955964-1 2022-04-28T19:53:48.043 INFO:tasks.rados.rados.0.smithi059.stdout:2961: finishing copy_from to smithi05955964-47 2022-04-28T19:53:48.043 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1319 (ObjNum 904 snap 271 seq_num 904) dirty exists 2022-04-28T19:53:48.044 INFO:tasks.rados.rados.0.smithi059.stdout:2961: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:53:48.044 INFO:tasks.rados.rados.0.smithi059.stdout:2962: finishing write tid 6 to smithi05955964-1 2022-04-28T19:53:48.044 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1106 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:53:48.044 INFO:tasks.rados.rados.0.smithi059.stdout:2962: left oid 1 (ObjNum 933 snap 276 seq_num 933) 2022-04-28T19:53:48.044 INFO:tasks.rados.rados.0.smithi059.stdout:2963: expect (ObjNum 905 snap 271 seq_num 905) 2022-04-28T19:53:48.069 INFO:tasks.rados.rados.0.smithi059.stdout:2964: expect (ObjNum 821 snap 240 seq_num 821) 2022-04-28T19:53:48.226 INFO:tasks.rados.rados.0.smithi059.stdout:2965: finishing write tid 1 to smithi05955964-6 2022-04-28T19:53:48.226 INFO:tasks.rados.rados.0.smithi059.stdout:2965: finishing write tid 2 to smithi05955964-6 2022-04-28T19:53:48.226 INFO:tasks.rados.rados.0.smithi059.stdout:2965: finishing write tid 3 to smithi05955964-6 2022-04-28T19:53:48.226 INFO:tasks.rados.rados.0.smithi059.stdout:2965: finishing write tid 5 to smithi05955964-6 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2965: finishing write tid 6 to smithi05955964-6 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1226 (ObjNum 934 snap 276 seq_num 934) dirty exists 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2965: left oid 6 (ObjNum 934 snap 276 seq_num 934) 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2966: finishing write tid 1 to smithi05955964-31 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2966: finishing write tid 2 to smithi05955964-31 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2966: finishing write tid 3 to smithi05955964-31 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2966: finishing write tid 5 to smithi05955964-31 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2966: finishing write tid 6 to smithi05955964-31 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1327 (ObjNum 935 snap 276 seq_num 935) dirty exists 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2966: left oid 31 (ObjNum 935 snap 276 seq_num 935) 2022-04-28T19:53:48.227 INFO:tasks.rados.rados.0.smithi059.stdout:2969: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2969: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2969: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1178 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2968: finishing write tid 1 to smithi05955964-12 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2968: finishing write tid 2 to smithi05955964-12 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2968: finishing write tid 3 to smithi05955964-12 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2968: finishing write tid 5 to smithi05955964-12 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2968: finishing write tid 6 to smithi05955964-12 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1331 (ObjNum 936 snap 276 seq_num 936) dirty exists 2022-04-28T19:53:48.228 INFO:tasks.rados.rados.0.smithi059.stdout:2968: left oid 12 (ObjNum 936 snap 276 seq_num 936) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2961: done (9 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2962: done (8 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2963: done (7 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2964: done (6 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2965: done (5 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2966: done (4 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2968: done (3 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2969: done (2 left) 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2975: copy_from oid 43 from oid 44 current snap is 276 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2972: finishing write tid 1 to smithi05955964-25 2022-04-28T19:53:48.229 INFO:tasks.rados.rados.0.smithi059.stdout:2972: finishing write tid 2 to smithi05955964-25 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2972: finishing write tid 3 to smithi05955964-25 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2972: finishing write tid 5 to smithi05955964-25 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2972: finishing write tid 6 to smithi05955964-25 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1417 (ObjNum 937 snap 276 seq_num 937) dirty exists 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2972: left oid 25 (ObjNum 937 snap 276 seq_num 937) 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2974: finishing copy_from to smithi05955964-18 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1226 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2974: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2972: done (2 left) 2022-04-28T19:53:48.230 INFO:tasks.rados.rados.0.smithi059.stdout:2974: done (1 left) 2022-04-28T19:53:48.231 INFO:tasks.rados.rados.0.smithi059.stdout:2976: setattr oid 21 current snap is 276 2022-04-28T19:53:48.232 INFO:tasks.rados.rados.0.smithi059.stdout:2977: delete oid 13 current snap is 276 2022-04-28T19:53:48.233 INFO:tasks.rados.rados.0.smithi059.stdout:2975: finishing copy_from to smithi05955964-43 2022-04-28T19:53:48.233 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 964 (ObjNum 1 snap 21893 seq_num 4163274728) dirty exists 2022-04-28T19:53:48.235 INFO:tasks.rados.rados.0.smithi059.stdout:2975: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:53:48.236 INFO:tasks.rados.rados.0.smithi059.stdout:2975: done (2 left) 2022-04-28T19:53:48.236 INFO:tasks.rados.rados.0.smithi059.stdout:2977: done (1 left) 2022-04-28T19:53:48.236 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 276 2022-04-28T19:53:48.237 INFO:tasks.rados.rados.0.smithi059.stdout:2978: seq_num 939 ranges {4530176=532480,5062656=704512,5767168=98304} 2022-04-28T19:53:48.265 INFO:tasks.rados.rados.0.smithi059.stdout:2978: writing smithi05955964-14 from 4530176 to 5062656 tid 1 2022-04-28T19:53:48.269 INFO:tasks.rados.rados.0.smithi059.stdout:2978: writing smithi05955964-14 from 5062656 to 5767168 tid 2 2022-04-28T19:53:48.270 INFO:tasks.rados.rados.0.smithi059.stdout:2978: writing smithi05955964-14 from 5767168 to 5865472 tid 3 2022-04-28T19:53:48.270 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 276 2022-04-28T19:53:48.270 INFO:tasks.rados.rados.0.smithi059.stdout:2979: seq_num 940 ranges {6029312=548864,6578176=589824,7168000=212992} 2022-04-28T19:53:48.307 INFO:tasks.rados.rados.0.smithi059.stdout:2979: writing smithi05955964-47 from 6029312 to 6578176 tid 1 2022-04-28T19:53:48.311 INFO:tasks.rados.rados.0.smithi059.stdout:2979: writing smithi05955964-47 from 6578176 to 7168000 tid 2 2022-04-28T19:53:48.313 INFO:tasks.rados.rados.0.smithi059.stdout:2979: writing smithi05955964-47 from 7168000 to 7380992 tid 3 2022-04-28T19:53:48.313 INFO:tasks.rados.rados.0.smithi059.stdout:2980: setattr oid 35 current snap is 276 2022-04-28T19:53:48.313 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1155 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:53:48.313 INFO:tasks.rados.rados.0.smithi059.stdout:2978: finishing write tid 1 to smithi05955964-14 2022-04-28T19:53:48.313 INFO:tasks.rados.rados.0.smithi059.stdout:2978: finishing write tid 2 to smithi05955964-14 2022-04-28T19:53:48.314 INFO:tasks.rados.rados.0.smithi059.stdout:2978: finishing write tid 3 to smithi05955964-14 2022-04-28T19:53:48.314 INFO:tasks.rados.rados.0.smithi059.stdout:2978: finishing write tid 5 to smithi05955964-14 2022-04-28T19:53:48.314 INFO:tasks.rados.rados.0.smithi059.stdout:2978: finishing write tid 6 to smithi05955964-14 2022-04-28T19:53:48.314 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1169 (ObjNum 939 snap 276 seq_num 939) dirty exists 2022-04-28T19:53:48.314 INFO:tasks.rados.rados.0.smithi059.stdout:2978: left oid 14 (ObjNum 939 snap 276 seq_num 939) 2022-04-28T19:53:48.314 INFO:tasks.rados.rados.0.smithi059.stdout:2979: finishing write tid 1 to smithi05955964-47 2022-04-28T19:53:48.315 INFO:tasks.rados.rados.0.smithi059.stdout:2976: done (3 left) 2022-04-28T19:53:48.315 INFO:tasks.rados.rados.0.smithi059.stdout:2978: done (2 left) 2022-04-28T19:53:48.315 INFO:tasks.rados.rados.0.smithi059.stdout:2981: rollback oid 14 current snap is 276 2022-04-28T19:53:48.315 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 272 2022-04-28T19:53:48.315 INFO:tasks.rados.rados.0.smithi059.stdout:2982: setattr oid 22 current snap is 276 2022-04-28T19:53:48.316 INFO:tasks.rados.rados.0.smithi059.stdout:2983: snap_create 2022-04-28T19:53:48.317 INFO:tasks.rados.rados.0.smithi059.stdout:2979: finishing write tid 2 to smithi05955964-47 2022-04-28T19:53:48.317 INFO:tasks.rados.rados.0.smithi059.stdout:2979: finishing write tid 3 to smithi05955964-47 2022-04-28T19:53:48.317 INFO:tasks.rados.rados.0.smithi059.stdout:2979: finishing write tid 5 to smithi05955964-47 2022-04-28T19:53:48.318 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1317 (ObjNum 919 snap 274 seq_num 919) dirty exists 2022-04-28T19:53:48.319 INFO:tasks.rados.rados.0.smithi059.stdout:2981: finishing rollback tid 0 to smithi05955964-14 2022-04-28T19:53:48.319 INFO:tasks.rados.rados.0.smithi059.stdout:2981: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:53:48.319 INFO:tasks.rados.rados.0.smithi059.stdout:2981: finishing rollback tid 2 to smithi05955964-14 2022-04-28T19:53:48.319 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1172 (ObjNum 800 snap 235 seq_num 800) dirty exists 2022-04-28T19:53:48.320 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1173 (ObjNum -133551408 snap 21893 seq_num 4161426368) dirty exists 2022-04-28T19:53:48.320 INFO:tasks.rados.rados.0.smithi059.stdout:2979: finishing write tid 6 to smithi05955964-47 2022-04-28T19:53:48.320 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1323 (ObjNum 940 snap 276 seq_num 940) dirty exists 2022-04-28T19:53:48.320 INFO:tasks.rados.rados.0.smithi059.stdout:2979: left oid 47 (ObjNum 940 snap 276 seq_num 940) 2022-04-28T19:53:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[61152]: cluster 2022-04-28T19:53:46.958107+0000 mgr.y (mgr.24961) 244 : cluster [DBG] pgmap v396: 89 pgs: 89 active+clean; 626 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 11 op/s; 9.7 MiB/s, 2 objects/s recovering 2022-04-28T19:53:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[61152]: cluster 2022-04-28T19:53:47.034772+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2022-04-28T19:53:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[61152]: cluster 2022-04-28T19:53:47.916124+0000 mon.a (mon.0) 883 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1092 objects degraded (5.220%), 3 pgs degraded) 2022-04-28T19:53:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[61152]: cluster 2022-04-28T19:53:47.916198+0000 mon.a (mon.0) 884 : cluster [INF] Cluster is now healthy 2022-04-28T19:53:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[59745]: cluster 2022-04-28T19:53:46.958107+0000 mgr.y (mgr.24961) 244 : cluster [DBG] pgmap v396: 89 pgs: 89 active+clean; 626 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 11 op/s; 9.7 MiB/s, 2 objects/s recovering 2022-04-28T19:53:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[59745]: cluster 2022-04-28T19:53:47.034772+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2022-04-28T19:53:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[59745]: cluster 2022-04-28T19:53:47.916124+0000 mon.a (mon.0) 883 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1092 objects degraded (5.220%), 3 pgs degraded) 2022-04-28T19:53:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:48 smithi059 bash[59745]: cluster 2022-04-28T19:53:47.916198+0000 mon.a (mon.0) 884 : cluster [INF] Cluster is now healthy 2022-04-28T19:53:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:48 smithi139 bash[43301]: cluster 2022-04-28T19:53:46.958107+0000 mgr.y (mgr.24961) 244 : cluster [DBG] pgmap v396: 89 pgs: 89 active+clean; 626 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 11 op/s; 9.7 MiB/s, 2 objects/s recovering 2022-04-28T19:53:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:48 smithi139 bash[43301]: cluster 2022-04-28T19:53:47.034772+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2022-04-28T19:53:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:48 smithi139 bash[43301]: cluster 2022-04-28T19:53:47.916124+0000 mon.a (mon.0) 883 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1092 objects degraded (5.220%), 3 pgs degraded) 2022-04-28T19:53:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:48 smithi139 bash[43301]: cluster 2022-04-28T19:53:47.916198+0000 mon.a (mon.0) 884 : cluster [INF] Cluster is now healthy 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:2979: done (4 left) 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:2980: done (3 left) 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:2981: done (2 left) 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:2982: done (1 left) 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:2983: done (0 left) 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 277 2022-04-28T19:53:49.072 INFO:tasks.rados.rados.0.smithi059.stdout:2984: seq_num 941 ranges {12705792=565248,13271040=475136,13746176=327680} 2022-04-28T19:53:49.146 INFO:tasks.rados.rados.0.smithi059.stdout:2984: writing smithi05955964-45 from 12705792 to 13271040 tid 1 2022-04-28T19:53:49.149 INFO:tasks.rados.rados.0.smithi059.stdout:2984: writing smithi05955964-45 from 13271040 to 13746176 tid 2 2022-04-28T19:53:49.152 INFO:tasks.rados.rados.0.smithi059.stdout:2984: writing smithi05955964-45 from 13746176 to 14073856 tid 3 2022-04-28T19:53:49.152 INFO:tasks.rados.rados.0.smithi059.stdout:2985: read oid 41 snap 270 2022-04-28T19:53:49.152 INFO:tasks.rados.rados.0.smithi059.stdout:2985: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:49.152 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 277 2022-04-28T19:53:49.152 INFO:tasks.rados.rados.0.smithi059.stdout:2986: seq_num 942 ranges {8970240=581632,9551872=761856,10313728=40960} 2022-04-28T19:53:49.207 INFO:tasks.rados.rados.0.smithi059.stdout:2986: writing smithi05955964-35 from 8970240 to 9551872 tid 1 2022-04-28T19:53:49.211 INFO:tasks.rados.rados.0.smithi059.stdout:2986: writing smithi05955964-35 from 9551872 to 10313728 tid 2 2022-04-28T19:53:49.211 INFO:tasks.rados.rados.0.smithi059.stdout:2986: writing smithi05955964-35 from 10313728 to 10354688 tid 3 2022-04-28T19:53:49.212 INFO:tasks.rados.rados.0.smithi059.stdout:2987: delete oid 40 current snap is 277 2022-04-28T19:53:49.212 INFO:tasks.rados.rados.0.smithi059.stdout:2984: finishing write tid 1 to smithi05955964-45 2022-04-28T19:53:49.212 INFO:tasks.rados.rados.0.smithi059.stdout:2984: finishing write tid 2 to smithi05955964-45 2022-04-28T19:53:49.212 INFO:tasks.rados.rados.0.smithi059.stdout:2984: finishing write tid 3 to smithi05955964-45 2022-04-28T19:53:49.212 INFO:tasks.rados.rados.0.smithi059.stdout:2984: finishing write tid 5 to smithi05955964-45 2022-04-28T19:53:49.212 INFO:tasks.rados.rados.0.smithi059.stdout:2985: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:53:49.249 INFO:tasks.rados.rados.0.smithi059.stdout:2984: finishing write tid 6 to smithi05955964-45 2022-04-28T19:53:49.346 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1230 (ObjNum 941 snap 277 seq_num 941) dirty exists 2022-04-28T19:53:49.346 INFO:tasks.rados.rados.0.smithi059.stdout:2984: left oid 45 (ObjNum 941 snap 277 seq_num 941) 2022-04-28T19:53:49.346 INFO:tasks.rados.rados.0.smithi059.stdout:2986: finishing write tid 1 to smithi05955964-35 2022-04-28T19:53:49.346 INFO:tasks.rados.rados.0.smithi059.stdout:2986: finishing write tid 2 to smithi05955964-35 2022-04-28T19:53:49.346 INFO:tasks.rados.rados.0.smithi059.stdout:2986: finishing write tid 3 to smithi05955964-35 2022-04-28T19:53:49.346 INFO:tasks.rados.rados.0.smithi059.stdout:2986: finishing write tid 5 to smithi05955964-35 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2984: done (3 left) 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2985: done (2 left) 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2987: done (1 left) 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2988: copy_from oid 42 from oid 3 current snap is 277 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2986: finishing write tid 6 to smithi05955964-35 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1321 (ObjNum 942 snap 277 seq_num 942) dirty exists 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2986: left oid 35 (ObjNum 942 snap 277 seq_num 942) 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2986: done (1 left) 2022-04-28T19:53:49.347 INFO:tasks.rados.rados.0.smithi059.stdout:2989: setattr oid 33 current snap is 277 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:2990: rollback oid 36 current snap is 277 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 253 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:2991: rmattr oid 21 current snap is 277 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:2992: rollback oid 15 current snap is 277 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 276 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:2993: copy_from oid 46 from oid 9 current snap is 277 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:2994: rollback oid 20 current snap is 277 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 270 2022-04-28T19:53:49.352 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 277 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2995: seq_num 944 ranges {2367488=614400,2981888=573440,3555328=229376} 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2995: writing smithi05955964-11 from 2367488 to 2981888 tid 1 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2995: writing smithi05955964-11 from 2981888 to 3555328 tid 2 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2995: writing smithi05955964-11 from 3555328 to 3784704 tid 3 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2996: read oid 22 snap 252 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2996: expect (ObjNum 812 snap 240 seq_num 812) 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1156 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2997: read oid 44 snap -1 2022-04-28T19:53:49.353 INFO:tasks.rados.rados.0.smithi059.stdout:2997: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2991: done (9 left) 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2998: delete oid 5 current snap is 277 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2990: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2990: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1432 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2990: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1110 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2992: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2992: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:53:49.354 INFO:tasks.rados.rados.0.smithi059.stdout:2992: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1334 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2993: finishing copy_from to smithi05955964-46 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 967 (ObjNum 896 snap 268 seq_num 896) dirty exists 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2993: finishing copy_from racing read to smithi05955964-46 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2994: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2994: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2994: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1233 (ObjNum 821 snap 240 seq_num 821) dirty exists 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2995: finishing write tid 1 to smithi05955964-11 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2997: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:53:49.355 INFO:tasks.rados.rados.0.smithi059.stdout:2995: finishing write tid 2 to smithi05955964-11 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2989: done (9 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2990: done (8 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2992: done (7 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2993: done (6 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2994: done (5 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2997: done (4 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2998: done (3 left) 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:2999: rmattr oid 27 current snap is 277 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 277 2022-04-28T19:53:49.356 INFO:tasks.rados.rados.0.smithi059.stdout:3000: seq_num 946 ranges {1777664=647168,2424832=745472,3170304=57344} 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:3000: writing smithi05955964-15 from 1777664 to 2424832 tid 1 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:3000: writing smithi05955964-15 from 2424832 to 3170304 tid 2 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:3000: writing smithi05955964-15 from 3170304 to 3227648 tid 3 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:3001: snap_create 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:2995: finishing write tid 3 to smithi05955964-11 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1324 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:2995: finishing write tid 5 to smithi05955964-11 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:2995: finishing write tid 6 to smithi05955964-11 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1182 (ObjNum 944 snap 277 seq_num 944) dirty exists 2022-04-28T19:53:49.357 INFO:tasks.rados.rados.0.smithi059.stdout:2995: left oid 11 (ObjNum 944 snap 277 seq_num 944) 2022-04-28T19:53:49.358 INFO:tasks.rados.rados.0.smithi059.stdout:3000: finishing write tid 1 to smithi05955964-15 2022-04-28T19:53:49.358 INFO:tasks.rados.rados.0.smithi059.stdout:2988: finishing copy_from to smithi05955964-42 2022-04-28T19:53:49.358 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 986 (ObjNum 874 snap 262 seq_num 874) dirty exists 2022-04-28T19:53:49.358 INFO:tasks.rados.rados.0.smithi059.stdout:2996: expect (ObjNum 812 snap 240 seq_num 812) 2022-04-28T19:53:49.358 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:49 smithi059 bash[61152]: cluster 2022-04-28T19:53:48.038342+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2022-04-28T19:53:49.358 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:49 smithi059 bash[59745]: cluster 2022-04-28T19:53:48.038342+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2022-04-28T19:53:49.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:49 smithi139 bash[43301]: cluster 2022-04-28T19:53:48.038342+0000 mon.a (mon.0) 885 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2022-04-28T19:53:49.772 INFO:tasks.rados.rados.0.smithi059.stdout:2988: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:53:49.868 INFO:tasks.rados.rados.0.smithi059.stdout:3000: finishing write tid 2 to smithi05955964-15 2022-04-28T19:53:49.869 INFO:tasks.rados.rados.0.smithi059.stdout:3000: finishing write tid 3 to smithi05955964-15 2022-04-28T19:53:49.869 INFO:tasks.rados.rados.0.smithi059.stdout:3000: finishing write tid 5 to smithi05955964-15 2022-04-28T19:53:49.869 INFO:tasks.rados.rados.0.smithi059.stdout:3000: finishing write tid 6 to smithi05955964-15 2022-04-28T19:53:49.873 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1338 (ObjNum 946 snap 277 seq_num 946) dirty exists 2022-04-28T19:53:49.874 INFO:tasks.rados.rados.0.smithi059.stdout:3000: left oid 15 (ObjNum 946 snap 277 seq_num 946) 2022-04-28T19:53:50.075 INFO:tasks.rados.rados.0.smithi059.stdout:2988: done (5 left) 2022-04-28T19:53:50.075 INFO:tasks.rados.rados.0.smithi059.stdout:2995: done (4 left) 2022-04-28T19:53:50.075 INFO:tasks.rados.rados.0.smithi059.stdout:2996: done (3 left) 2022-04-28T19:53:50.075 INFO:tasks.rados.rados.0.smithi059.stdout:2999: done (2 left) 2022-04-28T19:53:50.075 INFO:tasks.rados.rados.0.smithi059.stdout:3000: done (1 left) 2022-04-28T19:53:50.075 INFO:tasks.rados.rados.0.smithi059.stdout:3001: done (0 left) 2022-04-28T19:53:50.076 INFO:tasks.rados.rados.0.smithi059.stdout:3002: delete oid 25 current snap is 278 2022-04-28T19:53:50.083 INFO:tasks.rados.rados.0.smithi059.stdout:3002: done (0 left) 2022-04-28T19:53:50.083 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 278 2022-04-28T19:53:50.083 INFO:tasks.rados.rados.0.smithi059.stdout:3003: seq_num 948 ranges {14073856=679936,14753792=516096,15269888=286720} 2022-04-28T19:53:50.160 INFO:tasks.rados.rados.0.smithi059.stdout:3003: writing smithi05955964-45 from 14073856 to 14753792 tid 1 2022-04-28T19:53:50.163 INFO:tasks.rados.rados.0.smithi059.stdout:3003: writing smithi05955964-45 from 14753792 to 15269888 tid 2 2022-04-28T19:53:50.165 INFO:tasks.rados.rados.0.smithi059.stdout:3003: writing smithi05955964-45 from 15269888 to 15556608 tid 3 2022-04-28T19:53:50.165 INFO:tasks.rados.rados.0.smithi059.stdout:3004: read oid 17 snap -1 2022-04-28T19:53:50.165 INFO:tasks.rados.rados.0.smithi059.stdout:3004: expect (ObjNum 919 snap 274 seq_num 919) 2022-04-28T19:53:50.165 INFO:tasks.rados.rados.0.smithi059.stdout:3005: rollback oid 5 current snap is 278 2022-04-28T19:53:50.165 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 276 2022-04-28T19:53:50.166 INFO:tasks.rados.rados.0.smithi059.stdout:3006: snap_create 2022-04-28T19:53:50.171 INFO:tasks.rados.rados.0.smithi059.stdout:3003: finishing write tid 1 to smithi05955964-45 2022-04-28T19:53:50.172 INFO:tasks.rados.rados.0.smithi059.stdout:3005: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:53:50.172 INFO:tasks.rados.rados.0.smithi059.stdout:3003: finishing write tid 2 to smithi05955964-45 2022-04-28T19:53:50.172 INFO:tasks.rados.rados.0.smithi059.stdout:3005: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:53:50.172 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1430 (ObjNum 875 snap 262 seq_num 875) dirty exists 2022-04-28T19:53:50.172 INFO:tasks.rados.rados.0.smithi059.stdout:3003: finishing write tid 3 to smithi05955964-45 2022-04-28T19:53:50.175 INFO:tasks.rados.rados.0.smithi059.stdout:3003: finishing write tid 5 to smithi05955964-45 2022-04-28T19:53:50.176 INFO:tasks.rados.rados.0.smithi059.stdout:3003: finishing write tid 6 to smithi05955964-45 2022-04-28T19:53:50.176 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1237 (ObjNum 948 snap 278 seq_num 948) dirty exists 2022-04-28T19:53:50.176 INFO:tasks.rados.rados.0.smithi059.stdout:3003: left oid 45 (ObjNum 948 snap 278 seq_num 948) 2022-04-28T19:53:50.209 INFO:tasks.rados.rados.0.smithi059.stdout:3004: expect (ObjNum 919 snap 274 seq_num 919) 2022-04-28T19:53:50.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:50 smithi059 bash[61152]: cluster 2022-04-28T19:53:48.959000+0000 mgr.y (mgr.24961) 245 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 629 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.5 MiB/s wr, 12 op/s; 4.1 MiB/s, 1 objects/s recovering 2022-04-28T19:53:50.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:50 smithi059 bash[61152]: cluster 2022-04-28T19:53:49.069611+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2022-04-28T19:53:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:50 smithi059 bash[59745]: cluster 2022-04-28T19:53:48.959000+0000 mgr.y (mgr.24961) 245 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 629 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.5 MiB/s wr, 12 op/s; 4.1 MiB/s, 1 objects/s recovering 2022-04-28T19:53:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:50 smithi059 bash[59745]: cluster 2022-04-28T19:53:49.069611+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2022-04-28T19:53:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:50 smithi139 bash[43301]: cluster 2022-04-28T19:53:48.959000+0000 mgr.y (mgr.24961) 245 : cluster [DBG] pgmap v399: 89 pgs: 89 active+clean; 629 MiB data, 1.3 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.5 MiB/s wr, 12 op/s; 4.1 MiB/s, 1 objects/s recovering 2022-04-28T19:53:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:50 smithi139 bash[43301]: cluster 2022-04-28T19:53:49.069611+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2022-04-28T19:53:51.095 INFO:tasks.rados.rados.0.smithi059.stdout:3003: done (3 left) 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3004: done (2 left) 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3005: done (1 left) 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3006: done (0 left) 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3007: read oid 15 snap -1 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3007: expect (ObjNum 946 snap 277 seq_num 946) 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3008: read oid 13 snap -1 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3008: expect deleted 2022-04-28T19:53:51.096 INFO:tasks.rados.rados.0.smithi059.stdout:3009: copy_from oid 10 from oid 19 current snap is 279 2022-04-28T19:53:51.097 INFO:tasks.rados.rados.0.smithi059.stdout:3010: read oid 32 snap -1 2022-04-28T19:53:51.097 INFO:tasks.rados.rados.0.smithi059.stdout:3010: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:53:51.097 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 279 2022-04-28T19:53:51.097 INFO:tasks.rados.rados.0.smithi059.stdout:3011: seq_num 949 ranges {581632=704512,1286144=794624,2080768=8192} 2022-04-28T19:53:51.106 INFO:tasks.rados.rados.0.smithi059.stdout:3011: writing smithi05955964-2 from 581632 to 1286144 tid 1 2022-04-28T19:53:51.112 INFO:tasks.rados.rados.0.smithi059.stdout:3011: writing smithi05955964-2 from 1286144 to 2080768 tid 2 2022-04-28T19:53:51.112 INFO:tasks.rados.rados.0.smithi059.stdout:3011: writing smithi05955964-2 from 2080768 to 2088960 tid 3 2022-04-28T19:53:51.112 INFO:tasks.rados.rados.0.smithi059.stdout:3012: setattr oid 46 current snap is 279 2022-04-28T19:53:51.114 INFO:tasks.rados.rados.0.smithi059.stdout:3008: done (5 left) 2022-04-28T19:53:51.114 INFO:tasks.rados.rados.0.smithi059.stdout:3013: rmattr oid 3 current snap is 279 2022-04-28T19:53:51.114 INFO:tasks.rados.rados.0.smithi059.stdout:3014: copy_from oid 16 from oid 5 current snap is 279 2022-04-28T19:53:51.114 INFO:tasks.rados.rados.0.smithi059.stdout:3015: read oid 49 snap -1 2022-04-28T19:53:51.115 INFO:tasks.rados.rados.0.smithi059.stdout:3015: expect (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:53:51.115 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 30 current snap is 279 2022-04-28T19:53:51.115 INFO:tasks.rados.rados.0.smithi059.stdout:3016: seq_num 950 ranges {0=720896,720896=679936,1400832=122880} 2022-04-28T19:53:51.119 INFO:tasks.rados.rados.0.smithi059.stdout:3016: writing smithi05955964-30 from 0 to 720896 tid 1 2022-04-28T19:53:51.124 INFO:tasks.rados.rados.0.smithi059.stdout:3016: writing smithi05955964-30 from 720896 to 1400832 tid 2 2022-04-28T19:53:51.125 INFO:tasks.rados.rados.0.smithi059.stdout:3016: writing smithi05955964-30 from 1400832 to 1523712 tid 3 2022-04-28T19:53:51.125 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 279 2022-04-28T19:53:51.125 INFO:tasks.rados.rados.0.smithi059.stdout:3017: seq_num 951 ranges {0=737280,737280=614400,1351680=188416} 2022-04-28T19:53:51.129 INFO:tasks.rados.rados.0.smithi059.stdout:3017: writing smithi05955964-7 from 0 to 737280 tid 1 2022-04-28T19:53:51.133 INFO:tasks.rados.rados.0.smithi059.stdout:3017: writing smithi05955964-7 from 737280 to 1351680 tid 2 2022-04-28T19:53:51.134 INFO:tasks.rados.rados.0.smithi059.stdout:3017: writing smithi05955964-7 from 1351680 to 1540096 tid 3 2022-04-28T19:53:51.135 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 279 2022-04-28T19:53:51.135 INFO:tasks.rados.rados.0.smithi059.stdout:3018: seq_num 952 ranges {7815168=753664,8568832=499712,9068544=303104} 2022-04-28T19:53:51.183 INFO:tasks.rados.rados.0.smithi059.stdout:3018: writing smithi05955964-42 from 7815168 to 8568832 tid 1 2022-04-28T19:53:51.185 INFO:tasks.rados.rados.0.smithi059.stdout:3018: writing smithi05955964-42 from 8568832 to 9068544 tid 2 2022-04-28T19:53:51.187 INFO:tasks.rados.rados.0.smithi059.stdout:3018: writing smithi05955964-42 from 9068544 to 9371648 tid 3 2022-04-28T19:53:51.187 INFO:tasks.rados.rados.0.smithi059.stdout:3019: snap_remove snap 277 2022-04-28T19:53:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:51 smithi059 bash[61152]: cluster 2022-04-28T19:53:50.074322+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2022-04-28T19:53:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:51 smithi059 bash[59745]: cluster 2022-04-28T19:53:50.074322+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2022-04-28T19:53:51.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:51 smithi139 bash[43301]: cluster 2022-04-28T19:53:50.074322+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2022-04-28T19:53:51.926 INFO:tasks.rados.rados.0.smithi059.stdout:3019: done (11 left) 2022-04-28T19:53:51.927 INFO:tasks.rados.rados.0.smithi059.stdout:3020: read oid 12 snap -1 2022-04-28T19:53:51.927 INFO:tasks.rados.rados.0.smithi059.stdout:3020: expect (ObjNum 936 snap 276 seq_num 936) 2022-04-28T19:53:51.927 INFO:tasks.rados.rados.0.smithi059.stdout:3021: read oid 36 snap -1 2022-04-28T19:53:51.927 INFO:tasks.rados.rados.0.smithi059.stdout:3021: expect (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:53:51.927 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 279 2022-04-28T19:53:51.927 INFO:tasks.rados.rados.0.smithi059.stdout:3022: seq_num 953 ranges {0=770048,770048=786432,1556480=16384} 2022-04-28T19:53:51.931 INFO:tasks.rados.rados.0.smithi059.stdout:3022: writing smithi05955964-8 from 0 to 770048 tid 1 2022-04-28T19:53:51.936 INFO:tasks.rados.rados.0.smithi059.stdout:3022: writing smithi05955964-8 from 770048 to 1556480 tid 2 2022-04-28T19:53:51.936 INFO:tasks.rados.rados.0.smithi059.stdout:3022: writing smithi05955964-8 from 1556480 to 1572864 tid 3 2022-04-28T19:53:51.936 INFO:tasks.rados.rados.0.smithi059.stdout:3023: read oid 45 snap -1 2022-04-28T19:53:51.936 INFO:tasks.rados.rados.0.smithi059.stdout:3023: expect (ObjNum 948 snap 278 seq_num 948) 2022-04-28T19:53:51.936 INFO:tasks.rados.rados.0.smithi059.stdout:3024: read oid 23 snap 271 2022-04-28T19:53:51.937 INFO:tasks.rados.rados.0.smithi059.stdout:3024: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:53:51.938 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 16 2022-04-28T19:53:51.938 INFO:tasks.rados.rados.0.smithi059.stdout:3011: finishing write tid 1 to smithi05955964-2 2022-04-28T19:53:51.938 INFO:tasks.rados.rados.0.smithi059.stdout:3011: finishing write tid 2 to smithi05955964-2 2022-04-28T19:53:51.938 INFO:tasks.rados.rados.0.smithi059.stdout:3007: expect (ObjNum 946 snap 277 seq_num 946) 2022-04-28T19:53:52.063 INFO:tasks.rados.rados.0.smithi059.stdout:3011: finishing write tid 3 to smithi05955964-2 2022-04-28T19:53:52.064 INFO:tasks.rados.rados.0.smithi059.stdout:3011: finishing write tid 5 to smithi05955964-2 2022-04-28T19:53:52.064 INFO:tasks.rados.rados.0.smithi059.stdout:3010: expect (ObjNum 819 snap 240 seq_num 819) 2022-04-28T19:53:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:51.634Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:51.634Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:51.634Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:53:52.188 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1432 (ObjNum 896 snap 268 seq_num 896) dirty exists 2022-04-28T19:53:52.228 INFO:tasks.rados.rados.0.smithi059.stdout:3011: finishing write tid 6 to smithi05955964-2 2022-04-28T19:53:52.228 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1329 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:53:52.229 INFO:tasks.rados.rados.0.smithi059.stdout:3011: left oid 2 (ObjNum 949 snap 279 seq_num 949) 2022-04-28T19:53:52.229 INFO:tasks.rados.rados.0.smithi059.stdout:3016: finishing write tid 1 to smithi05955964-30 2022-04-28T19:53:52.229 INFO:tasks.rados.rados.0.smithi059.stdout:3016: finishing write tid 2 to smithi05955964-30 2022-04-28T19:53:52.229 INFO:tasks.rados.rados.0.smithi059.stdout:3016: finishing write tid 3 to smithi05955964-30 2022-04-28T19:53:52.229 INFO:tasks.rados.rados.0.smithi059.stdout:3016: finishing write tid 5 to smithi05955964-30 2022-04-28T19:53:52.230 INFO:tasks.rados.rados.0.smithi059.stdout:3016: finishing write tid 6 to smithi05955964-30 2022-04-28T19:53:52.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1324 (ObjNum 950 snap 279 seq_num 950) dirty exists 2022-04-28T19:53:52.230 INFO:tasks.rados.rados.0.smithi059.stdout:3016: left oid 30 (ObjNum 950 snap 279 seq_num 950) 2022-04-28T19:53:52.230 INFO:tasks.rados.rados.0.smithi059.stdout:3017: finishing write tid 1 to smithi05955964-7 2022-04-28T19:53:52.230 INFO:tasks.rados.rados.0.smithi059.stdout:3014: finishing copy_from to smithi05955964-16 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1430 (ObjNum 875 snap 262 seq_num 875) dirty exists 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3014: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1238 (ObjNum 874 snap 262 seq_num 874) dirty exists 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3017: finishing write tid 2 to smithi05955964-7 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3007: done (15 left) 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3010: done (14 left) 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3011: done (13 left) 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3012: done (12 left) 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3013: done (11 left) 2022-04-28T19:53:52.231 INFO:tasks.rados.rados.0.smithi059.stdout:3014: done (10 left) 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3016: done (9 left) 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3025: read oid 34 snap -1 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3025: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3026: snap_create 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3017: finishing write tid 3 to smithi05955964-7 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3017: finishing write tid 5 to smithi05955964-7 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3009: finishing copy_from to smithi05955964-10 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1162 (ObjNum 917 snap 273 seq_num 917) dirty exists 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3009: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:53:52.232 INFO:tasks.rados.rados.0.smithi059.stdout:3017: finishing write tid 6 to smithi05955964-7 2022-04-28T19:53:52.233 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 1342 (ObjNum 951 snap 279 seq_num 951) dirty exists 2022-04-28T19:53:52.233 INFO:tasks.rados.rados.0.smithi059.stdout:3017: left oid 7 (ObjNum 951 snap 279 seq_num 951) 2022-04-28T19:53:52.233 INFO:tasks.rados.rados.0.smithi059.stdout:3015: expect (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:53:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:52 smithi139 bash[43301]: cluster 2022-04-28T19:53:50.959931+0000 mgr.y (mgr.24961) 246 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 667 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 9.8 MiB/s wr, 44 op/s 2022-04-28T19:53:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:52 smithi139 bash[43301]: cluster 2022-04-28T19:53:51.093341+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2022-04-28T19:53:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:52 smithi139 bash[43301]: audit 2022-04-28T19:53:51.653860+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:52 smithi139 bash[43301]: cluster 2022-04-28T19:53:51.923500+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2022-04-28T19:53:52.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[61152]: cluster 2022-04-28T19:53:50.959931+0000 mgr.y (mgr.24961) 246 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 667 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 9.8 MiB/s wr, 44 op/s 2022-04-28T19:53:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[61152]: cluster 2022-04-28T19:53:51.093341+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2022-04-28T19:53:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[61152]: audit 2022-04-28T19:53:51.653860+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[61152]: cluster 2022-04-28T19:53:51.923500+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2022-04-28T19:53:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[59745]: cluster 2022-04-28T19:53:50.959931+0000 mgr.y (mgr.24961) 246 : cluster [DBG] pgmap v402: 89 pgs: 89 active+clean; 667 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 9.8 MiB/s wr, 44 op/s 2022-04-28T19:53:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[59745]: cluster 2022-04-28T19:53:51.093341+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2022-04-28T19:53:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[59745]: audit 2022-04-28T19:53:51.653860+0000 mon.a (mon.0) 889 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:52 smithi059 bash[59745]: cluster 2022-04-28T19:53:51.923500+0000 mon.a (mon.0) 890 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2022-04-28T19:53:53.284 INFO:tasks.rados.rados.0.smithi059.stdout:3018: finishing write tid 1 to smithi05955964-42 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3018: finishing write tid 2 to smithi05955964-42 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3018: finishing write tid 3 to smithi05955964-42 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3018: finishing write tid 5 to smithi05955964-42 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3018: finishing write tid 6 to smithi05955964-42 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1328 (ObjNum 952 snap 279 seq_num 952) dirty exists 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3018: left oid 42 (ObjNum 952 snap 279 seq_num 952) 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3022: finishing write tid 1 to smithi05955964-8 2022-04-28T19:53:53.285 INFO:tasks.rados.rados.0.smithi059.stdout:3022: finishing write tid 2 to smithi05955964-8 2022-04-28T19:53:53.286 INFO:tasks.rados.rados.0.smithi059.stdout:3022: finishing write tid 3 to smithi05955964-8 2022-04-28T19:53:53.286 INFO:tasks.rados.rados.0.smithi059.stdout:3022: finishing write tid 5 to smithi05955964-8 2022-04-28T19:53:53.286 INFO:tasks.rados.rados.0.smithi059.stdout:3021: expect (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:53:53.323 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:53 smithi139 bash[43301]: audit 2022-04-28T19:53:51.654246+0000 mgr.y (mgr.24961) 247 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:53.323 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:53 smithi139 bash[43301]: cephadm 2022-04-28T19:53:51.656531+0000 mgr.y (mgr.24961) 248 : cephadm [INF] Upgrade: osd.7 is safe to restart 2022-04-28T19:53:53.324 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:53 smithi139 bash[43301]: audit 2022-04-28T19:53:52.401535+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:53.324 INFO:tasks.rados.rados.0.smithi059.stdout:3022: finishing write tid 6 to smithi05955964-8 2022-04-28T19:53:53.324 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1440 (ObjNum 953 snap 279 seq_num 953) dirty exists 2022-04-28T19:53:53.324 INFO:tasks.rados.rados.0.smithi059.stdout:3022: left oid 8 (ObjNum 953 snap 279 seq_num 953) 2022-04-28T19:53:53.324 INFO:tasks.rados.rados.0.smithi059.stdout:3020: expect (ObjNum 936 snap 276 seq_num 936) 2022-04-28T19:53:53.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[59745]: audit 2022-04-28T19:53:51.654246+0000 mgr.y (mgr.24961) 247 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:53.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[59745]: cephadm 2022-04-28T19:53:51.656531+0000 mgr.y (mgr.24961) 248 : cephadm [INF] Upgrade: osd.7 is safe to restart 2022-04-28T19:53:53.528 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[59745]: audit 2022-04-28T19:53:52.401535+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:53.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[61152]: audit 2022-04-28T19:53:51.654246+0000 mgr.y (mgr.24961) 247 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2022-04-28T19:53:53.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[61152]: cephadm 2022-04-28T19:53:51.656531+0000 mgr.y (mgr.24961) 248 : cephadm [INF] Upgrade: osd.7 is safe to restart 2022-04-28T19:53:53.529 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[61152]: audit 2022-04-28T19:53:52.401535+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:53.528Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:53:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:53.528Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:53:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:53:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:53:53.528Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:53:54.153 INFO:tasks.rados.rados.0.smithi059.stdout:3024: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:53:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:54 smithi139 bash[43301]: cluster 2022-04-28T19:53:52.960495+0000 mgr.y (mgr.24961) 249 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 672 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 49 op/s 2022-04-28T19:53:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:54 smithi139 bash[43301]: cluster 2022-04-28T19:53:53.112111+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2022-04-28T19:53:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:54 smithi139 bash[43301]: audit 2022-04-28T19:53:53.919289+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:54 smithi139 bash[43301]: audit 2022-04-28T19:53:53.921996+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "container_image"}]: dispatch 2022-04-28T19:53:54.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[59745]: cluster 2022-04-28T19:53:52.960495+0000 mgr.y (mgr.24961) 249 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 672 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 49 op/s 2022-04-28T19:53:54.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[59745]: cluster 2022-04-28T19:53:53.112111+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2022-04-28T19:53:54.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[59745]: audit 2022-04-28T19:53:53.919289+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[59745]: audit 2022-04-28T19:53:53.921996+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "container_image"}]: dispatch 2022-04-28T19:53:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[61152]: cluster 2022-04-28T19:53:52.960495+0000 mgr.y (mgr.24961) 249 : cluster [DBG] pgmap v405: 89 pgs: 89 active+clean; 672 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 49 op/s 2022-04-28T19:53:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[61152]: cluster 2022-04-28T19:53:53.112111+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2022-04-28T19:53:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[61152]: audit 2022-04-28T19:53:53.919289+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:53:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:54 smithi059 bash[61152]: audit 2022-04-28T19:53:53.921996+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "container_image"}]: dispatch 2022-04-28T19:53:54.754 INFO:tasks.rados.rados.0.smithi059.stdout:3023: expect (ObjNum 948 snap 278 seq_num 948) 2022-04-28T19:53:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:55 smithi139 bash[43301]: cephadm 2022-04-28T19:53:53.910224+0000 mgr.y (mgr.24961) 250 : cephadm [INF] Upgrade: Updating osd.7 2022-04-28T19:53:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:55 smithi139 bash[43301]: cephadm 2022-04-28T19:53:53.921234+0000 mgr.y (mgr.24961) 251 : cephadm [INF] Deploying daemon osd.7 on smithi139 2022-04-28T19:53:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:55 smithi059 bash[61152]: cephadm 2022-04-28T19:53:53.910224+0000 mgr.y (mgr.24961) 250 : cephadm [INF] Upgrade: Updating osd.7 2022-04-28T19:53:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:55 smithi059 bash[61152]: cephadm 2022-04-28T19:53:53.921234+0000 mgr.y (mgr.24961) 251 : cephadm [INF] Deploying daemon osd.7 on smithi139 2022-04-28T19:53:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:55 smithi059 bash[59745]: cephadm 2022-04-28T19:53:53.910224+0000 mgr.y (mgr.24961) 250 : cephadm [INF] Upgrade: Updating osd.7 2022-04-28T19:53:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:55 smithi059 bash[59745]: cephadm 2022-04-28T19:53:53.921234+0000 mgr.y (mgr.24961) 251 : cephadm [INF] Deploying daemon osd.7 on smithi139 2022-04-28T19:53:56.307 INFO:tasks.rados.rados.0.smithi059.stdout:3025: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:53:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:56 smithi059 bash[61152]: cluster 2022-04-28T19:53:54.961329+0000 mgr.y (mgr.24961) 252 : cluster [DBG] pgmap v407: 89 pgs: 89 active+clean; 692 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 4.9 MiB/s wr, 33 op/s 2022-04-28T19:53:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:56 smithi059 bash[59745]: cluster 2022-04-28T19:53:54.961329+0000 mgr.y (mgr.24961) 252 : cluster [DBG] pgmap v407: 89 pgs: 89 active+clean; 692 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 4.9 MiB/s wr, 33 op/s 2022-04-28T19:53:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:56 smithi139 bash[43301]: cluster 2022-04-28T19:53:54.961329+0000 mgr.y (mgr.24961) 252 : cluster [DBG] pgmap v407: 89 pgs: 89 active+clean; 692 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 4.9 MiB/s wr, 33 op/s 2022-04-28T19:53:57.141 INFO:tasks.rados.rados.0.smithi059.stdout:3009: done (10 left) 2022-04-28T19:53:57.141 INFO:tasks.rados.rados.0.smithi059.stdout:3015: done (9 left) 2022-04-28T19:53:57.141 INFO:tasks.rados.rados.0.smithi059.stdout:3017: done (8 left) 2022-04-28T19:53:57.141 INFO:tasks.rados.rados.0.smithi059.stdout:3018: done (7 left) 2022-04-28T19:53:57.141 INFO:tasks.rados.rados.0.smithi059.stdout:3020: done (6 left) 2022-04-28T19:53:57.141 INFO:tasks.rados.rados.0.smithi059.stdout:3021: done (5 left) 2022-04-28T19:53:57.142 INFO:tasks.rados.rados.0.smithi059.stdout:3022: done (4 left) 2022-04-28T19:53:57.142 INFO:tasks.rados.rados.0.smithi059.stdout:3023: done (3 left) 2022-04-28T19:53:57.143 INFO:tasks.rados.rados.0.smithi059.stdout:3024: done (2 left) 2022-04-28T19:53:57.143 INFO:tasks.rados.rados.0.smithi059.stdout:3025: done (1 left) 2022-04-28T19:53:57.143 INFO:tasks.rados.rados.0.smithi059.stdout:3026: done (0 left) 2022-04-28T19:53:57.143 INFO:tasks.rados.rados.0.smithi059.stdout:3027: snap_remove snap 276 2022-04-28T19:53:57.944 INFO:tasks.rados.rados.0.smithi059.stdout:3027: done (0 left) 2022-04-28T19:53:57.944 INFO:tasks.rados.rados.0.smithi059.stdout:3028: read oid 24 snap 261 2022-04-28T19:53:57.944 INFO:tasks.rados.rados.0.smithi059.stdout:3028: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:57.944 INFO:tasks.rados.rados.0.smithi059.stdout:3029: read oid 34 snap -1 2022-04-28T19:53:57.945 INFO:tasks.rados.rados.0.smithi059.stdout:3029: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:53:57.945 INFO:tasks.rados.rados.0.smithi059.stdout:3030: snap_remove snap 278 2022-04-28T19:53:58.265 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:57 smithi139 bash[43301]: cluster 2022-04-28T19:53:56.924724+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2022-04-28T19:53:58.288 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:57 smithi139 bash[43301]: cluster 2022-04-28T19:53:56.961833+0000 mgr.y (mgr.24961) 253 : cluster [DBG] pgmap v409: 89 pgs: 89 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 4.5 MiB/s wr, 23 op/s 2022-04-28T19:53:58.288 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:58 smithi139 systemd[1]: Stopping Ceph osd.7 for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:53:58.288 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[32027]: debug 2022-04-28T19:53:58.148+0000 7ff641b37700 -1 received signal: Terminated from Kernel ( Could be generated by pthread_kill(), raise(), abort(), alarm() ) UID: 0 2022-04-28T19:53:58.289 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[32027]: debug 2022-04-28T19:53:58.148+0000 7ff641b37700 -1 osd.7 615 *** Got signal Terminated *** 2022-04-28T19:53:58.289 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[32027]: debug 2022-04-28T19:53:58.148+0000 7ff641b37700 -1 osd.7 615 *** Immediate shutdown (osd_fast_shutdown=true) *** 2022-04-28T19:53:58.315 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:53:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:57 smithi059 bash[61152]: cluster 2022-04-28T19:53:56.924724+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2022-04-28T19:53:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:57 smithi059 bash[61152]: cluster 2022-04-28T19:53:56.961833+0000 mgr.y (mgr.24961) 253 : cluster [DBG] pgmap v409: 89 pgs: 89 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 4.5 MiB/s wr, 23 op/s 2022-04-28T19:53:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:57 smithi059 bash[59745]: cluster 2022-04-28T19:53:56.924724+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2022-04-28T19:53:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:57 smithi059 bash[59745]: cluster 2022-04-28T19:53:56.961833+0000 mgr.y (mgr.24961) 253 : cluster [DBG] pgmap v409: 89 pgs: 89 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 4.5 MiB/s wr, 23 op/s 2022-04-28T19:53:58.642 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:53:58.643 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (12m) 112s ago 13m 14.7M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:53:58.643 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (13m) 22s ago 13m 26.3M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:53:58.643 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (11m) 112s ago 11m 14.9M - 15.2.16-79-gb51d8665 6d41e73600e1 44062639a78b 2022-04-28T19:53:58.643 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (11m) 22s ago 11m 19.2M - 15.2.16-79-gb51d8665 6d41e73600e1 3516bf4894c9 2022-04-28T19:53:58.643 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (5m) 22s ago 17m 377M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:53:58.644 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (5m) 112s ago 19m 415M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:53:58.644 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (4m) 112s ago 19m 66.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:53:58.644 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (4m) 22s ago 17m 50.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:53:58.644 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (4m) 112s ago 17m 52.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:53:58.645 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (13m) 112s ago 13m 11.2M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:53:58.645 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (13m) 22s ago 13m 11.3M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:53:58.645 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (3m) 112s ago 16m 294M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:53:58.645 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (2m) 112s ago 16m 215M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:53:58.645 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (2m) 112s ago 16m 256M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:53:58.645 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (114s) 112s ago 15m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:53:58.646 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (85s) 22s ago 15m 269M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:53:58.646 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (55s) 22s ago 14m 327M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 4d55ca87a6a9 2022-04-28T19:53:58.646 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (24s) 22s ago 14m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 2cf53c9fc29e 2022-04-28T19:53:58.646 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (14m) 22s ago 14m 224M 4096M 15.2.16-79-gb51d8665 6d41e73600e1 0fc2e2c015f5 2022-04-28T19:53:58.647 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (6m) 22s ago 13m 40.7M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:53:58.958 INFO:tasks.rados.rados.0.smithi059.stdout:3030: done (2 left) 2022-04-28T19:53:58.958 INFO:tasks.rados.rados.0.smithi059.stdout:3031: snap_create 2022-04-28T19:53:58.958 INFO:tasks.rados.rados.0.smithi059.stdout:3028: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:53:59.027 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:53:59.109 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:53:59.110 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 7 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 15.2.16-79-gb51d8665 (b51d866559c7cde15d3b75ee17bbba8fdf2584be) octopus (stable)": 2, 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 12 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:53:59.111 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:53:59.223 INFO:tasks.rados.rados.0.smithi059.stdout:3029: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:57.940668+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.302929+0000 mgr.y (mgr.24961) 254 : audit [DBG] from='client.54283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.456372+0000 mon.a (mon.0) 897 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.456511+0000 mon.a (mon.0) 898 : cluster [INF] osd.7 failed (root=default,host=smithi139) (connection refused reported by osd.2) 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.457185+0000 mon.a (mon.0) 899 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.457718+0000 mon.a (mon.0) 900 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.458194+0000 mon.a (mon.0) 901 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.458769+0000 mon.a (mon.0) 902 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.459263+0000 mon.a (mon.0) 903 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.459643+0000 mon.a (mon.0) 904 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.459883+0000 mon.a (mon.0) 905 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.460150+0000 mon.a (mon.0) 906 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.460374+0000 mon.a (mon.0) 907 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.460639+0000 mon.a (mon.0) 908 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.460868+0000 mon.a (mon.0) 909 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.461115+0000 mon.a (mon.0) 910 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.461338+0000 mon.a (mon.0) 911 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.617082+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.618005+0000 mon.a (mon.0) 913 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.618846+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.619739+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.620719+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:57.940668+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.302929+0000 mgr.y (mgr.24961) 254 : audit [DBG] from='client.54283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:59.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.456372+0000 mon.a (mon.0) 897 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.456511+0000 mon.a (mon.0) 898 : cluster [INF] osd.7 failed (root=default,host=smithi139) (connection refused reported by osd.2) 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.457185+0000 mon.a (mon.0) 899 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.457718+0000 mon.a (mon.0) 900 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.458194+0000 mon.a (mon.0) 901 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.458769+0000 mon.a (mon.0) 902 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.459263+0000 mon.a (mon.0) 903 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.459643+0000 mon.a (mon.0) 904 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.459883+0000 mon.a (mon.0) 905 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.460150+0000 mon.a (mon.0) 906 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.460374+0000 mon.a (mon.0) 907 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.460639+0000 mon.a (mon.0) 908 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.460868+0000 mon.a (mon.0) 909 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.461115+0000 mon.a (mon.0) 910 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.461338+0000 mon.a (mon.0) 911 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.617082+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.618005+0000 mon.a (mon.0) 913 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.618846+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.619739+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.620719+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.621623+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.625707+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.626554+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.627197+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.627950+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: audit 2022-04-28T19:53:58.628695+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.860750+0000 mon.a (mon.0) 923 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.861289+0000 mon.a (mon.0) 924 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.861561+0000 mon.a (mon.0) 925 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.861826+0000 mon.a (mon.0) 926 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.862176+0000 mon.a (mon.0) 927 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.862377+0000 mon.a (mon.0) 928 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.862623+0000 mon.a (mon.0) 929 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.862853+0000 mon.a (mon.0) 930 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.863177+0000 mon.a (mon.0) 931 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.863532+0000 mon.a (mon.0) 932 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.863803+0000 mon.a (mon.0) 933 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.864126+0000 mon.a (mon.0) 934 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.864377+0000 mon.a (mon.0) 935 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.864694+0000 mon.a (mon.0) 936 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.621623+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.625707+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.626554+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.627197+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.627950+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: audit 2022-04-28T19:53:58.628695+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.860750+0000 mon.a (mon.0) 923 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.861289+0000 mon.a (mon.0) 924 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.861561+0000 mon.a (mon.0) 925 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.861826+0000 mon.a (mon.0) 926 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.862176+0000 mon.a (mon.0) 927 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.862377+0000 mon.a (mon.0) 928 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.862623+0000 mon.a (mon.0) 929 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.862853+0000 mon.a (mon.0) 930 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.863177+0000 mon.a (mon.0) 931 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.863532+0000 mon.a (mon.0) 932 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.863803+0000 mon.a (mon.0) 933 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.864126+0000 mon.a (mon.0) 934 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.864377+0000 mon.a (mon.0) 935 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:58 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.864694+0000 mon.a (mon.0) 936 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.368 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[50137]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.7 2022-04-28T19:53:59.368 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[50137]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd-7 2022-04-28T19:53:59.369 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[50137]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-osd.7 2022-04-28T19:53:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:57.940668+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2022-04-28T19:53:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.302929+0000 mgr.y (mgr.24961) 254 : audit [DBG] from='client.54283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:53:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.456372+0000 mon.a (mon.0) 897 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.456511+0000 mon.a (mon.0) 898 : cluster [INF] osd.7 failed (root=default,host=smithi139) (connection refused reported by osd.2) 2022-04-28T19:53:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.457185+0000 mon.a (mon.0) 899 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.457718+0000 mon.a (mon.0) 900 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.458194+0000 mon.a (mon.0) 901 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.458769+0000 mon.a (mon.0) 902 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.459263+0000 mon.a (mon.0) 903 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.459643+0000 mon.a (mon.0) 904 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.459883+0000 mon.a (mon.0) 905 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.460150+0000 mon.a (mon.0) 906 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.460374+0000 mon.a (mon.0) 907 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.460639+0000 mon.a (mon.0) 908 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.460868+0000 mon.a (mon.0) 909 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.461115+0000 mon.a (mon.0) 910 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.461338+0000 mon.a (mon.0) 911 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.617082+0000 mon.a (mon.0) 912 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.618005+0000 mon.a (mon.0) 913 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.618846+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.619739+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.620719+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.621623+0000 mon.a (mon.0) 917 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.625707+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:53:59.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.626554+0000 mon.a (mon.0) 919 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.627197+0000 mon.a (mon.0) 920 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.627950+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: audit 2022-04-28T19:53:58.628695+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.860750+0000 mon.a (mon.0) 923 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.861289+0000 mon.a (mon.0) 924 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.861561+0000 mon.a (mon.0) 925 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.861826+0000 mon.a (mon.0) 926 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.862176+0000 mon.a (mon.0) 927 : cluster [DBG] osd.7 reported immediately failed by osd.6 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.862377+0000 mon.a (mon.0) 928 : cluster [DBG] osd.7 reported immediately failed by osd.2 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.862623+0000 mon.a (mon.0) 929 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.862853+0000 mon.a (mon.0) 930 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.863177+0000 mon.a (mon.0) 931 : cluster [DBG] osd.7 reported immediately failed by osd.1 2022-04-28T19:53:59.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.863532+0000 mon.a (mon.0) 932 : cluster [DBG] osd.7 reported immediately failed by osd.4 2022-04-28T19:53:59.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.863803+0000 mon.a (mon.0) 933 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:53:59.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.864126+0000 mon.a (mon.0) 934 : cluster [DBG] osd.7 reported immediately failed by osd.5 2022-04-28T19:53:59.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.864377+0000 mon.a (mon.0) 935 : cluster [DBG] osd.7 reported immediately failed by osd.0 2022-04-28T19:53:59.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:58 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.864694+0000 mon.a (mon.0) 936 : cluster [DBG] osd.7 reported immediately failed by osd.3 2022-04-28T19:54:00.056 INFO:tasks.rados.rados.0.smithi059.stdout:3028: done (2 left) 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3029: done (1 left) 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3031: done (0 left) 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3032: read oid 37 snap -1 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3032: expect (ObjNum 904 snap 271 seq_num 904) 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3033: read oid 5 snap -1 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3033: expect (ObjNum 875 snap 262 seq_num 875) 2022-04-28T19:54:00.057 INFO:tasks.rados.rados.0.smithi059.stdout:3034: read oid 6 snap -1 2022-04-28T19:54:00.058 INFO:tasks.rados.rados.0.smithi059.stdout:3034: expect (ObjNum 934 snap 276 seq_num 934) 2022-04-28T19:54:00.058 INFO:tasks.rados.rados.0.smithi059.stdout:3035: read oid 11 snap 246 2022-04-28T19:54:00.058 INFO:tasks.rados.rados.0.smithi059.stdout:3035: expect deleted 2022-04-28T19:54:00.058 INFO:tasks.rados.rados.0.smithi059.stdout:3036: snap_remove snap 236 2022-04-28T19:54:00.231 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[43301]: audit 2022-04-28T19:53:58.613550+0000 mgr.y (mgr.24961) 255 : audit [DBG] from='client.44343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:00.231 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.938237+0000 mon.a (mon.0) 937 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:54:00.231 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.938327+0000 mon.a (mon.0) 938 : cluster [WRN] Health check failed: all OSDs are running pacific or later but require_osd_release < pacific (OSD_UPGRADE_FINISHED) 2022-04-28T19:54:00.231 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.954572+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e617: 8 total, 7 up, 8 in 2022-04-28T19:54:00.232 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[43301]: cluster 2022-04-28T19:53:58.962285+0000 mgr.y (mgr.24961) 256 : cluster [DBG] pgmap v412: 89 pgs: 11 stale+active+clean, 78 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 4.1 MiB/s wr, 13 op/s 2022-04-28T19:54:00.232 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:53:59 smithi139 bash[43301]: audit 2022-04-28T19:53:59.027055+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.59:0/2582834493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:00.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[61152]: audit 2022-04-28T19:53:58.613550+0000 mgr.y (mgr.24961) 255 : audit [DBG] from='client.44343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.938237+0000 mon.a (mon.0) 937 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.938327+0000 mon.a (mon.0) 938 : cluster [WRN] Health check failed: all OSDs are running pacific or later but require_osd_release < pacific (OSD_UPGRADE_FINISHED) 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.954572+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e617: 8 total, 7 up, 8 in 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[61152]: cluster 2022-04-28T19:53:58.962285+0000 mgr.y (mgr.24961) 256 : cluster [DBG] pgmap v412: 89 pgs: 11 stale+active+clean, 78 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 4.1 MiB/s wr, 13 op/s 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[61152]: audit 2022-04-28T19:53:59.027055+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.59:0/2582834493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[59745]: audit 2022-04-28T19:53:58.613550+0000 mgr.y (mgr.24961) 255 : audit [DBG] from='client.44343 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.938237+0000 mon.a (mon.0) 937 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2022-04-28T19:54:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.938327+0000 mon.a (mon.0) 938 : cluster [WRN] Health check failed: all OSDs are running pacific or later but require_osd_release < pacific (OSD_UPGRADE_FINISHED) 2022-04-28T19:54:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.954572+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e617: 8 total, 7 up, 8 in 2022-04-28T19:54:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[59745]: cluster 2022-04-28T19:53:58.962285+0000 mgr.y (mgr.24961) 256 : cluster [DBG] pgmap v412: 89 pgs: 11 stale+active+clean, 78 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 4.1 MiB/s wr, 13 op/s 2022-04-28T19:54:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:53:59 smithi059 bash[59745]: audit 2022-04-28T19:53:59.027055+0000 mon.a (mon.0) 940 : audit [DBG] from='client.? 172.21.15.59:0/2582834493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:00.977 INFO:tasks.rados.rados.0.smithi059.stdout:3036: done (4 left) 2022-04-28T19:54:00.977 INFO:tasks.rados.rados.0.smithi059.stdout:3037: snap_create 2022-04-28T19:54:00.977 INFO:tasks.rados.rados.0.smithi059.stdout:3033: expect (ObjNum 875 snap 262 seq_num 875) 2022-04-28T19:54:01.083 INFO:tasks.rados.rados.0.smithi059.stdout:3032: expect (ObjNum 904 snap 271 seq_num 904) 2022-04-28T19:54:01.228 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:00 smithi139 bash[43301]: cluster 2022-04-28T19:53:59.961075+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e618: 8 total, 7 up, 8 in 2022-04-28T19:54:01.228 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:01 smithi139 systemd[1]: Stopped Ceph osd.7 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:01.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:00 smithi059 bash[61152]: cluster 2022-04-28T19:53:59.961075+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e618: 8 total, 7 up, 8 in 2022-04-28T19:54:01.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:00 smithi059 bash[59745]: cluster 2022-04-28T19:53:59.961075+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e618: 8 total, 7 up, 8 in 2022-04-28T19:54:01.459 INFO:tasks.rados.rados.0.smithi059.stdout:3034: expect (ObjNum 934 snap 276 seq_num 934) 2022-04-28T19:54:01.618 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:01 smithi139 systemd[1]: Started Ceph osd.7 for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:01.971 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:01.634Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:01.971 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:01.634Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:01.971 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:01.634Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:54:01.971 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: cluster 2022-04-28T19:54:00.963366+0000 mgr.y (mgr.24961) 257 : cluster [DBG] pgmap v414: 89 pgs: 4 activating+undersized, 2 active+undersized+degraded, 7 peering, 4 stale+active+clean, 14 active+undersized, 58 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 0 op/s; 28/1188 objects degraded (2.357%) 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: cluster 2022-04-28T19:54:00.973171+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e619: 8 total, 7 up, 8 in 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: audit 2022-04-28T19:54:01.412509+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: audit 2022-04-28T19:54:01.413654+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: audit 2022-04-28T19:54:01.415625+0000 mon.a (mon.0) 945 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: audit 2022-04-28T19:54:01.416843+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: audit 2022-04-28T19:54:01.418905+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: cluster 2022-04-28T19:54:01.929454+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e620: 8 total, 7 up, 8 in 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: cluster 2022-04-28T19:54:01.959693+0000 mon.a (mon.0) 949 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2022-04-28T19:54:01.972 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[59745]: cluster 2022-04-28T19:54:01.959756+0000 mon.a (mon.0) 950 : cluster [WRN] Health check failed: Degraded data redundancy: 28/1188 objects degraded (2.357%), 2 pgs degraded (PG_DEGRADED) 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: cluster 2022-04-28T19:54:00.963366+0000 mgr.y (mgr.24961) 257 : cluster [DBG] pgmap v414: 89 pgs: 4 activating+undersized, 2 active+undersized+degraded, 7 peering, 4 stale+active+clean, 14 active+undersized, 58 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 0 op/s; 28/1188 objects degraded (2.357%) 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: cluster 2022-04-28T19:54:00.973171+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e619: 8 total, 7 up, 8 in 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: audit 2022-04-28T19:54:01.412509+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: audit 2022-04-28T19:54:01.413654+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: audit 2022-04-28T19:54:01.415625+0000 mon.a (mon.0) 945 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: audit 2022-04-28T19:54:01.416843+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: audit 2022-04-28T19:54:01.418905+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:01.973 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: cluster 2022-04-28T19:54:01.929454+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e620: 8 total, 7 up, 8 in 2022-04-28T19:54:01.974 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: cluster 2022-04-28T19:54:01.959693+0000 mon.a (mon.0) 949 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2022-04-28T19:54:01.974 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:01 smithi059 bash[61152]: cluster 2022-04-28T19:54:01.959756+0000 mon.a (mon.0) 950 : cluster [WRN] Health check failed: Degraded data redundancy: 28/1188 objects degraded (2.357%), 2 pgs degraded (PG_DEGRADED) 2022-04-28T19:54:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: cluster 2022-04-28T19:54:00.963366+0000 mgr.y (mgr.24961) 257 : cluster [DBG] pgmap v414: 89 pgs: 4 activating+undersized, 2 active+undersized+degraded, 7 peering, 4 stale+active+clean, 14 active+undersized, 58 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 0 op/s; 28/1188 objects degraded (2.357%) 2022-04-28T19:54:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: cluster 2022-04-28T19:54:00.973171+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e619: 8 total, 7 up, 8 in 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: audit 2022-04-28T19:54:01.412509+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: audit 2022-04-28T19:54:01.413654+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: audit 2022-04-28T19:54:01.415625+0000 mon.a (mon.0) 945 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: audit 2022-04-28T19:54:01.416843+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: audit 2022-04-28T19:54:01.418905+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: cluster 2022-04-28T19:54:01.929454+0000 mon.a (mon.0) 948 : cluster [DBG] osdmap e620: 8 total, 7 up, 8 in 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: cluster 2022-04-28T19:54:01.959693+0000 mon.a (mon.0) 949 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2022-04-28T19:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:01 smithi139 bash[43301]: cluster 2022-04-28T19:54:01.959756+0000 mon.a (mon.0) 950 : cluster [WRN] Health check failed: Degraded data redundancy: 28/1188 objects degraded (2.357%), 2 pgs degraded (PG_DEGRADED) 2022-04-28T19:54:03.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:02 smithi059 bash[61152]: cephadm 2022-04-28T19:54:01.417955+0000 mgr.y (mgr.24961) 258 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:03.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:02 smithi059 bash[59745]: cephadm 2022-04-28T19:54:01.417955+0000 mgr.y (mgr.24961) 258 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:02 smithi139 bash[43301]: cephadm 2022-04-28T19:54:01.417955+0000 mgr.y (mgr.24961) 258 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:03.369 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2022-04-28T19:54:03.369 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2022-04-28T19:54:03.369 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2022-04-28T19:54:03.369 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2022-04-28T19:54:03.370 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2022-04-28T19:54:03.370 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2022-04-28T19:54:03.370 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[50386]: --> ceph-volume lvm activate successful for osd ID: 7 2022-04-28T19:54:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:03.528Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:03.528Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:03.528Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:54:03.930 INFO:tasks.rados.rados.0.smithi059.stdout:3032: done (4 left) 2022-04-28T19:54:03.931 INFO:tasks.rados.rados.0.smithi059.stdout:3033: done (3 left) 2022-04-28T19:54:03.931 INFO:tasks.rados.rados.0.smithi059.stdout:3034: done (2 left) 2022-04-28T19:54:03.933 INFO:tasks.rados.rados.0.smithi059.stdout:3035: done (1 left) 2022-04-28T19:54:03.933 INFO:tasks.rados.rados.0.smithi059.stdout:3037: done (0 left) 2022-04-28T19:54:03.933 INFO:tasks.rados.rados.0.smithi059.stdout:3038: read oid 48 snap 229 2022-04-28T19:54:03.933 INFO:tasks.rados.rados.0.smithi059.stdout:3038: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:54:03.933 INFO:tasks.rados.rados.0.smithi059.stdout:3039: delete oid 46 current snap is 282 2022-04-28T19:54:03.939 INFO:tasks.rados.rados.0.smithi059.stdout:3039: done (1 left) 2022-04-28T19:54:03.939 INFO:tasks.rados.rados.0.smithi059.stdout:3040: setattr oid 16 current snap is 282 2022-04-28T19:54:03.942 INFO:tasks.rados.rados.0.smithi059.stdout:3041: snap_remove snap 261 2022-04-28T19:54:03.986 INFO:tasks.rados.rados.0.smithi059.stdout:3041: done (2 left) 2022-04-28T19:54:04.060 INFO:tasks.rados.rados.0.smithi059.stdout:3042: snap_create 2022-04-28T19:54:04.060 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1431 (ObjNum 875 snap 262 seq_num 875) dirty exists 2022-04-28T19:54:04.062 INFO:tasks.rados.rados.0.smithi059.stdout:3038: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:54:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:03 smithi059 bash[61152]: cluster 2022-04-28T19:54:02.964160+0000 mgr.y (mgr.24961) 259 : cluster [DBG] pgmap v417: 89 pgs: 4 activating+undersized, 2 active+undersized+degraded, 7 peering, 4 stale+active+clean, 14 active+undersized, 58 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 0 op/s; 28/1188 objects degraded (2.357%) 2022-04-28T19:54:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:03 smithi059 bash[59745]: cluster 2022-04-28T19:54:02.964160+0000 mgr.y (mgr.24961) 259 : cluster [DBG] pgmap v417: 89 pgs: 4 activating+undersized, 2 active+undersized+degraded, 7 peering, 4 stale+active+clean, 14 active+undersized, 58 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 0 op/s; 28/1188 objects degraded (2.357%) 2022-04-28T19:54:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:03 smithi139 bash[43301]: cluster 2022-04-28T19:54:02.964160+0000 mgr.y (mgr.24961) 259 : cluster [DBG] pgmap v417: 89 pgs: 4 activating+undersized, 2 active+undersized+degraded, 7 peering, 4 stale+active+clean, 14 active+undersized, 58 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 0 op/s; 28/1188 objects degraded (2.357%) 2022-04-28T19:54:05.001 INFO:tasks.rados.rados.0.smithi059.stdout:3038: done (2 left) 2022-04-28T19:54:05.001 INFO:tasks.rados.rados.0.smithi059.stdout:3040: done (1 left) 2022-04-28T19:54:05.001 INFO:tasks.rados.rados.0.smithi059.stdout:3042: done (0 left) 2022-04-28T19:54:05.001 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 283 2022-04-28T19:54:05.001 INFO:tasks.rados.rados.0.smithi059.stdout:3043: seq_num 955 ranges {0=802816,802816=557056,1359872=245760} 2022-04-28T19:54:05.004 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:04 smithi139 bash[50386]: debug 2022-04-28T19:54:04.824+0000 7fc059899700 1 -- 172.21.15.139:0/3692842210 <== mon.2 v2:172.21.15.139:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559634c904e0 con 0x559634d09000 2022-04-28T19:54:05.011 INFO:tasks.rados.rados.0.smithi059.stdout:3043: writing smithi05955964-46 from 0 to 802816 tid 1 2022-04-28T19:54:05.013 INFO:tasks.rados.rados.0.smithi059.stdout:3043: writing smithi05955964-46 from 802816 to 1359872 tid 2 2022-04-28T19:54:05.015 INFO:tasks.rados.rados.0.smithi059.stdout:3043: writing smithi05955964-46 from 1359872 to 1605632 tid 3 2022-04-28T19:54:05.015 INFO:tasks.rados.rados.0.smithi059.stdout:3044: read oid 11 snap -1 2022-04-28T19:54:05.015 INFO:tasks.rados.rados.0.smithi059.stdout:3044: expect (ObjNum 944 snap 277 seq_num 944) 2022-04-28T19:54:05.015 INFO:tasks.rados.rados.0.smithi059.stdout:3045: setattr oid 38 current snap is 283 2022-04-28T19:54:05.016 INFO:tasks.rados.rados.0.smithi059.stdout:3046: snap_remove snap 281 2022-04-28T19:54:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:05 smithi059 bash[61152]: cluster 2022-04-28T19:54:03.983088+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e621: 8 total, 7 up, 8 in 2022-04-28T19:54:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:05 smithi059 bash[59745]: cluster 2022-04-28T19:54:03.983088+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e621: 8 total, 7 up, 8 in 2022-04-28T19:54:05.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:05 smithi139 bash[43301]: cluster 2022-04-28T19:54:03.983088+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e621: 8 total, 7 up, 8 in 2022-04-28T19:54:05.868 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:05 smithi139 bash[50386]: debug 2022-04-28T19:54:05.376+0000 7fc0610f4200 -1 Falling back to public interface 2022-04-28T19:54:06.014 INFO:tasks.rados.rados.0.smithi059.stdout:3046: done (3 left) 2022-04-28T19:54:06.163 INFO:tasks.rados.rados.0.smithi059.stdout:3047: snap_remove snap 279 2022-04-28T19:54:06.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:06 smithi059 bash[61152]: cluster 2022-04-28T19:54:04.965124+0000 mgr.y (mgr.24961) 260 : cluster [DBG] pgmap v419: 89 pgs: 4 activating+undersized, 10 active+undersized+degraded, 20 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2 op/s; 79/1188 objects degraded (6.650%) 2022-04-28T19:54:06.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:06 smithi059 bash[61152]: cluster 2022-04-28T19:54:04.988089+0000 mon.a (mon.0) 952 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2022-04-28T19:54:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:06 smithi059 bash[61152]: cluster 2022-04-28T19:54:04.996613+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e622: 8 total, 7 up, 8 in 2022-04-28T19:54:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:06 smithi059 bash[59745]: cluster 2022-04-28T19:54:04.965124+0000 mgr.y (mgr.24961) 260 : cluster [DBG] pgmap v419: 89 pgs: 4 activating+undersized, 10 active+undersized+degraded, 20 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2 op/s; 79/1188 objects degraded (6.650%) 2022-04-28T19:54:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:06 smithi059 bash[59745]: cluster 2022-04-28T19:54:04.988089+0000 mon.a (mon.0) 952 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2022-04-28T19:54:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:06 smithi059 bash[59745]: cluster 2022-04-28T19:54:04.996613+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e622: 8 total, 7 up, 8 in 2022-04-28T19:54:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:06 smithi139 bash[43301]: cluster 2022-04-28T19:54:04.965124+0000 mgr.y (mgr.24961) 260 : cluster [DBG] pgmap v419: 89 pgs: 4 activating+undersized, 10 active+undersized+degraded, 20 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2 op/s; 79/1188 objects degraded (6.650%) 2022-04-28T19:54:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:06 smithi139 bash[43301]: cluster 2022-04-28T19:54:04.988089+0000 mon.a (mon.0) 952 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2022-04-28T19:54:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:06 smithi139 bash[43301]: cluster 2022-04-28T19:54:04.996613+0000 mon.a (mon.0) 953 : cluster [DBG] osdmap e622: 8 total, 7 up, 8 in 2022-04-28T19:54:07.040 INFO:tasks.rados.rados.0.smithi059.stdout:3047: done (3 left) 2022-04-28T19:54:07.040 INFO:tasks.rados.rados.0.smithi059.stdout:3048: snap_create 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:3043: finishing write tid 1 to smithi05955964-46 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:3043: finishing write tid 2 to smithi05955964-46 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:3043: finishing write tid 3 to smithi05955964-46 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:3043: finishing write tid 5 to smithi05955964-46 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:3043: finishing write tid 6 to smithi05955964-46 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1446 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:3043: left oid 46 (ObjNum 955 snap 283 seq_num 955) 2022-04-28T19:54:07.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1183 (ObjNum 924 snap 276 seq_num 924) dirty exists 2022-04-28T19:54:07.042 INFO:tasks.rados.rados.0.smithi059.stdout:3044: expect (ObjNum 944 snap 277 seq_num 944) 2022-04-28T19:54:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: cluster 2022-04-28T19:54:06.011717+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2022-04-28T19:54:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.232645+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.240606+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.251196+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.254909+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.256877+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.258479+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.260576+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: cephadm 2022-04-28T19:54:06.261609+0000 mgr.y (mgr.24961) 261 : cephadm [INF] Upgrade: Setting container_image for all osd 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.269032+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.270606+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.278202+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2022-04-28T19:54:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.279799+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-04-28T19:54:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.285437+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2022-04-28T19:54:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.287547+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-04-28T19:54:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.292651+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2022-04-28T19:54:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.293897+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-04-28T19:54:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.299568+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2022-04-28T19:54:07.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: cluster 2022-04-28T19:54:06.011717+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.232645+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.240606+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.251196+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.254909+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.256877+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.258479+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.260576+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: cephadm 2022-04-28T19:54:06.261609+0000 mgr.y (mgr.24961) 261 : cephadm [INF] Upgrade: Setting container_image for all osd 2022-04-28T19:54:07.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.269032+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.270606+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.278202+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.279799+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.285437+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.287547+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.292651+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.293897+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-04-28T19:54:07.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.299568+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.300877+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.305533+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.306951+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.313253+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.314540+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.319795+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.321168+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.327685+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2022-04-28T19:54:07.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: cephadm 2022-04-28T19:54:06.330520+0000 mgr.y (mgr.24961) 262 : cephadm [INF] Upgrade: Setting require_osd_release to 16 pacific 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[61152]: audit 2022-04-28T19:54:06.330975+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "pacific"}]: dispatch 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.300877+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.305533+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.306951+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.313253+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.314540+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-04-28T19:54:07.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.319795+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2022-04-28T19:54:07.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.321168+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-04-28T19:54:07.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.327685+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2022-04-28T19:54:07.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: cephadm 2022-04-28T19:54:06.330520+0000 mgr.y (mgr.24961) 262 : cephadm [INF] Upgrade: Setting require_osd_release to 16 pacific 2022-04-28T19:54:07.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:07 smithi059 bash[59745]: audit 2022-04-28T19:54:06.330975+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "pacific"}]: dispatch 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: cluster 2022-04-28T19:54:06.011717+0000 mon.a (mon.0) 954 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.232645+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.240606+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.251196+0000 mon.a (mon.0) 957 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.254909+0000 mon.a (mon.0) 958 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.256877+0000 mon.a (mon.0) 959 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.258479+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.260576+0000 mon.a (mon.0) 961 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: cephadm 2022-04-28T19:54:06.261609+0000 mgr.y (mgr.24961) 261 : cephadm [INF] Upgrade: Setting container_image for all osd 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.269032+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.270606+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.278202+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.279799+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.285437+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.287547+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.292651+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2022-04-28T19:54:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.293897+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.299568+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.300877+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.305533+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.306951+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.313253+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.314540+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.319795+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.321168+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.327685+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: cephadm 2022-04-28T19:54:06.330520+0000 mgr.y (mgr.24961) 262 : cephadm [INF] Upgrade: Setting require_osd_release to 16 pacific 2022-04-28T19:54:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:07 smithi139 bash[43301]: audit 2022-04-28T19:54:06.330975+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "pacific"}]: dispatch 2022-04-28T19:54:08.039 INFO:tasks.rados.rados.0.smithi059.stdout:3043: done (3 left) 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3044: done (2 left) 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3045: done (1 left) 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3048: done (0 left) 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3049: rmattr oid 20 current snap is 284 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3050: rollback oid 4 current snap is 284 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 275 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3051: rmattr oid 41 current snap is 284 2022-04-28T19:54:08.040 INFO:tasks.rados.rados.0.smithi059.stdout:3052: read oid 27 snap 252 2022-04-28T19:54:08.041 INFO:tasks.rados.rados.0.smithi059.stdout:3052: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:54:08.041 INFO:tasks.rados.rados.0.smithi059.stdout:3053: snap_remove snap 264 2022-04-28T19:54:08.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: cluster 2022-04-28T19:54:06.965993+0000 mgr.y (mgr.24961) 263 : cluster [DBG] pgmap v422: 89 pgs: 11 active+undersized+degraded, 23 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 0 B/s wr, 12 op/s; 80/1197 objects degraded (6.683%) 2022-04-28T19:54:08.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: cluster 2022-04-28T19:54:07.018644+0000 mon.a (mon.0) 980 : cluster [INF] Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running pacific or later but require_osd_release < pacific) 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: audit 2022-04-28T19:54:07.035826+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "pacific"}]': finished 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: cluster 2022-04-28T19:54:07.035940+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: audit 2022-04-28T19:54:07.037914+0000 mon.a (mon.0) 983 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foo.smithi059.tnizcj"]}]: dispatch 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: cephadm 2022-04-28T19:54:07.039157+0000 mgr.y (mgr.24961) 264 : cephadm [INF] Upgrade: It appears safe to stop mds.foo.smithi059.tnizcj 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[59745]: cluster 2022-04-28T19:54:07.923945+0000 mon.a (mon.0) 984 : cluster [WRN] Health check update: Degraded data redundancy: 80/1197 objects degraded (6.683%), 11 pgs degraded (PG_DEGRADED) 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: cluster 2022-04-28T19:54:06.965993+0000 mgr.y (mgr.24961) 263 : cluster [DBG] pgmap v422: 89 pgs: 11 active+undersized+degraded, 23 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 0 B/s wr, 12 op/s; 80/1197 objects degraded (6.683%) 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: cluster 2022-04-28T19:54:07.018644+0000 mon.a (mon.0) 980 : cluster [INF] Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running pacific or later but require_osd_release < pacific) 2022-04-28T19:54:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: audit 2022-04-28T19:54:07.035826+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "pacific"}]': finished 2022-04-28T19:54:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: cluster 2022-04-28T19:54:07.035940+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2022-04-28T19:54:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: audit 2022-04-28T19:54:07.037914+0000 mon.a (mon.0) 983 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foo.smithi059.tnizcj"]}]: dispatch 2022-04-28T19:54:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: cephadm 2022-04-28T19:54:07.039157+0000 mgr.y (mgr.24961) 264 : cephadm [INF] Upgrade: It appears safe to stop mds.foo.smithi059.tnizcj 2022-04-28T19:54:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:08 smithi059 bash[61152]: cluster 2022-04-28T19:54:07.923945+0000 mon.a (mon.0) 984 : cluster [WRN] Health check update: Degraded data redundancy: 80/1197 objects degraded (6.683%), 11 pgs degraded (PG_DEGRADED) 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: cluster 2022-04-28T19:54:06.965993+0000 mgr.y (mgr.24961) 263 : cluster [DBG] pgmap v422: 89 pgs: 11 active+undersized+degraded, 23 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 0 B/s wr, 12 op/s; 80/1197 objects degraded (6.683%) 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: cluster 2022-04-28T19:54:07.018644+0000 mon.a (mon.0) 980 : cluster [INF] Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running pacific or later but require_osd_release < pacific) 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: audit 2022-04-28T19:54:07.035826+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "pacific"}]': finished 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: cluster 2022-04-28T19:54:07.035940+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: audit 2022-04-28T19:54:07.037914+0000 mon.a (mon.0) 983 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foo.smithi059.tnizcj"]}]: dispatch 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: cephadm 2022-04-28T19:54:07.039157+0000 mgr.y (mgr.24961) 264 : cephadm [INF] Upgrade: It appears safe to stop mds.foo.smithi059.tnizcj 2022-04-28T19:54:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[43301]: cluster 2022-04-28T19:54:07.923945+0000 mon.a (mon.0) 984 : cluster [WRN] Health check update: Degraded data redundancy: 80/1197 objects degraded (6.683%), 11 pgs degraded (PG_DEGRADED) 2022-04-28T19:54:09.037 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:08 smithi139 bash[50386]: debug 2022-04-28T19:54:08.708+0000 7fc0610f4200 -1 osd.7 615 log_to_monitors {default=true} 2022-04-28T19:54:09.051 INFO:tasks.rados.rados.0.smithi059.stdout:3053: done (4 left) 2022-04-28T19:54:09.051 INFO:tasks.rados.rados.0.smithi059.stdout:3054: rmattr oid 2 current snap is 284 2022-04-28T19:54:09.051 INFO:tasks.rados.rados.0.smithi059.stdout:3054: done (4 left) 2022-04-28T19:54:09.051 INFO:tasks.rados.rados.0.smithi059.stdout:3055: rollback oid 22 current snap is 284 2022-04-28T19:54:09.051 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 272 2022-04-28T19:54:09.051 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 284 2022-04-28T19:54:09.052 INFO:tasks.rados.rados.0.smithi059.stdout:3056: seq_num 956 ranges {581632=417792,999424=434176,1433600=770048} 2022-04-28T19:54:09.057 INFO:tasks.rados.rados.0.smithi059.stdout:3056: writing smithi05955964-39 from 581632 to 999424 tid 1 2022-04-28T19:54:09.060 INFO:tasks.rados.rados.0.smithi059.stdout:3056: writing smithi05955964-39 from 999424 to 1433600 tid 2 2022-04-28T19:54:09.066 INFO:tasks.rados.rados.0.smithi059.stdout:3056: writing smithi05955964-39 from 1433600 to 2203648 tid 3 2022-04-28T19:54:09.066 INFO:tasks.rados.rados.0.smithi059.stdout:3057: read oid 5 snap 272 2022-04-28T19:54:09.066 INFO:tasks.rados.rados.0.smithi059.stdout:3057: expect (ObjNum 875 snap 262 seq_num 875) 2022-04-28T19:54:09.066 INFO:tasks.rados.rados.0.smithi059.stdout:3058: copy_from oid 16 from oid 25 current snap is 284 2022-04-28T19:54:09.067 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 284 2022-04-28T19:54:09.067 INFO:tasks.rados.rados.0.smithi059.stdout:3059: seq_num 957 ranges {4530176=434176,4964352=720896,5685248=483328} 2022-04-28T19:54:09.095 INFO:tasks.rados.rados.0.smithi059.stdout:3059: writing smithi05955964-19 from 4530176 to 4964352 tid 1 2022-04-28T19:54:09.100 INFO:tasks.rados.rados.0.smithi059.stdout:3059: writing smithi05955964-19 from 4964352 to 5685248 tid 2 2022-04-28T19:54:09.103 INFO:tasks.rados.rados.0.smithi059.stdout:3059: writing smithi05955964-19 from 5685248 to 6168576 tid 3 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:3060: rollback oid 37 current snap is 284 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 37 to 268 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1017 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:3050: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:3050: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:3050: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1186 (ObjNum 809 snap 240 seq_num 809) dirty exists 2022-04-28T19:54:09.104 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1239 (ObjNum 821 snap 240 seq_num 821) dirty exists 2022-04-28T19:54:09.105 INFO:tasks.rados.rados.0.smithi059.stdout:3052: expect (ObjNum 578 snap 165 seq_num 578) 2022-04-28T19:54:09.207 INFO:tasks.rados.rados.0.smithi059.stdout:3055: finishing rollback tid 0 to smithi05955964-22 2022-04-28T19:54:09.207 INFO:tasks.rados.rados.0.smithi059.stdout:3055: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:54:09.207 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1176 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty dne 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3056: finishing write tid 1 to smithi05955964-39 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3056: finishing write tid 2 to smithi05955964-39 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3056: finishing write tid 3 to smithi05955964-39 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3056: finishing write tid 5 to smithi05955964-39 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3049: done (9 left) 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3050: done (8 left) 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3051: done (7 left) 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3052: done (6 left) 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:3055: done (5 left) 2022-04-28T19:54:09.208 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 284 2022-04-28T19:54:09.209 INFO:tasks.rados.rados.0.smithi059.stdout:3061: seq_num 958 ranges {0=450560,450560=655360,1105920=548864} 2022-04-28T19:54:09.213 INFO:tasks.rados.rados.0.smithi059.stdout:3061: writing smithi05955964-22 from 0 to 450560 tid 1 2022-04-28T19:54:09.222 INFO:tasks.rados.rados.0.smithi059.stdout:3061: writing smithi05955964-22 from 450560 to 1105920 tid 2 2022-04-28T19:54:09.226 INFO:tasks.rados.rados.0.smithi059.stdout:3061: writing smithi05955964-22 from 1105920 to 1654784 tid 3 2022-04-28T19:54:09.226 INFO:tasks.rados.rados.0.smithi059.stdout:3062: snap_create 2022-04-28T19:54:09.227 INFO:tasks.rados.rados.0.smithi059.stdout:3058: finishing copy_from to smithi05955964-16 2022-04-28T19:54:09.227 INFO:tasks.rados.rados.0.smithi059.stdout:3058: got expected ENOENT (src dne) 2022-04-28T19:54:09.227 INFO:tasks.rados.rados.0.smithi059.stdout:3058: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:54:09.227 INFO:tasks.rados.rados.0.smithi059.stdout:3056: finishing write tid 6 to smithi05955964-39 2022-04-28T19:54:09.227 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1437 (ObjNum 956 snap 284 seq_num 956) dirty exists 2022-04-28T19:54:09.227 INFO:tasks.rados.rados.0.smithi059.stdout:3056: left oid 39 (ObjNum 956 snap 284 seq_num 956) 2022-04-28T19:54:09.228 INFO:tasks.rados.rados.0.smithi059.stdout:3057: expect (ObjNum 875 snap 262 seq_num 875) 2022-04-28T19:54:09.332 INFO:tasks.rados.rados.0.smithi059.stdout:3059: finishing write tid 1 to smithi05955964-19 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3060: finishing rollback tid 0 to smithi05955964-37 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3060: finishing rollback tid 1 to smithi05955964-37 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3060: finishing rollback tid 2 to smithi05955964-37 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1331 (ObjNum 898 snap 268 seq_num 898) dirty exists 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3059: finishing write tid 2 to smithi05955964-19 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3059: finishing write tid 3 to smithi05955964-19 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3059: finishing write tid 5 to smithi05955964-19 2022-04-28T19:54:09.335 INFO:tasks.rados.rados.0.smithi059.stdout:3059: finishing write tid 6 to smithi05955964-19 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1181 (ObjNum 957 snap 284 seq_num 957) dirty exists 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3059: left oid 19 (ObjNum 957 snap 284 seq_num 957) 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3061: finishing write tid 1 to smithi05955964-22 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3061: finishing write tid 2 to smithi05955964-22 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3061: finishing write tid 3 to smithi05955964-22 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3061: finishing write tid 5 to smithi05955964-22 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3061: finishing write tid 6 to smithi05955964-22 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1185 (ObjNum 958 snap 284 seq_num 958) dirty exists 2022-04-28T19:54:09.336 INFO:tasks.rados.rados.0.smithi059.stdout:3061: left oid 22 (ObjNum 958 snap 284 seq_num 958) 2022-04-28T19:54:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:09 smithi059 bash[61152]: cluster 2022-04-28T19:54:08.038987+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2022-04-28T19:54:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:09 smithi059 bash[61152]: audit 2022-04-28T19:54:08.712113+0000 mon.b (mon.2) 9 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:54:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:09 smithi059 bash[61152]: audit 2022-04-28T19:54:08.714833+0000 mon.a (mon.0) 986 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:54:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:09 smithi059 bash[59745]: cluster 2022-04-28T19:54:08.038987+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2022-04-28T19:54:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:09 smithi059 bash[59745]: audit 2022-04-28T19:54:08.712113+0000 mon.b (mon.2) 9 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:54:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:09 smithi059 bash[59745]: audit 2022-04-28T19:54:08.714833+0000 mon.a (mon.0) 986 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:54:09.368 INFO:journalctl@ceph.osd.7.smithi139.stdout:Apr 28 19:54:09 smithi139 bash[50386]: debug 2022-04-28T19:54:09.076+0000 7fc054011700 -1 osd.7 615 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2022-04-28T19:54:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:09 smithi139 bash[43301]: cluster 2022-04-28T19:54:08.038987+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2022-04-28T19:54:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:09 smithi139 bash[43301]: audit 2022-04-28T19:54:08.712113+0000 mon.b (mon.2) 9 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:54:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:09 smithi139 bash[43301]: audit 2022-04-28T19:54:08.714833+0000 mon.a (mon.0) 986 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2022-04-28T19:54:10.056 INFO:tasks.rados.rados.0.smithi059.stdout:3056: done (6 left) 2022-04-28T19:54:10.056 INFO:tasks.rados.rados.0.smithi059.stdout:3057: done (5 left) 2022-04-28T19:54:10.056 INFO:tasks.rados.rados.0.smithi059.stdout:3058: done (4 left) 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3059: done (3 left) 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3060: done (2 left) 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3061: done (1 left) 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3062: done (0 left) 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3063: rmattr oid 26 current snap is 285 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3063: done (0 left) 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3064: rollback oid 26 current snap is 285 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 253 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 285 2022-04-28T19:54:10.057 INFO:tasks.rados.rados.0.smithi059.stdout:3065: seq_num 959 ranges {614400=466944,1081344=540672,1622016=557056,2179072=106496} 2022-04-28T19:54:10.063 INFO:tasks.rados.rados.0.smithi059.stdout:3065: writing smithi05955964-9 from 614400 to 1081344 tid 1 2022-04-28T19:54:10.066 INFO:tasks.rados.rados.0.smithi059.stdout:3065: writing smithi05955964-9 from 1081344 to 1622016 tid 2 2022-04-28T19:54:10.070 INFO:tasks.rados.rados.0.smithi059.stdout:3065: writing smithi05955964-9 from 1622016 to 2179072 tid 3 2022-04-28T19:54:10.071 INFO:tasks.rados.rados.0.smithi059.stdout:3065: writing smithi05955964-9 from 2179072 to 2285568 tid 4 2022-04-28T19:54:10.071 INFO:tasks.rados.rados.0.smithi059.stdout:3066: snap_remove snap 268 2022-04-28T19:54:10.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: cluster 2022-04-28T19:54:08.967035+0000 mgr.y (mgr.24961) 265 : cluster [DBG] pgmap v425: 89 pgs: 11 active+undersized+degraded, 23 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 0 B/s wr, 13 op/s; 80/1197 objects degraded (6.683%) 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.049466+0000 mon.b (mon.2) 10 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.049703+0000 mon.a (mon.0) 987 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: cluster 2022-04-28T19:54:09.049912+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.055839+0000 mon.a (mon.0) 989 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: cephadm 2022-04-28T19:54:09.289302+0000 mgr.y (mgr.24961) 266 : cephadm [INF] Upgrade: Updating mds.foo.smithi059.tnizcj 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.296798+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.298155+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.299434+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: cephadm 2022-04-28T19:54:09.300467+0000 mgr.y (mgr.24961) 267 : cephadm [INF] Deploying daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[59745]: audit 2022-04-28T19:54:09.301108+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "container_image"}]: dispatch 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: cluster 2022-04-28T19:54:08.967035+0000 mgr.y (mgr.24961) 265 : cluster [DBG] pgmap v425: 89 pgs: 11 active+undersized+degraded, 23 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 0 B/s wr, 13 op/s; 80/1197 objects degraded (6.683%) 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.049466+0000 mon.b (mon.2) 10 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.049703+0000 mon.a (mon.0) 987 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: cluster 2022-04-28T19:54:09.049912+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2022-04-28T19:54:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.055839+0000 mon.a (mon.0) 989 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:54:10.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: cephadm 2022-04-28T19:54:09.289302+0000 mgr.y (mgr.24961) 266 : cephadm [INF] Upgrade: Updating mds.foo.smithi059.tnizcj 2022-04-28T19:54:10.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.296798+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:10.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.298155+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:54:10.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.299434+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:10.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: cephadm 2022-04-28T19:54:09.300467+0000 mgr.y (mgr.24961) 267 : cephadm [INF] Deploying daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:54:10.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:10 smithi059 bash[61152]: audit 2022-04-28T19:54:09.301108+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "container_image"}]: dispatch 2022-04-28T19:54:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: cluster 2022-04-28T19:54:08.967035+0000 mgr.y (mgr.24961) 265 : cluster [DBG] pgmap v425: 89 pgs: 11 active+undersized+degraded, 23 active+undersized, 55 active+clean; 696 MiB data, 1.4 GiB used, 713 GiB / 715 GiB avail; 8.6 MiB/s rd, 0 B/s wr, 13 op/s; 80/1197 objects degraded (6.683%) 2022-04-28T19:54:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.049466+0000 mon.b (mon.2) 10 : audit [INF] from='osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:54:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.049703+0000 mon.a (mon.0) 987 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: cluster 2022-04-28T19:54:09.049912+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.055839+0000 mon.a (mon.0) 989 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi139", "root=default"]}]: dispatch 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: cephadm 2022-04-28T19:54:09.289302+0000 mgr.y (mgr.24961) 266 : cephadm [INF] Upgrade: Updating mds.foo.smithi059.tnizcj 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.296798+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.298155+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi059.tnizcj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.299434+0000 mon.a (mon.0) 992 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: cephadm 2022-04-28T19:54:09.300467+0000 mgr.y (mgr.24961) 267 : cephadm [INF] Deploying daemon mds.foo.smithi059.tnizcj on smithi059 2022-04-28T19:54:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:10 smithi139 bash[43301]: audit 2022-04-28T19:54:09.301108+0000 mon.a (mon.0) 993 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "container_image"}]: dispatch 2022-04-28T19:54:11.056 INFO:tasks.rados.rados.0.smithi059.stdout:3066: done (2 left) 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:3067: setattr oid 5 current snap is 285 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:3065: finishing write tid 1 to smithi05955964-9 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:3064: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:3064: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1187 (ObjNum 839 snap 246 seq_num 839) dirty exists 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:3065: finishing write tid 2 to smithi05955964-9 2022-04-28T19:54:11.057 INFO:tasks.rados.rados.0.smithi059.stdout:3065: finishing write tid 3 to smithi05955964-9 2022-04-28T19:54:11.058 INFO:tasks.rados.rados.0.smithi059.stdout:3065: finishing write tid 4 to smithi05955964-9 2022-04-28T19:54:11.058 INFO:tasks.rados.rados.0.smithi059.stdout:3065: finishing write tid 6 to smithi05955964-9 2022-04-28T19:54:11.058 INFO:tasks.rados.rados.0.smithi059.stdout:3065: finishing write tid 7 to smithi05955964-9 2022-04-28T19:54:11.058 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 972 (ObjNum 959 snap 285 seq_num 959) dirty exists 2022-04-28T19:54:11.058 INFO:tasks.rados.rados.0.smithi059.stdout:3065: left oid 9 (ObjNum 959 snap 285 seq_num 959) 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3064: done (2 left) 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3065: done (1 left) 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3068: read oid 43 snap -1 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3068: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3069: copy_from oid 9 from oid 24 current snap is 285 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3070: rmattr oid 39 current snap is 285 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3070: done (3 left) 2022-04-28T19:54:11.060 INFO:tasks.rados.rados.0.smithi059.stdout:3071: snap_remove snap 280 2022-04-28T19:54:11.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[59745]: cluster 2022-04-28T19:54:10.044370+0000 mon.a (mon.0) 994 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:54:11.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[59745]: cluster 2022-04-28T19:54:10.054339+0000 mon.a (mon.0) 995 : cluster [INF] osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116] boot 2022-04-28T19:54:11.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[59745]: cluster 2022-04-28T19:54:10.054437+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2022-04-28T19:54:11.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[59745]: audit 2022-04-28T19:54:10.056227+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:54:11.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[59745]: cluster 2022-04-28T19:54:11.056226+0000 mon.a (mon.0) 998 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2022-04-28T19:54:11.328 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[61152]: cluster 2022-04-28T19:54:10.044370+0000 mon.a (mon.0) 994 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:54:11.328 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[61152]: cluster 2022-04-28T19:54:10.054339+0000 mon.a (mon.0) 995 : cluster [INF] osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116] boot 2022-04-28T19:54:11.329 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[61152]: cluster 2022-04-28T19:54:10.054437+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2022-04-28T19:54:11.329 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[61152]: audit 2022-04-28T19:54:10.056227+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:54:11.329 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[61152]: cluster 2022-04-28T19:54:11.056226+0000 mon.a (mon.0) 998 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2022-04-28T19:54:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:11 smithi139 bash[43301]: cluster 2022-04-28T19:54:10.044370+0000 mon.a (mon.0) 994 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2022-04-28T19:54:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:11 smithi139 bash[43301]: cluster 2022-04-28T19:54:10.054339+0000 mon.a (mon.0) 995 : cluster [INF] osd.7 [v2:172.21.15.139:6824/1889887116,v1:172.21.15.139:6825/1889887116] boot 2022-04-28T19:54:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:11 smithi139 bash[43301]: cluster 2022-04-28T19:54:10.054437+0000 mon.a (mon.0) 996 : cluster [DBG] osdmap e627: 8 total, 8 up, 8 in 2022-04-28T19:54:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:11 smithi139 bash[43301]: audit 2022-04-28T19:54:10.056227+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2022-04-28T19:54:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:11 smithi139 bash[43301]: cluster 2022-04-28T19:54:11.056226+0000 mon.a (mon.0) 998 : cluster [DBG] osdmap e628: 8 total, 8 up, 8 in 2022-04-28T19:54:11.624 INFO:tasks.rados.rados.0.smithi059.stdout:3071: done (3 left) 2022-04-28T19:54:11.624 INFO:tasks.rados.rados.0.smithi059.stdout:3072: setattr oid 34 current snap is 285 2022-04-28T19:54:11.624 INFO:tasks.rados.rados.0.smithi059.stdout:3069: finishing copy_from to smithi05955964-9 2022-04-28T19:54:11.624 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1171 (ObjNum 921 snap 275 seq_num 921) dirty exists 2022-04-28T19:54:11.624 INFO:tasks.rados.rados.0.smithi059.stdout:3068: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3069: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1448 (ObjNum 875 snap 262 seq_num 875) dirty exists 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3067: done (3 left) 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3068: done (2 left) 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3069: done (1 left) 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3073: copy_from oid 3 from oid 9 current snap is 285 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3074: read oid 44 snap -1 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3074: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:11.629 INFO:tasks.rados.rados.0.smithi059.stdout:3075: snap_remove snap 284 2022-04-28T19:54:12.086 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:11.635Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:12.086 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:11.635Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:12.086 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:11.635Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:54:12.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:12 smithi059 bash[59745]: cluster 2022-04-28T19:54:10.968209+0000 mgr.y (mgr.24961) 268 : cluster [DBG] pgmap v428: 89 pgs: 12 peering, 10 active+undersized+degraded, 12 active+undersized, 55 active+clean; 704 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 81/1215 objects degraded (6.667%) 2022-04-28T19:54:12.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:12 smithi059 bash[59745]: cluster 2022-04-28T19:54:11.614888+0000 mon.a (mon.0) 999 : cluster [WRN] daemon mds.foo.smithi059.tnizcj compat changed unexpectedly 2022-04-28T19:54:12.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:12 smithi059 bash[59745]: cluster 2022-04-28T19:54:11.622427+0000 mon.a (mon.0) 1000 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2022-04-28T19:54:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:12 smithi059 bash[61152]: cluster 2022-04-28T19:54:10.968209+0000 mgr.y (mgr.24961) 268 : cluster [DBG] pgmap v428: 89 pgs: 12 peering, 10 active+undersized+degraded, 12 active+undersized, 55 active+clean; 704 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 81/1215 objects degraded (6.667%) 2022-04-28T19:54:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:12 smithi059 bash[61152]: cluster 2022-04-28T19:54:11.614888+0000 mon.a (mon.0) 999 : cluster [WRN] daemon mds.foo.smithi059.tnizcj compat changed unexpectedly 2022-04-28T19:54:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:12 smithi059 bash[61152]: cluster 2022-04-28T19:54:11.622427+0000 mon.a (mon.0) 1000 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2022-04-28T19:54:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:12 smithi139 bash[43301]: cluster 2022-04-28T19:54:10.968209+0000 mgr.y (mgr.24961) 268 : cluster [DBG] pgmap v428: 89 pgs: 12 peering, 10 active+undersized+degraded, 12 active+undersized, 55 active+clean; 704 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 81/1215 objects degraded (6.667%) 2022-04-28T19:54:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:12 smithi139 bash[43301]: cluster 2022-04-28T19:54:11.614888+0000 mon.a (mon.0) 999 : cluster [WRN] daemon mds.foo.smithi059.tnizcj compat changed unexpectedly 2022-04-28T19:54:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:12 smithi139 bash[43301]: cluster 2022-04-28T19:54:11.622427+0000 mon.a (mon.0) 1000 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2022-04-28T19:54:12.627 INFO:tasks.rados.rados.0.smithi059.stdout:3075: done (3 left) 2022-04-28T19:54:12.628 INFO:tasks.rados.rados.0.smithi059.stdout:3076: snap_remove snap 229 2022-04-28T19:54:13.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[61152]: cluster 2022-04-28T19:54:12.075281+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2022-04-28T19:54:13.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[61152]: cluster 2022-04-28T19:54:12.081842+0000 mon.a (mon.0) 1002 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 2022-04-28T19:54:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[61152]: cluster 2022-04-28T19:54:12.625520+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2022-04-28T19:54:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[61152]: audit 2022-04-28T19:54:12.817858+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[59745]: cluster 2022-04-28T19:54:12.075281+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2022-04-28T19:54:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[59745]: cluster 2022-04-28T19:54:12.081842+0000 mon.a (mon.0) 1002 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 2022-04-28T19:54:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[59745]: cluster 2022-04-28T19:54:12.625520+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2022-04-28T19:54:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[59745]: audit 2022-04-28T19:54:12.817858+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:13 smithi139 bash[43301]: cluster 2022-04-28T19:54:12.075281+0000 mon.a (mon.0) 1001 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2022-04-28T19:54:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:13 smithi139 bash[43301]: cluster 2022-04-28T19:54:12.081842+0000 mon.a (mon.0) 1002 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 2022-04-28T19:54:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:13 smithi139 bash[43301]: cluster 2022-04-28T19:54:12.625520+0000 mon.a (mon.0) 1003 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2022-04-28T19:54:13.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:13 smithi139 bash[43301]: audit 2022-04-28T19:54:12.817858+0000 mon.a (mon.0) 1004 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:13.635 INFO:tasks.rados.rados.0.smithi059.stdout:3076: done (3 left) 2022-04-28T19:54:13.635 INFO:tasks.rados.rados.0.smithi059.stdout:3077: snap_remove snap 283 2022-04-28T19:54:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:13.529Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:13.529Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:13.529Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:54:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:14 smithi059 bash[61152]: cluster 2022-04-28T19:54:12.969108+0000 mgr.y (mgr.24961) 269 : cluster [DBG] pgmap v432: 89 pgs: 12 peering, 10 active+undersized+degraded, 12 active+undersized, 55 active+clean; 704 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 81/1215 objects degraded (6.667%) 2022-04-28T19:54:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:14 smithi059 bash[61152]: cluster 2022-04-28T19:54:13.634817+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2022-04-28T19:54:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:14 smithi059 bash[59745]: cluster 2022-04-28T19:54:12.969108+0000 mgr.y (mgr.24961) 269 : cluster [DBG] pgmap v432: 89 pgs: 12 peering, 10 active+undersized+degraded, 12 active+undersized, 55 active+clean; 704 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 81/1215 objects degraded (6.667%) 2022-04-28T19:54:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:14 smithi059 bash[59745]: cluster 2022-04-28T19:54:13.634817+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2022-04-28T19:54:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:14 smithi139 bash[43301]: cluster 2022-04-28T19:54:12.969108+0000 mgr.y (mgr.24961) 269 : cluster [DBG] pgmap v432: 89 pgs: 12 peering, 10 active+undersized+degraded, 12 active+undersized, 55 active+clean; 704 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 11 op/s; 81/1215 objects degraded (6.667%) 2022-04-28T19:54:14.378 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:14 smithi139 bash[43301]: cluster 2022-04-28T19:54:13.634817+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2022-04-28T19:54:14.639 INFO:tasks.rados.rados.0.smithi059.stdout:3077: done (3 left) 2022-04-28T19:54:14.639 INFO:tasks.rados.rados.0.smithi059.stdout:3078: read oid 8 snap -1 2022-04-28T19:54:14.639 INFO:tasks.rados.rados.0.smithi059.stdout:3078: expect (ObjNum 953 snap 279 seq_num 953) 2022-04-28T19:54:14.640 INFO:tasks.rados.rados.0.smithi059.stdout:3074: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:14.642 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1332 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:54:14.642 INFO:tasks.rados.rados.0.smithi059.stdout:3073: finishing copy_from to smithi05955964-3 2022-04-28T19:54:14.642 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1171 (ObjNum 921 snap 275 seq_num 921) dirty exists 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3073: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3072: done (3 left) 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3073: done (2 left) 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3074: done (1 left) 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3079: read oid 34 snap -1 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3079: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 285 2022-04-28T19:54:14.643 INFO:tasks.rados.rados.0.smithi059.stdout:3080: seq_num 960 ranges {8970240=483328,9453568=425984,9879552=761856,10641408=16384} 2022-04-28T19:54:14.697 INFO:tasks.rados.rados.0.smithi059.stdout:3080: writing smithi05955964-17 from 8970240 to 9453568 tid 1 2022-04-28T19:54:14.699 INFO:tasks.rados.rados.0.smithi059.stdout:3080: writing smithi05955964-17 from 9453568 to 9879552 tid 2 2022-04-28T19:54:14.705 INFO:tasks.rados.rados.0.smithi059.stdout:3080: writing smithi05955964-17 from 9879552 to 10641408 tid 3 2022-04-28T19:54:14.705 INFO:tasks.rados.rados.0.smithi059.stdout:3080: writing smithi05955964-17 from 10641408 to 10657792 tid 4 2022-04-28T19:54:14.705 INFO:tasks.rados.rados.0.smithi059.stdout:3081: copy_from oid 42 from oid 49 current snap is 285 2022-04-28T19:54:14.705 INFO:tasks.rados.rados.0.smithi059.stdout:3078: expect (ObjNum 953 snap 279 seq_num 953) 2022-04-28T19:54:14.742 INFO:tasks.rados.rados.0.smithi059.stdout:3079: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:54:15.585 INFO:tasks.rados.rados.0.smithi059.stdout:3080: finishing write tid 1 to smithi05955964-17 2022-04-28T19:54:15.585 INFO:tasks.rados.rados.0.smithi059.stdout:3080: finishing write tid 2 to smithi05955964-17 2022-04-28T19:54:15.585 INFO:tasks.rados.rados.0.smithi059.stdout:3080: finishing write tid 3 to smithi05955964-17 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3080: finishing write tid 4 to smithi05955964-17 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3080: finishing write tid 6 to smithi05955964-17 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3080: finishing write tid 7 to smithi05955964-17 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 1062 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3080: left oid 17 (ObjNum 960 snap 285 seq_num 960) 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3081: finishing copy_from to smithi05955964-42 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1099 (ObjNum 929 snap 276 seq_num 929) dirty exists 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3081: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:54:15.586 INFO:tasks.rados.rados.0.smithi059.stdout:3078: done (3 left) 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3079: done (2 left) 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3080: done (1 left) 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3081: done (0 left) 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3082: rollback oid 48 current snap is 285 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 253 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3083: copy_from oid 42 from oid 3 current snap is 285 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3084: read oid 12 snap -1 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3084: expect (ObjNum 936 snap 276 seq_num 936) 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3085: copy_from oid 4 from oid 23 current snap is 285 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3086: copy_from oid 19 from oid 8 current snap is 285 2022-04-28T19:54:15.587 INFO:tasks.rados.rados.0.smithi059.stdout:3087: read oid 6 snap -1 2022-04-28T19:54:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:3087: expect (ObjNum 934 snap 276 seq_num 934) 2022-04-28T19:54:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:3088: setattr oid 34 current snap is 285 2022-04-28T19:54:15.588 INFO:tasks.rados.rados.0.smithi059.stdout:3089: snap_remove snap 271 2022-04-28T19:54:15.643 INFO:tasks.rados.rados.0.smithi059.stdout:3089: done (7 left) 2022-04-28T19:54:15.643 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 285 2022-04-28T19:54:15.643 INFO:tasks.rados.rados.0.smithi059.stdout:3090: seq_num 961 ranges {10354688=499712,10854400=712704,11567104=491520} 2022-04-28T19:54:15.704 INFO:tasks.rados.rados.0.smithi059.stdout:3090: writing smithi05955964-35 from 10354688 to 10854400 tid 1 2022-04-28T19:54:15.708 INFO:tasks.rados.rados.0.smithi059.stdout:3090: writing smithi05955964-35 from 10854400 to 11567104 tid 2 2022-04-28T19:54:15.711 INFO:tasks.rados.rados.0.smithi059.stdout:3090: writing smithi05955964-35 from 11567104 to 12058624 tid 3 2022-04-28T19:54:15.711 INFO:tasks.rados.rados.0.smithi059.stdout:3091: snap_remove snap 270 2022-04-28T19:54:15.909 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:15 smithi059 bash[61152]: cluster 2022-04-28T19:54:14.638801+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2022-04-28T19:54:15.910 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:15 smithi059 bash[61152]: cluster 2022-04-28T19:54:14.969608+0000 mgr.y (mgr.24961) 270 : cluster [DBG] pgmap v435: 89 pgs: 12 peering, 3 active+undersized, 74 active+clean; 709 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 1.6 MiB/s wr, 13 op/s; 1.8 MiB/s, 1 objects/s recovering 2022-04-28T19:54:15.910 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:15 smithi059 bash[59745]: cluster 2022-04-28T19:54:14.638801+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2022-04-28T19:54:15.910 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:15 smithi059 bash[59745]: cluster 2022-04-28T19:54:14.969608+0000 mgr.y (mgr.24961) 270 : cluster [DBG] pgmap v435: 89 pgs: 12 peering, 3 active+undersized, 74 active+clean; 709 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 1.6 MiB/s wr, 13 op/s; 1.8 MiB/s, 1 objects/s recovering 2022-04-28T19:54:16.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:15 smithi139 bash[43301]: cluster 2022-04-28T19:54:14.638801+0000 mon.a (mon.0) 1006 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2022-04-28T19:54:16.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:15 smithi139 bash[43301]: cluster 2022-04-28T19:54:14.969608+0000 mgr.y (mgr.24961) 270 : cluster [DBG] pgmap v435: 89 pgs: 12 peering, 3 active+undersized, 74 active+clean; 709 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 1.6 MiB/s wr, 13 op/s; 1.8 MiB/s, 1 objects/s recovering 2022-04-28T19:54:16.650 INFO:tasks.rados.rados.0.smithi059.stdout:3091: done (8 left) 2022-04-28T19:54:16.650 INFO:tasks.rados.rados.0.smithi059.stdout:3092: read oid 1 snap 252 2022-04-28T19:54:16.650 INFO:tasks.rados.rados.0.smithi059.stdout:3092: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3093: copy_from oid 29 from oid 2 current snap is 285 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3082: finishing rollback tid 0 to smithi05955964-48 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3082: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3082: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1191 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3082: done (9 left) 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3094: setattr oid 9 current snap is 285 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3086: finishing copy_from to smithi05955964-19 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1440 (ObjNum 953 snap 279 seq_num 953) dirty exists 2022-04-28T19:54:16.651 INFO:tasks.rados.rados.0.smithi059.stdout:3086: finishing copy_from racing read to smithi05955964-19 2022-04-28T19:54:16.652 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1333 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:54:16.652 INFO:tasks.rados.rados.0.smithi059.stdout:3085: finishing copy_from to smithi05955964-4 2022-04-28T19:54:16.652 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1146 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:54:16.652 INFO:tasks.rados.rados.0.smithi059.stdout:3085: finishing copy_from racing read to smithi05955964-4 2022-04-28T19:54:16.652 INFO:tasks.rados.rados.0.smithi059.stdout:3084: expect (ObjNum 936 snap 276 seq_num 936) 2022-04-28T19:54:17.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: cluster 2022-04-28T19:54:15.631916+0000 mon.a (mon.0) 1007 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/1215 objects degraded (6.667%), 10 pgs degraded) 2022-04-28T19:54:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: cluster 2022-04-28T19:54:15.642079+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2022-04-28T19:54:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: audit 2022-04-28T19:54:16.160804+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: audit 2022-04-28T19:54:16.161993+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: audit 2022-04-28T19:54:16.163645+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: audit 2022-04-28T19:54:16.164621+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: cephadm 2022-04-28T19:54:16.166085+0000 mgr.y (mgr.24961) 271 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[59745]: audit 2022-04-28T19:54:16.167016+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: cluster 2022-04-28T19:54:15.631916+0000 mon.a (mon.0) 1007 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/1215 objects degraded (6.667%), 10 pgs degraded) 2022-04-28T19:54:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: cluster 2022-04-28T19:54:15.642079+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2022-04-28T19:54:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: audit 2022-04-28T19:54:16.160804+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: audit 2022-04-28T19:54:16.161993+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: audit 2022-04-28T19:54:16.163645+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: audit 2022-04-28T19:54:16.164621+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: cephadm 2022-04-28T19:54:16.166085+0000 mgr.y (mgr.24961) 271 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:16 smithi059 bash[61152]: audit 2022-04-28T19:54:16.167016+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:17.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: cluster 2022-04-28T19:54:15.631916+0000 mon.a (mon.0) 1007 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 81/1215 objects degraded (6.667%), 10 pgs degraded) 2022-04-28T19:54:17.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: cluster 2022-04-28T19:54:15.642079+0000 mon.a (mon.0) 1008 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2022-04-28T19:54:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: audit 2022-04-28T19:54:16.160804+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: audit 2022-04-28T19:54:16.161993+0000 mon.a (mon.0) 1010 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: audit 2022-04-28T19:54:16.163645+0000 mon.a (mon.0) 1011 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: audit 2022-04-28T19:54:16.164621+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: cephadm 2022-04-28T19:54:16.166085+0000 mgr.y (mgr.24961) 271 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:16 smithi139 bash[43301]: audit 2022-04-28T19:54:16.167016+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:17.492 INFO:tasks.rados.rados.0.smithi059.stdout:3083: finishing copy_from to smithi05955964-42 2022-04-28T19:54:17.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1171 (ObjNum 921 snap 275 seq_num 921) dirty exists 2022-04-28T19:54:17.492 INFO:tasks.rados.rados.0.smithi059.stdout:3083: finishing copy_from racing read to smithi05955964-42 2022-04-28T19:54:17.492 INFO:tasks.rados.rados.0.smithi059.stdout:3087: expect (ObjNum 934 snap 276 seq_num 934) 2022-04-28T19:54:17.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: cluster 2022-04-28T19:54:16.647545+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: audit 2022-04-28T19:54:16.914495+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: audit 2022-04-28T19:54:16.915165+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: audit 2022-04-28T19:54:16.915866+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: cluster 2022-04-28T19:54:16.932564+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: cluster 2022-04-28T19:54:16.969982+0000 mgr.y (mgr.24961) 272 : cluster [DBG] pgmap v439: 89 pgs: 89 active+clean; 717 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.1 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 objects/s recovering 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: audit 2022-04-28T19:54:17.144366+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:54:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[59745]: audit 2022-04-28T19:54:17.149105+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: cluster 2022-04-28T19:54:16.647545+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: audit 2022-04-28T19:54:16.914495+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: audit 2022-04-28T19:54:16.915165+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: audit 2022-04-28T19:54:16.915866+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: cluster 2022-04-28T19:54:16.932564+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: cluster 2022-04-28T19:54:16.969982+0000 mgr.y (mgr.24961) 272 : cluster [DBG] pgmap v439: 89 pgs: 89 active+clean; 717 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.1 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 objects/s recovering 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: audit 2022-04-28T19:54:17.144366+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:54:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:17 smithi059 bash[61152]: audit 2022-04-28T19:54:17.149105+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:54:18.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: cluster 2022-04-28T19:54:16.647545+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: audit 2022-04-28T19:54:16.914495+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: audit 2022-04-28T19:54:16.915165+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: audit 2022-04-28T19:54:16.915866+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: cluster 2022-04-28T19:54:16.932564+0000 mon.a (mon.0) 1018 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: cluster 2022-04-28T19:54:16.969982+0000 mgr.y (mgr.24961) 272 : cluster [DBG] pgmap v439: 89 pgs: 89 active+clean; 717 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.1 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 objects/s recovering 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: audit 2022-04-28T19:54:17.144366+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:54:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:17 smithi139 bash[43301]: audit 2022-04-28T19:54:17.149105+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:54:19.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[59745]: cluster 2022-04-28T19:54:17.937433+0000 mon.a (mon.0) 1021 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[59745]: cluster 2022-04-28T19:54:17.937483+0000 mon.a (mon.0) 1022 : cluster [INF] Cluster is now healthy 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[59745]: cluster 2022-04-28T19:54:17.948328+0000 mon.a (mon.0) 1023 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:boot 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[59745]: cluster 2022-04-28T19:54:17.948368+0000 mon.a (mon.0) 1024 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[59745]: audit 2022-04-28T19:54:17.948515+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[61152]: cluster 2022-04-28T19:54:17.937433+0000 mon.a (mon.0) 1021 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[61152]: cluster 2022-04-28T19:54:17.937483+0000 mon.a (mon.0) 1022 : cluster [INF] Cluster is now healthy 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[61152]: cluster 2022-04-28T19:54:17.948328+0000 mon.a (mon.0) 1023 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:boot 2022-04-28T19:54:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[61152]: cluster 2022-04-28T19:54:17.948368+0000 mon.a (mon.0) 1024 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:54:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:18 smithi059 bash[61152]: audit 2022-04-28T19:54:17.948515+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:19.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:18 smithi139 bash[43301]: cluster 2022-04-28T19:54:17.937433+0000 mon.a (mon.0) 1021 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2022-04-28T19:54:19.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:18 smithi139 bash[43301]: cluster 2022-04-28T19:54:17.937483+0000 mon.a (mon.0) 1022 : cluster [INF] Cluster is now healthy 2022-04-28T19:54:19.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:18 smithi139 bash[43301]: cluster 2022-04-28T19:54:17.948328+0000 mon.a (mon.0) 1023 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:boot 2022-04-28T19:54:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:18 smithi139 bash[43301]: cluster 2022-04-28T19:54:17.948368+0000 mon.a (mon.0) 1024 : cluster [DBG] fsmap foo:1 {0=foo.smithi139.oefiza=up:active} 1 up:standby 2022-04-28T19:54:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:18 smithi139 bash[43301]: audit 2022-04-28T19:54:17.948515+0000 mon.a (mon.0) 1025 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:19.960 INFO:tasks.rados.rados.0.smithi059.stdout:3090: finishing write tid 1 to smithi05955964-35 2022-04-28T19:54:19.960 INFO:tasks.rados.rados.0.smithi059.stdout:3090: finishing write tid 2 to smithi05955964-35 2022-04-28T19:54:19.961 INFO:tasks.rados.rados.0.smithi059.stdout:3090: finishing write tid 3 to smithi05955964-35 2022-04-28T19:54:19.961 INFO:tasks.rados.rados.0.smithi059.stdout:3090: finishing write tid 5 to smithi05955964-35 2022-04-28T19:54:19.961 INFO:tasks.rados.rados.0.smithi059.stdout:3090: finishing write tid 6 to smithi05955964-35 2022-04-28T19:54:19.961 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1325 (ObjNum 961 snap 285 seq_num 961) dirty exists 2022-04-28T19:54:19.961 INFO:tasks.rados.rados.0.smithi059.stdout:3090: left oid 35 (ObjNum 961 snap 285 seq_num 961) 2022-04-28T19:54:19.961 INFO:tasks.rados.rados.0.smithi059.stdout:3092: expect (ObjNum 792 snap 235 seq_num 792) 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:3093: finishing copy_from to smithi05955964-29 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1329 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:3093: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:3083: done (9 left) 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:3084: done (8 left) 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:3085: done (7 left) 2022-04-28T19:54:20.021 INFO:tasks.rados.rados.0.smithi059.stdout:3086: done (6 left) 2022-04-28T19:54:20.022 INFO:tasks.rados.rados.0.smithi059.stdout:3087: done (5 left) 2022-04-28T19:54:20.024 INFO:tasks.rados.rados.0.smithi059.stdout:3088: done (4 left) 2022-04-28T19:54:20.024 INFO:tasks.rados.rados.0.smithi059.stdout:3090: done (3 left) 2022-04-28T19:54:20.024 INFO:tasks.rados.rados.0.smithi059.stdout:3092: done (2 left) 2022-04-28T19:54:20.024 INFO:tasks.rados.rados.0.smithi059.stdout:3093: done (1 left) 2022-04-28T19:54:20.024 INFO:tasks.rados.rados.0.smithi059.stdout:3095: snap_create 2022-04-28T19:54:20.025 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1172 (ObjNum 921 snap 275 seq_num 921) dirty exists 2022-04-28T19:54:20.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:19 smithi059 bash[61152]: cluster 2022-04-28T19:54:18.970712+0000 mgr.y (mgr.24961) 273 : cluster [DBG] pgmap v440: 89 pgs: 89 active+clean; 720 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 3.9 MiB/s wr, 6 op/s; 520 KiB/s, 0 objects/s recovering 2022-04-28T19:54:20.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:19 smithi059 bash[59745]: cluster 2022-04-28T19:54:18.970712+0000 mgr.y (mgr.24961) 273 : cluster [DBG] pgmap v440: 89 pgs: 89 active+clean; 720 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 3.9 MiB/s wr, 6 op/s; 520 KiB/s, 0 objects/s recovering 2022-04-28T19:54:20.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:19 smithi139 bash[43301]: cluster 2022-04-28T19:54:18.970712+0000 mgr.y (mgr.24961) 273 : cluster [DBG] pgmap v440: 89 pgs: 89 active+clean; 720 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 3.9 MiB/s wr, 6 op/s; 520 KiB/s, 0 objects/s recovering 2022-04-28T19:54:20.676 INFO:tasks.rados.rados.0.smithi059.stdout:3094: done (1 left) 2022-04-28T19:54:20.677 INFO:tasks.rados.rados.0.smithi059.stdout:3095: done (0 left) 2022-04-28T19:54:20.677 INFO:tasks.rados.rados.0.smithi059.stdout:3096: delete oid 34 current snap is 286 2022-04-28T19:54:20.684 INFO:tasks.rados.rados.0.smithi059.stdout:3096: done (0 left) 2022-04-28T19:54:20.684 INFO:tasks.rados.rados.0.smithi059.stdout:3097: rmattr oid 6 current snap is 286 2022-04-28T19:54:20.684 INFO:tasks.rados.rados.0.smithi059.stdout:3098: read oid 37 snap -1 2022-04-28T19:54:20.684 INFO:tasks.rados.rados.0.smithi059.stdout:3098: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:54:20.684 INFO:tasks.rados.rados.0.smithi059.stdout:3099: snap_create 2022-04-28T19:54:20.688 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1240 (ObjNum 934 snap 276 seq_num 934) dirty exists 2022-04-28T19:54:20.709 INFO:tasks.rados.rados.0.smithi059.stdout:3098: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:54:21.686 INFO:tasks.rados.rados.0.smithi059.stdout:3097: done (2 left) 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3098: done (1 left) 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3099: done (0 left) 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3100: read oid 28 snap -1 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3100: expect (ObjNum 924 snap 276 seq_num 924) 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3101: rollback oid 39 current snap is 287 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 39 to 272 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3102: read oid 10 snap -1 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3102: expect (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:54:21.687 INFO:tasks.rados.rados.0.smithi059.stdout:3103: snap_create 2022-04-28T19:54:21.697 INFO:tasks.rados.rados.0.smithi059.stdout:3101: finishing rollback tid 0 to smithi05955964-39 2022-04-28T19:54:21.698 INFO:tasks.rados.rados.0.smithi059.stdout:3101: finishing rollback tid 1 to smithi05955964-39 2022-04-28T19:54:21.699 INFO:tasks.rados.rados.0.smithi059.stdout:3101: finishing rollback tid 2 to smithi05955964-39 2022-04-28T19:54:21.699 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1441 (ObjNum 894 snap 267 seq_num 894) dirty exists 2022-04-28T19:54:21.706 INFO:tasks.rados.rados.0.smithi059.stdout:3100: expect (ObjNum 924 snap 276 seq_num 924) 2022-04-28T19:54:21.780 INFO:tasks.rados.rados.0.smithi059.stdout:3102: expect (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:54:22.035 INFO:tasks.rados.rados.0.smithi059.stdout:3100: done (3 left) 2022-04-28T19:54:22.035 INFO:tasks.rados.rados.0.smithi059.stdout:3101: done (2 left) 2022-04-28T19:54:22.036 INFO:tasks.rados.rados.0.smithi059.stdout:3102: done (1 left) 2022-04-28T19:54:22.036 INFO:tasks.rados.rados.0.smithi059.stdout:3103: done (0 left) 2022-04-28T19:54:22.036 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 288 2022-04-28T19:54:22.036 INFO:tasks.rados.rados.0.smithi059.stdout:3104: seq_num 963 ranges {4210688=532480,4743168=475136,5218304=532480,5750784=196608} 2022-04-28T19:54:22.070 INFO:tasks.rados.rados.0.smithi059.stdout:3104: writing smithi05955964-1 from 4210688 to 4743168 tid 1 2022-04-28T19:54:22.072 INFO:tasks.rados.rados.0.smithi059.stdout:3104: writing smithi05955964-1 from 4743168 to 5218304 tid 2 2022-04-28T19:54:22.076 INFO:tasks.rados.rados.0.smithi059.stdout:3104: writing smithi05955964-1 from 5218304 to 5750784 tid 3 2022-04-28T19:54:22.077 INFO:tasks.rados.rados.0.smithi059.stdout:3104: writing smithi05955964-1 from 5750784 to 5947392 tid 4 2022-04-28T19:54:22.077 INFO:tasks.rados.rados.0.smithi059.stdout:3105: delete oid 50 current snap is 288 2022-04-28T19:54:22.078 INFO:tasks.rados.rados.0.smithi059.stdout:3104: finishing write tid 1 to smithi05955964-1 2022-04-28T19:54:22.078 INFO:tasks.rados.rados.0.smithi059.stdout:3104: finishing write tid 2 to smithi05955964-1 2022-04-28T19:54:22.079 INFO:tasks.rados.rados.0.smithi059.stdout:3104: finishing write tid 3 to smithi05955964-1 2022-04-28T19:54:22.080 INFO:tasks.rados.rados.0.smithi059.stdout:3104: finishing write tid 4 to smithi05955964-1 2022-04-28T19:54:22.082 INFO:tasks.rados.rados.0.smithi059.stdout:3104: finishing write tid 6 to smithi05955964-1 2022-04-28T19:54:22.083 INFO:tasks.rados.rados.0.smithi059.stdout:3104: finishing write tid 7 to smithi05955964-1 2022-04-28T19:54:22.083 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1115 (ObjNum 963 snap 288 seq_num 963) dirty exists 2022-04-28T19:54:22.083 INFO:tasks.rados.rados.0.smithi059.stdout:3104: left oid 1 (ObjNum 963 snap 288 seq_num 963) 2022-04-28T19:54:22.084 INFO:tasks.rados.rados.0.smithi059.stdout:3104: done (1 left) 2022-04-28T19:54:22.084 INFO:tasks.rados.rados.0.smithi059.stdout:3105: done (0 left) 2022-04-28T19:54:22.084 INFO:tasks.rados.rados.0.smithi059.stdout:3106: snap_create 2022-04-28T19:54:22.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: cluster 2022-04-28T19:54:20.676584+0000 mon.a (mon.0) 1026 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2022-04-28T19:54:22.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: cluster 2022-04-28T19:54:20.971231+0000 mgr.y (mgr.24961) 274 : cluster [DBG] pgmap v442: 89 pgs: 89 active+clean; 724 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 5.1 MiB/s wr, 11 op/s 2022-04-28T19:54:22.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.269380+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:22.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.275929+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:22.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.286123+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.289807+0000 mon.a (mon.0) 1030 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.291462+0000 mon.a (mon.0) 1031 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.292848+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.294917+0000 mon.a (mon.0) 1033 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: audit 2022-04-28T19:54:21.297746+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foo.smithi139.oefiza"]}]: dispatch 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[59745]: cephadm 2022-04-28T19:54:21.298745+0000 mgr.y (mgr.24961) 275 : cephadm [INF] Upgrade: It appears safe to stop mds.foo.smithi139.oefiza 2022-04-28T19:54:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: cluster 2022-04-28T19:54:20.676584+0000 mon.a (mon.0) 1026 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: cluster 2022-04-28T19:54:20.971231+0000 mgr.y (mgr.24961) 274 : cluster [DBG] pgmap v442: 89 pgs: 89 active+clean; 724 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 5.1 MiB/s wr, 11 op/s 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.269380+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.275929+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.286123+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.289807+0000 mon.a (mon.0) 1030 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.291462+0000 mon.a (mon.0) 1031 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.292848+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.294917+0000 mon.a (mon.0) 1033 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: audit 2022-04-28T19:54:21.297746+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foo.smithi139.oefiza"]}]: dispatch 2022-04-28T19:54:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[61152]: cephadm 2022-04-28T19:54:21.298745+0000 mgr.y (mgr.24961) 275 : cephadm [INF] Upgrade: It appears safe to stop mds.foo.smithi139.oefiza 2022-04-28T19:54:22.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:21.635Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:22.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:21.635Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:22.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:21 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:21.635Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:54:22.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: cluster 2022-04-28T19:54:20.676584+0000 mon.a (mon.0) 1026 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2022-04-28T19:54:22.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: cluster 2022-04-28T19:54:20.971231+0000 mgr.y (mgr.24961) 274 : cluster [DBG] pgmap v442: 89 pgs: 89 active+clean; 724 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 5.1 MiB/s wr, 11 op/s 2022-04-28T19:54:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.269380+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.275929+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.286123+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.289807+0000 mon.a (mon.0) 1030 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.291462+0000 mon.a (mon.0) 1031 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.292848+0000 mon.a (mon.0) 1032 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.294917+0000 mon.a (mon.0) 1033 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: audit 2022-04-28T19:54:21.297746+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foo.smithi139.oefiza"]}]: dispatch 2022-04-28T19:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:21 smithi139 bash[43301]: cephadm 2022-04-28T19:54:21.298745+0000 mgr.y (mgr.24961) 275 : cephadm [INF] Upgrade: It appears safe to stop mds.foo.smithi139.oefiza 2022-04-28T19:54:22.952 INFO:tasks.rados.rados.0.smithi059.stdout:3106: done (0 left) 2022-04-28T19:54:22.952 INFO:tasks.rados.rados.0.smithi059.stdout:3107: rmattr oid 15 current snap is 289 2022-04-28T19:54:22.952 INFO:tasks.rados.rados.0.smithi059.stdout:3108: read oid 31 snap -1 2022-04-28T19:54:22.952 INFO:tasks.rados.rados.0.smithi059.stdout:3108: expect (ObjNum 935 snap 276 seq_num 935) 2022-04-28T19:54:22.952 INFO:tasks.rados.rados.0.smithi059.stdout:3109: delete oid 6 current snap is 289 2022-04-28T19:54:22.956 INFO:tasks.rados.rados.0.smithi059.stdout:3109: done (2 left) 2022-04-28T19:54:22.956 INFO:tasks.rados.rados.0.smithi059.stdout:3110: rollback oid 32 current snap is 289 2022-04-28T19:54:22.956 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 282 2022-04-28T19:54:22.956 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 289 2022-04-28T19:54:22.957 INFO:tasks.rados.rados.0.smithi059.stdout:3111: seq_num 966 ranges {2064384=581632,2646016=581632,3227648=622592} 2022-04-28T19:54:22.971 INFO:tasks.rados.rados.0.smithi059.stdout:3111: writing smithi05955964-26 from 2064384 to 2646016 tid 1 2022-04-28T19:54:22.975 INFO:tasks.rados.rados.0.smithi059.stdout:3111: writing smithi05955964-26 from 2646016 to 3227648 tid 2 2022-04-28T19:54:22.979 INFO:tasks.rados.rados.0.smithi059.stdout:3111: writing smithi05955964-26 from 3227648 to 3850240 tid 3 2022-04-28T19:54:22.979 INFO:tasks.rados.rados.0.smithi059.stdout:3112: delete oid 1 current snap is 289 2022-04-28T19:54:22.979 INFO:tasks.rados.rados.0.smithi059.stdout:3110: finishing rollback tid 0 to smithi05955964-32 2022-04-28T19:54:22.979 INFO:tasks.rados.rados.0.smithi059.stdout:3110: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:54:22.979 INFO:tasks.rados.rados.0.smithi059.stdout:3110: finishing rollback tid 2 to smithi05955964-32 2022-04-28T19:54:22.980 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1447 (ObjNum 819 snap 240 seq_num 819) dirty exists 2022-04-28T19:54:22.980 INFO:tasks.rados.rados.0.smithi059.stdout:3111: finishing write tid 1 to smithi05955964-26 2022-04-28T19:54:22.980 INFO:tasks.rados.rados.0.smithi059.stdout:3111: finishing write tid 2 to smithi05955964-26 2022-04-28T19:54:22.985 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1343 (ObjNum 946 snap 277 seq_num 946) dirty exists 2022-04-28T19:54:22.985 INFO:tasks.rados.rados.0.smithi059.stdout:3111: finishing write tid 3 to smithi05955964-26 2022-04-28T19:54:22.985 INFO:tasks.rados.rados.0.smithi059.stdout:3111: finishing write tid 5 to smithi05955964-26 2022-04-28T19:54:22.985 INFO:tasks.rados.rados.0.smithi059.stdout:3107: done (4 left) 2022-04-28T19:54:22.986 INFO:tasks.rados.rados.0.smithi059.stdout:3110: done (3 left) 2022-04-28T19:54:22.986 INFO:tasks.rados.rados.0.smithi059.stdout:3112: done (2 left) 2022-04-28T19:54:22.986 INFO:tasks.rados.rados.0.smithi059.stdout:3113: snap_remove snap 285 2022-04-28T19:54:23.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:22 smithi059 bash[59745]: cluster 2022-04-28T19:54:21.685711+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2022-04-28T19:54:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:22 smithi059 bash[59745]: cluster 2022-04-28T19:54:21.933939+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2022-04-28T19:54:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:22 smithi059 bash[59745]: audit 2022-04-28T19:54:22.441781+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:22 smithi059 bash[61152]: cluster 2022-04-28T19:54:21.685711+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2022-04-28T19:54:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:22 smithi059 bash[61152]: cluster 2022-04-28T19:54:21.933939+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2022-04-28T19:54:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:22 smithi059 bash[61152]: audit 2022-04-28T19:54:22.441781+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:23.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:22 smithi139 bash[43301]: cluster 2022-04-28T19:54:21.685711+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e637: 8 total, 8 up, 8 in 2022-04-28T19:54:23.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:22 smithi139 bash[43301]: cluster 2022-04-28T19:54:21.933939+0000 mon.a (mon.0) 1036 : cluster [DBG] osdmap e638: 8 total, 8 up, 8 in 2022-04-28T19:54:23.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:22 smithi139 bash[43301]: audit 2022-04-28T19:54:22.441781+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:23.529Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:23.529Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:23.529Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:54:23.964 INFO:tasks.rados.rados.0.smithi059.stdout:3113: done (2 left) 2022-04-28T19:54:23.965 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 289 2022-04-28T19:54:23.965 INFO:tasks.rados.rados.0.smithi059.stdout:3114: seq_num 968 ranges {12058624=614400,12673024=745472,13418496=458752} 2022-04-28T19:54:24.036 INFO:tasks.rados.rados.0.smithi059.stdout:3114: writing smithi05955964-35 from 12058624 to 12673024 tid 1 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3114: writing smithi05955964-35 from 12673024 to 13418496 tid 2 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3114: writing smithi05955964-35 from 13418496 to 13877248 tid 3 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 289 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3115: seq_num 969 ranges {7380992=638976,8019968=630784,8650752=532480,9183232=40960} 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3115: writing smithi05955964-47 from 7380992 to 8019968 tid 1 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3115: writing smithi05955964-47 from 8019968 to 8650752 tid 2 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3115: writing smithi05955964-47 from 8650752 to 9183232 tid 3 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3115: writing smithi05955964-47 from 9183232 to 9224192 tid 4 2022-04-28T19:54:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 289 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3116: seq_num 970 ranges {0=655360,655360=516096,1171456=688128} 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3116: writing smithi05955964-40 from 0 to 655360 tid 1 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3116: writing smithi05955964-40 from 655360 to 1171456 tid 2 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3116: writing smithi05955964-40 from 1171456 to 1859584 tid 3 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 289 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3117: seq_num 971 ranges {2539520=671744,3211264=802816,4014080=401408} 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3117: writing smithi05955964-9 from 2539520 to 3211264 tid 1 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3117: writing smithi05955964-9 from 3211264 to 4014080 tid 2 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3117: writing smithi05955964-9 from 4014080 to 4415488 tid 3 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 289 2022-04-28T19:54:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3118: seq_num 972 ranges {3751936=688128,4440064=688128,5128192=516096} 2022-04-28T19:54:24.159 INFO:tasks.rados.rados.0.smithi059.stdout:3118: writing smithi05955964-20 from 3751936 to 4440064 tid 1 2022-04-28T19:54:24.163 INFO:tasks.rados.rados.0.smithi059.stdout:3118: writing smithi05955964-20 from 4440064 to 5128192 tid 2 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:3118: writing smithi05955964-20 from 5128192 to 5644288 tid 3 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:3119: rmattr oid 45 current snap is 289 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:3111: finishing write tid 6 to smithi05955964-26 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1444 (ObjNum 966 snap 289 seq_num 966) dirty exists 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:3111: left oid 26 (ObjNum 966 snap 289 seq_num 966) 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:3111: done (7 left) 2022-04-28T19:54:24.167 INFO:tasks.rados.rados.0.smithi059.stdout:3120: rollback oid 11 current snap is 289 2022-04-28T19:54:24.168 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 253 2022-04-28T19:54:24.168 INFO:tasks.rados.rados.0.smithi059.stdout:3108: expect (ObjNum 935 snap 276 seq_num 935) 2022-04-28T19:54:24.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[61152]: cluster 2022-04-28T19:54:22.950386+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e639: 8 total, 8 up, 8 in 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[61152]: cluster 2022-04-28T19:54:22.971664+0000 mgr.y (mgr.24961) 276 : cluster [DBG] pgmap v446: 89 pgs: 89 active+clean; 724 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.2 MiB/s wr, 12 op/s 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[61152]: audit 2022-04-28T19:54:23.769944+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[61152]: audit 2022-04-28T19:54:23.771936+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[61152]: audit 2022-04-28T19:54:23.773952+0000 mon.a (mon.0) 1041 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[61152]: audit 2022-04-28T19:54:23.776673+0000 mon.a (mon.0) 1042 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "container_image"}]: dispatch 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[59745]: cluster 2022-04-28T19:54:22.950386+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e639: 8 total, 8 up, 8 in 2022-04-28T19:54:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[59745]: cluster 2022-04-28T19:54:22.971664+0000 mgr.y (mgr.24961) 276 : cluster [DBG] pgmap v446: 89 pgs: 89 active+clean; 724 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.2 MiB/s wr, 12 op/s 2022-04-28T19:54:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[59745]: audit 2022-04-28T19:54:23.769944+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[59745]: audit 2022-04-28T19:54:23.771936+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:54:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[59745]: audit 2022-04-28T19:54:23.773952+0000 mon.a (mon.0) 1041 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:23 smithi059 bash[59745]: audit 2022-04-28T19:54:23.776673+0000 mon.a (mon.0) 1042 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "container_image"}]: dispatch 2022-04-28T19:54:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:23 smithi139 bash[43301]: cluster 2022-04-28T19:54:22.950386+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e639: 8 total, 8 up, 8 in 2022-04-28T19:54:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:23 smithi139 bash[43301]: cluster 2022-04-28T19:54:22.971664+0000 mgr.y (mgr.24961) 276 : cluster [DBG] pgmap v446: 89 pgs: 89 active+clean; 724 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 4.2 MiB/s wr, 12 op/s 2022-04-28T19:54:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:23 smithi139 bash[43301]: audit 2022-04-28T19:54:23.769944+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:23 smithi139 bash[43301]: audit 2022-04-28T19:54:23.771936+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi139.oefiza", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2022-04-28T19:54:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:23 smithi139 bash[43301]: audit 2022-04-28T19:54:23.773952+0000 mon.a (mon.0) 1041 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:23 smithi139 bash[43301]: audit 2022-04-28T19:54:23.776673+0000 mon.a (mon.0) 1042 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "container_image"}]: dispatch 2022-04-28T19:54:24.722 INFO:tasks.rados.rados.0.smithi059.stdout:3114: finishing write tid 1 to smithi05955964-35 2022-04-28T19:54:24.722 INFO:tasks.rados.rados.0.smithi059.stdout:3114: finishing write tid 2 to smithi05955964-35 2022-04-28T19:54:24.722 INFO:tasks.rados.rados.0.smithi059.stdout:3114: finishing write tid 3 to smithi05955964-35 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3114: finishing write tid 5 to smithi05955964-35 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3114: finishing write tid 6 to smithi05955964-35 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1329 (ObjNum 968 snap 289 seq_num 968) dirty exists 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3114: left oid 35 (ObjNum 968 snap 289 seq_num 968) 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3115: finishing write tid 1 to smithi05955964-47 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3115: finishing write tid 2 to smithi05955964-47 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3115: finishing write tid 3 to smithi05955964-47 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3115: finishing write tid 4 to smithi05955964-47 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3115: finishing write tid 6 to smithi05955964-47 2022-04-28T19:54:24.723 INFO:tasks.rados.rados.0.smithi059.stdout:3115: finishing write tid 7 to smithi05955964-47 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1329 (ObjNum 969 snap 289 seq_num 969) dirty exists 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3115: left oid 47 (ObjNum 969 snap 289 seq_num 969) 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3116: finishing write tid 1 to smithi05955964-40 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3116: finishing write tid 2 to smithi05955964-40 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3116: finishing write tid 3 to smithi05955964-40 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3108: done (7 left) 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3114: done (6 left) 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3115: done (5 left) 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3121: rmattr oid 49 current snap is 289 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3116: finishing write tid 5 to smithi05955964-40 2022-04-28T19:54:24.724 INFO:tasks.rados.rados.0.smithi059.stdout:3116: finishing write tid 6 to smithi05955964-40 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1120 (ObjNum 970 snap 289 seq_num 970) dirty exists 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3116: left oid 40 (ObjNum 970 snap 289 seq_num 970) 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3117: finishing write tid 1 to smithi05955964-9 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3117: finishing write tid 2 to smithi05955964-9 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3117: finishing write tid 3 to smithi05955964-9 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3117: finishing write tid 5 to smithi05955964-9 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3117: finishing write tid 6 to smithi05955964-9 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1176 (ObjNum 971 snap 289 seq_num 971) dirty exists 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3117: left oid 9 (ObjNum 971 snap 289 seq_num 971) 2022-04-28T19:54:24.725 INFO:tasks.rados.rados.0.smithi059.stdout:3118: finishing write tid 1 to smithi05955964-20 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3118: finishing write tid 2 to smithi05955964-20 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3116: done (5 left) 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3117: done (4 left) 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3122: rmattr oid 9 current snap is 289 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3120: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3120: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3120: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1189 (ObjNum 857 snap 253 seq_num 857) dirty exists 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3118: finishing write tid 3 to smithi05955964-20 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:3118: finishing write tid 5 to smithi05955964-20 2022-04-28T19:54:24.726 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1246 (ObjNum 948 snap 278 seq_num 948) dirty exists 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3118: finishing write tid 6 to smithi05955964-20 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1245 (ObjNum 972 snap 289 seq_num 972) dirty exists 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3118: left oid 20 (ObjNum 972 snap 289 seq_num 972) 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3118: done (4 left) 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3119: done (3 left) 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3120: done (2 left) 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3123: read oid 27 snap -1 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3123: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:3124: rollback oid 12 current snap is 289 2022-04-28T19:54:24.727 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 12 to 288 2022-04-28T19:54:24.728 INFO:tasks.rados.rados.0.smithi059.stdout:3125: rollback oid 43 current snap is 289 2022-04-28T19:54:24.728 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 43 to 282 2022-04-28T19:54:24.728 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 289 2022-04-28T19:54:24.728 INFO:tasks.rados.rados.0.smithi059.stdout:3126: seq_num 973 ranges {5644288=704512,6348800=573440,6922240=507904,7430144=122880} 2022-04-28T19:54:24.762 INFO:tasks.rados.rados.0.smithi059.stdout:3126: writing smithi05955964-20 from 5644288 to 6348800 tid 1 2022-04-28T19:54:24.765 INFO:tasks.rados.rados.0.smithi059.stdout:3126: writing smithi05955964-20 from 6348800 to 6922240 tid 2 2022-04-28T19:54:24.769 INFO:tasks.rados.rados.0.smithi059.stdout:3126: writing smithi05955964-20 from 6922240 to 7430144 tid 3 2022-04-28T19:54:24.770 INFO:tasks.rados.rados.0.smithi059.stdout:3126: writing smithi05955964-20 from 7430144 to 7553024 tid 4 2022-04-28T19:54:24.770 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 289 2022-04-28T19:54:24.770 INFO:tasks.rados.rados.0.smithi059.stdout:3127: seq_num 974 ranges {1859584=720896,2580480=507904,3088384=671744,3760128=24576} 2022-04-28T19:54:24.785 INFO:tasks.rados.rados.0.smithi059.stdout:3127: writing smithi05955964-40 from 1859584 to 2580480 tid 1 2022-04-28T19:54:24.787 INFO:tasks.rados.rados.0.smithi059.stdout:3127: writing smithi05955964-40 from 2580480 to 3088384 tid 2 2022-04-28T19:54:24.792 INFO:tasks.rados.rados.0.smithi059.stdout:3127: writing smithi05955964-40 from 3088384 to 3760128 tid 3 2022-04-28T19:54:24.792 INFO:tasks.rados.rados.0.smithi059.stdout:3127: writing smithi05955964-40 from 3760128 to 3784704 tid 4 2022-04-28T19:54:24.792 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 289 2022-04-28T19:54:24.792 INFO:tasks.rados.rados.0.smithi059.stdout:3128: seq_num 975 ranges {2236416=737280,2973696=786432,3760128=417792} 2022-04-28T19:54:24.808 INFO:tasks.rados.rados.0.smithi059.stdout:3128: writing smithi05955964-38 from 2236416 to 2973696 tid 1 2022-04-28T19:54:24.813 INFO:tasks.rados.rados.0.smithi059.stdout:3128: writing smithi05955964-38 from 2973696 to 3760128 tid 2 2022-04-28T19:54:24.816 INFO:tasks.rados.rados.0.smithi059.stdout:3128: writing smithi05955964-38 from 3760128 to 4177920 tid 3 2022-04-28T19:54:24.816 INFO:tasks.rados.rados.0.smithi059.stdout:3129: snap_remove snap 272 2022-04-28T19:54:24.961 INFO:tasks.rados.rados.0.smithi059.stdout:3129: done (8 left) 2022-04-28T19:54:24.961 INFO:tasks.rados.rados.0.smithi059.stdout:3130: read oid 41 snap -1 2022-04-28T19:54:24.961 INFO:tasks.rados.rados.0.smithi059.stdout:3130: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:54:24.961 INFO:tasks.rados.rados.0.smithi059.stdout:3131: rollback oid 44 current snap is 289 2022-04-28T19:54:24.961 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 44 to 282 2022-04-28T19:54:24.961 INFO:tasks.rados.rados.0.smithi059.stdout:3123: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:24.965 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1177 (ObjNum 971 snap 289 seq_num 971) dirty exists 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1121 (ObjNum 929 snap 276 seq_num 929) dirty exists 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:3124: finishing rollback tid 0 to smithi05955964-12 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:3125: finishing rollback tid 0 to smithi05955964-43 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:3124: finishing rollback tid 1 to smithi05955964-12 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:3124: finishing rollback tid 2 to smithi05955964-12 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1346 (ObjNum 936 snap 276 seq_num 936) dirty exists 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:3125: finishing rollback tid 1 to smithi05955964-43 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:3125: finishing rollback tid 2 to smithi05955964-43 2022-04-28T19:54:24.966 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1451 (ObjNum 1498765668 snap 1717328995 seq_num 1917869913) dirty exists 2022-04-28T19:54:24.969 INFO:tasks.rados.rados.0.smithi059.stdout:3126: finishing write tid 1 to smithi05955964-20 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3126: finishing write tid 2 to smithi05955964-20 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3126: finishing write tid 3 to smithi05955964-20 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3126: finishing write tid 4 to smithi05955964-20 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3126: finishing write tid 6 to smithi05955964-20 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3126: finishing write tid 7 to smithi05955964-20 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1251 (ObjNum 973 snap 289 seq_num 973) dirty exists 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3126: left oid 20 (ObjNum 973 snap 289 seq_num 973) 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3127: finishing write tid 1 to smithi05955964-40 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3127: finishing write tid 2 to smithi05955964-40 2022-04-28T19:54:24.970 INFO:tasks.rados.rados.0.smithi059.stdout:3127: finishing write tid 3 to smithi05955964-40 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3127: finishing write tid 4 to smithi05955964-40 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3127: finishing write tid 6 to smithi05955964-40 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3127: finishing write tid 7 to smithi05955964-40 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1126 (ObjNum 974 snap 289 seq_num 974) dirty exists 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3127: left oid 40 (ObjNum 974 snap 289 seq_num 974) 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3128: finishing write tid 1 to smithi05955964-38 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3128: finishing write tid 2 to smithi05955964-38 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3128: finishing write tid 3 to smithi05955964-38 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3128: finishing write tid 5 to smithi05955964-38 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:3128: finishing write tid 6 to smithi05955964-38 2022-04-28T19:54:24.971 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1193 (ObjNum 975 snap 289 seq_num 975) dirty exists 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3128: left oid 38 (ObjNum 975 snap 289 seq_num 975) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3121: done (9 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3122: done (8 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3123: done (7 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3124: done (6 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3125: done (5 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3126: done (4 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3127: done (3 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3128: done (2 left) 2022-04-28T19:54:24.972 INFO:tasks.rados.rados.0.smithi059.stdout:3132: read oid 1 snap 286 2022-04-28T19:54:24.973 INFO:tasks.rados.rados.0.smithi059.stdout:3132: expect (ObjNum 933 snap 276 seq_num 933) 2022-04-28T19:54:24.973 INFO:tasks.rados.rados.0.smithi059.stdout:3133: delete oid 25 current snap is 289 2022-04-28T19:54:24.973 INFO:tasks.rados.rados.0.smithi059.stdout:3133: done (3 left) 2022-04-28T19:54:24.973 INFO:tasks.rados.rados.0.smithi059.stdout:3134: copy_from oid 38 from oid 23 current snap is 289 2022-04-28T19:54:24.973 INFO:tasks.rados.rados.0.smithi059.stdout:3135: delete oid 28 current snap is 289 2022-04-28T19:54:24.974 INFO:tasks.rados.rados.0.smithi059.stdout:3130: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:54:25.018 INFO:tasks.rados.rados.0.smithi059.stdout:3131: finishing rollback tid 0 to smithi05955964-44 2022-04-28T19:54:25.018 INFO:tasks.rados.rados.0.smithi059.stdout:3131: finishing rollback tid 1 to smithi05955964-44 2022-04-28T19:54:25.019 INFO:tasks.rados.rados.0.smithi059.stdout:3130: done (4 left) 2022-04-28T19:54:25.019 INFO:tasks.rados.rados.0.smithi059.stdout:3135: done (3 left) 2022-04-28T19:54:25.019 INFO:tasks.rados.rados.0.smithi059.stdout:3136: snap_create 2022-04-28T19:54:25.019 INFO:tasks.rados.rados.0.smithi059.stdout:3131: finishing rollback tid 2 to smithi05955964-44 2022-04-28T19:54:25.019 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1180 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:54:25.019 INFO:tasks.rados.rados.0.smithi059.stdout:3132: expect (ObjNum 933 snap 276 seq_num 933) 2022-04-28T19:54:25.207 INFO:tasks.rados.rados.0.smithi059.stdout:3134: finishing copy_from to smithi05955964-38 2022-04-28T19:54:25.208 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1146 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:54:25.208 INFO:tasks.rados.rados.0.smithi059.stdout:3134: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:54:25.276 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:24 smithi139 bash[43301]: cephadm 2022-04-28T19:54:23.760803+0000 mgr.y (mgr.24961) 277 : cephadm [INF] Upgrade: Updating mds.foo.smithi139.oefiza 2022-04-28T19:54:25.276 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:24 smithi139 bash[43301]: cephadm 2022-04-28T19:54:23.775620+0000 mgr.y (mgr.24961) 278 : cephadm [INF] Deploying daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:54:25.276 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:24 smithi139 bash[43301]: cluster 2022-04-28T19:54:23.959151+0000 mon.a (mon.0) 1043 : cluster [DBG] osdmap e640: 8 total, 8 up, 8 in 2022-04-28T19:54:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:24 smithi059 bash[61152]: cephadm 2022-04-28T19:54:23.760803+0000 mgr.y (mgr.24961) 277 : cephadm [INF] Upgrade: Updating mds.foo.smithi139.oefiza 2022-04-28T19:54:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:24 smithi059 bash[61152]: cephadm 2022-04-28T19:54:23.775620+0000 mgr.y (mgr.24961) 278 : cephadm [INF] Deploying daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:54:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:24 smithi059 bash[61152]: cluster 2022-04-28T19:54:23.959151+0000 mon.a (mon.0) 1043 : cluster [DBG] osdmap e640: 8 total, 8 up, 8 in 2022-04-28T19:54:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:24 smithi059 bash[59745]: cephadm 2022-04-28T19:54:23.760803+0000 mgr.y (mgr.24961) 277 : cephadm [INF] Upgrade: Updating mds.foo.smithi139.oefiza 2022-04-28T19:54:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:24 smithi059 bash[59745]: cephadm 2022-04-28T19:54:23.775620+0000 mgr.y (mgr.24961) 278 : cephadm [INF] Deploying daemon mds.foo.smithi139.oefiza on smithi139 2022-04-28T19:54:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:24 smithi059 bash[59745]: cluster 2022-04-28T19:54:23.959151+0000 mon.a (mon.0) 1043 : cluster [DBG] osdmap e640: 8 total, 8 up, 8 in 2022-04-28T19:54:25.982 INFO:tasks.rados.rados.0.smithi059.stdout:3131: done (3 left) 2022-04-28T19:54:25.982 INFO:tasks.rados.rados.0.smithi059.stdout:3132: done (2 left) 2022-04-28T19:54:25.982 INFO:tasks.rados.rados.0.smithi059.stdout:3134: done (1 left) 2022-04-28T19:54:25.982 INFO:tasks.rados.rados.0.smithi059.stdout:3136: done (0 left) 2022-04-28T19:54:25.982 INFO:tasks.rados.rados.0.smithi059.stdout:3137: snap_remove snap 253 2022-04-28T19:54:26.121 INFO:tasks.rados.rados.0.smithi059.stdout:3137: done (0 left) 2022-04-28T19:54:26.122 INFO:tasks.rados.rados.0.smithi059.stdout:3138: read oid 8 snap -1 2022-04-28T19:54:26.122 INFO:tasks.rados.rados.0.smithi059.stdout:3138: expect (ObjNum 953 snap 279 seq_num 953) 2022-04-28T19:54:26.122 INFO:tasks.rados.rados.0.smithi059.stdout:3139: copy_from oid 2 from oid 5 current snap is 290 2022-04-28T19:54:26.122 INFO:tasks.rados.rados.0.smithi059.stdout:3140: copy_from oid 15 from oid 11 current snap is 290 2022-04-28T19:54:26.123 INFO:tasks.rados.rados.0.smithi059.stdout:3141: delete oid 22 current snap is 290 2022-04-28T19:54:26.126 INFO:tasks.rados.rados.0.smithi059.stdout:3141: done (3 left) 2022-04-28T19:54:26.126 INFO:tasks.rados.rados.0.smithi059.stdout:3142: rollback oid 24 current snap is 290 2022-04-28T19:54:26.126 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 252 2022-04-28T19:54:26.127 INFO:tasks.rados.rados.0.smithi059.stdout:3143: rollback oid 1 current snap is 290 2022-04-28T19:54:26.127 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 287 2022-04-28T19:54:26.127 INFO:tasks.rados.rados.0.smithi059.stdout:3144: delete oid 7 current snap is 290 2022-04-28T19:54:26.131 INFO:tasks.rados.rados.0.smithi059.stdout:3143: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:54:26.131 INFO:tasks.rados.rados.0.smithi059.stdout:3143: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:54:26.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1128 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:54:26.132 INFO:tasks.rados.rados.0.smithi059.stdout:3143: done (5 left) 2022-04-28T19:54:26.132 INFO:tasks.rados.rados.0.smithi059.stdout:3144: done (4 left) 2022-04-28T19:54:26.132 INFO:tasks.rados.rados.0.smithi059.stdout:3145: copy_from oid 25 from oid 6 current snap is 290 2022-04-28T19:54:26.132 INFO:tasks.rados.rados.0.smithi059.stdout:3146: snap_create 2022-04-28T19:54:26.132 INFO:tasks.rados.rados.0.smithi059.stdout:3142: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:54:26.135 INFO:tasks.rados.rados.0.smithi059.stdout:3142: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:54:26.135 INFO:tasks.rados.rados.0.smithi059.stdout:3142: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:54:26.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1196 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:54:26.137 INFO:tasks.rados.rados.0.smithi059.stdout:3138: expect (ObjNum 953 snap 279 seq_num 953) 2022-04-28T19:54:26.175 INFO:tasks.rados.rados.0.smithi059.stdout:3145: finishing copy_from to smithi05955964-25 2022-04-28T19:54:26.175 INFO:tasks.rados.rados.0.smithi059.stdout:3145: got expected ENOENT (src dne) 2022-04-28T19:54:26.175 INFO:tasks.rados.rados.0.smithi059.stdout:3145: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:54:26.175 INFO:tasks.rados.rados.0.smithi059.stdout:3139: finishing copy_from to smithi05955964-2 2022-04-28T19:54:26.175 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1448 (ObjNum 875 snap 262 seq_num 875) dirty exists 2022-04-28T19:54:26.176 INFO:tasks.rados.rados.0.smithi059.stdout:3140: finishing copy_from to smithi05955964-15 2022-04-28T19:54:26.176 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1189 (ObjNum 857 snap 253 seq_num 857) dirty exists 2022-04-28T19:54:26.176 INFO:tasks.rados.rados.0.smithi059.stdout:3139: finishing copy_from racing read to smithi05955964-2 2022-04-28T19:54:26.176 INFO:tasks.rados.rados.0.smithi059.stdout:3140: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:54:26.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:25 smithi059 bash[59745]: cluster 2022-04-28T19:54:24.958901+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e641: 8 total, 8 up, 8 in 2022-04-28T19:54:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:25 smithi059 bash[59745]: cluster 2022-04-28T19:54:24.972095+0000 mgr.y (mgr.24961) 279 : cluster [DBG] pgmap v449: 89 pgs: 89 active+clean; 737 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 41 op/s 2022-04-28T19:54:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:25 smithi059 bash[61152]: cluster 2022-04-28T19:54:24.958901+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e641: 8 total, 8 up, 8 in 2022-04-28T19:54:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:25 smithi059 bash[61152]: cluster 2022-04-28T19:54:24.972095+0000 mgr.y (mgr.24961) 279 : cluster [DBG] pgmap v449: 89 pgs: 89 active+clean; 737 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 41 op/s 2022-04-28T19:54:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:25 smithi139 bash[43301]: cluster 2022-04-28T19:54:24.958901+0000 mon.a (mon.0) 1044 : cluster [DBG] osdmap e641: 8 total, 8 up, 8 in 2022-04-28T19:54:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:25 smithi139 bash[43301]: cluster 2022-04-28T19:54:24.972095+0000 mgr.y (mgr.24961) 279 : cluster [DBG] pgmap v449: 89 pgs: 89 active+clean; 737 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 41 op/s 2022-04-28T19:54:27.122 INFO:tasks.rados.rados.0.smithi059.stdout:3138: done (5 left) 2022-04-28T19:54:27.122 INFO:tasks.rados.rados.0.smithi059.stdout:3139: done (4 left) 2022-04-28T19:54:27.123 INFO:tasks.rados.rados.0.smithi059.stdout:3140: done (3 left) 2022-04-28T19:54:27.123 INFO:tasks.rados.rados.0.smithi059.stdout:3142: done (2 left) 2022-04-28T19:54:27.123 INFO:tasks.rados.rados.0.smithi059.stdout:3145: done (1 left) 2022-04-28T19:54:27.123 INFO:tasks.rados.rados.0.smithi059.stdout:3146: done (0 left) 2022-04-28T19:54:27.123 INFO:tasks.rados.rados.0.smithi059.stdout:3147: delete oid 13 current snap is 291 2022-04-28T19:54:27.124 INFO:tasks.rados.rados.0.smithi059.stdout:3147: done (0 left) 2022-04-28T19:54:27.124 INFO:tasks.rados.rados.0.smithi059.stdout:3148: delete oid 2 current snap is 291 2022-04-28T19:54:27.130 INFO:tasks.rados.rados.0.smithi059.stdout:3148: done (0 left) 2022-04-28T19:54:27.130 INFO:tasks.rados.rados.0.smithi059.stdout:3149: snap_create 2022-04-28T19:54:27.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:25.979892+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.109399+0000 mon.a (mon.0) 1046 : cluster [WRN] daemon mds.foo.smithi139.oefiza compat changed unexpectedly 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.120154+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.928810+0000 mon.a (mon.0) 1048 : cluster [INF] Standby daemon mds.foo.smithi059.tnizcj assigned to filesystem foo as rank 0 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.929141+0000 mon.a (mon.0) 1049 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.929200+0000 mon.a (mon.0) 1050 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.936574+0000 mon.a (mon.0) 1051 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:replay} 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:25.979892+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2022-04-28T19:54:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.109399+0000 mon.a (mon.0) 1046 : cluster [WRN] daemon mds.foo.smithi139.oefiza compat changed unexpectedly 2022-04-28T19:54:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.120154+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2022-04-28T19:54:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.928810+0000 mon.a (mon.0) 1048 : cluster [INF] Standby daemon mds.foo.smithi059.tnizcj assigned to filesystem foo as rank 0 2022-04-28T19:54:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.929141+0000 mon.a (mon.0) 1049 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2022-04-28T19:54:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.929200+0000 mon.a (mon.0) 1050 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2022-04-28T19:54:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:26 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.936574+0000 mon.a (mon.0) 1051 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:replay} 2022-04-28T19:54:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:25.979892+0000 mon.a (mon.0) 1045 : cluster [DBG] osdmap e642: 8 total, 8 up, 8 in 2022-04-28T19:54:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.109399+0000 mon.a (mon.0) 1046 : cluster [WRN] daemon mds.foo.smithi139.oefiza compat changed unexpectedly 2022-04-28T19:54:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.120154+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2022-04-28T19:54:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.928810+0000 mon.a (mon.0) 1048 : cluster [INF] Standby daemon mds.foo.smithi059.tnizcj assigned to filesystem foo as rank 0 2022-04-28T19:54:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.929141+0000 mon.a (mon.0) 1049 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2022-04-28T19:54:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.929200+0000 mon.a (mon.0) 1050 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2022-04-28T19:54:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:26 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.936574+0000 mon.a (mon.0) 1051 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:replay} 2022-04-28T19:54:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:3149: done (0 left) 2022-04-28T19:54:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:3150: read oid 49 snap -1 2022-04-28T19:54:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:3150: expect (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:54:28.128 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 292 2022-04-28T19:54:28.129 INFO:tasks.rados.rados.0.smithi059.stdout:3151: seq_num 982 ranges {0=450560,450560=425984,876544=679936,1556480=466944,2023424=32768} 2022-04-28T19:54:28.131 INFO:tasks.rados.rados.0.smithi059.stdout:3151: writing smithi05955964-25 from 0 to 450560 tid 1 2022-04-28T19:54:28.134 INFO:tasks.rados.rados.0.smithi059.stdout:3151: writing smithi05955964-25 from 450560 to 876544 tid 2 2022-04-28T19:54:28.142 INFO:tasks.rados.rados.0.smithi059.stdout:3151: writing smithi05955964-25 from 876544 to 1556480 tid 3 2022-04-28T19:54:28.143 INFO:tasks.rados.rados.0.smithi059.stdout:3151: writing smithi05955964-25 from 1556480 to 2023424 tid 4 2022-04-28T19:54:28.143 INFO:tasks.rados.rados.0.smithi059.stdout:3151: writing smithi05955964-25 from 2023424 to 2056192 tid 5 2022-04-28T19:54:28.143 INFO:tasks.rados.rados.0.smithi059.stdout:3152: delete oid 16 current snap is 292 2022-04-28T19:54:28.144 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 1 to smithi05955964-25 2022-04-28T19:54:28.144 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 2 to smithi05955964-25 2022-04-28T19:54:28.144 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 3 to smithi05955964-25 2022-04-28T19:54:28.149 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 4 to smithi05955964-25 2022-04-28T19:54:28.149 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 5 to smithi05955964-25 2022-04-28T19:54:28.149 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 7 to smithi05955964-25 2022-04-28T19:54:28.151 INFO:tasks.rados.rados.0.smithi059.stdout:3151: finishing write tid 8 to smithi05955964-25 2022-04-28T19:54:28.151 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1457 (ObjNum 982 snap 292 seq_num 982) dirty exists 2022-04-28T19:54:28.151 INFO:tasks.rados.rados.0.smithi059.stdout:3151: left oid 25 (ObjNum 982 snap 292 seq_num 982) 2022-04-28T19:54:28.151 INFO:tasks.rados.rados.0.smithi059.stdout:3151: done (2 left) 2022-04-28T19:54:28.151 INFO:tasks.rados.rados.0.smithi059.stdout:3152: done (1 left) 2022-04-28T19:54:28.151 INFO:tasks.rados.rados.0.smithi059.stdout:3153: rmattr oid 8 current snap is 292 2022-04-28T19:54:28.152 INFO:tasks.rados.rados.0.smithi059.stdout:3153: done (1 left) 2022-04-28T19:54:28.152 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 292 2022-04-28T19:54:28.152 INFO:tasks.rados.rados.0.smithi059.stdout:3154: seq_num 984 ranges {13877248=483328,14360576=598016,14958592=647168,15605760=360448} 2022-04-28T19:54:28.232 INFO:tasks.rados.rados.0.smithi059.stdout:3154: writing smithi05955964-35 from 13877248 to 14360576 tid 1 2022-04-28T19:54:28.235 INFO:tasks.rados.rados.0.smithi059.stdout:3154: writing smithi05955964-35 from 14360576 to 14958592 tid 2 2022-04-28T19:54:28.240 INFO:tasks.rados.rados.0.smithi059.stdout:3154: writing smithi05955964-35 from 14958592 to 15605760 tid 3 2022-04-28T19:54:28.242 INFO:tasks.rados.rados.0.smithi059.stdout:3154: writing smithi05955964-35 from 15605760 to 15966208 tid 4 2022-04-28T19:54:28.242 INFO:tasks.rados.rados.0.smithi059.stdout:3155: rollback oid 29 current snap is 292 2022-04-28T19:54:28.242 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 291 2022-04-28T19:54:28.242 INFO:tasks.rados.rados.0.smithi059.stdout:3150: expect (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:54:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[59745]: cluster 2022-04-28T19:54:26.972558+0000 mgr.y (mgr.24961) 280 : cluster [DBG] pgmap v452: 89 pgs: 89 active+clean; 746 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.8 MiB/s wr, 38 op/s 2022-04-28T19:54:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[59745]: cluster 2022-04-28T19:54:27.120919+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2022-04-28T19:54:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[59745]: audit 2022-04-28T19:54:27.831138+0000 mon.a (mon.0) 1053 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[59745]: cluster 2022-04-28T19:54:27.952122+0000 mon.a (mon.0) 1054 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:reconnect 2022-04-28T19:54:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[59745]: cluster 2022-04-28T19:54:27.952216+0000 mon.a (mon.0) 1055 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:reconnect} 2022-04-28T19:54:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[61152]: cluster 2022-04-28T19:54:26.972558+0000 mgr.y (mgr.24961) 280 : cluster [DBG] pgmap v452: 89 pgs: 89 active+clean; 746 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.8 MiB/s wr, 38 op/s 2022-04-28T19:54:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[61152]: cluster 2022-04-28T19:54:27.120919+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2022-04-28T19:54:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[61152]: audit 2022-04-28T19:54:27.831138+0000 mon.a (mon.0) 1053 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[61152]: cluster 2022-04-28T19:54:27.952122+0000 mon.a (mon.0) 1054 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:reconnect 2022-04-28T19:54:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:28 smithi059 bash[61152]: cluster 2022-04-28T19:54:27.952216+0000 mon.a (mon.0) 1055 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:reconnect} 2022-04-28T19:54:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:28 smithi139 bash[43301]: cluster 2022-04-28T19:54:26.972558+0000 mgr.y (mgr.24961) 280 : cluster [DBG] pgmap v452: 89 pgs: 89 active+clean; 746 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 5.8 MiB/s wr, 38 op/s 2022-04-28T19:54:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:28 smithi139 bash[43301]: cluster 2022-04-28T19:54:27.120919+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2022-04-28T19:54:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:28 smithi139 bash[43301]: audit 2022-04-28T19:54:27.831138+0000 mon.a (mon.0) 1053 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:28 smithi139 bash[43301]: cluster 2022-04-28T19:54:27.952122+0000 mon.a (mon.0) 1054 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:reconnect 2022-04-28T19:54:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:28 smithi139 bash[43301]: cluster 2022-04-28T19:54:27.952216+0000 mon.a (mon.0) 1055 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:reconnect} 2022-04-28T19:54:29.215 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:29 smithi139 bash[43301]: cluster 2022-04-28T19:54:28.125631+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2022-04-28T19:54:29.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:29 smithi139 bash[43301]: cluster 2022-04-28T19:54:28.954605+0000 mon.a (mon.0) 1057 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:rejoin 2022-04-28T19:54:29.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:29 smithi139 bash[43301]: cluster 2022-04-28T19:54:28.954786+0000 mon.a (mon.0) 1058 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:rejoin} 2022-04-28T19:54:29.255 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:29 smithi139 bash[43301]: cluster 2022-04-28T19:54:28.957236+0000 mon.a (mon.0) 1059 : cluster [INF] daemon mds.foo.smithi059.tnizcj is now active in filesystem foo as rank 0 2022-04-28T19:54:29.350 INFO:tasks.rados.rados.0.smithi059.stdout:3154: finishing write tid 1 to smithi05955964-35 2022-04-28T19:54:29.350 INFO:tasks.rados.rados.0.smithi059.stdout:3154: finishing write tid 2 to smithi05955964-35 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3155: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3155: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3155: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1461 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3154: finishing write tid 3 to smithi05955964-35 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3154: finishing write tid 4 to smithi05955964-35 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3150: done (2 left) 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3155: done (1 left) 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3156: rmattr oid 1 current snap is 292 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3156: done (1 left) 2022-04-28T19:54:29.351 INFO:tasks.rados.rados.0.smithi059.stdout:3157: read oid 27 snap -1 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:3157: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:3158: setattr oid 39 current snap is 292 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:3154: finishing write tid 6 to smithi05955964-35 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:3154: finishing write tid 7 to smithi05955964-35 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1334 (ObjNum 984 snap 292 seq_num 984) dirty exists 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:3154: left oid 35 (ObjNum 984 snap 292 seq_num 984) 2022-04-28T19:54:29.352 INFO:tasks.rados.rados.0.smithi059.stdout:3157: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:54:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:3154: done (2 left) 2022-04-28T19:54:29.354 INFO:tasks.rados.rados.0.smithi059.stdout:3157: done (1 left) 2022-04-28T19:54:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:3159: read oid 16 snap -1 2022-04-28T19:54:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:3159: expect deleted 2022-04-28T19:54:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 292 2022-04-28T19:54:29.355 INFO:tasks.rados.rados.0.smithi059.stdout:3160: seq_num 985 ranges {15556608=499712,16056320=483328,16539648=450560,16990208=581632,17571840=90112} 2022-04-28T19:54:29.360 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:54:29.449 INFO:tasks.rados.rados.0.smithi059.stdout:3160: writing smithi05955964-45 from 15556608 to 16056320 tid 1 2022-04-28T19:54:29.452 INFO:tasks.rados.rados.0.smithi059.stdout:3160: writing smithi05955964-45 from 16056320 to 16539648 tid 2 2022-04-28T19:54:29.455 INFO:tasks.rados.rados.0.smithi059.stdout:3160: writing smithi05955964-45 from 16539648 to 16990208 tid 3 2022-04-28T19:54:29.459 INFO:tasks.rados.rados.0.smithi059.stdout:3160: writing smithi05955964-45 from 16990208 to 17571840 tid 4 2022-04-28T19:54:29.460 INFO:tasks.rados.rados.0.smithi059.stdout:3160: writing smithi05955964-45 from 17571840 to 17661952 tid 5 2022-04-28T19:54:29.460 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 292 2022-04-28T19:54:29.460 INFO:tasks.rados.rados.0.smithi059.stdout:3161: seq_num 986 ranges {2539520=516096,3055616=770048,3825664=655360,4481024=180224} 2022-04-28T19:54:29.477 INFO:tasks.rados.rados.0.smithi059.stdout:3161: writing smithi05955964-42 from 2539520 to 3055616 tid 1 2022-04-28T19:54:29.482 INFO:tasks.rados.rados.0.smithi059.stdout:3161: writing smithi05955964-42 from 3055616 to 3825664 tid 2 2022-04-28T19:54:29.486 INFO:tasks.rados.rados.0.smithi059.stdout:3161: writing smithi05955964-42 from 3825664 to 4481024 tid 3 2022-04-28T19:54:29.487 INFO:tasks.rados.rados.0.smithi059.stdout:3161: writing smithi05955964-42 from 4481024 to 4661248 tid 4 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3162: read oid 24 snap -1 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3162: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3163: snap_create 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1453 (ObjNum 894 snap 267 seq_num 894) dirty exists 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 1 to smithi05955964-45 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 2 to smithi05955964-45 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 3 to smithi05955964-45 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 4 to smithi05955964-45 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 5 to smithi05955964-45 2022-04-28T19:54:29.488 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 7 to smithi05955964-45 2022-04-28T19:54:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:3160: finishing write tid 8 to smithi05955964-45 2022-04-28T19:54:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1257 (ObjNum 985 snap 292 seq_num 985) dirty exists 2022-04-28T19:54:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:3160: left oid 45 (ObjNum 985 snap 292 seq_num 985) 2022-04-28T19:54:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:3161: finishing write tid 1 to smithi05955964-42 2022-04-28T19:54:29.489 INFO:tasks.rados.rados.0.smithi059.stdout:3161: finishing write tid 2 to smithi05955964-42 2022-04-28T19:54:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:3161: finishing write tid 3 to smithi05955964-42 2022-04-28T19:54:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:3161: finishing write tid 4 to smithi05955964-42 2022-04-28T19:54:29.493 INFO:tasks.rados.rados.0.smithi059.stdout:3161: finishing write tid 6 to smithi05955964-42 2022-04-28T19:54:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:3161: finishing write tid 7 to smithi05955964-42 2022-04-28T19:54:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1339 (ObjNum 986 snap 292 seq_num 986) dirty exists 2022-04-28T19:54:29.494 INFO:tasks.rados.rados.0.smithi059.stdout:3161: left oid 42 (ObjNum 986 snap 292 seq_num 986) 2022-04-28T19:54:29.503 INFO:tasks.rados.rados.0.smithi059.stdout:3162: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:54:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[61152]: cluster 2022-04-28T19:54:28.125631+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2022-04-28T19:54:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[61152]: cluster 2022-04-28T19:54:28.954605+0000 mon.a (mon.0) 1057 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:rejoin 2022-04-28T19:54:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[61152]: cluster 2022-04-28T19:54:28.954786+0000 mon.a (mon.0) 1058 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:rejoin} 2022-04-28T19:54:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[61152]: cluster 2022-04-28T19:54:28.957236+0000 mon.a (mon.0) 1059 : cluster [INF] daemon mds.foo.smithi059.tnizcj is now active in filesystem foo as rank 0 2022-04-28T19:54:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[59745]: cluster 2022-04-28T19:54:28.125631+0000 mon.a (mon.0) 1056 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2022-04-28T19:54:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[59745]: cluster 2022-04-28T19:54:28.954605+0000 mon.a (mon.0) 1057 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:rejoin 2022-04-28T19:54:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[59745]: cluster 2022-04-28T19:54:28.954786+0000 mon.a (mon.0) 1058 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi059.tnizcj=up:rejoin} 2022-04-28T19:54:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:29 smithi059 bash[59745]: cluster 2022-04-28T19:54:28.957236+0000 mon.a (mon.0) 1059 : cluster [INF] daemon mds.foo.smithi059.tnizcj is now active in filesystem foo as rank 0 2022-04-28T19:54:29.685 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:54:29.685 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (13m) 8s ago 13m 14.8M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:54:29.685 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (13m) 23s ago 13m 26.3M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:54:29.685 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (13s) 8s ago 11m 11.8M - 16.2.7-969-g4fa079ba 863db77b5b5e 0b1d79d85003 2022-04-28T19:54:29.685 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 starting - - - - 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (5m) 23s ago 17m 377M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (6m) 8s ago 19m 418M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (5m) 8s ago 20m 79.2M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (4m) 23s ago 17m 52.5M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (5m) 8s ago 18m 63.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (13m) 8s ago 14m 11.3M - 0.18.1 e5a616e4b9cf 82b17de25899 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (14m) 23s ago 14m 11.4M - 0.18.1 e5a616e4b9cf fa8e68662cb4 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (3m) 8s ago 17m 341M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (3m) 8s ago 16m 242M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:54:29.686 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (2m) 8s ago 16m 318M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:54:29.687 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (2m) 8s ago 16m 242M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:54:29.687 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (116s) 23s ago 15m 289M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:54:29.687 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (86s) 23s ago 15m 346M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 4d55ca87a6a9 2022-04-28T19:54:29.687 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (55s) 23s ago 14m 215M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 2cf53c9fc29e 2022-04-28T19:54:29.687 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (25s) 23s ago 14m - 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 8655092f1b14 2022-04-28T19:54:29.687 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (6m) 23s ago 14m 41.3M - 2.18.1 de242295e225 95641c36c5ae 2022-04-28T19:54:30.045 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 8 2022-04-28T19:54:30.147 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 1 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 14 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:54:30.148 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3158: done (5 left) 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3159: done (4 left) 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3160: done (3 left) 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3161: done (2 left) 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3162: done (1 left) 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3163: done (0 left) 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 293 2022-04-28T19:54:30.287 INFO:tasks.rados.rados.0.smithi059.stdout:3164: seq_num 987 ranges {0=532480,532480=647168,1179648=417792,1597440=409600,2007040=131072} 2022-04-28T19:54:30.288 INFO:tasks.rados.rados.0.smithi059.stdout:3164: writing smithi05955964-13 from 0 to 532480 tid 1 2022-04-28T19:54:30.290 INFO:tasks.rados.rados.0.smithi059.stdout:3164: writing smithi05955964-13 from 532480 to 1179648 tid 2 2022-04-28T19:54:30.292 INFO:tasks.rados.rados.0.smithi059.stdout:3164: writing smithi05955964-13 from 1179648 to 1597440 tid 3 2022-04-28T19:54:30.295 INFO:tasks.rados.rados.0.smithi059.stdout:3164: writing smithi05955964-13 from 1597440 to 2007040 tid 4 2022-04-28T19:54:30.300 INFO:tasks.rados.rados.0.smithi059.stdout:3164: writing smithi05955964-13 from 2007040 to 2138112 tid 5 2022-04-28T19:54:30.300 INFO:tasks.rados.rados.0.smithi059.stdout:3165: read oid 26 snap -1 2022-04-28T19:54:30.300 INFO:tasks.rados.rados.0.smithi059.stdout:3165: expect (ObjNum 966 snap 289 seq_num 966) 2022-04-28T19:54:30.301 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 293 2022-04-28T19:54:30.301 INFO:tasks.rados.rados.0.smithi059.stdout:3166: seq_num 988 ranges {0=557056,557056=532480,1089536=622592,1712128=450560} 2022-04-28T19:54:30.301 INFO:tasks.rados.rados.0.smithi059.stdout:3166: writing smithi05955964-22 from 0 to 557056 tid 1 2022-04-28T19:54:30.303 INFO:tasks.rados.rados.0.smithi059.stdout:3166: writing smithi05955964-22 from 557056 to 1089536 tid 2 2022-04-28T19:54:30.307 INFO:tasks.rados.rados.0.smithi059.stdout:3166: writing smithi05955964-22 from 1089536 to 1712128 tid 3 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3166: writing smithi05955964-22 from 1712128 to 2162688 tid 4 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3167: rollback oid 23 current snap is 293 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 282 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 1 to smithi05955964-13 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 2 to smithi05955964-13 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 3 to smithi05955964-13 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 4 to smithi05955964-13 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 5 to smithi05955964-13 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 7 to smithi05955964-13 2022-04-28T19:54:30.311 INFO:tasks.rados.rados.0.smithi059.stdout:3168: setattr oid 35 current snap is 293 2022-04-28T19:54:30.312 INFO:tasks.rados.rados.0.smithi059.stdout:3165: expect (ObjNum 966 snap 289 seq_num 966) 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3164: finishing write tid 8 to smithi05955964-13 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1068 (ObjNum 987 snap 293 seq_num 987) dirty exists 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3164: left oid 13 (ObjNum 987 snap 293 seq_num 987) 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3166: finishing write tid 1 to smithi05955964-22 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3166: finishing write tid 2 to smithi05955964-22 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3167: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3167: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:54:30.437 INFO:tasks.rados.rados.0.smithi059.stdout:3167: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1332 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3166: finishing write tid 3 to smithi05955964-22 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3166: finishing write tid 4 to smithi05955964-22 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3166: finishing write tid 6 to smithi05955964-22 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3166: finishing write tid 7 to smithi05955964-22 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1450 (ObjNum 988 snap 293 seq_num 988) dirty exists 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3166: left oid 22 (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3164: done (4 left) 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3165: done (3 left) 2022-04-28T19:54:30.438 INFO:tasks.rados.rados.0.smithi059.stdout:3166: done (2 left) 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3167: done (1 left) 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3169: read oid 6 snap -1 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3169: expect deleted 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3170: rollback oid 15 current snap is 293 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 275 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3171: read oid 38 snap -1 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3171: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3172: read oid 50 snap -1 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3172: expect deleted 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3173: read oid 10 snap -1 2022-04-28T19:54:30.439 INFO:tasks.rados.rados.0.smithi059.stdout:3173: expect (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:54:30.440 INFO:tasks.rados.rados.0.smithi059.stdout:3174: snap_create 2022-04-28T19:54:30.447 INFO:tasks.rados.rados.0.smithi059.stdout:3170: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:54:30.447 INFO:tasks.rados.rados.0.smithi059.stdout:3170: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:54:30.447 INFO:tasks.rados.rados.0.smithi059.stdout:3170: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:54:30.447 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1350 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:54:30.448 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1335 (ObjNum 984 snap 292 seq_num 984) dirty exists 2022-04-28T19:54:30.460 INFO:tasks.rados.rados.0.smithi059.stdout:3173: expect (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: cluster 2022-04-28T19:54:28.973183+0000 mgr.y (mgr.24961) 281 : cluster [DBG] pgmap v455: 89 pgs: 89 active+clean; 749 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.6 MiB/s wr, 8 op/s 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.269110+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.270774+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.273022+0000 mon.a (mon.0) 1062 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.274347+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: cephadm 2022-04-28T19:54:29.275623+0000 mgr.y (mgr.24961) 282 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.276737+0000 mon.a (mon.0) 1064 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.346870+0000 mgr.y (mgr.24961) 283 : audit [DBG] from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.654613+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.655262+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.655767+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.656203+0000 mon.a (mon.0) 1068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.656633+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.657045+0000 mon.a (mon.0) 1070 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.659170+0000 mon.a (mon.0) 1071 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.659755+0000 mon.a (mon.0) 1072 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: cluster 2022-04-28T19:54:28.973183+0000 mgr.y (mgr.24961) 281 : cluster [DBG] pgmap v455: 89 pgs: 89 active+clean; 749 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.6 MiB/s wr, 8 op/s 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.269110+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.270774+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.273022+0000 mon.a (mon.0) 1062 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.274347+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: cephadm 2022-04-28T19:54:29.275623+0000 mgr.y (mgr.24961) 282 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.276737+0000 mon.a (mon.0) 1064 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.346870+0000 mgr.y (mgr.24961) 283 : audit [DBG] from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.654613+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.655262+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.655767+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.656203+0000 mon.a (mon.0) 1068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.656633+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.657045+0000 mon.a (mon.0) 1070 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.659170+0000 mon.a (mon.0) 1071 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.659755+0000 mon.a (mon.0) 1072 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.660422+0000 mon.a (mon.0) 1073 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.661037+0000 mon.a (mon.0) 1074 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:29.661649+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: cluster 2022-04-28T19:54:29.950869+0000 mon.a (mon.0) 1076 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: cluster 2022-04-28T19:54:29.962725+0000 mon.a (mon.0) 1077 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:active 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: cluster 2022-04-28T19:54:29.962846+0000 mon.a (mon.0) 1078 : cluster [DBG] fsmap foo:1 {0=foo.smithi059.tnizcj=up:active} 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[59745]: audit 2022-04-28T19:54:30.045011+0000 mon.a (mon.0) 1079 : audit [DBG] from='client.? 172.21.15.59:0/1151351661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.660422+0000 mon.a (mon.0) 1073 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.661037+0000 mon.a (mon.0) 1074 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:29.661649+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: cluster 2022-04-28T19:54:29.950869+0000 mon.a (mon.0) 1076 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: cluster 2022-04-28T19:54:29.962725+0000 mon.a (mon.0) 1077 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:active 2022-04-28T19:54:30.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: cluster 2022-04-28T19:54:29.962846+0000 mon.a (mon.0) 1078 : cluster [DBG] fsmap foo:1 {0=foo.smithi059.tnizcj=up:active} 2022-04-28T19:54:30.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:30 smithi059 bash[61152]: audit 2022-04-28T19:54:30.045011+0000 mon.a (mon.0) 1079 : audit [DBG] from='client.? 172.21.15.59:0/1151351661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: cluster 2022-04-28T19:54:28.973183+0000 mgr.y (mgr.24961) 281 : cluster [DBG] pgmap v455: 89 pgs: 89 active+clean; 749 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.6 MiB/s wr, 8 op/s 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.269110+0000 mon.a (mon.0) 1060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.270774+0000 mon.a (mon.0) 1061 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.273022+0000 mon.a (mon.0) 1062 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.274347+0000 mon.a (mon.0) 1063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: cephadm 2022-04-28T19:54:29.275623+0000 mgr.y (mgr.24961) 282 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.276737+0000 mon.a (mon.0) 1064 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.346870+0000 mgr.y (mgr.24961) 283 : audit [DBG] from='client.44364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.654613+0000 mon.a (mon.0) 1065 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.655262+0000 mon.a (mon.0) 1066 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.655767+0000 mon.a (mon.0) 1067 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.656203+0000 mon.a (mon.0) 1068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.656633+0000 mon.a (mon.0) 1069 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.657045+0000 mon.a (mon.0) 1070 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.659170+0000 mon.a (mon.0) 1071 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:54:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.659755+0000 mon.a (mon.0) 1072 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.660422+0000 mon.a (mon.0) 1073 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.661037+0000 mon.a (mon.0) 1074 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:29.661649+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: cluster 2022-04-28T19:54:29.950869+0000 mon.a (mon.0) 1076 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: cluster 2022-04-28T19:54:29.962725+0000 mon.a (mon.0) 1077 : cluster [DBG] mds.? [v2:172.21.15.59:6834/1523628264,v1:172.21.15.59:6835/1523628264] up:active 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: cluster 2022-04-28T19:54:29.962846+0000 mon.a (mon.0) 1078 : cluster [DBG] fsmap foo:1 {0=foo.smithi059.tnizcj=up:active} 2022-04-28T19:54:30.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:30 smithi139 bash[43301]: audit 2022-04-28T19:54:30.045011+0000 mon.a (mon.0) 1079 : audit [DBG] from='client.? 172.21.15.59:0/1151351661' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:30.716 INFO:tasks.rados.rados.0.smithi059.stdout:3171: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3168: done (6 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3169: done (5 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3170: done (4 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3171: done (3 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3172: done (2 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3173: done (1 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3174: done (0 left) 2022-04-28T19:54:31.319 INFO:tasks.rados.rados.0.smithi059.stdout:3175: copy_from oid 28 from oid 26 current snap is 294 2022-04-28T19:54:31.320 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 294 2022-04-28T19:54:31.320 INFO:tasks.rados.rados.0.smithi059.stdout:3176: seq_num 989 ranges {7553024=573440,8126464=466944,8593408=434176,9027584=638976,9666560=65536} 2022-04-28T19:54:31.373 INFO:tasks.rados.rados.0.smithi059.stdout:3176: writing smithi05955964-20 from 7553024 to 8126464 tid 1 2022-04-28T19:54:31.376 INFO:tasks.rados.rados.0.smithi059.stdout:3176: writing smithi05955964-20 from 8126464 to 8593408 tid 2 2022-04-28T19:54:31.378 INFO:tasks.rados.rados.0.smithi059.stdout:3176: writing smithi05955964-20 from 8593408 to 9027584 tid 3 2022-04-28T19:54:31.383 INFO:tasks.rados.rados.0.smithi059.stdout:3176: writing smithi05955964-20 from 9027584 to 9666560 tid 4 2022-04-28T19:54:31.383 INFO:tasks.rados.rados.0.smithi059.stdout:3176: writing smithi05955964-20 from 9666560 to 9732096 tid 5 2022-04-28T19:54:31.384 INFO:tasks.rados.rados.0.smithi059.stdout:3177: snap_create 2022-04-28T19:54:31.384 INFO:tasks.rados.rados.0.smithi059.stdout:3175: finishing copy_from to smithi05955964-28 2022-04-28T19:54:31.384 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1444 (ObjNum 966 snap 289 seq_num 966) dirty exists 2022-04-28T19:54:31.384 INFO:tasks.rados.rados.0.smithi059.stdout:3175: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:54:31.384 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 1 to smithi05955964-20 2022-04-28T19:54:31.384 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 2 to smithi05955964-20 2022-04-28T19:54:31.385 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 3 to smithi05955964-20 2022-04-28T19:54:31.386 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 4 to smithi05955964-20 2022-04-28T19:54:31.387 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 5 to smithi05955964-20 2022-04-28T19:54:31.387 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 7 to smithi05955964-20 2022-04-28T19:54:31.388 INFO:tasks.rados.rados.0.smithi059.stdout:3176: finishing write tid 8 to smithi05955964-20 2022-04-28T19:54:31.388 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1263 (ObjNum 989 snap 294 seq_num 989) dirty exists 2022-04-28T19:54:31.388 INFO:tasks.rados.rados.0.smithi059.stdout:3176: left oid 20 (ObjNum 989 snap 294 seq_num 989) 2022-04-28T19:54:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[61152]: audit 2022-04-28T19:54:29.651397+0000 mgr.y (mgr.24961) 284 : audit [DBG] from='client.54313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[61152]: cluster 2022-04-28T19:54:30.281789+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2022-04-28T19:54:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[59745]: audit 2022-04-28T19:54:29.651397+0000 mgr.y (mgr.24961) 284 : audit [DBG] from='client.54313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[59745]: cluster 2022-04-28T19:54:30.281789+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2022-04-28T19:54:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:31 smithi139 bash[43301]: audit 2022-04-28T19:54:29.651397+0000 mgr.y (mgr.24961) 284 : audit [DBG] from='client.54313 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:54:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:31 smithi139 bash[43301]: cluster 2022-04-28T19:54:30.281789+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e646: 8 total, 8 up, 8 in 2022-04-28T19:54:31.943 INFO:tasks.rados.rados.0.smithi059.stdout:3175: done (2 left) 2022-04-28T19:54:31.943 INFO:tasks.rados.rados.0.smithi059.stdout:3176: done (1 left) 2022-04-28T19:54:31.943 INFO:tasks.rados.rados.0.smithi059.stdout:3177: done (0 left) 2022-04-28T19:54:31.944 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 5 current snap is 295 2022-04-28T19:54:31.944 INFO:tasks.rados.rados.0.smithi059.stdout:3178: seq_num 990 ranges {2646016=589824,3235840=753664,3989504=638976,4628480=212992} 2022-04-28T19:54:31.961 INFO:tasks.rados.rados.0.smithi059.stdout:3178: writing smithi05955964-5 from 2646016 to 3235840 tid 1 2022-04-28T19:54:32.062 INFO:tasks.rados.rados.0.smithi059.stdout:3178: writing smithi05955964-5 from 3235840 to 3989504 tid 2 2022-04-28T19:54:32.062 INFO:tasks.rados.rados.0.smithi059.stdout:3178: writing smithi05955964-5 from 3989504 to 4628480 tid 3 2022-04-28T19:54:32.063 INFO:tasks.rados.rados.0.smithi059.stdout:3178: writing smithi05955964-5 from 4628480 to 4841472 tid 4 2022-04-28T19:54:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3179: delete oid 35 current snap is 295 2022-04-28T19:54:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3178: finishing write tid 1 to smithi05955964-5 2022-04-28T19:54:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3178: finishing write tid 2 to smithi05955964-5 2022-04-28T19:54:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3178: finishing write tid 3 to smithi05955964-5 2022-04-28T19:54:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3178: finishing write tid 4 to smithi05955964-5 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3178: finishing write tid 6 to smithi05955964-5 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3178: finishing write tid 7 to smithi05955964-5 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1467 (ObjNum 990 snap 295 seq_num 990) dirty exists 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3178: left oid 5 (ObjNum 990 snap 295 seq_num 990) 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3178: done (1 left) 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3179: done (0 left) 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3180: rmattr oid 4 current snap is 295 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3180: done (0 left) 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3181: read oid 3 snap -1 2022-04-28T19:54:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3181: expect (ObjNum 921 snap 275 seq_num 921) 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3182: read oid 16 snap -1 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3182: expect deleted 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 295 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3183: seq_num 992 ranges {2162688=622592,2785280=524288,3309568=606208,3915776=475136} 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3183: writing smithi05955964-22 from 2162688 to 2785280 tid 1 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3183: writing smithi05955964-22 from 2785280 to 3309568 tid 2 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3183: writing smithi05955964-22 from 3309568 to 3915776 tid 3 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3183: writing smithi05955964-22 from 3915776 to 4390912 tid 4 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3184: snap_create 2022-04-28T19:54:32.067 INFO:tasks.rados.rados.0.smithi059.stdout:3181: expect (ObjNum 921 snap 275 seq_num 921) 2022-04-28T19:54:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:31.635Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:31.635Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:31 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:31.635Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:54:32.093 INFO:tasks.rados.rados.0.smithi059.stdout:3183: finishing write tid 1 to smithi05955964-22 2022-04-28T19:54:32.093 INFO:tasks.rados.rados.0.smithi059.stdout:3183: finishing write tid 2 to smithi05955964-22 2022-04-28T19:54:32.093 INFO:tasks.rados.rados.0.smithi059.stdout:3183: finishing write tid 3 to smithi05955964-22 2022-04-28T19:54:32.093 INFO:tasks.rados.rados.0.smithi059.stdout:3183: finishing write tid 4 to smithi05955964-22 2022-04-28T19:54:32.093 INFO:tasks.rados.rados.0.smithi059.stdout:3183: finishing write tid 6 to smithi05955964-22 2022-04-28T19:54:32.094 INFO:tasks.rados.rados.0.smithi059.stdout:3183: finishing write tid 7 to smithi05955964-22 2022-04-28T19:54:32.094 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1455 (ObjNum 992 snap 295 seq_num 992) dirty exists 2022-04-28T19:54:32.094 INFO:tasks.rados.rados.0.smithi059.stdout:3183: left oid 22 (ObjNum 992 snap 295 seq_num 992) 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:30.974060+0000 mgr.y (mgr.24961) 285 : cluster [DBG] pgmap v457: 89 pgs: 89 active+clean; 766 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:31.279386+0000 mon.a (mon.0) 1081 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:31.279445+0000 mon.a (mon.0) 1082 : cluster [INF] Cluster is now healthy 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:31.295867+0000 mon.a (mon.0) 1083 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1824139607,v1:172.21.15.139:6833/1824139607] up:boot 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:31.296032+0000 mon.a (mon.0) 1084 : cluster [DBG] fsmap foo:1 {0=foo.smithi059.tnizcj=up:active} 1 up:standby 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:31.296204+0000 mon.a (mon.0) 1085 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: audit 2022-04-28T19:54:31.296443+0000 mon.a (mon.0) 1086 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[59745]: cluster 2022-04-28T19:54:31.940028+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:30.974060+0000 mgr.y (mgr.24961) 285 : cluster [DBG] pgmap v457: 89 pgs: 89 active+clean; 766 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:31.279386+0000 mon.a (mon.0) 1081 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:31.279445+0000 mon.a (mon.0) 1082 : cluster [INF] Cluster is now healthy 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:31.295867+0000 mon.a (mon.0) 1083 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1824139607,v1:172.21.15.139:6833/1824139607] up:boot 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:31.296032+0000 mon.a (mon.0) 1084 : cluster [DBG] fsmap foo:1 {0=foo.smithi059.tnizcj=up:active} 1 up:standby 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:31.296204+0000 mon.a (mon.0) 1085 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2022-04-28T19:54:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: audit 2022-04-28T19:54:31.296443+0000 mon.a (mon.0) 1086 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:32.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:32 smithi059 bash[61152]: cluster 2022-04-28T19:54:31.940028+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2022-04-28T19:54:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:30.974060+0000 mgr.y (mgr.24961) 285 : cluster [DBG] pgmap v457: 89 pgs: 89 active+clean; 766 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.6 MiB/s wr, 18 op/s 2022-04-28T19:54:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:31.279386+0000 mon.a (mon.0) 1081 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2022-04-28T19:54:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:31.279445+0000 mon.a (mon.0) 1082 : cluster [INF] Cluster is now healthy 2022-04-28T19:54:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:31.295867+0000 mon.a (mon.0) 1083 : cluster [DBG] mds.? [v2:172.21.15.139:6832/1824139607,v1:172.21.15.139:6833/1824139607] up:boot 2022-04-28T19:54:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:31.296032+0000 mon.a (mon.0) 1084 : cluster [DBG] fsmap foo:1 {0=foo.smithi059.tnizcj=up:active} 1 up:standby 2022-04-28T19:54:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:31.296204+0000 mon.a (mon.0) 1085 : cluster [DBG] osdmap e647: 8 total, 8 up, 8 in 2022-04-28T19:54:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: audit 2022-04-28T19:54:31.296443+0000 mon.a (mon.0) 1086 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:32 smithi139 bash[43301]: cluster 2022-04-28T19:54:31.940028+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e648: 8 total, 8 up, 8 in 2022-04-28T19:54:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:3181: done (3 left) 2022-04-28T19:54:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:3182: done (2 left) 2022-04-28T19:54:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:3183: done (1 left) 2022-04-28T19:54:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:3184: done (0 left) 2022-04-28T19:54:32.955 INFO:tasks.rados.rados.0.smithi059.stdout:3185: delete oid 15 current snap is 296 2022-04-28T19:54:32.961 INFO:tasks.rados.rados.0.smithi059.stdout:3185: done (0 left) 2022-04-28T19:54:32.961 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 296 2022-04-28T19:54:32.961 INFO:tasks.rados.rados.0.smithi059.stdout:3186: seq_num 994 ranges {1572864=655360,2228224=688128,2916352=622592,3538944=294912} 2022-04-28T19:54:32.976 INFO:tasks.rados.rados.0.smithi059.stdout:3186: writing smithi05955964-8 from 1572864 to 2228224 tid 1 2022-04-28T19:54:32.979 INFO:tasks.rados.rados.0.smithi059.stdout:3186: writing smithi05955964-8 from 2228224 to 2916352 tid 2 2022-04-28T19:54:32.983 INFO:tasks.rados.rados.0.smithi059.stdout:3186: writing smithi05955964-8 from 2916352 to 3538944 tid 3 2022-04-28T19:54:32.985 INFO:tasks.rados.rados.0.smithi059.stdout:3186: writing smithi05955964-8 from 3538944 to 3833856 tid 4 2022-04-28T19:54:32.985 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 296 2022-04-28T19:54:32.985 INFO:tasks.rados.rados.0.smithi059.stdout:3187: seq_num 995 ranges {9732096=671744,10403840=573440,10977280=778240,11755520=253952} 2022-04-28T19:54:33.042 INFO:tasks.rados.rados.0.smithi059.stdout:3187: writing smithi05955964-20 from 9732096 to 10403840 tid 1 2022-04-28T19:54:33.046 INFO:tasks.rados.rados.0.smithi059.stdout:3187: writing smithi05955964-20 from 10403840 to 10977280 tid 2 2022-04-28T19:54:33.051 INFO:tasks.rados.rados.0.smithi059.stdout:3187: writing smithi05955964-20 from 10977280 to 11755520 tid 3 2022-04-28T19:54:33.052 INFO:tasks.rados.rados.0.smithi059.stdout:3187: writing smithi05955964-20 from 11755520 to 12009472 tid 4 2022-04-28T19:54:33.053 INFO:tasks.rados.rados.0.smithi059.stdout:3188: snap_remove snap 291 2022-04-28T19:54:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:33.529Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:33.529Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:33.530Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:54:33.965 INFO:tasks.rados.rados.0.smithi059.stdout:3188: done (2 left) 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3189: delete oid 19 current snap is 296 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3186: finishing write tid 1 to smithi05955964-8 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3186: finishing write tid 2 to smithi05955964-8 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3186: finishing write tid 3 to smithi05955964-8 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3186: finishing write tid 4 to smithi05955964-8 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3186: finishing write tid 6 to smithi05955964-8 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:3186: finishing write tid 7 to smithi05955964-8 2022-04-28T19:54:33.966 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1473 (ObjNum 994 snap 296 seq_num 994) dirty exists 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3186: left oid 8 (ObjNum 994 snap 296 seq_num 994) 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: finishing write tid 1 to smithi05955964-20 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: finishing write tid 2 to smithi05955964-20 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: finishing write tid 3 to smithi05955964-20 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: finishing write tid 4 to smithi05955964-20 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: finishing write tid 6 to smithi05955964-20 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: finishing write tid 7 to smithi05955964-20 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1268 (ObjNum 995 snap 296 seq_num 995) dirty exists 2022-04-28T19:54:33.967 INFO:tasks.rados.rados.0.smithi059.stdout:3187: left oid 20 (ObjNum 995 snap 296 seq_num 995) 2022-04-28T19:54:33.968 INFO:tasks.rados.rados.0.smithi059.stdout:3186: done (2 left) 2022-04-28T19:54:33.968 INFO:tasks.rados.rados.0.smithi059.stdout:3187: done (1 left) 2022-04-28T19:54:33.968 INFO:tasks.rados.rados.0.smithi059.stdout:3189: done (0 left) 2022-04-28T19:54:33.968 INFO:tasks.rados.rados.0.smithi059.stdout:3190: snap_create 2022-04-28T19:54:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[59745]: cluster 2022-04-28T19:54:32.952481+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2022-04-28T19:54:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[59745]: cluster 2022-04-28T19:54:32.974522+0000 mgr.y (mgr.24961) 286 : cluster [DBG] pgmap v461: 89 pgs: 89 active+clean; 766 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.2 MiB/s wr, 19 op/s 2022-04-28T19:54:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[61152]: cluster 2022-04-28T19:54:32.952481+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2022-04-28T19:54:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:33 smithi059 bash[61152]: cluster 2022-04-28T19:54:32.974522+0000 mgr.y (mgr.24961) 286 : cluster [DBG] pgmap v461: 89 pgs: 89 active+clean; 766 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.2 MiB/s wr, 19 op/s 2022-04-28T19:54:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:33 smithi139 bash[43301]: cluster 2022-04-28T19:54:32.952481+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2022-04-28T19:54:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:33 smithi139 bash[43301]: cluster 2022-04-28T19:54:32.974522+0000 mgr.y (mgr.24961) 286 : cluster [DBG] pgmap v461: 89 pgs: 89 active+clean; 766 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.2 MiB/s wr, 19 op/s 2022-04-28T19:54:34.970 INFO:tasks.rados.rados.0.smithi059.stdout:3190: done (0 left) 2022-04-28T19:54:34.970 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 297 2022-04-28T19:54:34.970 INFO:tasks.rados.rados.0.smithi059.stdout:3191: seq_num 997 ranges {3833856=704512,4538368=794624,5332992=794624,6127616=16384} 2022-04-28T19:54:35.001 INFO:tasks.rados.rados.0.smithi059.stdout:3191: writing smithi05955964-8 from 3833856 to 4538368 tid 1 2022-04-28T19:54:35.005 INFO:tasks.rados.rados.0.smithi059.stdout:3191: writing smithi05955964-8 from 4538368 to 5332992 tid 2 2022-04-28T19:54:35.010 INFO:tasks.rados.rados.0.smithi059.stdout:3191: writing smithi05955964-8 from 5332992 to 6127616 tid 3 2022-04-28T19:54:35.010 INFO:tasks.rados.rados.0.smithi059.stdout:3191: writing smithi05955964-8 from 6127616 to 6144000 tid 4 2022-04-28T19:54:35.010 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 297 2022-04-28T19:54:35.011 INFO:tasks.rados.rados.0.smithi059.stdout:3192: seq_num 998 ranges {2088960=720896,2809856=679936,3489792=598016,4087808=327680} 2022-04-28T19:54:35.026 INFO:tasks.rados.rados.0.smithi059.stdout:3192: writing smithi05955964-29 from 2088960 to 2809856 tid 1 2022-04-28T19:54:35.030 INFO:tasks.rados.rados.0.smithi059.stdout:3192: writing smithi05955964-29 from 2809856 to 3489792 tid 2 2022-04-28T19:54:35.034 INFO:tasks.rados.rados.0.smithi059.stdout:3192: writing smithi05955964-29 from 3489792 to 4087808 tid 3 2022-04-28T19:54:35.037 INFO:tasks.rados.rados.0.smithi059.stdout:3192: writing smithi05955964-29 from 4087808 to 4415488 tid 4 2022-04-28T19:54:35.037 INFO:tasks.rados.rados.0.smithi059.stdout:3193: copy_from oid 12 from oid 17 current snap is 297 2022-04-28T19:54:35.037 INFO:tasks.rados.rados.0.smithi059.stdout:3191: finishing write tid 1 to smithi05955964-8 2022-04-28T19:54:35.037 INFO:tasks.rados.rados.0.smithi059.stdout:3191: finishing write tid 2 to smithi05955964-8 2022-04-28T19:54:35.037 INFO:tasks.rados.rados.0.smithi059.stdout:3191: finishing write tid 3 to smithi05955964-8 2022-04-28T19:54:35.037 INFO:tasks.rados.rados.0.smithi059.stdout:3191: finishing write tid 4 to smithi05955964-8 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3191: finishing write tid 6 to smithi05955964-8 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3191: finishing write tid 7 to smithi05955964-8 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1479 (ObjNum 997 snap 297 seq_num 997) dirty exists 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3191: left oid 8 (ObjNum 997 snap 297 seq_num 997) 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3192: finishing write tid 1 to smithi05955964-29 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3192: finishing write tid 2 to smithi05955964-29 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3191: done (2 left) 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 22 current snap is 297 2022-04-28T19:54:35.038 INFO:tasks.rados.rados.0.smithi059.stdout:3194: seq_num 999 ranges {4390912=737280,5128192=565248,5693440=761856,6455296=278528} 2022-04-28T19:54:35.066 INFO:tasks.rados.rados.0.smithi059.stdout:3194: writing smithi05955964-22 from 4390912 to 5128192 tid 1 2022-04-28T19:54:35.069 INFO:tasks.rados.rados.0.smithi059.stdout:3194: writing smithi05955964-22 from 5128192 to 5693440 tid 2 2022-04-28T19:54:35.075 INFO:tasks.rados.rados.0.smithi059.stdout:3194: writing smithi05955964-22 from 5693440 to 6455296 tid 3 2022-04-28T19:54:35.077 INFO:tasks.rados.rados.0.smithi059.stdout:3194: writing smithi05955964-22 from 6455296 to 6733824 tid 4 2022-04-28T19:54:35.077 INFO:tasks.rados.rados.0.smithi059.stdout:3195: snap_remove snap 293 2022-04-28T19:54:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: cluster 2022-04-28T19:54:33.964830+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2022-04-28T19:54:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.721204+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.727097+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.734944+0000 mon.a (mon.0) 1092 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.738013+0000 mon.a (mon.0) 1093 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.739905+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.741360+0000 mon.a (mon.0) 1095 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.744041+0000 mon.a (mon.0) 1096 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.746492+0000 mon.a (mon.0) 1097 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.753143+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.754578+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.761362+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi059.tnizcj"}]': finished 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.762664+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.768163+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi139.oefiza"}]': finished 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.769625+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.776717+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.778035+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.784571+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.785955+0000 mon.a (mon.0) 1107 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.787811+0000 mon.a (mon.0) 1108 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.797648+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.799137+0000 mon.a (mon.0) 1110 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[59745]: audit 2022-04-28T19:54:34.808864+0000 mon.a (mon.0) 1111 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: cluster 2022-04-28T19:54:33.964830+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2022-04-28T19:54:35.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.721204+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.727097+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.734944+0000 mon.a (mon.0) 1092 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.738013+0000 mon.a (mon.0) 1093 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.739905+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.741360+0000 mon.a (mon.0) 1095 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.744041+0000 mon.a (mon.0) 1096 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.746492+0000 mon.a (mon.0) 1097 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.753143+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.754578+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.761362+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi059.tnizcj"}]': finished 2022-04-28T19:54:35.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.762664+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.768163+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi139.oefiza"}]': finished 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.769625+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.776717+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.778035+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.784571+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.785955+0000 mon.a (mon.0) 1107 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.787811+0000 mon.a (mon.0) 1108 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.797648+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.799137+0000 mon.a (mon.0) 1110 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:34 smithi059 bash[61152]: audit 2022-04-28T19:54:34.808864+0000 mon.a (mon.0) 1111 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: cluster 2022-04-28T19:54:33.964830+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2022-04-28T19:54:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.721204+0000 mon.a (mon.0) 1090 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.727097+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.734944+0000 mon.a (mon.0) 1092 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.738013+0000 mon.a (mon.0) 1093 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.739905+0000 mon.a (mon.0) 1094 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.741360+0000 mon.a (mon.0) 1095 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.744041+0000 mon.a (mon.0) 1096 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.746492+0000 mon.a (mon.0) 1097 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.753143+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.754578+0000 mon.a (mon.0) 1099 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi059.tnizcj"}]: dispatch 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.761362+0000 mon.a (mon.0) 1100 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi059.tnizcj"}]': finished 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.762664+0000 mon.a (mon.0) 1101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi139.oefiza"}]: dispatch 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.768163+0000 mon.a (mon.0) 1102 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foo.smithi139.oefiza"}]': finished 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.769625+0000 mon.a (mon.0) 1103 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.776717+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.778035+0000 mon.a (mon.0) 1105 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.784571+0000 mon.a (mon.0) 1106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.785955+0000 mon.a (mon.0) 1107 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.787811+0000 mon.a (mon.0) 1108 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.797648+0000 mon.a (mon.0) 1109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.799137+0000 mon.a (mon.0) 1110 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:35.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:34 smithi139 bash[43301]: audit 2022-04-28T19:54:34.808864+0000 mon.a (mon.0) 1111 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:3195: done (3 left) 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:3196: rollback oid 40 current snap is 297 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 40 to 292 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:3192: finishing write tid 3 to smithi05955964-29 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:3192: finishing write tid 4 to smithi05955964-29 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:3192: finishing write tid 6 to smithi05955964-29 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:3192: finishing write tid 7 to smithi05955964-29 2022-04-28T19:54:35.990 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1485 (ObjNum 998 snap 297 seq_num 998) dirty exists 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3192: left oid 29 (ObjNum 998 snap 297 seq_num 998) 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3194: finishing write tid 1 to smithi05955964-22 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3194: finishing write tid 2 to smithi05955964-22 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3194: finishing write tid 3 to smithi05955964-22 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3193: finishing copy_from to smithi05955964-12 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1062 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3194: finishing write tid 4 to smithi05955964-22 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3194: finishing write tid 6 to smithi05955964-22 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3193: finishing copy_from racing read to smithi05955964-12 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:3194: finishing write tid 7 to smithi05955964-22 2022-04-28T19:54:35.991 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1461 (ObjNum 999 snap 297 seq_num 999) dirty exists 2022-04-28T19:54:35.992 INFO:tasks.rados.rados.0.smithi059.stdout:3194: left oid 22 (ObjNum 999 snap 297 seq_num 999) 2022-04-28T19:54:35.992 INFO:tasks.rados.rados.0.smithi059.stdout:3192: done (3 left) 2022-04-28T19:54:35.992 INFO:tasks.rados.rados.0.smithi059.stdout:3193: done (2 left) 2022-04-28T19:54:35.992 INFO:tasks.rados.rados.0.smithi059.stdout:3194: done (1 left) 2022-04-28T19:54:35.992 INFO:tasks.rados.rados.0.smithi059.stdout:3197: snap_remove snap 282 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cephadm 2022-04-28T19:54:34.747743+0000 mgr.y (mgr.24961) 287 : cephadm [INF] Upgrade: Setting container_image for all mds 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cephadm 2022-04-28T19:54:34.770916+0000 mgr.y (mgr.24961) 288 : cephadm [INF] Upgrade: Setting container_image for all rgw 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cephadm 2022-04-28T19:54:34.779164+0000 mgr.y (mgr.24961) 289 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cephadm 2022-04-28T19:54:34.789022+0000 mgr.y (mgr.24961) 290 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cephadm 2022-04-28T19:54:34.800448+0000 mgr.y (mgr.24961) 291 : cephadm [INF] Upgrade: Setting container_image for all nfs 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cluster 2022-04-28T19:54:34.967113+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2022-04-28T19:54:36.274 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[59745]: cluster 2022-04-28T19:54:34.974964+0000 mgr.y (mgr.24961) 292 : cluster [DBG] pgmap v464: 89 pgs: 89 active+clean; 780 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 29 op/s 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cephadm 2022-04-28T19:54:34.747743+0000 mgr.y (mgr.24961) 287 : cephadm [INF] Upgrade: Setting container_image for all mds 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cephadm 2022-04-28T19:54:34.770916+0000 mgr.y (mgr.24961) 288 : cephadm [INF] Upgrade: Setting container_image for all rgw 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cephadm 2022-04-28T19:54:34.779164+0000 mgr.y (mgr.24961) 289 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cephadm 2022-04-28T19:54:34.789022+0000 mgr.y (mgr.24961) 290 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cephadm 2022-04-28T19:54:34.800448+0000 mgr.y (mgr.24961) 291 : cephadm [INF] Upgrade: Setting container_image for all nfs 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cluster 2022-04-28T19:54:34.967113+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2022-04-28T19:54:36.275 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:35 smithi059 bash[61152]: cluster 2022-04-28T19:54:34.974964+0000 mgr.y (mgr.24961) 292 : cluster [DBG] pgmap v464: 89 pgs: 89 active+clean; 780 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 29 op/s 2022-04-28T19:54:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cephadm 2022-04-28T19:54:34.747743+0000 mgr.y (mgr.24961) 287 : cephadm [INF] Upgrade: Setting container_image for all mds 2022-04-28T19:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cephadm 2022-04-28T19:54:34.770916+0000 mgr.y (mgr.24961) 288 : cephadm [INF] Upgrade: Setting container_image for all rgw 2022-04-28T19:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cephadm 2022-04-28T19:54:34.779164+0000 mgr.y (mgr.24961) 289 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2022-04-28T19:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cephadm 2022-04-28T19:54:34.789022+0000 mgr.y (mgr.24961) 290 : cephadm [INF] Upgrade: Setting container_image for all iscsi 2022-04-28T19:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cephadm 2022-04-28T19:54:34.800448+0000 mgr.y (mgr.24961) 291 : cephadm [INF] Upgrade: Setting container_image for all nfs 2022-04-28T19:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cluster 2022-04-28T19:54:34.967113+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2022-04-28T19:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:35 smithi139 bash[43301]: cluster 2022-04-28T19:54:34.974964+0000 mgr.y (mgr.24961) 292 : cluster [DBG] pgmap v464: 89 pgs: 89 active+clean; 780 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.4 MiB/s wr, 29 op/s 2022-04-28T19:54:36.993 INFO:tasks.rados.rados.0.smithi059.stdout:3197: done (1 left) 2022-04-28T19:54:36.994 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 297 2022-04-28T19:54:36.994 INFO:tasks.rados.rados.0.smithi059.stdout:3198: seq_num 1000 ranges {4415488=753664,5169152=442368,5611520=565248,6176768=458752,6635520=139264} 2022-04-28T19:54:36.994 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:36 smithi059 bash[59745]: cluster 2022-04-28T19:54:35.986711+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2022-04-28T19:54:37.029 INFO:tasks.rados.rados.0.smithi059.stdout:3198: writing smithi05955964-9 from 4415488 to 5169152 tid 1 2022-04-28T19:54:37.030 INFO:tasks.rados.rados.0.smithi059.stdout:3198: writing smithi05955964-9 from 5169152 to 5611520 tid 2 2022-04-28T19:54:37.034 INFO:tasks.rados.rados.0.smithi059.stdout:3198: writing smithi05955964-9 from 5611520 to 6176768 tid 3 2022-04-28T19:54:37.037 INFO:tasks.rados.rados.0.smithi059.stdout:3198: writing smithi05955964-9 from 6176768 to 6635520 tid 4 2022-04-28T19:54:37.038 INFO:tasks.rados.rados.0.smithi059.stdout:3198: writing smithi05955964-9 from 6635520 to 6774784 tid 5 2022-04-28T19:54:37.038 INFO:tasks.rados.rados.0.smithi059.stdout:3199: snap_create 2022-04-28T19:54:37.038 INFO:tasks.rados.rados.0.smithi059.stdout:3196: finishing rollback tid 0 to smithi05955964-40 2022-04-28T19:54:37.038 INFO:tasks.rados.rados.0.smithi059.stdout:3196: finishing rollback tid 1 to smithi05955964-40 2022-04-28T19:54:37.038 INFO:tasks.rados.rados.0.smithi059.stdout:3196: finishing rollback tid 2 to smithi05955964-40 2022-04-28T19:54:37.038 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1131 (ObjNum 974 snap 289 seq_num 974) dirty exists 2022-04-28T19:54:37.039 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 1 to smithi05955964-9 2022-04-28T19:54:37.039 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 2 to smithi05955964-9 2022-04-28T19:54:37.040 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 3 to smithi05955964-9 2022-04-28T19:54:37.040 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 4 to smithi05955964-9 2022-04-28T19:54:37.041 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 5 to smithi05955964-9 2022-04-28T19:54:37.042 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 7 to smithi05955964-9 2022-04-28T19:54:37.043 INFO:tasks.rados.rados.0.smithi059.stdout:3198: finishing write tid 8 to smithi05955964-9 2022-04-28T19:54:37.043 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1186 (ObjNum 1000 snap 297 seq_num 1000) dirty exists 2022-04-28T19:54:37.043 INFO:tasks.rados.rados.0.smithi059.stdout:3198: left oid 9 (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:54:37.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:37 smithi059 bash[61152]: cluster 2022-04-28T19:54:35.986711+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2022-04-28T19:54:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:36 smithi139 bash[43301]: cluster 2022-04-28T19:54:35.986711+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2022-04-28T19:54:37.808 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:37 smithi059 systemd[1]: Stopping Ceph node-exporter.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:54:38.004 INFO:tasks.rados.rados.0.smithi059.stdout:3196: done (2 left) 2022-04-28T19:54:38.005 INFO:tasks.rados.rados.0.smithi059.stdout:3198: done (1 left) 2022-04-28T19:54:38.005 INFO:tasks.rados.rados.0.smithi059.stdout:3199: done (0 left) 2022-04-28T19:54:38.005 INFO:tasks.rados.rados.0.smithi059.stdout:3200: read oid 31 snap -1 2022-04-28T19:54:38.005 INFO:tasks.rados.rados.0.smithi059.stdout:3200: expect (ObjNum 935 snap 276 seq_num 935) 2022-04-28T19:54:38.005 INFO:tasks.rados.rados.0.smithi059.stdout:3201: delete oid 23 current snap is 298 2022-04-28T19:54:38.008 INFO:tasks.rados.rados.0.smithi059.stdout:3201: done (1 left) 2022-04-28T19:54:38.008 INFO:tasks.rados.rados.0.smithi059.stdout:3202: read oid 20 snap 296 2022-04-28T19:54:38.008 INFO:tasks.rados.rados.0.smithi059.stdout:3202: expect (ObjNum 995 snap 296 seq_num 995) 2022-04-28T19:54:38.008 INFO:tasks.rados.rados.0.smithi059.stdout:3203: snap_create 2022-04-28T19:54:38.048 INFO:tasks.rados.rados.0.smithi059.stdout:3200: expect (ObjNum 935 snap 276 seq_num 935) 2022-04-28T19:54:38.155 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[61152]: cluster 2022-04-28T19:54:36.975740+0000 mgr.y (mgr.24961) 293 : cluster [DBG] pgmap v466: 89 pgs: 89 active+clean; 788 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2022-04-28T19:54:38.155 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[61152]: cluster 2022-04-28T19:54:36.989345+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2022-04-28T19:54:38.158 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[61152]: cephadm 2022-04-28T19:54:37.278518+0000 mgr.y (mgr.24961) 294 : cephadm [INF] Upgrade: Updating node-exporter.a 2022-04-28T19:54:38.158 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[61152]: cephadm 2022-04-28T19:54:37.278853+0000 mgr.y (mgr.24961) 295 : cephadm [INF] Deploying daemon node-exporter.a on smithi059 2022-04-28T19:54:38.159 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[59745]: cluster 2022-04-28T19:54:36.975740+0000 mgr.y (mgr.24961) 293 : cluster [DBG] pgmap v466: 89 pgs: 89 active+clean; 788 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2022-04-28T19:54:38.159 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[59745]: cluster 2022-04-28T19:54:36.989345+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2022-04-28T19:54:38.159 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[59745]: cephadm 2022-04-28T19:54:37.278518+0000 mgr.y (mgr.24961) 294 : cephadm [INF] Upgrade: Updating node-exporter.a 2022-04-28T19:54:38.159 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[59745]: cephadm 2022-04-28T19:54:37.278853+0000 mgr.y (mgr.24961) 295 : cephadm [INF] Deploying daemon node-exporter.a on smithi059 2022-04-28T19:54:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:38 smithi139 bash[43301]: cluster 2022-04-28T19:54:36.975740+0000 mgr.y (mgr.24961) 293 : cluster [DBG] pgmap v466: 89 pgs: 89 active+clean; 788 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 5.1 MiB/s wr, 31 op/s 2022-04-28T19:54:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:38 smithi139 bash[43301]: cluster 2022-04-28T19:54:36.989345+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2022-04-28T19:54:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:38 smithi139 bash[43301]: cephadm 2022-04-28T19:54:37.278518+0000 mgr.y (mgr.24961) 294 : cephadm [INF] Upgrade: Updating node-exporter.a 2022-04-28T19:54:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:38 smithi139 bash[43301]: cephadm 2022-04-28T19:54:37.278853+0000 mgr.y (mgr.24961) 295 : cephadm [INF] Deploying daemon node-exporter.a on smithi059 2022-04-28T19:54:38.610 INFO:tasks.rados.rados.0.smithi059.stdout:3202: expect (ObjNum 995 snap 296 seq_num 995) 2022-04-28T19:54:38.761 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[70759]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-node-exporter.a 2022-04-28T19:54:38.762 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@node-exporter.a.service: Main process exited, code=exited, status=2/INVALIDARGUMENT 2022-04-28T19:54:38.762 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[70759]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-node-exporter-a 2022-04-28T19:54:38.762 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[70759]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-node-exporter.a 2022-04-28T19:54:38.762 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@node-exporter.a.service: Failed with result 'exit-code'. 2022-04-28T19:54:38.762 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 systemd[1]: Stopped Ceph node-exporter.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:38.762 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 systemd[1]: Started Ceph node-exporter.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:39.015 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[59745]: cluster 2022-04-28T19:54:38.004271+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2022-04-28T19:54:39.016 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:38 smithi059 bash[70870]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2022-04-28T19:54:39.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[61152]: cluster 2022-04-28T19:54:38.004271+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2022-04-28T19:54:39.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[61152]: audit 2022-04-28T19:54:38.795620+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:39.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[61152]: audit 2022-04-28T19:54:38.796623+0000 mon.a (mon.0) 1117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:39.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[61152]: audit 2022-04-28T19:54:38.798509+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:39.334 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[61152]: audit 2022-04-28T19:54:38.799558+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:39.335 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[61152]: audit 2022-04-28T19:54:38.802040+0000 mon.a (mon.0) 1120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:39.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[59745]: audit 2022-04-28T19:54:38.795620+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:39.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[59745]: audit 2022-04-28T19:54:38.796623+0000 mon.a (mon.0) 1117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:39.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[59745]: audit 2022-04-28T19:54:38.798509+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:39.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[59745]: audit 2022-04-28T19:54:38.799558+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:39.335 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[59745]: audit 2022-04-28T19:54:38.802040+0000 mon.a (mon.0) 1120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:39 smithi139 bash[43301]: cluster 2022-04-28T19:54:38.004271+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2022-04-28T19:54:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:39 smithi139 bash[43301]: audit 2022-04-28T19:54:38.795620+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:39 smithi139 bash[43301]: audit 2022-04-28T19:54:38.796623+0000 mon.a (mon.0) 1117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:39.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:39 smithi139 bash[43301]: audit 2022-04-28T19:54:38.798509+0000 mon.a (mon.0) 1118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:39 smithi139 bash[43301]: audit 2022-04-28T19:54:38.799558+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:39 smithi139 bash[43301]: audit 2022-04-28T19:54:38.802040+0000 mon.a (mon.0) 1120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:39.437 INFO:tasks.rados.rados.0.smithi059.stdout:3200: done (2 left) 2022-04-28T19:54:39.438 INFO:tasks.rados.rados.0.smithi059.stdout:3202: done (1 left) 2022-04-28T19:54:39.438 INFO:tasks.rados.rados.0.smithi059.stdout:3203: done (0 left) 2022-04-28T19:54:39.438 INFO:tasks.rados.rados.0.smithi059.stdout:3204: snap_remove snap 292 2022-04-28T19:54:39.591 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: v1.3.1: Pulling from prometheus/node-exporter 2022-04-28T19:54:39.591 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: aa2a8d90b84c: Pulling fs layer 2022-04-28T19:54:39.592 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: b45d31ee2d7f: Pulling fs layer 2022-04-28T19:54:39.592 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: b5db1e299295: Pulling fs layer 2022-04-28T19:54:39.973 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: b45d31ee2d7f: Verifying Checksum 2022-04-28T19:54:39.974 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: b45d31ee2d7f: Download complete 2022-04-28T19:54:39.974 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: aa2a8d90b84c: Verifying Checksum 2022-04-28T19:54:39.974 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: aa2a8d90b84c: Download complete 2022-04-28T19:54:39.974 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: b5db1e299295: Verifying Checksum 2022-04-28T19:54:39.975 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: b5db1e299295: Download complete 2022-04-28T19:54:40.041 INFO:tasks.rados.rados.0.smithi059.stdout:3204: done (0 left) 2022-04-28T19:54:40.041 INFO:tasks.rados.rados.0.smithi059.stdout:3205: delete oid 49 current snap is 299 2022-04-28T19:54:40.046 INFO:tasks.rados.rados.0.smithi059.stdout:3205: done (0 left) 2022-04-28T19:54:40.046 INFO:tasks.rados.rados.0.smithi059.stdout:3206: snap_remove snap 255 2022-04-28T19:54:40.340 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:39 smithi059 bash[70870]: aa2a8d90b84c: Pull complete 2022-04-28T19:54:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[59745]: cephadm 2022-04-28T19:54:38.800593+0000 mgr.y (mgr.24961) 296 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[59745]: cluster 2022-04-28T19:54:38.976568+0000 mgr.y (mgr.24961) 297 : cluster [DBG] pgmap v469: 89 pgs: 89 active+clean; 788 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2022-04-28T19:54:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[59745]: cluster 2022-04-28T19:54:39.021014+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2022-04-28T19:54:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[61152]: cephadm 2022-04-28T19:54:38.800593+0000 mgr.y (mgr.24961) 296 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[61152]: cluster 2022-04-28T19:54:38.976568+0000 mgr.y (mgr.24961) 297 : cluster [DBG] pgmap v469: 89 pgs: 89 active+clean; 788 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2022-04-28T19:54:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[61152]: cluster 2022-04-28T19:54:39.021014+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2022-04-28T19:54:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:40 smithi139 bash[43301]: cephadm 2022-04-28T19:54:38.800593+0000 mgr.y (mgr.24961) 296 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:40 smithi139 bash[43301]: cluster 2022-04-28T19:54:38.976568+0000 mgr.y (mgr.24961) 297 : cluster [DBG] pgmap v469: 89 pgs: 89 active+clean; 788 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.0 MiB/s wr, 5 op/s 2022-04-28T19:54:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:40 smithi139 bash[43301]: cluster 2022-04-28T19:54:39.021014+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2022-04-28T19:54:40.840 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[70870]: b45d31ee2d7f: Pull complete 2022-04-28T19:54:41.051 INFO:tasks.rados.rados.0.smithi059.stdout:3206: done (0 left) 2022-04-28T19:54:41.052 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 299 2022-04-28T19:54:41.052 INFO:tasks.rados.rados.0.smithi059.stdout:3207: seq_num 1003 ranges {0=802816,802816=499712,1302528=737280,2039808=368640} 2022-04-28T19:54:41.056 INFO:tasks.rados.rados.0.smithi059.stdout:3207: writing smithi05955964-50 from 0 to 802816 tid 1 2022-04-28T19:54:41.059 INFO:tasks.rados.rados.0.smithi059.stdout:3207: writing smithi05955964-50 from 802816 to 1302528 tid 2 2022-04-28T19:54:41.064 INFO:tasks.rados.rados.0.smithi059.stdout:3207: writing smithi05955964-50 from 1302528 to 2039808 tid 3 2022-04-28T19:54:41.066 INFO:tasks.rados.rados.0.smithi059.stdout:3207: writing smithi05955964-50 from 2039808 to 2408448 tid 4 2022-04-28T19:54:41.066 INFO:tasks.rados.rados.0.smithi059.stdout:3208: delete oid 42 current snap is 299 2022-04-28T19:54:41.066 INFO:tasks.rados.rados.0.smithi059.stdout:3207: finishing write tid 1 to smithi05955964-50 2022-04-28T19:54:41.067 INFO:tasks.rados.rados.0.smithi059.stdout:3207: finishing write tid 2 to smithi05955964-50 2022-04-28T19:54:41.071 INFO:tasks.rados.rados.0.smithi059.stdout:3207: finishing write tid 3 to smithi05955964-50 2022-04-28T19:54:41.071 INFO:tasks.rados.rados.0.smithi059.stdout:3207: finishing write tid 4 to smithi05955964-50 2022-04-28T19:54:41.072 INFO:tasks.rados.rados.0.smithi059.stdout:3207: finishing write tid 6 to smithi05955964-50 2022-04-28T19:54:41.072 INFO:tasks.rados.rados.0.smithi059.stdout:3208: done (1 left) 2022-04-28T19:54:41.072 INFO:tasks.rados.rados.0.smithi059.stdout:3209: snap_create 2022-04-28T19:54:41.073 INFO:tasks.rados.rados.0.smithi059.stdout:3207: finishing write tid 7 to smithi05955964-50 2022-04-28T19:54:41.073 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1458 (ObjNum 1003 snap 299 seq_num 1003) dirty exists 2022-04-28T19:54:41.073 INFO:tasks.rados.rados.0.smithi059.stdout:3207: left oid 50 (ObjNum 1003 snap 299 seq_num 1003) 2022-04-28T19:54:41.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[61152]: cluster 2022-04-28T19:54:40.037473+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2022-04-28T19:54:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[59745]: cluster 2022-04-28T19:54:40.037473+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2022-04-28T19:54:41.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:40 smithi059 bash[70870]: b5db1e299295: Pull complete 2022-04-28T19:54:41.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[70870]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2022-04-28T19:54:41.341 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[70870]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2022-04-28T19:54:41.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:41 smithi139 bash[43301]: cluster 2022-04-28T19:54:40.037473+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2022-04-28T19:54:41.946 INFO:tasks.rados.rados.0.smithi059.stdout:3207: done (1 left) 2022-04-28T19:54:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:3209: done (0 left) 2022-04-28T19:54:41.947 INFO:tasks.rados.rados.0.smithi059.stdout:3210: snap_create 2022-04-28T19:54:42.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[59745]: cluster 2022-04-28T19:54:40.977577+0000 mgr.y (mgr.24961) 298 : cluster [DBG] pgmap v472: 89 pgs: 89 active+clean; 807 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.8 MiB/s wr, 9 op/s 2022-04-28T19:54:42.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[59745]: cluster 2022-04-28T19:54:41.051624+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2022-04-28T19:54:42.047 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[59745]: cluster 2022-04-28T19:54:41.942264+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2022-04-28T19:54:42.048 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:41.636Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:42.048 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:41.636Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:42.048 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:41 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:41.636Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:54:42.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[61152]: cluster 2022-04-28T19:54:40.977577+0000 mgr.y (mgr.24961) 298 : cluster [DBG] pgmap v472: 89 pgs: 89 active+clean; 807 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.8 MiB/s wr, 9 op/s 2022-04-28T19:54:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[61152]: cluster 2022-04-28T19:54:41.051624+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2022-04-28T19:54:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[61152]: cluster 2022-04-28T19:54:41.942264+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2022-04-28T19:54:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:42 smithi139 bash[43301]: cluster 2022-04-28T19:54:40.977577+0000 mgr.y (mgr.24961) 298 : cluster [DBG] pgmap v472: 89 pgs: 89 active+clean; 807 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.8 MiB/s wr, 9 op/s 2022-04-28T19:54:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:42 smithi139 bash[43301]: cluster 2022-04-28T19:54:41.051624+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2022-04-28T19:54:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:42 smithi139 bash[43301]: cluster 2022-04-28T19:54:41.942264+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2022-04-28T19:54:42.957 INFO:tasks.rados.rados.0.smithi059.stdout:3210: done (0 left) 2022-04-28T19:54:42.958 INFO:tasks.rados.rados.0.smithi059.stdout:3211: rollback oid 11 current snap is 301 2022-04-28T19:54:42.958 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 299 2022-04-28T19:54:42.958 INFO:tasks.rados.rados.0.smithi059.stdout:3212: rollback oid 32 current snap is 301 2022-04-28T19:54:42.958 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 32 to 294 2022-04-28T19:54:42.958 INFO:tasks.rados.rados.0.smithi059.stdout:3213: delete oid 15 current snap is 301 2022-04-28T19:54:42.964 INFO:tasks.rados.rados.0.smithi059.stdout:3211: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:54:42.964 INFO:tasks.rados.rados.0.smithi059.stdout:3211: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:54:42.965 INFO:tasks.rados.rados.0.smithi059.stdout:3211: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:54:42.965 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1199 (ObjNum 857 snap 253 seq_num 857) dirty exists 2022-04-28T19:54:42.967 INFO:tasks.rados.rados.0.smithi059.stdout:3211: done (2 left) 2022-04-28T19:54:42.967 INFO:tasks.rados.rados.0.smithi059.stdout:3213: done (1 left) 2022-04-28T19:54:42.967 INFO:tasks.rados.rados.0.smithi059.stdout:3214: rollback oid 20 current snap is 301 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 296 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:3212: finishing rollback tid 0 to smithi05955964-32 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 301 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:3215: seq_num 1006 ranges {0=49152} 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:3215: writing smithi05955964-16 from 0 to 49152 tid 1 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:3216: snap_create 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:3212: finishing rollback tid 1 to smithi05955964-32 2022-04-28T19:54:42.968 INFO:tasks.rados.rados.0.smithi059.stdout:3212: finishing rollback tid 2 to smithi05955964-32 2022-04-28T19:54:42.969 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1462 (ObjNum 819 snap 240 seq_num 819) dirty exists 2022-04-28T19:54:42.970 INFO:tasks.rados.rados.0.smithi059.stdout:3215: finishing write tid 1 to smithi05955964-16 2022-04-28T19:54:42.970 INFO:tasks.rados.rados.0.smithi059.stdout:3215: finishing write tid 3 to smithi05955964-16 2022-04-28T19:54:42.973 INFO:tasks.rados.rados.0.smithi059.stdout:3214: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:54:42.973 INFO:tasks.rados.rados.0.smithi059.stdout:3215: finishing write tid 4 to smithi05955964-16 2022-04-28T19:54:42.973 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1446 (ObjNum 1006 snap 301 seq_num 1006) dirty exists 2022-04-28T19:54:42.974 INFO:tasks.rados.rados.0.smithi059.stdout:3215: left oid 16 (ObjNum 1006 snap 301 seq_num 1006) 2022-04-28T19:54:42.974 INFO:tasks.rados.rados.0.smithi059.stdout:3214: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:54:42.974 INFO:tasks.rados.rados.0.smithi059.stdout:3214: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:54:42.974 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1271 (ObjNum 995 snap 296 seq_num 995) dirty exists 2022-04-28T19:54:43.091 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.812Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2022-04-28T19:54:43.091 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.812Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2022-04-28T19:54:43.091 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.812Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2022-04-28T19:54:43.091 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.812Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2022-04-28T19:54:43.091 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2022-04-28T19:54:43.091 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=arp 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=bcache 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=bonding 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=btrfs 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=conntrack 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=cpu 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=cpufreq 2022-04-28T19:54:43.092 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=diskstats 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=dmi 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=edac 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=entropy 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=fibrechannel 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=filefd 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=filesystem 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=hwmon 2022-04-28T19:54:43.093 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=infiniband 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=ipvs 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=loadavg 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=mdadm 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=meminfo 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.813Z caller=node_exporter.go:115 level=info collector=netclass 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=netdev 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=netstat 2022-04-28T19:54:43.094 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=nfs 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=nfsd 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=nvme 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=os 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=pressure 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=rapl 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=schedstat 2022-04-28T19:54:43.095 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=sockstat 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=softnet 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=stat 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=tapestats 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=textfile 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=thermal_zone 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=time 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=udp_queues 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=uname 2022-04-28T19:54:43.096 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=vmstat 2022-04-28T19:54:43.097 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=xfs 2022-04-28T19:54:43.097 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:115 level=info collector=zfs 2022-04-28T19:54:43.097 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2022-04-28T19:54:43.097 INFO:journalctl@ceph.node-exporter.a.smithi059.stdout:Apr 28 19:54:42 smithi059 bash[70870]: ts=2022-04-28T19:54:42.814Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2022-04-28T19:54:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:43.530Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:43.530Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:43.530Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:54:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:3212: done (3 left) 2022-04-28T19:54:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:3214: done (2 left) 2022-04-28T19:54:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:3215: done (1 left) 2022-04-28T19:54:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:3216: done (0 left) 2022-04-28T19:54:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 302 2022-04-28T19:54:43.969 INFO:tasks.rados.rados.0.smithi059.stdout:3217: seq_num 1007 ranges {2138112=65536} 2022-04-28T19:54:43.982 INFO:tasks.rados.rados.0.smithi059.stdout:3217: writing smithi05955964-13 from 2138112 to 2203648 tid 1 2022-04-28T19:54:43.982 INFO:tasks.rados.rados.0.smithi059.stdout:3218: rollback oid 30 current snap is 302 2022-04-28T19:54:43.982 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 30 to 295 2022-04-28T19:54:43.982 INFO:tasks.rados.rados.0.smithi059.stdout:3219: snap_create 2022-04-28T19:54:43.985 INFO:tasks.rados.rados.0.smithi059.stdout:3217: finishing write tid 1 to smithi05955964-13 2022-04-28T19:54:43.986 INFO:tasks.rados.rados.0.smithi059.stdout:3217: finishing write tid 3 to smithi05955964-13 2022-04-28T19:54:43.988 INFO:tasks.rados.rados.0.smithi059.stdout:3217: finishing write tid 4 to smithi05955964-13 2022-04-28T19:54:43.988 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1070 (ObjNum 1007 snap 302 seq_num 1007) dirty exists 2022-04-28T19:54:43.989 INFO:tasks.rados.rados.0.smithi059.stdout:3217: left oid 13 (ObjNum 1007 snap 302 seq_num 1007) 2022-04-28T19:54:43.990 INFO:tasks.rados.rados.0.smithi059.stdout:3218: finishing rollback tid 0 to smithi05955964-30 2022-04-28T19:54:43.990 INFO:tasks.rados.rados.0.smithi059.stdout:3218: finishing rollback tid 1 to smithi05955964-30 2022-04-28T19:54:43.990 INFO:tasks.rados.rados.0.smithi059.stdout:3218: finishing rollback tid 2 to smithi05955964-30 2022-04-28T19:54:43.991 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1343 (ObjNum 950 snap 279 seq_num 950) dirty exists 2022-04-28T19:54:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: cluster 2022-04-28T19:54:42.954433+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2022-04-28T19:54:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: cluster 2022-04-28T19:54:42.978042+0000 mgr.y (mgr.24961) 299 : cluster [DBG] pgmap v476: 89 pgs: 89 active+clean; 807 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.2 MiB/s wr, 8 op/s 2022-04-28T19:54:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.482266+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.787671+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.799446+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.801922+0000 mon.a (mon.0) 1129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.803281+0000 mon.a (mon.0) 1130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.804399+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.805971+0000 mon.a (mon.0) 1132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.807693+0000 mon.a (mon.0) 1133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.808849+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.809933+0000 mon.a (mon.0) 1135 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.810957+0000 mon.a (mon.0) 1136 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.812000+0000 mon.a (mon.0) 1137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[59745]: audit 2022-04-28T19:54:43.813055+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: cluster 2022-04-28T19:54:42.954433+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2022-04-28T19:54:44.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: cluster 2022-04-28T19:54:42.978042+0000 mgr.y (mgr.24961) 299 : cluster [DBG] pgmap v476: 89 pgs: 89 active+clean; 807 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.2 MiB/s wr, 8 op/s 2022-04-28T19:54:44.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.482266+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:44.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.787671+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.799446+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.801922+0000 mon.a (mon.0) 1129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.803281+0000 mon.a (mon.0) 1130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.804399+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.805971+0000 mon.a (mon.0) 1132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.807693+0000 mon.a (mon.0) 1133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.808849+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.809933+0000 mon.a (mon.0) 1135 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.810957+0000 mon.a (mon.0) 1136 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.812000+0000 mon.a (mon.0) 1137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:43 smithi059 bash[61152]: audit 2022-04-28T19:54:43.813055+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: cluster 2022-04-28T19:54:42.954433+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: cluster 2022-04-28T19:54:42.978042+0000 mgr.y (mgr.24961) 299 : cluster [DBG] pgmap v476: 89 pgs: 89 active+clean; 807 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.2 MiB/s wr, 8 op/s 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.482266+0000 mon.a (mon.0) 1126 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.787671+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.799446+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.801922+0000 mon.a (mon.0) 1129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.803281+0000 mon.a (mon.0) 1130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.804399+0000 mon.a (mon.0) 1131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.805971+0000 mon.a (mon.0) 1132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.807693+0000 mon.a (mon.0) 1133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.808849+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.809933+0000 mon.a (mon.0) 1135 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.810957+0000 mon.a (mon.0) 1136 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.812000+0000 mon.a (mon.0) 1137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:43 smithi139 bash[43301]: audit 2022-04-28T19:54:43.813055+0000 mon.a (mon.0) 1138 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:44.973 INFO:tasks.rados.rados.0.smithi059.stdout:3217: done (2 left) 2022-04-28T19:54:44.973 INFO:tasks.rados.rados.0.smithi059.stdout:3218: done (1 left) 2022-04-28T19:54:44.973 INFO:tasks.rados.rados.0.smithi059.stdout:3219: done (0 left) 2022-04-28T19:54:44.973 INFO:tasks.rados.rados.0.smithi059.stdout:3220: read oid 41 snap -1 2022-04-28T19:54:44.973 INFO:tasks.rados.rados.0.smithi059.stdout:3220: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:54:44.973 INFO:tasks.rados.rados.0.smithi059.stdout:3221: copy_from oid 25 from oid 40 current snap is 303 2022-04-28T19:54:44.974 INFO:tasks.rados.rados.0.smithi059.stdout:3222: rmattr oid 2 current snap is 303 2022-04-28T19:54:44.974 INFO:tasks.rados.rados.0.smithi059.stdout:3222: done (2 left) 2022-04-28T19:54:44.974 INFO:tasks.rados.rados.0.smithi059.stdout:3223: delete oid 10 current snap is 303 2022-04-28T19:54:44.978 INFO:tasks.rados.rados.0.smithi059.stdout:3220: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:54:45.009 INFO:tasks.rados.rados.0.smithi059.stdout:3221: finishing copy_from to smithi05955964-25 2022-04-28T19:54:45.010 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1131 (ObjNum 974 snap 289 seq_num 974) dirty exists 2022-04-28T19:54:45.010 INFO:tasks.rados.rados.0.smithi059.stdout:3221: finishing copy_from racing read to smithi05955964-25 2022-04-28T19:54:45.010 INFO:tasks.rados.rados.0.smithi059.stdout:3220: done (2 left) 2022-04-28T19:54:45.010 INFO:tasks.rados.rados.0.smithi059.stdout:3221: done (1 left) 2022-04-28T19:54:45.011 INFO:tasks.rados.rados.0.smithi059.stdout:3223: done (0 left) 2022-04-28T19:54:45.011 INFO:tasks.rados.rados.0.smithi059.stdout:3224: delete oid 35 current snap is 303 2022-04-28T19:54:45.012 INFO:tasks.rados.rados.0.smithi059.stdout:3224: done (0 left) 2022-04-28T19:54:45.012 INFO:tasks.rados.rados.0.smithi059.stdout:3225: snap_create 2022-04-28T19:54:45.275 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:44 smithi139 bash[43301]: cluster 2022-04-28T19:54:43.968925+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2022-04-28T19:54:45.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:44 smithi059 bash[59745]: cluster 2022-04-28T19:54:43.968925+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2022-04-28T19:54:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:44 smithi059 bash[61152]: cluster 2022-04-28T19:54:43.968925+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2022-04-28T19:54:45.992 INFO:tasks.rados.rados.0.smithi059.stdout:3225: done (0 left) 2022-04-28T19:54:45.992 INFO:tasks.rados.rados.0.smithi059.stdout:3226: snap_remove snap 302 2022-04-28T19:54:46.338 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:45 smithi139 bash[43301]: cluster 2022-04-28T19:54:44.969002+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2022-04-28T19:54:46.338 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:45 smithi139 bash[43301]: cluster 2022-04-28T19:54:44.978495+0000 mgr.y (mgr.24961) 300 : cluster [DBG] pgmap v479: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 520 B/s rd, 610 KiB/s wr, 4 op/s 2022-04-28T19:54:46.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:45 smithi059 bash[61152]: cluster 2022-04-28T19:54:44.969002+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2022-04-28T19:54:46.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:45 smithi059 bash[61152]: cluster 2022-04-28T19:54:44.978495+0000 mgr.y (mgr.24961) 300 : cluster [DBG] pgmap v479: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 520 B/s rd, 610 KiB/s wr, 4 op/s 2022-04-28T19:54:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:45 smithi059 bash[59745]: cluster 2022-04-28T19:54:44.969002+0000 mon.a (mon.0) 1140 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2022-04-28T19:54:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:45 smithi059 bash[59745]: cluster 2022-04-28T19:54:44.978495+0000 mgr.y (mgr.24961) 300 : cluster [DBG] pgmap v479: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 520 B/s rd, 610 KiB/s wr, 4 op/s 2022-04-28T19:54:46.618 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:46 smithi139 systemd[1]: Stopping Ceph node-exporter.b for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:54:47.003 INFO:tasks.rados.rados.0.smithi059.stdout:3226: done (0 left) 2022-04-28T19:54:47.003 INFO:tasks.rados.rados.0.smithi059.stdout:3227: snap_create 2022-04-28T19:54:47.293 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:46 smithi139 bash[43301]: cluster 2022-04-28T19:54:45.991871+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2022-04-28T19:54:47.355 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:46 smithi139 bash[43301]: cephadm 2022-04-28T19:54:46.132577+0000 mgr.y (mgr.24961) 301 : cephadm [INF] Upgrade: Updating node-exporter.b 2022-04-28T19:54:47.356 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:46 smithi139 bash[43301]: cephadm 2022-04-28T19:54:46.132924+0000 mgr.y (mgr.24961) 302 : cephadm [INF] Deploying daemon node-exporter.b on smithi139 2022-04-28T19:54:47.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:46 smithi059 bash[61152]: cluster 2022-04-28T19:54:45.991871+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2022-04-28T19:54:47.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:46 smithi059 bash[61152]: cephadm 2022-04-28T19:54:46.132577+0000 mgr.y (mgr.24961) 301 : cephadm [INF] Upgrade: Updating node-exporter.b 2022-04-28T19:54:47.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:46 smithi059 bash[61152]: cephadm 2022-04-28T19:54:46.132924+0000 mgr.y (mgr.24961) 302 : cephadm [INF] Deploying daemon node-exporter.b on smithi139 2022-04-28T19:54:47.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:46 smithi059 bash[59745]: cluster 2022-04-28T19:54:45.991871+0000 mon.a (mon.0) 1141 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2022-04-28T19:54:47.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:46 smithi059 bash[59745]: cephadm 2022-04-28T19:54:46.132577+0000 mgr.y (mgr.24961) 301 : cephadm [INF] Upgrade: Updating node-exporter.b 2022-04-28T19:54:47.357 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:46 smithi059 bash[59745]: cephadm 2022-04-28T19:54:46.132924+0000 mgr.y (mgr.24961) 302 : cephadm [INF] Deploying daemon node-exporter.b on smithi139 2022-04-28T19:54:47.560 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[52475]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-node-exporter.b 2022-04-28T19:54:47.561 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@node-exporter.b.service: Main process exited, code=exited, status=2/INVALIDARGUMENT 2022-04-28T19:54:47.561 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[52475]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-node-exporter-b 2022-04-28T19:54:47.561 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[52475]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-node-exporter.b 2022-04-28T19:54:47.561 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 systemd[1]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c@node-exporter.b.service: Failed with result 'exit-code'. 2022-04-28T19:54:47.561 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 systemd[1]: Stopped Ceph node-exporter.b for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:47.562 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 systemd[1]: Started Ceph node-exporter.b for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:47.868 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[52578]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2022-04-28T19:54:48.012 INFO:tasks.rados.rados.0.smithi059.stdout:3227: done (0 left) 2022-04-28T19:54:48.012 INFO:tasks.rados.rados.0.smithi059.stdout:3228: copy_from oid 13 from oid 23 current snap is 305 2022-04-28T19:54:48.012 INFO:tasks.rados.rados.0.smithi059.stdout:3229: delete oid 7 current snap is 305 2022-04-28T19:54:48.013 INFO:tasks.rados.rados.0.smithi059.stdout:3228: finishing copy_from to smithi05955964-13 2022-04-28T19:54:48.013 INFO:tasks.rados.rados.0.smithi059.stdout:3228: got expected ENOENT (src dne) 2022-04-28T19:54:48.013 INFO:tasks.rados.rados.0.smithi059.stdout:3228: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:54:48.016 INFO:tasks.rados.rados.0.smithi059.stdout:3228: done (1 left) 2022-04-28T19:54:48.017 INFO:tasks.rados.rados.0.smithi059.stdout:3229: done (0 left) 2022-04-28T19:54:48.017 INFO:tasks.rados.rados.0.smithi059.stdout:3230: snap_create 2022-04-28T19:54:48.311 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: cluster 2022-04-28T19:54:46.979201+0000 mgr.y (mgr.24961) 303 : cluster [DBG] pgmap v481: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 508 B/s rd, 596 KiB/s wr, 4 op/s 2022-04-28T19:54:48.311 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: cluster 2022-04-28T19:54:46.998801+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2022-04-28T19:54:48.311 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: audit 2022-04-28T19:54:47.601517+0000 mon.a (mon.0) 1143 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:48.311 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: audit 2022-04-28T19:54:47.603307+0000 mon.a (mon.0) 1144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:48.311 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: audit 2022-04-28T19:54:47.605910+0000 mon.a (mon.0) 1145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:48.311 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: audit 2022-04-28T19:54:47.607663+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:48.312 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:47 smithi139 bash[43301]: audit 2022-04-28T19:54:47.610650+0000 mon.a (mon.0) 1147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:48.312 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: v1.3.1: Pulling from prometheus/node-exporter 2022-04-28T19:54:48.449 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:47 smithi059 bash[59745]: cluster 2022-04-28T19:54:46.979201+0000 mgr.y (mgr.24961) 303 : cluster [DBG] pgmap v481: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 508 B/s rd, 596 KiB/s wr, 4 op/s 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:47 smithi059 bash[59745]: cluster 2022-04-28T19:54:46.998801+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[59745]: audit 2022-04-28T19:54:47.601517+0000 mon.a (mon.0) 1143 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[59745]: audit 2022-04-28T19:54:47.603307+0000 mon.a (mon.0) 1144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[59745]: audit 2022-04-28T19:54:47.605910+0000 mon.a (mon.0) 1145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[59745]: audit 2022-04-28T19:54:47.607663+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[59745]: audit 2022-04-28T19:54:47.610650+0000 mon.a (mon.0) 1147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: cluster 2022-04-28T19:54:46.979201+0000 mgr.y (mgr.24961) 303 : cluster [DBG] pgmap v481: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 508 B/s rd, 596 KiB/s wr, 4 op/s 2022-04-28T19:54:48.450 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: cluster 2022-04-28T19:54:46.998801+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2022-04-28T19:54:48.451 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: audit 2022-04-28T19:54:47.601517+0000 mon.a (mon.0) 1143 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:48.451 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: audit 2022-04-28T19:54:47.603307+0000 mon.a (mon.0) 1144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:48.451 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: audit 2022-04-28T19:54:47.605910+0000 mon.a (mon.0) 1145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:54:48.451 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: audit 2022-04-28T19:54:47.607663+0000 mon.a (mon.0) 1146 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:54:48.451 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:48 smithi059 bash[61152]: audit 2022-04-28T19:54:47.610650+0000 mon.a (mon.0) 1147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:54:48.599 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: aa2a8d90b84c: Pulling fs layer 2022-04-28T19:54:48.599 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: b45d31ee2d7f: Pulling fs layer 2022-04-28T19:54:48.599 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: b5db1e299295: Pulling fs layer 2022-04-28T19:54:48.599 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: aa2a8d90b84c: Verifying Checksum 2022-04-28T19:54:48.599 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: aa2a8d90b84c: Download complete 2022-04-28T19:54:48.600 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: b45d31ee2d7f: Download complete 2022-04-28T19:54:48.868 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: b5db1e299295: Verifying Checksum 2022-04-28T19:54:48.868 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:48 smithi139 bash[52578]: aa2a8d90b84c: Pull complete 2022-04-28T19:54:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:3230: done (0 left) 2022-04-28T19:54:49.018 INFO:tasks.rados.rados.0.smithi059.stdout:3231: snap_create 2022-04-28T19:54:49.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:49 smithi059 bash[61152]: cephadm 2022-04-28T19:54:47.609201+0000 mgr.y (mgr.24961) 304 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:49.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:49 smithi059 bash[61152]: cluster 2022-04-28T19:54:48.010281+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2022-04-28T19:54:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:49 smithi059 bash[59745]: cephadm 2022-04-28T19:54:47.609201+0000 mgr.y (mgr.24961) 304 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:49 smithi059 bash[59745]: cluster 2022-04-28T19:54:48.010281+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2022-04-28T19:54:49.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:49 smithi139 bash[43301]: cephadm 2022-04-28T19:54:47.609201+0000 mgr.y (mgr.24961) 304 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:54:49.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:49 smithi139 bash[43301]: cluster 2022-04-28T19:54:48.010281+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2022-04-28T19:54:49.369 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:49 smithi139 bash[52578]: b45d31ee2d7f: Pull complete 2022-04-28T19:54:49.868 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:49 smithi139 bash[52578]: b5db1e299295: Pull complete 2022-04-28T19:54:49.868 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:49 smithi139 bash[52578]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2022-04-28T19:54:49.868 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:49 smithi139 bash[52578]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2022-04-28T19:54:50.038 INFO:tasks.rados.rados.0.smithi059.stdout:3231: done (0 left) 2022-04-28T19:54:50.038 INFO:tasks.rados.rados.0.smithi059.stdout:3232: snap_create 2022-04-28T19:54:50.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:50 smithi059 bash[59745]: cluster 2022-04-28T19:54:48.980085+0000 mgr.y (mgr.24961) 305 : cluster [DBG] pgmap v484: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 510 B/s rd, 28 KiB/s wr, 2 op/s 2022-04-28T19:54:50.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:50 smithi059 bash[59745]: cluster 2022-04-28T19:54:49.015699+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2022-04-28T19:54:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:50 smithi059 bash[61152]: cluster 2022-04-28T19:54:48.980085+0000 mgr.y (mgr.24961) 305 : cluster [DBG] pgmap v484: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 510 B/s rd, 28 KiB/s wr, 2 op/s 2022-04-28T19:54:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:50 smithi059 bash[61152]: cluster 2022-04-28T19:54:49.015699+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2022-04-28T19:54:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:50 smithi139 bash[43301]: cluster 2022-04-28T19:54:48.980085+0000 mgr.y (mgr.24961) 305 : cluster [DBG] pgmap v484: 89 pgs: 89 active+clean; 810 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 510 B/s rd, 28 KiB/s wr, 2 op/s 2022-04-28T19:54:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:50 smithi139 bash[43301]: cluster 2022-04-28T19:54:49.015699+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2022-04-28T19:54:51.042 INFO:tasks.rados.rados.0.smithi059.stdout:3232: done (0 left) 2022-04-28T19:54:51.042 INFO:tasks.rados.rados.0.smithi059.stdout:3233: snap_create 2022-04-28T19:54:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:51 smithi059 bash[61152]: cluster 2022-04-28T19:54:50.036544+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2022-04-28T19:54:51.383 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:51 smithi059 bash[59745]: cluster 2022-04-28T19:54:50.036544+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2022-04-28T19:54:51.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[43301]: cluster 2022-04-28T19:54:50.036544+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2022-04-28T19:54:51.869 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.390Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2022-04-28T19:54:51.959 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.390Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2022-04-28T19:54:51.959 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2022-04-28T19:54:51.960 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2022-04-28T19:54:51.960 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2022-04-28T19:54:51.960 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=arp 2022-04-28T19:54:51.960 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=bcache 2022-04-28T19:54:51.960 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=bonding 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=btrfs 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=conntrack 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=cpu 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=cpufreq 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=diskstats 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=dmi 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=edac 2022-04-28T19:54:51.965 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=entropy 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=fibrechannel 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.392Z caller=node_exporter.go:115 level=info collector=filefd 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=filesystem 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=hwmon 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=infiniband 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=ipvs 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=loadavg 2022-04-28T19:54:51.966 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=mdadm 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=meminfo 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=netclass 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=netdev 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=netstat 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=nfs 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=nfsd 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=nvme 2022-04-28T19:54:51.967 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=os 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=pressure 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=rapl 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=schedstat 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=sockstat 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=softnet 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=stat 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=tapestats 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=textfile 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=thermal_zone 2022-04-28T19:54:51.968 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=time 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=udp_queues 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=uname 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=vmstat 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=xfs 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:115 level=info collector=zfs 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2022-04-28T19:54:51.969 INFO:journalctl@ceph.node-exporter.b.smithi139.stdout:Apr 28 19:54:51 smithi139 bash[52578]: ts=2022-04-28T19:54:51.393Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2022-04-28T19:54:51.970 INFO:tasks.rados.rados.0.smithi059.stdout:3233: done (0 left) 2022-04-28T19:54:51.970 INFO:tasks.rados.rados.0.smithi059.stdout:3234: read oid 3 snap 296 2022-04-28T19:54:51.970 INFO:tasks.rados.rados.0.smithi059.stdout:3234: expect (ObjNum 921 snap 275 seq_num 921) 2022-04-28T19:54:51.971 INFO:tasks.rados.rados.0.smithi059.stdout:3235: snap_create 2022-04-28T19:54:51.971 INFO:tasks.rados.rados.0.smithi059.stdout:3234: expect (ObjNum 921 snap 275 seq_num 921) 2022-04-28T19:54:52.054 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:51.636Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:52.054 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:51.636Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:52.055 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:51 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:51.636Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:54:52.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:52 smithi059 bash[59745]: cluster 2022-04-28T19:54:50.981183+0000 mgr.y (mgr.24961) 306 : cluster [DBG] pgmap v487: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 854 KiB/s rd, 940 KiB/s wr, 3 op/s 2022-04-28T19:54:52.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:52 smithi059 bash[59745]: cluster 2022-04-28T19:54:51.041937+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2022-04-28T19:54:52.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:52 smithi059 bash[59745]: cluster 2022-04-28T19:54:51.950293+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2022-04-28T19:54:52.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:52 smithi059 bash[61152]: cluster 2022-04-28T19:54:50.981183+0000 mgr.y (mgr.24961) 306 : cluster [DBG] pgmap v487: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 854 KiB/s rd, 940 KiB/s wr, 3 op/s 2022-04-28T19:54:52.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:52 smithi059 bash[61152]: cluster 2022-04-28T19:54:51.041937+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2022-04-28T19:54:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:52 smithi059 bash[61152]: cluster 2022-04-28T19:54:51.950293+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2022-04-28T19:54:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:52 smithi139 bash[43301]: cluster 2022-04-28T19:54:50.981183+0000 mgr.y (mgr.24961) 306 : cluster [DBG] pgmap v487: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 854 KiB/s rd, 940 KiB/s wr, 3 op/s 2022-04-28T19:54:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:52 smithi139 bash[43301]: cluster 2022-04-28T19:54:51.041937+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2022-04-28T19:54:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:52 smithi139 bash[43301]: cluster 2022-04-28T19:54:51.950293+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2022-04-28T19:54:52.961 INFO:tasks.rados.rados.0.smithi059.stdout:3234: done (1 left) 2022-04-28T19:54:52.962 INFO:tasks.rados.rados.0.smithi059.stdout:3235: done (0 left) 2022-04-28T19:54:52.962 INFO:tasks.rados.rados.0.smithi059.stdout:3236: read oid 48 snap 304 2022-04-28T19:54:52.962 INFO:tasks.rados.rados.0.smithi059.stdout:3236: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:54:52.962 INFO:tasks.rados.rados.0.smithi059.stdout:3237: snap_create 2022-04-28T19:54:52.982 INFO:tasks.rados.rados.0.smithi059.stdout:3236: expect (ObjNum 646 snap 184 seq_num 646) 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.049749+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.064223+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.069843+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.072087+0000 mon.a (mon.0) 1156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.073288+0000 mon.a (mon.0) 1157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.074203+0000 mon.a (mon.0) 1158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.075728+0000 mon.a (mon.0) 1159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.077452+0000 mon.a (mon.0) 1160 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.078382+0000 mon.a (mon.0) 1161 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.079302+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.080207+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.081141+0000 mon.a (mon.0) 1164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.082088+0000 mon.a (mon.0) 1165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: audit 2022-04-28T19:54:52.083279+0000 mon.a (mon.0) 1166 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[61152]: cluster 2022-04-28T19:54:52.959866+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2022-04-28T19:54:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.049749+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.064223+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.069843+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.072087+0000 mon.a (mon.0) 1156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.073288+0000 mon.a (mon.0) 1157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.074203+0000 mon.a (mon.0) 1158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.075728+0000 mon.a (mon.0) 1159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.077452+0000 mon.a (mon.0) 1160 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.078382+0000 mon.a (mon.0) 1161 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.079302+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.080207+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.081141+0000 mon.a (mon.0) 1164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.082088+0000 mon.a (mon.0) 1165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: audit 2022-04-28T19:54:52.083279+0000 mon.a (mon.0) 1166 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[59745]: cluster 2022-04-28T19:54:52.959866+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2022-04-28T19:54:53.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.049749+0000 mon.a (mon.0) 1153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:53.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.064223+0000 mon.a (mon.0) 1154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:54:53.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.069843+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:54:53.364 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.072087+0000 mon.a (mon.0) 1156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.073288+0000 mon.a (mon.0) 1157 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.074203+0000 mon.a (mon.0) 1158 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.075728+0000 mon.a (mon.0) 1159 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.077452+0000 mon.a (mon.0) 1160 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.078382+0000 mon.a (mon.0) 1161 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.079302+0000 mon.a (mon.0) 1162 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.080207+0000 mon.a (mon.0) 1163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.081141+0000 mon.a (mon.0) 1164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.082088+0000 mon.a (mon.0) 1165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.365 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: audit 2022-04-28T19:54:52.083279+0000 mon.a (mon.0) 1166 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:54:53.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:53 smithi139 bash[43301]: cluster 2022-04-28T19:54:52.959866+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2022-04-28T19:54:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:53.530Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:54:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:53.530Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:54:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:54:53 smithi059 bash[42244]: level=error ts=2022-04-28T19:54:53.530Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:54:53.970 INFO:tasks.rados.rados.0.smithi059.stdout:3236: done (1 left) 2022-04-28T19:54:53.970 INFO:tasks.rados.rados.0.smithi059.stdout:3237: done (0 left) 2022-04-28T19:54:53.970 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 311 2022-04-28T19:54:53.971 INFO:tasks.rados.rados.0.smithi059.stdout:3238: seq_num 1011 ranges {17661952=139264} 2022-04-28T19:54:54.075 INFO:tasks.rados.rados.0.smithi059.stdout:3238: writing smithi05955964-45 from 17661952 to 17801216 tid 1 2022-04-28T19:54:54.075 INFO:tasks.rados.rados.0.smithi059.stdout:3239: rollback oid 38 current snap is 311 2022-04-28T19:54:54.075 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 310 2022-04-28T19:54:54.076 INFO:tasks.rados.rados.0.smithi059.stdout:3240: rollback oid 1 current snap is 311 2022-04-28T19:54:54.076 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 301 2022-04-28T19:54:54.076 INFO:tasks.rados.rados.0.smithi059.stdout:3241: setattr oid 22 current snap is 311 2022-04-28T19:54:54.078 INFO:tasks.rados.rados.0.smithi059.stdout:3242: snap_create 2022-04-28T19:54:54.083 INFO:tasks.rados.rados.0.smithi059.stdout:3239: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:54:54.083 INFO:tasks.rados.rados.0.smithi059.stdout:3239: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:54:54.083 INFO:tasks.rados.rados.0.smithi059.stdout:3239: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:54:54.083 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1202 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:54:54.084 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1462 (ObjNum 999 snap 297 seq_num 999) dirty exists 2022-04-28T19:54:54.085 INFO:tasks.rados.rados.0.smithi059.stdout:3238: finishing write tid 1 to smithi05955964-45 2022-04-28T19:54:54.086 INFO:tasks.rados.rados.0.smithi059.stdout:3238: finishing write tid 3 to smithi05955964-45 2022-04-28T19:54:54.087 INFO:tasks.rados.rados.0.smithi059.stdout:3240: finishing rollback tid 0 to smithi05955964-1 2022-04-28T19:54:54.087 INFO:tasks.rados.rados.0.smithi059.stdout:3240: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:54:54.087 INFO:tasks.rados.rados.0.smithi059.stdout:3240: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:54:54.087 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1135 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:54:54.088 INFO:tasks.rados.rados.0.smithi059.stdout:3238: finishing write tid 4 to smithi05955964-45 2022-04-28T19:54:54.088 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1273 (ObjNum 1011 snap 311 seq_num 1011) dirty exists 2022-04-28T19:54:54.088 INFO:tasks.rados.rados.0.smithi059.stdout:3238: left oid 45 (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:54:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:54 smithi059 bash[61152]: cluster 2022-04-28T19:54:52.981867+0000 mgr.y (mgr.24961) 307 : cluster [DBG] pgmap v491: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 861 KiB/s rd, 931 KiB/s wr, 1 op/s 2022-04-28T19:54:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:54 smithi059 bash[61152]: cluster 2022-04-28T19:54:53.967512+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2022-04-28T19:54:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:54 smithi059 bash[59745]: cluster 2022-04-28T19:54:52.981867+0000 mgr.y (mgr.24961) 307 : cluster [DBG] pgmap v491: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 861 KiB/s rd, 931 KiB/s wr, 1 op/s 2022-04-28T19:54:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:54 smithi059 bash[59745]: cluster 2022-04-28T19:54:53.967512+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2022-04-28T19:54:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:54 smithi139 bash[43301]: cluster 2022-04-28T19:54:52.981867+0000 mgr.y (mgr.24961) 307 : cluster [DBG] pgmap v491: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 861 KiB/s rd, 931 KiB/s wr, 1 op/s 2022-04-28T19:54:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:54 smithi139 bash[43301]: cluster 2022-04-28T19:54:53.967512+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2022-04-28T19:54:55.079 INFO:tasks.rados.rados.0.smithi059.stdout:3238: done (4 left) 2022-04-28T19:54:55.079 INFO:tasks.rados.rados.0.smithi059.stdout:3239: done (3 left) 2022-04-28T19:54:55.079 INFO:tasks.rados.rados.0.smithi059.stdout:3240: done (2 left) 2022-04-28T19:54:55.080 INFO:tasks.rados.rados.0.smithi059.stdout:3241: done (1 left) 2022-04-28T19:54:55.080 INFO:tasks.rados.rados.0.smithi059.stdout:3242: done (0 left) 2022-04-28T19:54:55.080 INFO:tasks.rados.rados.0.smithi059.stdout:3243: rollback oid 29 current snap is 312 2022-04-28T19:54:55.080 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 287 2022-04-28T19:54:55.081 INFO:tasks.rados.rados.0.smithi059.stdout:3244: snap_create 2022-04-28T19:54:55.086 INFO:tasks.rados.rados.0.smithi059.stdout:3243: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:54:55.086 INFO:tasks.rados.rados.0.smithi059.stdout:3243: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:54:55.087 INFO:tasks.rados.rados.0.smithi059.stdout:3243: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:54:55.087 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1491 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:54:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:55 smithi059 bash[61152]: cephadm 2022-04-28T19:54:54.322731+0000 mgr.y (mgr.24961) 308 : cephadm [INF] Upgrade: Updating prometheus.a 2022-04-28T19:54:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:55 smithi059 bash[61152]: cephadm 2022-04-28T19:54:54.344275+0000 mgr.y (mgr.24961) 309 : cephadm [INF] Deploying daemon prometheus.a on smithi139 2022-04-28T19:54:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:55 smithi059 bash[59745]: cephadm 2022-04-28T19:54:54.322731+0000 mgr.y (mgr.24961) 308 : cephadm [INF] Upgrade: Updating prometheus.a 2022-04-28T19:54:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:55 smithi059 bash[59745]: cephadm 2022-04-28T19:54:54.344275+0000 mgr.y (mgr.24961) 309 : cephadm [INF] Deploying daemon prometheus.a on smithi139 2022-04-28T19:54:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:55 smithi139 bash[43301]: cephadm 2022-04-28T19:54:54.322731+0000 mgr.y (mgr.24961) 308 : cephadm [INF] Upgrade: Updating prometheus.a 2022-04-28T19:54:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:55 smithi139 bash[43301]: cephadm 2022-04-28T19:54:54.344275+0000 mgr.y (mgr.24961) 309 : cephadm [INF] Deploying daemon prometheus.a on smithi139 2022-04-28T19:54:56.081 INFO:tasks.rados.rados.0.smithi059.stdout:3243: done (1 left) 2022-04-28T19:54:56.081 INFO:tasks.rados.rados.0.smithi059.stdout:3244: done (0 left) 2022-04-28T19:54:56.081 INFO:tasks.rados.rados.0.smithi059.stdout:3245: delete oid 19 current snap is 313 2022-04-28T19:54:56.084 INFO:tasks.rados.rados.0.smithi059.stdout:3245: done (0 left) 2022-04-28T19:54:56.084 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 313 2022-04-28T19:54:56.084 INFO:tasks.rados.rados.0.smithi059.stdout:3246: seq_num 1013 ranges {0=172032} 2022-04-28T19:54:56.085 INFO:tasks.rados.rados.0.smithi059.stdout:3246: writing smithi05955964-44 from 0 to 172032 tid 1 2022-04-28T19:54:56.085 INFO:tasks.rados.rados.0.smithi059.stdout:3247: snap_create 2022-04-28T19:54:56.090 INFO:tasks.rados.rados.0.smithi059.stdout:3246: finishing write tid 1 to smithi05955964-44 2022-04-28T19:54:56.090 INFO:tasks.rados.rados.0.smithi059.stdout:3246: finishing write tid 3 to smithi05955964-44 2022-04-28T19:54:56.091 INFO:tasks.rados.rados.0.smithi059.stdout:3246: finishing write tid 4 to smithi05955964-44 2022-04-28T19:54:56.091 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1188 (ObjNum 1013 snap 313 seq_num 1013) dirty exists 2022-04-28T19:54:56.091 INFO:tasks.rados.rados.0.smithi059.stdout:3246: left oid 44 (ObjNum 1013 snap 313 seq_num 1013) 2022-04-28T19:54:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:56 smithi059 bash[61152]: cluster 2022-04-28T19:54:54.982767+0000 mgr.y (mgr.24961) 310 : cluster [DBG] pgmap v493: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 34 KiB/s wr, 2 op/s 2022-04-28T19:54:56.460 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:56 smithi059 bash[61152]: cluster 2022-04-28T19:54:55.075236+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2022-04-28T19:54:56.461 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:56 smithi059 bash[59745]: cluster 2022-04-28T19:54:54.982767+0000 mgr.y (mgr.24961) 310 : cluster [DBG] pgmap v493: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 34 KiB/s wr, 2 op/s 2022-04-28T19:54:56.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:56 smithi059 bash[59745]: cluster 2022-04-28T19:54:55.075236+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2022-04-28T19:54:56.554 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:56 smithi139 bash[43301]: cluster 2022-04-28T19:54:54.982767+0000 mgr.y (mgr.24961) 310 : cluster [DBG] pgmap v493: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 34 KiB/s wr, 2 op/s 2022-04-28T19:54:56.554 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:56 smithi139 bash[43301]: cluster 2022-04-28T19:54:55.075236+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2022-04-28T19:54:57.096 INFO:tasks.rados.rados.0.smithi059.stdout:3246: done (1 left) 2022-04-28T19:54:57.097 INFO:tasks.rados.rados.0.smithi059.stdout:3247: done (0 left) 2022-04-28T19:54:57.097 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 314 2022-04-28T19:54:57.097 INFO:tasks.rados.rados.0.smithi059.stdout:3248: seq_num 1014 ranges {0=188416} 2022-04-28T19:54:57.097 INFO:tasks.rados.rados.0.smithi059.stdout:3248: writing smithi05955964-23 from 0 to 188416 tid 1 2022-04-28T19:54:57.098 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 314 2022-04-28T19:54:57.098 INFO:tasks.rados.rados.0.smithi059.stdout:3249: seq_num 1015 ranges {2539520=204800} 2022-04-28T19:54:57.118 INFO:tasks.rados.rados.0.smithi059.stdout:3249: writing smithi05955964-3 from 2539520 to 2744320 tid 1 2022-04-28T19:54:57.119 INFO:tasks.rados.rados.0.smithi059.stdout:3250: read oid 34 snap -1 2022-04-28T19:54:57.119 INFO:tasks.rados.rados.0.smithi059.stdout:3250: expect deleted 2022-04-28T19:54:57.119 INFO:tasks.rados.rados.0.smithi059.stdout:3251: rollback oid 19 current snap is 314 2022-04-28T19:54:57.119 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 19 to 308 2022-04-28T19:54:57.119 INFO:tasks.rados.rados.0.smithi059.stdout:3248: finishing write tid 1 to smithi05955964-23 2022-04-28T19:54:57.120 INFO:tasks.rados.rados.0.smithi059.stdout:3248: finishing write tid 3 to smithi05955964-23 2022-04-28T19:54:57.120 INFO:tasks.rados.rados.0.smithi059.stdout:3248: finishing write tid 4 to smithi05955964-23 2022-04-28T19:54:57.120 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1336 (ObjNum 1014 snap 314 seq_num 1014) dirty exists 2022-04-28T19:54:57.120 INFO:tasks.rados.rados.0.smithi059.stdout:3248: left oid 23 (ObjNum 1014 snap 314 seq_num 1014) 2022-04-28T19:54:57.121 INFO:tasks.rados.rados.0.smithi059.stdout:3248: done (3 left) 2022-04-28T19:54:57.121 INFO:tasks.rados.rados.0.smithi059.stdout:3252: delete oid 5 current snap is 314 2022-04-28T19:54:57.121 INFO:tasks.rados.rados.0.smithi059.stdout:3251: finishing rollback tid 1 to smithi05955964-19 2022-04-28T19:54:57.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1456 (ObjNum 288 snap 0 seq_num 33) dirty dne 2022-04-28T19:54:57.122 INFO:tasks.rados.rados.0.smithi059.stdout:3249: finishing write tid 1 to smithi05955964-3 2022-04-28T19:54:57.122 INFO:tasks.rados.rados.0.smithi059.stdout:3249: finishing write tid 3 to smithi05955964-3 2022-04-28T19:54:57.125 INFO:tasks.rados.rados.0.smithi059.stdout:3249: finishing write tid 4 to smithi05955964-3 2022-04-28T19:54:57.125 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1275 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2022-04-28T19:54:57.125 INFO:tasks.rados.rados.0.smithi059.stdout:3249: left oid 3 (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:54:57.125 INFO:tasks.rados.rados.0.smithi059.stdout:3249: done (3 left) 2022-04-28T19:54:57.126 INFO:tasks.rados.rados.0.smithi059.stdout:3250: done (2 left) 2022-04-28T19:54:57.126 INFO:tasks.rados.rados.0.smithi059.stdout:3251: done (1 left) 2022-04-28T19:54:57.126 INFO:tasks.rados.rados.0.smithi059.stdout:3252: done (0 left) 2022-04-28T19:54:57.126 INFO:tasks.rados.rados.0.smithi059.stdout:3253: rollback oid 45 current snap is 314 2022-04-28T19:54:57.127 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 312 2022-04-28T19:54:57.127 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 314 2022-04-28T19:54:57.127 INFO:tasks.rados.rados.0.smithi059.stdout:3254: seq_num 1017 ranges {1605632=237568} 2022-04-28T19:54:57.136 INFO:tasks.rados.rados.0.smithi059.stdout:3254: writing smithi05955964-46 from 1605632 to 1843200 tid 1 2022-04-28T19:54:57.136 INFO:tasks.rados.rados.0.smithi059.stdout:3255: rollback oid 12 current snap is 314 2022-04-28T19:54:57.136 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 12 to 303 2022-04-28T19:54:57.136 INFO:tasks.rados.rados.0.smithi059.stdout:3253: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:54:57.136 INFO:tasks.rados.rados.0.smithi059.stdout:3253: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:54:57.137 INFO:tasks.rados.rados.0.smithi059.stdout:3253: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:54:57.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1278 (ObjNum 1011 snap 311 seq_num 1011) dirty exists 2022-04-28T19:54:57.137 INFO:tasks.rados.rados.0.smithi059.stdout:3253: done (2 left) 2022-04-28T19:54:57.137 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 314 2022-04-28T19:54:57.137 INFO:tasks.rados.rados.0.smithi059.stdout:3256: seq_num 1018 ranges {7634944=253952} 2022-04-28T19:54:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:3256: writing smithi05955964-38 from 7634944 to 7888896 tid 1 2022-04-28T19:54:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:3257: snap_create 2022-04-28T19:54:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:3254: finishing write tid 1 to smithi05955964-46 2022-04-28T19:54:57.180 INFO:tasks.rados.rados.0.smithi059.stdout:3254: finishing write tid 3 to smithi05955964-46 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:3255: finishing rollback tid 0 to smithi05955964-12 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:3255: finishing rollback tid 1 to smithi05955964-12 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:3255: finishing rollback tid 2 to smithi05955964-12 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1354 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:3254: finishing write tid 4 to smithi05955964-46 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1496 (ObjNum 1017 snap 314 seq_num 1017) dirty exists 2022-04-28T19:54:57.181 INFO:tasks.rados.rados.0.smithi059.stdout:3254: left oid 46 (ObjNum 1017 snap 314 seq_num 1017) 2022-04-28T19:54:57.184 INFO:tasks.rados.rados.0.smithi059.stdout:3256: finishing write tid 1 to smithi05955964-38 2022-04-28T19:54:57.184 INFO:tasks.rados.rados.0.smithi059.stdout:3256: finishing write tid 3 to smithi05955964-38 2022-04-28T19:54:57.187 INFO:tasks.rados.rados.0.smithi059.stdout:3256: finishing write tid 4 to smithi05955964-38 2022-04-28T19:54:57.187 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1204 (ObjNum 1018 snap 314 seq_num 1018) dirty exists 2022-04-28T19:54:57.187 INFO:tasks.rados.rados.0.smithi059.stdout:3256: left oid 38 (ObjNum 1018 snap 314 seq_num 1018) 2022-04-28T19:54:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:57 smithi139 bash[43301]: cluster 2022-04-28T19:54:56.080689+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2022-04-28T19:54:57.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:57 smithi059 bash[61152]: cluster 2022-04-28T19:54:56.080689+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2022-04-28T19:54:57.608 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:57 smithi059 bash[59745]: cluster 2022-04-28T19:54:56.080689+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2022-04-28T19:54:58.112 INFO:tasks.rados.rados.0.smithi059.stdout:3254: done (3 left) 2022-04-28T19:54:58.113 INFO:tasks.rados.rados.0.smithi059.stdout:3255: done (2 left) 2022-04-28T19:54:58.113 INFO:tasks.rados.rados.0.smithi059.stdout:3256: done (1 left) 2022-04-28T19:54:58.113 INFO:tasks.rados.rados.0.smithi059.stdout:3257: done (0 left) 2022-04-28T19:54:58.113 INFO:tasks.rados.rados.0.smithi059.stdout:3258: delete oid 49 current snap is 315 2022-04-28T19:54:58.115 INFO:tasks.rados.rados.0.smithi059.stdout:3258: done (0 left) 2022-04-28T19:54:58.115 INFO:tasks.rados.rados.0.smithi059.stdout:3259: snap_remove snap 304 2022-04-28T19:54:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[43301]: cluster 2022-04-28T19:54:56.983537+0000 mgr.y (mgr.24961) 311 : cluster [DBG] pgmap v496: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 34 KiB/s wr, 8 op/s 2022-04-28T19:54:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[43301]: cluster 2022-04-28T19:54:57.093721+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2022-04-28T19:54:58.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:58 smithi059 bash[59745]: cluster 2022-04-28T19:54:56.983537+0000 mgr.y (mgr.24961) 311 : cluster [DBG] pgmap v496: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 34 KiB/s wr, 8 op/s 2022-04-28T19:54:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:58 smithi059 bash[59745]: cluster 2022-04-28T19:54:57.093721+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2022-04-28T19:54:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:58 smithi059 bash[61152]: cluster 2022-04-28T19:54:56.983537+0000 mgr.y (mgr.24961) 311 : cluster [DBG] pgmap v496: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 34 KiB/s wr, 8 op/s 2022-04-28T19:54:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:58 smithi059 bash[61152]: cluster 2022-04-28T19:54:57.093721+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2022-04-28T19:54:58.778 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 systemd[1]: Stopping Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:54:58.778 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=warn ts=2022-04-28T19:54:58.739Z caller=main.go:524 msg="Received SIGTERM, exiting gracefully..." 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=main.go:561 msg="Stopping notify discovery manager..." 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=main.go:583 msg="Stopping scrape manager..." 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=main.go:543 msg="Scrape discovery manager stopped" 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=main.go:557 msg="Notify discovery manager stopped" 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=main.go:577 msg="Scrape manager stopped" 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.739Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.740Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2022-04-28T19:54:58.779 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.740Z caller=main.go:749 msg="Notifier manager stopped" 2022-04-28T19:54:58.780 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:58 smithi139 bash[39417]: level=info ts=2022-04-28T19:54:58.740Z caller=main.go:761 msg="See you next time!" 2022-04-28T19:54:59.125 INFO:tasks.rados.rados.0.smithi059.stdout:3259: done (0 left) 2022-04-28T19:54:59.125 INFO:tasks.rados.rados.0.smithi059.stdout:3260: rollback oid 20 current snap is 315 2022-04-28T19:54:59.125 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 295 2022-04-28T19:54:59.126 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 315 2022-04-28T19:54:59.126 INFO:tasks.rados.rados.0.smithi059.stdout:3261: seq_num 1020 ranges {0=286720} 2022-04-28T19:54:59.126 INFO:tasks.rados.rados.0.smithi059.stdout:3261: writing smithi05955964-10 from 0 to 286720 tid 1 2022-04-28T19:54:59.127 INFO:tasks.rados.rados.0.smithi059.stdout:3262: copy_from oid 24 from oid 49 current snap is 315 2022-04-28T19:54:59.127 INFO:tasks.rados.rados.0.smithi059.stdout:3263: rollback oid 36 current snap is 315 2022-04-28T19:54:59.127 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 294 2022-04-28T19:54:59.127 INFO:tasks.rados.rados.0.smithi059.stdout:3264: snap_remove snap 297 2022-04-28T19:54:59.444 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:54:59 smithi139 bash[43301]: cluster 2022-04-28T19:54:58.112689+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2022-04-28T19:54:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:54:59 smithi059 bash[59745]: cluster 2022-04-28T19:54:58.112689+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2022-04-28T19:54:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:54:59 smithi059 bash[61152]: cluster 2022-04-28T19:54:58.112689+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2022-04-28T19:54:59.868 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:59 smithi139 bash[53562]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-prometheus.a 2022-04-28T19:54:59.868 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:59 smithi139 bash[53562]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-prometheus-a 2022-04-28T19:54:59.868 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:59 smithi139 bash[53562]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-prometheus.a 2022-04-28T19:54:59.868 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:59 smithi139 systemd[1]: Stopped Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:54:59.869 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:54:59 smithi139 systemd[1]: Started Ceph prometheus.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:55:00.133 INFO:tasks.rados.rados.0.smithi059.stdout:3264: done (4 left) 2022-04-28T19:55:00.133 INFO:tasks.rados.rados.0.smithi059.stdout:3265: delete oid 43 current snap is 315 2022-04-28T19:55:00.133 INFO:tasks.rados.rados.0.smithi059.stdout:3260: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:55:00.133 INFO:tasks.rados.rados.0.smithi059.stdout:3261: finishing write tid 1 to smithi05955964-10 2022-04-28T19:55:00.133 INFO:tasks.rados.rados.0.smithi059.stdout:3262: finishing copy_from to smithi05955964-24 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3262: got expected ENOENT (src dne) 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3261: finishing write tid 3 to smithi05955964-10 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3260: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3262: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3260: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1281 (ObjNum 989 snap 294 seq_num 989) dirty exists 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3263: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3263: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:3263: finishing rollback tid 2 to smithi05955964-36 2022-04-28T19:55:00.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1138 (ObjNum 678 snap 197 seq_num 678) dirty exists 2022-04-28T19:55:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:3261: finishing write tid 4 to smithi05955964-10 2022-04-28T19:55:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1338 (ObjNum 1020 snap 315 seq_num 1020) dirty exists 2022-04-28T19:55:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:3261: left oid 10 (ObjNum 1020 snap 315 seq_num 1020) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3260: done (4 left) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3261: done (3 left) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3262: done (2 left) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3263: done (1 left) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3265: done (0 left) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3266: read oid 27 snap -1 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3266: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:55:00.139 INFO:tasks.rados.rados.0.smithi059.stdout:3267: delete oid 48 current snap is 315 2022-04-28T19:55:00.140 INFO:tasks.rados.rados.0.smithi059.stdout:3266: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:55:00.144 INFO:tasks.rados.rados.0.smithi059.stdout:3266: done (1 left) 2022-04-28T19:55:00.144 INFO:tasks.rados.rados.0.smithi059.stdout:3267: done (0 left) 2022-04-28T19:55:00.144 INFO:tasks.rados.rados.0.smithi059.stdout:3268: setattr oid 49 current snap is 315 2022-04-28T19:55:00.145 INFO:tasks.rados.rados.0.smithi059.stdout:3269: setattr oid 12 current snap is 315 2022-04-28T19:55:00.146 INFO:tasks.rados.rados.0.smithi059.stdout:3270: read oid 8 snap -1 2022-04-28T19:55:00.146 INFO:tasks.rados.rados.0.smithi059.stdout:3270: expect (ObjNum 997 snap 297 seq_num 997) 2022-04-28T19:55:00.146 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 315 2022-04-28T19:55:00.146 INFO:tasks.rados.rados.0.smithi059.stdout:3271: seq_num 1023 ranges {8699904=335872} 2022-04-28T19:55:00.201 INFO:tasks.rados.rados.0.smithi059.stdout:3271: writing smithi05955964-31 from 8699904 to 9035776 tid 1 2022-04-28T19:55:00.533 INFO:tasks.rados.rados.0.smithi059.stdout:3272: read oid 40 snap -1 2022-04-28T19:55:00.533 INFO:tasks.rados.rados.0.smithi059.stdout:3272: expect (ObjNum 974 snap 289 seq_num 974) 2022-04-28T19:55:00.534 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 315 2022-04-28T19:55:00.534 INFO:tasks.rados.rados.0.smithi059.stdout:3273: seq_num 1024 ranges {172032=352256} 2022-04-28T19:55:00.535 INFO:tasks.rados.rados.0.smithi059.stdout:3273: writing smithi05955964-44 from 172032 to 524288 tid 1 2022-04-28T19:55:00.536 INFO:tasks.rados.rados.0.smithi059.stdout:3274: snap_create 2022-04-28T19:55:00.536 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1139 (ObjNum 1 snap 21893 seq_num 4162471664) dirty exists 2022-04-28T19:55:00.536 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1355 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:55:00.536 INFO:tasks.rados.rados.0.smithi059.stdout:3270: expect (ObjNum 997 snap 297 seq_num 997) 2022-04-28T19:55:00.537 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:55:00.537 INFO:tasks.rados.rados.0.smithi059.stdout:3271: finishing write tid 1 to smithi05955964-31 2022-04-28T19:55:00.538 INFO:tasks.rados.rados.0.smithi059.stdout:3271: finishing write tid 3 to smithi05955964-31 2022-04-28T19:55:00.538 INFO:tasks.rados.rados.0.smithi059.stdout:3271: finishing write tid 4 to smithi05955964-31 2022-04-28T19:55:00.538 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1357 (ObjNum 1023 snap 315 seq_num 1023) dirty exists 2022-04-28T19:55:00.538 INFO:tasks.rados.rados.0.smithi059.stdout:3271: left oid 31 (ObjNum 1023 snap 315 seq_num 1023) 2022-04-28T19:55:00.538 INFO:tasks.rados.rados.0.smithi059.stdout:3272: expect (ObjNum 974 snap 289 seq_num 974) 2022-04-28T19:55:00.562 INFO:tasks.rados.rados.0.smithi059.stdout:3273: finishing write tid 1 to smithi05955964-44 2022-04-28T19:55:00.562 INFO:tasks.rados.rados.0.smithi059.stdout:3273: finishing write tid 3 to smithi05955964-44 2022-04-28T19:55:00.562 INFO:tasks.rados.rados.0.smithi059.stdout:3273: finishing write tid 4 to smithi05955964-44 2022-04-28T19:55:00.562 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1190 (ObjNum 1024 snap 315 seq_num 1024) dirty exists 2022-04-28T19:55:00.562 INFO:tasks.rados.rados.0.smithi059.stdout:3273: left oid 44 (ObjNum 1024 snap 315 seq_num 1024) 2022-04-28T19:55:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: cluster 2022-04-28T19:54:58.984309+0000 mgr.y (mgr.24961) 312 : cluster [DBG] pgmap v499: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 0 B/s wr, 6 op/s 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: cluster 2022-04-28T19:54:59.124454+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: audit 2022-04-28T19:54:59.724484+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: audit 2022-04-28T19:54:59.726389+0000 mon.a (mon.0) 1175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: audit 2022-04-28T19:54:59.728898+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: audit 2022-04-28T19:54:59.730592+0000 mon.a (mon.0) 1177 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[61152]: audit 2022-04-28T19:54:59.733533+0000 mon.a (mon.0) 1178 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: cluster 2022-04-28T19:54:58.984309+0000 mgr.y (mgr.24961) 312 : cluster [DBG] pgmap v499: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 0 B/s wr, 6 op/s 2022-04-28T19:55:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: cluster 2022-04-28T19:54:59.124454+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2022-04-28T19:55:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: audit 2022-04-28T19:54:59.724484+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: audit 2022-04-28T19:54:59.726389+0000 mon.a (mon.0) 1175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: audit 2022-04-28T19:54:59.728898+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: audit 2022-04-28T19:54:59.730592+0000 mon.a (mon.0) 1177 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:00 smithi059 bash[59745]: audit 2022-04-28T19:54:59.733533+0000 mon.a (mon.0) 1178 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: cluster 2022-04-28T19:54:58.984309+0000 mgr.y (mgr.24961) 312 : cluster [DBG] pgmap v499: 89 pgs: 89 active+clean; 813 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 0 B/s wr, 6 op/s 2022-04-28T19:55:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: cluster 2022-04-28T19:54:59.124454+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2022-04-28T19:55:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: audit 2022-04-28T19:54:59.724484+0000 mon.a (mon.0) 1174 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: audit 2022-04-28T19:54:59.726389+0000 mon.a (mon.0) 1175 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: audit 2022-04-28T19:54:59.728898+0000 mon.a (mon.0) 1176 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: audit 2022-04-28T19:54:59.730592+0000 mon.a (mon.0) 1177 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:00 smithi139 bash[43301]: audit 2022-04-28T19:54:59.733533+0000 mon.a (mon.0) 1178 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:00.707 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:55:00.707 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (13m) 17s ago 14m 14.8M - 0.20.0 0881eb8f169f 8aadcbd03c70 2022-04-28T19:55:00.707 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (14m) 8s ago 14m 26.5M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:55:00.707 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (44s) 17s ago 12m 13.2M - 16.2.7-969-g4fa079ba 863db77b5b5e 0b1d79d85003 2022-04-28T19:55:00.707 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (31s) 8s ago 12m 11.8M - 16.2.7-969-g4fa079ba 863db77b5b5e 01b2328b32c7 2022-04-28T19:55:00.707 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (6m) 8s ago 18m 377M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (6m) 17s ago 20m 418M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (6m) 17s ago 20m 84.4M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (5m) 8s ago 18m 57.5M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (5m) 17s ago 18m 65.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (19s) 17s ago 14m - - 1dbe0e931976 c196770fa402 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (10s) 8s ago 14m - - 1dbe0e931976 97aaa15a2dd3 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (4m) 17s ago 17m 344M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (3m) 17s ago 17m 252M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (3m) 17s ago 17m 323M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:55:00.708 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (2m) 17s ago 16m 237M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:55:00.709 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (2m) 8s ago 16m 293M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:55:00.709 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (117s) 8s ago 15m 360M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 4d55ca87a6a9 2022-04-28T19:55:00.709 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (86s) 8s ago 15m 220M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 2cf53c9fc29e 2022-04-28T19:55:00.709 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (56s) 8s ago 15m 182M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 8655092f1b14 2022-04-28T19:55:00.709 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 starting - - - - 2022-04-28T19:55:01.086 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:55:01.145 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 8 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 15 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:55:01.146 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:55:01.147 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.034Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2022-04-28T19:55:01.147 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.034Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2022-04-28T19:55:01.147 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.034Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2022-04-28T19:55:01.147 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.034Z caller=main.go:518 level=info host_details="(Linux 4.15.0-176-generic #185-Ubuntu SMP Tue Mar 29 17:40:04 UTC 2022 x86_64 smithi139 (none))" 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.034Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.034Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.037Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.037Z caller=main.go:923 level=info msg="Starting TSDB ..." 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.037Z caller=dir_locker.go:77 level=warn component=tsdb msg="A lockfile from a previous execution already existed. It was replaced" file=/prometheus/lock 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.041Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.043Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.043Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.24µs 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.044Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2022-04-28T19:55:01.148 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.044Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2022-04-28T19:55:01.149 INFO:tasks.rados.rados.0.smithi059.stdout:3268: done (6 left) 2022-04-28T19:55:01.149 INFO:tasks.rados.rados.0.smithi059.stdout:3269: done (5 left) 2022-04-28T19:55:01.149 INFO:tasks.rados.rados.0.smithi059.stdout:3270: done (4 left) 2022-04-28T19:55:01.149 INFO:tasks.rados.rados.0.smithi059.stdout:3271: done (3 left) 2022-04-28T19:55:01.149 INFO:tasks.rados.rados.0.smithi059.stdout:3272: done (2 left) 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:3273: done (1 left) 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:3274: done (0 left) 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 316 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:3275: seq_num 1025 ranges {0=368640} 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:3275: writing smithi05955964-43 from 0 to 368640 tid 1 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:3276: setattr oid 29 current snap is 316 2022-04-28T19:55:01.150 INFO:tasks.rados.rados.0.smithi059.stdout:3277: snap_create 2022-04-28T19:55:01.155 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1498 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:55:01.156 INFO:tasks.rados.rados.0.smithi059.stdout:3275: finishing write tid 1 to smithi05955964-43 2022-04-28T19:55:01.157 INFO:tasks.rados.rados.0.smithi059.stdout:3275: finishing write tid 3 to smithi05955964-43 2022-04-28T19:55:01.158 INFO:tasks.rados.rados.0.smithi059.stdout:3275: finishing write tid 4 to smithi05955964-43 2022-04-28T19:55:01.158 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1466 (ObjNum 1025 snap 316 seq_num 1025) dirty exists 2022-04-28T19:55:01.159 INFO:tasks.rados.rados.0.smithi059.stdout:3275: left oid 43 (ObjNum 1025 snap 316 seq_num 1025) 2022-04-28T19:55:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: cephadm 2022-04-28T19:54:59.732055+0000 mgr.y (mgr.24961) 313 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: cluster 2022-04-28T19:55:00.133014+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2022-04-28T19:55:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.684449+0000 mon.a (mon.0) 1180 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.685300+0000 mon.a (mon.0) 1181 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.686201+0000 mon.a (mon.0) 1182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.687083+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.688020+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.688983+0000 mon.a (mon.0) 1185 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.692796+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.693598+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.694448+0000 mon.a (mon.0) 1188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.695296+0000 mon.a (mon.0) 1189 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:00.696048+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[59745]: audit 2022-04-28T19:55:01.086119+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.59:0/2691422053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: cephadm 2022-04-28T19:54:59.732055+0000 mgr.y (mgr.24961) 313 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: cluster 2022-04-28T19:55:00.133014+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.684449+0000 mon.a (mon.0) 1180 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.685300+0000 mon.a (mon.0) 1181 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.686201+0000 mon.a (mon.0) 1182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.687083+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.688020+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.688983+0000 mon.a (mon.0) 1185 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.692796+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.693598+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.694448+0000 mon.a (mon.0) 1188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.695296+0000 mon.a (mon.0) 1189 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:00.696048+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[61152]: audit 2022-04-28T19:55:01.086119+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.59:0/2691422053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:01.619 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.180Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2022-04-28T19:55:01.619 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.224Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2022-04-28T19:55:01.619 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.225Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2022-04-28T19:55:01.619 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.225Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=75.527µs wal_replay_duration=181.607476ms total_replay_duration=181.713887ms 2022-04-28T19:55:01.620 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.229Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2022-04-28T19:55:01.620 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.229Z caller=main.go:947 level=info msg="TSDB started" 2022-04-28T19:55:01.620 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.229Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2022-04-28T19:55:01.620 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.258Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=28.259632ms db_storage=1.238µs remote_storage=2.16µs web_handler=912ns query_engine=1.052µs scrape=439.271µs scrape_sd=43.688µs notify=51.658µs notify_sd=17.512µs rules=27.128153ms 2022-04-28T19:55:01.621 INFO:journalctl@ceph.prometheus.a.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[53671]: ts=2022-04-28T19:55:01.258Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2022-04-28T19:55:01.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: cephadm 2022-04-28T19:54:59.732055+0000 mgr.y (mgr.24961) 313 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:01.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: cluster 2022-04-28T19:55:00.133014+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2022-04-28T19:55:01.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.684449+0000 mon.a (mon.0) 1180 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.685300+0000 mon.a (mon.0) 1181 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.686201+0000 mon.a (mon.0) 1182 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.687083+0000 mon.a (mon.0) 1183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.688020+0000 mon.a (mon.0) 1184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.688983+0000 mon.a (mon.0) 1185 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.692796+0000 mon.a (mon.0) 1186 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:01.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.693598+0000 mon.a (mon.0) 1187 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.694448+0000 mon.a (mon.0) 1188 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.695296+0000 mon.a (mon.0) 1189 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:00.696048+0000 mon.a (mon.0) 1190 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:01.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:01 smithi139 bash[43301]: audit 2022-04-28T19:55:01.086119+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.59:0/2691422053' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:01.636Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:55:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:01.636Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:55:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:01 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:01.637Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:55:02.147 INFO:tasks.rados.rados.0.smithi059.stdout:3275: done (2 left) 2022-04-28T19:55:02.147 INFO:tasks.rados.rados.0.smithi059.stdout:3276: done (1 left) 2022-04-28T19:55:02.147 INFO:tasks.rados.rados.0.smithi059.stdout:3277: done (0 left) 2022-04-28T19:55:02.147 INFO:tasks.rados.rados.0.smithi059.stdout:3278: snap_create 2022-04-28T19:55:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[61152]: audit 2022-04-28T19:55:00.365535+0000 mgr.y (mgr.24961) 314 : audit [DBG] from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[61152]: audit 2022-04-28T19:55:00.678887+0000 mgr.y (mgr.24961) 315 : audit [DBG] from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[61152]: cluster 2022-04-28T19:55:00.985232+0000 mgr.y (mgr.24961) 316 : cluster [DBG] pgmap v502: 89 pgs: 89 active+clean; 815 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 328 KiB/s wr, 7 op/s 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[61152]: cluster 2022-04-28T19:55:01.142926+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[59745]: audit 2022-04-28T19:55:00.365535+0000 mgr.y (mgr.24961) 314 : audit [DBG] from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[59745]: audit 2022-04-28T19:55:00.678887+0000 mgr.y (mgr.24961) 315 : audit [DBG] from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[59745]: cluster 2022-04-28T19:55:00.985232+0000 mgr.y (mgr.24961) 316 : cluster [DBG] pgmap v502: 89 pgs: 89 active+clean; 815 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 328 KiB/s wr, 7 op/s 2022-04-28T19:55:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:02 smithi059 bash[59745]: cluster 2022-04-28T19:55:01.142926+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2022-04-28T19:55:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:02 smithi139 bash[43301]: audit 2022-04-28T19:55:00.365535+0000 mgr.y (mgr.24961) 314 : audit [DBG] from='client.54331 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:02 smithi139 bash[43301]: audit 2022-04-28T19:55:00.678887+0000 mgr.y (mgr.24961) 315 : audit [DBG] from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:02 smithi139 bash[43301]: cluster 2022-04-28T19:55:00.985232+0000 mgr.y (mgr.24961) 316 : cluster [DBG] pgmap v502: 89 pgs: 89 active+clean; 815 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 328 KiB/s wr, 7 op/s 2022-04-28T19:55:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:02 smithi139 bash[43301]: cluster 2022-04-28T19:55:01.142926+0000 mon.a (mon.0) 1191 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2022-04-28T19:55:03.166 INFO:tasks.rados.rados.0.smithi059.stdout:3278: done (0 left) 2022-04-28T19:55:03.167 INFO:tasks.rados.rados.0.smithi059.stdout:3279: delete oid 2 current snap is 318 2022-04-28T19:55:03.170 INFO:tasks.rados.rados.0.smithi059.stdout:3279: done (0 left) 2022-04-28T19:55:03.170 INFO:tasks.rados.rados.0.smithi059.stdout:3280: snap_remove snap 289 2022-04-28T19:55:03.531 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:03 smithi059 bash[61152]: cluster 2022-04-28T19:55:02.144837+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2022-04-28T19:55:03.531 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:03 smithi059 bash[59745]: cluster 2022-04-28T19:55:02.144837+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2022-04-28T19:55:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:03 smithi139 bash[43301]: cluster 2022-04-28T19:55:02.144837+0000 mon.a (mon.0) 1192 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2022-04-28T19:55:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:03.530Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:55:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:03.530Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:55:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:03 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:03.530Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:55:04.177 INFO:tasks.rados.rados.0.smithi059.stdout:3280: done (0 left) 2022-04-28T19:55:04.178 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 318 2022-04-28T19:55:04.178 INFO:tasks.rados.rados.0.smithi059.stdout:3281: seq_num 1027 ranges {0=409600} 2022-04-28T19:55:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:3281: writing smithi05955964-15 from 0 to 409600 tid 1 2022-04-28T19:55:04.180 INFO:tasks.rados.rados.0.smithi059.stdout:3282: read oid 39 snap -1 2022-04-28T19:55:04.181 INFO:tasks.rados.rados.0.smithi059.stdout:3282: expect (ObjNum 894 snap 267 seq_num 894) 2022-04-28T19:55:04.181 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 318 2022-04-28T19:55:04.181 INFO:tasks.rados.rados.0.smithi059.stdout:3283: seq_num 1028 ranges {1769472=425984} 2022-04-28T19:55:04.194 INFO:tasks.rados.rados.0.smithi059.stdout:3283: writing smithi05955964-36 from 1769472 to 2195456 tid 1 2022-04-28T19:55:04.194 INFO:tasks.rados.rados.0.smithi059.stdout:3284: delete oid 42 current snap is 318 2022-04-28T19:55:04.194 INFO:tasks.rados.rados.0.smithi059.stdout:3282: expect (ObjNum 894 snap 267 seq_num 894) 2022-04-28T19:55:04.213 INFO:tasks.rados.rados.0.smithi059.stdout:3281: finishing write tid 1 to smithi05955964-15 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3281: finishing write tid 3 to smithi05955964-15 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3281: finishing write tid 4 to smithi05955964-15 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1359 (ObjNum 1027 snap 318 seq_num 1027) dirty exists 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3281: left oid 15 (ObjNum 1027 snap 318 seq_num 1027) 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3283: finishing write tid 1 to smithi05955964-36 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3283: finishing write tid 3 to smithi05955964-36 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3281: done (3 left) 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3282: done (2 left) 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3284: done (1 left) 2022-04-28T19:55:04.214 INFO:tasks.rados.rados.0.smithi059.stdout:3285: read oid 7 snap 311 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:3285: expect deleted 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:3286: setattr oid 18 current snap is 318 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:3283: finishing write tid 4 to smithi05955964-36 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1141 (ObjNum 1028 snap 318 seq_num 1028) dirty exists 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:3283: left oid 36 (ObjNum 1028 snap 318 seq_num 1028) 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:3283: done (2 left) 2022-04-28T19:55:04.215 INFO:tasks.rados.rados.0.smithi059.stdout:3285: done (1 left) 2022-04-28T19:55:04.216 INFO:tasks.rados.rados.0.smithi059.stdout:3287: snap_create 2022-04-28T19:55:04.220 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1344 (ObjNum 701 snap 203 seq_num 701) dirty exists 2022-04-28T19:55:04.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:04 smithi059 bash[61152]: cluster 2022-04-28T19:55:02.985729+0000 mgr.y (mgr.24961) 317 : cluster [DBG] pgmap v505: 89 pgs: 89 active+clean; 815 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 328 KiB/s wr, 7 op/s 2022-04-28T19:55:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:04 smithi059 bash[61152]: cluster 2022-04-28T19:55:03.166394+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2022-04-28T19:55:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:04 smithi059 bash[59745]: cluster 2022-04-28T19:55:02.985729+0000 mgr.y (mgr.24961) 317 : cluster [DBG] pgmap v505: 89 pgs: 89 active+clean; 815 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 328 KiB/s wr, 7 op/s 2022-04-28T19:55:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:04 smithi059 bash[59745]: cluster 2022-04-28T19:55:03.166394+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2022-04-28T19:55:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:04 smithi139 bash[43301]: cluster 2022-04-28T19:55:02.985729+0000 mgr.y (mgr.24961) 317 : cluster [DBG] pgmap v505: 89 pgs: 89 active+clean; 815 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 328 KiB/s wr, 7 op/s 2022-04-28T19:55:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:04 smithi139 bash[43301]: cluster 2022-04-28T19:55:03.166394+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2022-04-28T19:55:05.186 INFO:tasks.rados.rados.0.smithi059.stdout:3286: done (1 left) 2022-04-28T19:55:05.186 INFO:tasks.rados.rados.0.smithi059.stdout:3287: done (0 left) 2022-04-28T19:55:05.186 INFO:tasks.rados.rados.0.smithi059.stdout:3288: rmattr oid 41 current snap is 319 2022-04-28T19:55:05.186 INFO:tasks.rados.rados.0.smithi059.stdout:3289: read oid 24 snap -1 2022-04-28T19:55:05.187 INFO:tasks.rados.rados.0.smithi059.stdout:3289: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:55:05.187 INFO:tasks.rados.rados.0.smithi059.stdout:3290: snap_remove snap 286 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: cluster 2022-04-28T19:55:04.177214+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.059053+0000 mon.a (mon.0) 1195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.066174+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.078151+0000 mon.a (mon.0) 1197 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.080985+0000 mon.a (mon.0) 1198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139.front.sepia.ceph.com:9095"}]: dispatch 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.089272+0000 mon.a (mon.0) 1199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.093073+0000 mon.a (mon.0) 1200 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.097112+0000 mon.a (mon.0) 1201 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.099331+0000 mon.a (mon.0) 1202 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.101390+0000 mon.a (mon.0) 1203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.104190+0000 mon.a (mon.0) 1204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.106960+0000 mon.a (mon.0) 1205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.108822+0000 mon.a (mon.0) 1206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.110510+0000 mon.a (mon.0) 1207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.112239+0000 mon.a (mon.0) 1208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.114073+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.115963+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.117930+0000 mon.a (mon.0) 1211 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[61152]: audit 2022-04-28T19:55:05.119533+0000 mon.a (mon.0) 1212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: cluster 2022-04-28T19:55:04.177214+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.059053+0000 mon.a (mon.0) 1195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.066174+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.078151+0000 mon.a (mon.0) 1197 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.080985+0000 mon.a (mon.0) 1198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139.front.sepia.ceph.com:9095"}]: dispatch 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.089272+0000 mon.a (mon.0) 1199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.093073+0000 mon.a (mon.0) 1200 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.097112+0000 mon.a (mon.0) 1201 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.099331+0000 mon.a (mon.0) 1202 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.101390+0000 mon.a (mon.0) 1203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.104190+0000 mon.a (mon.0) 1204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.106960+0000 mon.a (mon.0) 1205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.108822+0000 mon.a (mon.0) 1206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.110510+0000 mon.a (mon.0) 1207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.112239+0000 mon.a (mon.0) 1208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.114073+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.115963+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.117930+0000 mon.a (mon.0) 1211 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:05 smithi059 bash[59745]: audit 2022-04-28T19:55:05.119533+0000 mon.a (mon.0) 1212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: cluster 2022-04-28T19:55:04.177214+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.059053+0000 mon.a (mon.0) 1195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.066174+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.078151+0000 mon.a (mon.0) 1197 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.080985+0000 mon.a (mon.0) 1198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139.front.sepia.ceph.com:9095"}]: dispatch 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.089272+0000 mon.a (mon.0) 1199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.093073+0000 mon.a (mon.0) 1200 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.097112+0000 mon.a (mon.0) 1201 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.099331+0000 mon.a (mon.0) 1202 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.101390+0000 mon.a (mon.0) 1203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.104190+0000 mon.a (mon.0) 1204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.106960+0000 mon.a (mon.0) 1205 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.108822+0000 mon.a (mon.0) 1206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.110510+0000 mon.a (mon.0) 1207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.112239+0000 mon.a (mon.0) 1208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.114073+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.115963+0000 mon.a (mon.0) 1210 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.117930+0000 mon.a (mon.0) 1211 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:05 smithi139 bash[43301]: audit 2022-04-28T19:55:05.119533+0000 mon.a (mon.0) 1212 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:06.197 INFO:tasks.rados.rados.0.smithi059.stdout:3290: done (2 left) 2022-04-28T19:55:06.197 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 11 current snap is 319 2022-04-28T19:55:06.197 INFO:tasks.rados.rados.0.smithi059.stdout:3291: seq_num 1030 ranges {2367488=458752} 2022-04-28T19:55:06.213 INFO:tasks.rados.rados.0.smithi059.stdout:3291: writing smithi05955964-11 from 2367488 to 2826240 tid 1 2022-04-28T19:55:06.214 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 319 2022-04-28T19:55:06.214 INFO:tasks.rados.rados.0.smithi059.stdout:3292: seq_num 1031 ranges {10657792=475136} 2022-04-28T19:55:06.278 INFO:tasks.rados.rados.0.smithi059.stdout:3292: writing smithi05955964-12 from 10657792 to 11132928 tid 1 2022-04-28T19:55:06.279 INFO:tasks.rados.rados.0.smithi059.stdout:3293: read oid 1 snap 275 2022-04-28T19:55:06.279 INFO:tasks.rados.rados.0.smithi059.stdout:3293: expect (ObjNum 914 snap 272 seq_num 914) 2022-04-28T19:55:06.279 INFO:tasks.rados.rados.0.smithi059.stdout:3294: snap_create 2022-04-28T19:55:06.279 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1018 (ObjNum 785 snap 234 seq_num 785) dirty exists 2022-04-28T19:55:06.279 INFO:tasks.rados.rados.0.smithi059.stdout:3289: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:55:06.542 INFO:tasks.rados.rados.0.smithi059.stdout:3291: finishing write tid 1 to smithi05955964-11 2022-04-28T19:55:06.614 INFO:tasks.rados.rados.0.smithi059.stdout:3291: finishing write tid 3 to smithi05955964-11 2022-04-28T19:55:06.614 INFO:tasks.rados.rados.0.smithi059.stdout:3291: finishing write tid 4 to smithi05955964-11 2022-04-28T19:55:06.614 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1206 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:3291: left oid 11 (ObjNum 1030 snap 319 seq_num 1030) 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:3292: finishing write tid 1 to smithi05955964-12 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:3292: finishing write tid 3 to smithi05955964-12 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:3292: finishing write tid 4 to smithi05955964-12 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1361 (ObjNum 1031 snap 319 seq_num 1031) dirty exists 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:3292: left oid 12 (ObjNum 1031 snap 319 seq_num 1031) 2022-04-28T19:55:06.615 INFO:tasks.rados.rados.0.smithi059.stdout:3293: expect (ObjNum 914 snap 272 seq_num 914) 2022-04-28T19:55:06.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[61152]: cluster 2022-04-28T19:55:04.986519+0000 mgr.y (mgr.24961) 318 : cluster [DBG] pgmap v508: 89 pgs: 89 active+clean; 816 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 362 KiB/s wr, 21 op/s 2022-04-28T19:55:06.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[61152]: audit 2022-04-28T19:55:05.079098+0000 mgr.y (mgr.24961) 319 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:55:06.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[61152]: audit 2022-04-28T19:55:05.081465+0000 mgr.y (mgr.24961) 320 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139.front.sepia.ceph.com:9095"}]: dispatch 2022-04-28T19:55:06.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[61152]: cluster 2022-04-28T19:55:05.185632+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2022-04-28T19:55:06.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[59745]: cluster 2022-04-28T19:55:04.986519+0000 mgr.y (mgr.24961) 318 : cluster [DBG] pgmap v508: 89 pgs: 89 active+clean; 816 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 362 KiB/s wr, 21 op/s 2022-04-28T19:55:06.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[59745]: audit 2022-04-28T19:55:05.079098+0000 mgr.y (mgr.24961) 319 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:55:06.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[59745]: audit 2022-04-28T19:55:05.081465+0000 mgr.y (mgr.24961) 320 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139.front.sepia.ceph.com:9095"}]: dispatch 2022-04-28T19:55:06.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:06 smithi059 bash[59745]: cluster 2022-04-28T19:55:05.185632+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2022-04-28T19:55:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:06 smithi139 bash[43301]: cluster 2022-04-28T19:55:04.986519+0000 mgr.y (mgr.24961) 318 : cluster [DBG] pgmap v508: 89 pgs: 89 active+clean; 816 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 362 KiB/s wr, 21 op/s 2022-04-28T19:55:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:06 smithi139 bash[43301]: audit 2022-04-28T19:55:05.079098+0000 mgr.y (mgr.24961) 319 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2022-04-28T19:55:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:06 smithi139 bash[43301]: audit 2022-04-28T19:55:05.081465+0000 mgr.y (mgr.24961) 320 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi139.front.sepia.ceph.com:9095"}]: dispatch 2022-04-28T19:55:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:06 smithi139 bash[43301]: cluster 2022-04-28T19:55:05.185632+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2022-04-28T19:55:06.958 INFO:tasks.rados.rados.0.smithi059.stdout:3288: done (5 left) 2022-04-28T19:55:06.958 INFO:tasks.rados.rados.0.smithi059.stdout:3289: done (4 left) 2022-04-28T19:55:06.958 INFO:tasks.rados.rados.0.smithi059.stdout:3291: done (3 left) 2022-04-28T19:55:06.959 INFO:tasks.rados.rados.0.smithi059.stdout:3292: done (2 left) 2022-04-28T19:55:06.959 INFO:tasks.rados.rados.0.smithi059.stdout:3293: done (1 left) 2022-04-28T19:55:06.959 INFO:tasks.rados.rados.0.smithi059.stdout:3294: done (0 left) 2022-04-28T19:55:06.959 INFO:tasks.rados.rados.0.smithi059.stdout:3295: rollback oid 22 current snap is 320 2022-04-28T19:55:06.959 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 294 2022-04-28T19:55:06.959 INFO:tasks.rados.rados.0.smithi059.stdout:3296: snap_remove snap 317 2022-04-28T19:55:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:07 smithi059 bash[61152]: cluster 2022-04-28T19:55:06.196378+0000 mon.a (mon.0) 1214 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2022-04-28T19:55:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:07 smithi059 bash[61152]: cluster 2022-04-28T19:55:06.956378+0000 mon.a (mon.0) 1215 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2022-04-28T19:55:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:07 smithi059 bash[59745]: cluster 2022-04-28T19:55:06.196378+0000 mon.a (mon.0) 1214 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2022-04-28T19:55:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:07 smithi059 bash[59745]: cluster 2022-04-28T19:55:06.956378+0000 mon.a (mon.0) 1215 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2022-04-28T19:55:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:07 smithi139 bash[43301]: cluster 2022-04-28T19:55:06.196378+0000 mon.a (mon.0) 1214 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2022-04-28T19:55:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:07 smithi139 bash[43301]: cluster 2022-04-28T19:55:06.956378+0000 mon.a (mon.0) 1215 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2022-04-28T19:55:07.983 INFO:tasks.rados.rados.0.smithi059.stdout:3296: done (1 left) 2022-04-28T19:55:07.983 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 320 2022-04-28T19:55:07.983 INFO:tasks.rados.rados.0.smithi059.stdout:3297: seq_num 1032 ranges {2408448=491520} 2022-04-28T19:55:07.994 INFO:tasks.rados.rados.0.smithi059.stdout:3297: writing smithi05955964-50 from 2408448 to 2899968 tid 1 2022-04-28T19:55:07.994 INFO:tasks.rados.rados.0.smithi059.stdout:3298: delete oid 17 current snap is 320 2022-04-28T19:55:07.994 INFO:tasks.rados.rados.0.smithi059.stdout:3295: finishing rollback tid 0 to smithi05955964-22 2022-04-28T19:55:07.994 INFO:tasks.rados.rados.0.smithi059.stdout:3295: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:55:07.994 INFO:tasks.rados.rados.0.smithi059.stdout:3295: finishing rollback tid 2 to smithi05955964-22 2022-04-28T19:55:07.994 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1466 (ObjNum 988 snap 293 seq_num 988) dirty exists 2022-04-28T19:55:08.002 INFO:tasks.rados.rados.0.smithi059.stdout:3297: finishing write tid 1 to smithi05955964-50 2022-04-28T19:55:08.002 INFO:tasks.rados.rados.0.smithi059.stdout:3297: finishing write tid 3 to smithi05955964-50 2022-04-28T19:55:08.002 INFO:tasks.rados.rados.0.smithi059.stdout:3295: done (2 left) 2022-04-28T19:55:08.002 INFO:tasks.rados.rados.0.smithi059.stdout:3298: done (1 left) 2022-04-28T19:55:08.003 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 320 2022-04-28T19:55:08.003 INFO:tasks.rados.rados.0.smithi059.stdout:3299: seq_num 1034 ranges {3883008=524288} 2022-04-28T19:55:08.028 INFO:tasks.rados.rados.0.smithi059.stdout:3299: writing smithi05955964-14 from 3883008 to 4407296 tid 1 2022-04-28T19:55:08.028 INFO:tasks.rados.rados.0.smithi059.stdout:3300: read oid 44 snap -1 2022-04-28T19:55:08.028 INFO:tasks.rados.rados.0.smithi059.stdout:3300: expect (ObjNum 1024 snap 315 seq_num 1024) 2022-04-28T19:55:08.028 INFO:tasks.rados.rados.0.smithi059.stdout:3301: setattr oid 33 current snap is 320 2022-04-28T19:55:08.028 INFO:tasks.rados.rados.0.smithi059.stdout:3297: finishing write tid 4 to smithi05955964-50 2022-04-28T19:55:08.028 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1469 (ObjNum 1032 snap 320 seq_num 1032) dirty exists 2022-04-28T19:55:08.029 INFO:tasks.rados.rados.0.smithi059.stdout:3297: left oid 50 (ObjNum 1032 snap 320 seq_num 1032) 2022-04-28T19:55:08.029 INFO:tasks.rados.rados.0.smithi059.stdout:3297: done (3 left) 2022-04-28T19:55:08.029 INFO:tasks.rados.rados.0.smithi059.stdout:3302: delete oid 36 current snap is 320 2022-04-28T19:55:08.034 INFO:tasks.rados.rados.0.smithi059.stdout:3300: expect (ObjNum 1024 snap 315 seq_num 1024) 2022-04-28T19:55:08.053 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1471 (ObjNum 822 snap 241 seq_num 822) dirty exists 2022-04-28T19:55:08.053 INFO:tasks.rados.rados.0.smithi059.stdout:3299: finishing write tid 1 to smithi05955964-14 2022-04-28T19:55:08.053 INFO:tasks.rados.rados.0.smithi059.stdout:3299: finishing write tid 3 to smithi05955964-14 2022-04-28T19:55:08.053 INFO:tasks.rados.rados.0.smithi059.stdout:3300: done (3 left) 2022-04-28T19:55:08.053 INFO:tasks.rados.rados.0.smithi059.stdout:3301: done (2 left) 2022-04-28T19:55:08.054 INFO:tasks.rados.rados.0.smithi059.stdout:3302: done (1 left) 2022-04-28T19:55:08.054 INFO:tasks.rados.rados.0.smithi059.stdout:3303: delete oid 7 current snap is 320 2022-04-28T19:55:08.054 INFO:tasks.rados.rados.0.smithi059.stdout:3299: finishing write tid 4 to smithi05955964-14 2022-04-28T19:55:08.054 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1468 (ObjNum 1034 snap 320 seq_num 1034) dirty exists 2022-04-28T19:55:08.054 INFO:tasks.rados.rados.0.smithi059.stdout:3299: left oid 14 (ObjNum 1034 snap 320 seq_num 1034) 2022-04-28T19:55:08.055 INFO:tasks.rados.rados.0.smithi059.stdout:3299: done (1 left) 2022-04-28T19:55:08.055 INFO:tasks.rados.rados.0.smithi059.stdout:3303: done (0 left) 2022-04-28T19:55:08.055 INFO:tasks.rados.rados.0.smithi059.stdout:3304: snap_create 2022-04-28T19:55:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:08 smithi059 bash[61152]: cluster 2022-04-28T19:55:06.987202+0000 mgr.y (mgr.24961) 321 : cluster [DBG] pgmap v512: 89 pgs: 89 active+clean; 816 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 378 KiB/s wr, 23 op/s 2022-04-28T19:55:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:08 smithi059 bash[61152]: cluster 2022-04-28T19:55:07.977023+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2022-04-28T19:55:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:08 smithi059 bash[59745]: cluster 2022-04-28T19:55:06.987202+0000 mgr.y (mgr.24961) 321 : cluster [DBG] pgmap v512: 89 pgs: 89 active+clean; 816 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 378 KiB/s wr, 23 op/s 2022-04-28T19:55:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:08 smithi059 bash[59745]: cluster 2022-04-28T19:55:07.977023+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2022-04-28T19:55:08.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:08 smithi139 bash[43301]: cluster 2022-04-28T19:55:06.987202+0000 mgr.y (mgr.24961) 321 : cluster [DBG] pgmap v512: 89 pgs: 89 active+clean; 816 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 378 KiB/s wr, 23 op/s 2022-04-28T19:55:08.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:08 smithi139 bash[43301]: cluster 2022-04-28T19:55:07.977023+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2022-04-28T19:55:08.979 INFO:tasks.rados.rados.0.smithi059.stdout:3304: done (0 left) 2022-04-28T19:55:08.979 INFO:tasks.rados.rados.0.smithi059.stdout:3305: snap_remove snap 320 2022-04-28T19:55:09.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:09 smithi059 bash[61152]: cephadm 2022-04-28T19:55:07.573653+0000 mgr.y (mgr.24961) 322 : cephadm [INF] Upgrade: Updating alertmanager.a 2022-04-28T19:55:09.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:09 smithi059 bash[61152]: cephadm 2022-04-28T19:55:07.586867+0000 mgr.y (mgr.24961) 323 : cephadm [INF] Deploying daemon alertmanager.a on smithi059 2022-04-28T19:55:09.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:09 smithi059 bash[61152]: cluster 2022-04-28T19:55:08.978362+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2022-04-28T19:55:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:09 smithi059 bash[59745]: cephadm 2022-04-28T19:55:07.573653+0000 mgr.y (mgr.24961) 322 : cephadm [INF] Upgrade: Updating alertmanager.a 2022-04-28T19:55:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:09 smithi059 bash[59745]: cephadm 2022-04-28T19:55:07.586867+0000 mgr.y (mgr.24961) 323 : cephadm [INF] Deploying daemon alertmanager.a on smithi059 2022-04-28T19:55:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:09 smithi059 bash[59745]: cluster 2022-04-28T19:55:08.978362+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2022-04-28T19:55:09.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:09 smithi139 bash[43301]: cephadm 2022-04-28T19:55:07.573653+0000 mgr.y (mgr.24961) 322 : cephadm [INF] Upgrade: Updating alertmanager.a 2022-04-28T19:55:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:09 smithi139 bash[43301]: cephadm 2022-04-28T19:55:07.586867+0000 mgr.y (mgr.24961) 323 : cephadm [INF] Deploying daemon alertmanager.a on smithi059 2022-04-28T19:55:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:09 smithi139 bash[43301]: cluster 2022-04-28T19:55:08.978362+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2022-04-28T19:55:09.987 INFO:tasks.rados.rados.0.smithi059.stdout:3305: done (0 left) 2022-04-28T19:55:09.987 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 321 2022-04-28T19:55:09.987 INFO:tasks.rados.rados.0.smithi059.stdout:3306: seq_num 1037 ranges {0=573440} 2022-04-28T19:55:09.990 INFO:tasks.rados.rados.0.smithi059.stdout:3306: writing smithi05955964-7 from 0 to 573440 tid 1 2022-04-28T19:55:09.991 INFO:tasks.rados.rados.0.smithi059.stdout:3307: snap_remove snap 298 2022-04-28T19:55:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:10 smithi059 bash[61152]: cluster 2022-04-28T19:55:08.987727+0000 mgr.y (mgr.24961) 324 : cluster [DBG] pgmap v515: 89 pgs: 89 active+clean; 817 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 227 KiB/s wr, 8 op/s 2022-04-28T19:55:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:10 smithi059 bash[61152]: cluster 2022-04-28T19:55:09.984568+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2022-04-28T19:55:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:10 smithi059 bash[59745]: cluster 2022-04-28T19:55:08.987727+0000 mgr.y (mgr.24961) 324 : cluster [DBG] pgmap v515: 89 pgs: 89 active+clean; 817 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 227 KiB/s wr, 8 op/s 2022-04-28T19:55:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:10 smithi059 bash[59745]: cluster 2022-04-28T19:55:09.984568+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2022-04-28T19:55:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:10 smithi139 bash[43301]: cluster 2022-04-28T19:55:08.987727+0000 mgr.y (mgr.24961) 324 : cluster [DBG] pgmap v515: 89 pgs: 89 active+clean; 817 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 227 KiB/s wr, 8 op/s 2022-04-28T19:55:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:10 smithi139 bash[43301]: cluster 2022-04-28T19:55:09.984568+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2022-04-28T19:55:10.993 INFO:tasks.rados.rados.0.smithi059.stdout:3307: done (1 left) 2022-04-28T19:55:10.993 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 321 2022-04-28T19:55:10.994 INFO:tasks.rados.rados.0.smithi059.stdout:3308: seq_num 1038 ranges {1155072=589824} 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3308: writing smithi05955964-41 from 1155072 to 1744896 tid 1 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3309: read oid 43 snap 294 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3309: expect (ObjNum -133203856 snap 21893 seq_num 4161238064) 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3310: copy_from oid 29 from oid 23 current snap is 321 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3306: finishing write tid 1 to smithi05955964-7 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3306: finishing write tid 3 to smithi05955964-7 2022-04-28T19:55:11.004 INFO:tasks.rados.rados.0.smithi059.stdout:3306: finishing write tid 4 to smithi05955964-7 2022-04-28T19:55:11.005 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 1363 (ObjNum 1037 snap 321 seq_num 1037) dirty exists 2022-04-28T19:55:11.005 INFO:tasks.rados.rados.0.smithi059.stdout:3306: left oid 7 (ObjNum 1037 snap 321 seq_num 1037) 2022-04-28T19:55:11.005 INFO:tasks.rados.rados.0.smithi059.stdout:3306: done (3 left) 2022-04-28T19:55:11.005 INFO:tasks.rados.rados.0.smithi059.stdout:3311: read oid 42 snap 246 2022-04-28T19:55:11.005 INFO:tasks.rados.rados.0.smithi059.stdout:3311: expect (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:55:11.005 INFO:tasks.rados.rados.0.smithi059.stdout:3312: delete oid 33 current snap is 321 2022-04-28T19:55:11.008 INFO:tasks.rados.rados.0.smithi059.stdout:3309: expect (ObjNum -133203856 snap 21893 seq_num 4161238064) 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3308: finishing write tid 1 to smithi05955964-41 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3308: finishing write tid 3 to smithi05955964-41 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3308: finishing write tid 4 to smithi05955964-41 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1020 (ObjNum 1038 snap 321 seq_num 1038) dirty exists 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3308: left oid 41 (ObjNum 1038 snap 321 seq_num 1038) 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3308: done (4 left) 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3309: done (3 left) 2022-04-28T19:55:11.012 INFO:tasks.rados.rados.0.smithi059.stdout:3312: done (2 left) 2022-04-28T19:55:11.013 INFO:tasks.rados.rados.0.smithi059.stdout:3313: snap_remove snap 296 2022-04-28T19:55:11.962 INFO:tasks.rados.rados.0.smithi059.stdout:3313: done (2 left) 2022-04-28T19:55:11.962 INFO:tasks.rados.rados.0.smithi059.stdout:3314: delete oid 49 current snap is 321 2022-04-28T19:55:11.962 INFO:tasks.rados.rados.0.smithi059.stdout:3310: finishing copy_from to smithi05955964-29 2022-04-28T19:55:11.962 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1336 (ObjNum 1014 snap 314 seq_num 1014) dirty exists 2022-04-28T19:55:11.963 INFO:tasks.rados.rados.0.smithi059.stdout:3310: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:55:11.963 INFO:tasks.rados.rados.0.smithi059.stdout:3311: expect (ObjNum 756 snap 229 seq_num 756) 2022-04-28T19:55:11.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:11.637Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:55:11.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:11.637Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:55:11.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:11.637Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139; Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" 2022-04-28T19:55:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[61152]: cluster 2022-04-28T19:55:10.989508+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2022-04-28T19:55:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[61152]: cluster 2022-04-28T19:55:10.990506+0000 mgr.y (mgr.24961) 325 : cluster [DBG] pgmap v518: 89 pgs: 89 active+clean; 818 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 576 KiB/s wr, 19 op/s 2022-04-28T19:55:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[61152]: cluster 2022-04-28T19:55:11.958130+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2022-04-28T19:55:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[59745]: cluster 2022-04-28T19:55:10.989508+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2022-04-28T19:55:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[59745]: cluster 2022-04-28T19:55:10.990506+0000 mgr.y (mgr.24961) 325 : cluster [DBG] pgmap v518: 89 pgs: 89 active+clean; 818 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 576 KiB/s wr, 19 op/s 2022-04-28T19:55:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:11 smithi059 bash[59745]: cluster 2022-04-28T19:55:11.958130+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2022-04-28T19:55:12.363 INFO:tasks.rados.rados.0.smithi059.stdout:3310: done (2 left) 2022-04-28T19:55:12.363 INFO:tasks.rados.rados.0.smithi059.stdout:3311: done (1 left) 2022-04-28T19:55:12.363 INFO:tasks.rados.rados.0.smithi059.stdout:3314: done (0 left) 2022-04-28T19:55:12.364 INFO:tasks.rados.rados.0.smithi059.stdout:3315: rollback oid 8 current snap is 321 2022-04-28T19:55:12.364 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 8 to 287 2022-04-28T19:55:12.364 INFO:tasks.rados.rados.0.smithi059.stdout:3316: read oid 28 snap -1 2022-04-28T19:55:12.365 INFO:tasks.rados.rados.0.smithi059.stdout:3316: expect (ObjNum 966 snap 289 seq_num 966) 2022-04-28T19:55:12.365 INFO:tasks.rados.rados.0.smithi059.stdout:3317: delete oid 43 current snap is 321 2022-04-28T19:55:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:11 smithi139 bash[43301]: cluster 2022-04-28T19:55:10.989508+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2022-04-28T19:55:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:11 smithi139 bash[43301]: cluster 2022-04-28T19:55:10.990506+0000 mgr.y (mgr.24961) 325 : cluster [DBG] pgmap v518: 89 pgs: 89 active+clean; 818 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 576 KiB/s wr, 19 op/s 2022-04-28T19:55:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:11 smithi139 bash[43301]: cluster 2022-04-28T19:55:11.958130+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2022-04-28T19:55:12.370 INFO:tasks.rados.rados.0.smithi059.stdout:3317: done (2 left) 2022-04-28T19:55:12.371 INFO:tasks.rados.rados.0.smithi059.stdout:3318: snap_remove snap 318 2022-04-28T19:55:12.999 INFO:tasks.rados.rados.0.smithi059.stdout:3318: done (2 left) 2022-04-28T19:55:12.999 INFO:tasks.rados.rados.0.smithi059.stdout:3319: rollback oid 48 current snap is 321 2022-04-28T19:55:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 252 2022-04-28T19:55:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:3315: finishing rollback tid 0 to smithi05955964-8 2022-04-28T19:55:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:3315: finishing rollback tid 1 to smithi05955964-8 2022-04-28T19:55:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:3315: finishing rollback tid 2 to smithi05955964-8 2022-04-28T19:55:13.000 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1504 (ObjNum 953 snap 279 seq_num 953) dirty exists 2022-04-28T19:55:13.001 INFO:tasks.rados.rados.0.smithi059.stdout:3316: expect (ObjNum 966 snap 289 seq_num 966) 2022-04-28T19:55:13.130 INFO:tasks.rados.rados.0.smithi059.stdout:3319: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3319: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1470 (ObjNum 646 snap 184 seq_num 646) dirty exists 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3315: done (2 left) 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3316: done (1 left) 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3319: done (0 left) 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3320: rollback oid 30 current snap is 321 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 30 to 294 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3321: rollback oid 27 current snap is 321 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 307 2022-04-28T19:55:13.131 INFO:tasks.rados.rados.0.smithi059.stdout:3322: delete oid 24 current snap is 321 2022-04-28T19:55:13.135 INFO:tasks.rados.rados.0.smithi059.stdout:3321: finishing rollback tid 0 to smithi05955964-27 2022-04-28T19:55:13.135 INFO:tasks.rados.rados.0.smithi059.stdout:3321: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:55:13.136 INFO:tasks.rados.rados.0.smithi059.stdout:3321: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:55:13.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1341 (ObjNum -131500368 snap 21893 seq_num 4162291952) dirty exists 2022-04-28T19:55:13.137 INFO:tasks.rados.rados.0.smithi059.stdout:3320: finishing rollback tid 0 to smithi05955964-30 2022-04-28T19:55:13.137 INFO:tasks.rados.rados.0.smithi059.stdout:3320: finishing rollback tid 1 to smithi05955964-30 2022-04-28T19:55:13.137 INFO:tasks.rados.rados.0.smithi059.stdout:3320: finishing rollback tid 2 to smithi05955964-30 2022-04-28T19:55:13.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1347 (ObjNum 950 snap 279 seq_num 950) dirty exists 2022-04-28T19:55:13.137 INFO:tasks.rados.rados.0.smithi059.stdout:3320: done (2 left) 2022-04-28T19:55:13.138 INFO:tasks.rados.rados.0.smithi059.stdout:3321: done (1 left) 2022-04-28T19:55:13.138 INFO:tasks.rados.rados.0.smithi059.stdout:3322: done (0 left) 2022-04-28T19:55:13.138 INFO:tasks.rados.rados.0.smithi059.stdout:3323: delete oid 28 current snap is 321 2022-04-28T19:55:13.139 INFO:tasks.rados.rados.0.smithi059.stdout:3323: done (0 left) 2022-04-28T19:55:13.139 INFO:tasks.rados.rados.0.smithi059.stdout:3324: rmattr oid 44 current snap is 321 2022-04-28T19:55:13.140 INFO:tasks.rados.rados.0.smithi059.stdout:3325: read oid 37 snap -1 2022-04-28T19:55:13.140 INFO:tasks.rados.rados.0.smithi059.stdout:3325: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:55:13.140 INFO:tasks.rados.rados.0.smithi059.stdout:3326: snap_create 2022-04-28T19:55:13.141 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1191 (ObjNum 1024 snap 315 seq_num 1024) dirty exists 2022-04-28T19:55:13.165 INFO:tasks.rados.rados.0.smithi059.stdout:3325: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:55:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:13.531Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" context_err="context deadline exceeded" 2022-04-28T19:55:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:13.531Z caller=notify.go:372 component=dispatcher msg="Error on notify" err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059" context_err="context deadline exceeded" 2022-04-28T19:55:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:13 smithi059 bash[42244]: level=error ts=2022-04-28T19:55:13.531Z caller=dispatch.go:301 component=dispatcher msg="Notify for alerts failed" num_alerts=10 err="Post https://smithi059:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi059; Post https://smithi139:8443/api/prometheus_receiver: x509: certificate is valid for ceph-dashboard, not smithi139" 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3324: done (2 left) 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3325: done (1 left) 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3326: done (0 left) 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3327: rollback oid 49 current snap is 322 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 49 to 312 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3328: read oid 42 snap -1 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3328: expect deleted 2022-04-28T19:55:14.007 INFO:tasks.rados.rados.0.smithi059.stdout:3329: snap_create 2022-04-28T19:55:14.008 INFO:tasks.rados.rados.0.smithi059.stdout:3327: finishing rollback tid 1 to smithi05955964-49 2022-04-28T19:55:14.008 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1143 (ObjNum -132399008 snap 21893 seq_num 4162387664) dirty dne 2022-04-28T19:55:14.174 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:14 smithi059 bash[59745]: cluster 2022-04-28T19:55:12.990911+0000 mgr.y (mgr.24961) 326 : cluster [DBG] pgmap v520: 89 pgs: 89 active+clean; 818 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 574 KiB/s wr, 19 op/s 2022-04-28T19:55:14.174 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:14 smithi059 bash[59745]: cluster 2022-04-28T19:55:12.997126+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2022-04-28T19:55:14.174 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:14 smithi059 bash[61152]: cluster 2022-04-28T19:55:12.990911+0000 mgr.y (mgr.24961) 326 : cluster [DBG] pgmap v520: 89 pgs: 89 active+clean; 818 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 574 KiB/s wr, 19 op/s 2022-04-28T19:55:14.175 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:14 smithi059 bash[61152]: cluster 2022-04-28T19:55:12.997126+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2022-04-28T19:55:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:14 smithi139 bash[43301]: cluster 2022-04-28T19:55:12.990911+0000 mgr.y (mgr.24961) 326 : cluster [DBG] pgmap v520: 89 pgs: 89 active+clean; 818 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 574 KiB/s wr, 19 op/s 2022-04-28T19:55:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:14 smithi139 bash[43301]: cluster 2022-04-28T19:55:12.997126+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2022-04-28T19:55:15.014 INFO:tasks.rados.rados.0.smithi059.stdout:3327: done (2 left) 2022-04-28T19:55:15.014 INFO:tasks.rados.rados.0.smithi059.stdout:3328: done (1 left) 2022-04-28T19:55:15.014 INFO:tasks.rados.rados.0.smithi059.stdout:3329: done (0 left) 2022-04-28T19:55:15.015 INFO:tasks.rados.rados.0.smithi059.stdout:3330: read oid 49 snap 290 2022-04-28T19:55:15.015 INFO:tasks.rados.rados.0.smithi059.stdout:3330: expect (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:55:15.015 INFO:tasks.rados.rados.0.smithi059.stdout:3331: snap_remove snap 314 2022-04-28T19:55:15.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:15 smithi059 bash[59745]: cluster 2022-04-28T19:55:14.005741+0000 mon.a (mon.0) 1222 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2022-04-28T19:55:15.086 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:15 smithi059 bash[61152]: cluster 2022-04-28T19:55:14.005741+0000 mon.a (mon.0) 1222 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2022-04-28T19:55:15.086 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:14 smithi059 systemd[1]: Stopping Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:55:15.087 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:15 smithi059 bash[42244]: level=info ts=2022-04-28T19:55:15.022Z caller=main.go:536 msg="Received SIGTERM, exiting gracefully..." 2022-04-28T19:55:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:15 smithi139 bash[43301]: cluster 2022-04-28T19:55:14.005741+0000 mon.a (mon.0) 1222 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2022-04-28T19:55:15.973 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:15 smithi059 bash[71930]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-alertmanager.a 2022-04-28T19:55:15.973 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:15 smithi059 bash[71930]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-alertmanager-a 2022-04-28T19:55:15.974 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:15 smithi059 bash[71930]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-alertmanager.a 2022-04-28T19:55:15.974 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:15 smithi059 systemd[1]: Stopped Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:55:16.024 INFO:tasks.rados.rados.0.smithi059.stdout:3331: done (1 left) 2022-04-28T19:55:16.024 INFO:tasks.rados.rados.0.smithi059.stdout:3332: rollback oid 46 current snap is 323 2022-04-28T19:55:16.024 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 306 2022-04-28T19:55:16.024 INFO:tasks.rados.rados.0.smithi059.stdout:3330: expect (ObjNum 929 snap 276 seq_num 929) 2022-04-28T19:55:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:16 smithi059 bash[61152]: cluster 2022-04-28T19:55:14.991443+0000 mgr.y (mgr.24961) 327 : cluster [DBG] pgmap v523: 89 pgs: 89 active+clean; 820 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 329 KiB/s wr, 10 op/s 2022-04-28T19:55:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:16 smithi059 bash[61152]: cluster 2022-04-28T19:55:15.011689+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2022-04-28T19:55:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:16 smithi059 bash[59745]: cluster 2022-04-28T19:55:14.991443+0000 mgr.y (mgr.24961) 327 : cluster [DBG] pgmap v523: 89 pgs: 89 active+clean; 820 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 329 KiB/s wr, 10 op/s 2022-04-28T19:55:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:16 smithi059 bash[59745]: cluster 2022-04-28T19:55:15.011689+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2022-04-28T19:55:16.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:15 smithi059 systemd[1]: Started Ceph alertmanager.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:55:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:16 smithi139 bash[43301]: cluster 2022-04-28T19:55:14.991443+0000 mgr.y (mgr.24961) 327 : cluster [DBG] pgmap v523: 89 pgs: 89 active+clean; 820 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 329 KiB/s wr, 10 op/s 2022-04-28T19:55:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:16 smithi139 bash[43301]: cluster 2022-04-28T19:55:15.011689+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2022-04-28T19:55:17.137 INFO:tasks.rados.rados.0.smithi059.stdout:3330: done (1 left) 2022-04-28T19:55:17.137 INFO:tasks.rados.rados.0.smithi059.stdout:3333: delete oid 21 current snap is 323 2022-04-28T19:55:17.137 INFO:tasks.rados.rados.0.smithi059.stdout:3332: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:55:17.137 INFO:tasks.rados.rados.0.smithi059.stdout:3332: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:55:17.138 INFO:tasks.rados.rados.0.smithi059.stdout:3332: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:55:17.138 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1508 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:55:17.141 INFO:tasks.rados.rados.0.smithi059.stdout:3332: done (1 left) 2022-04-28T19:55:17.141 INFO:tasks.rados.rados.0.smithi059.stdout:3333: done (0 left) 2022-04-28T19:55:17.141 INFO:tasks.rados.rados.0.smithi059.stdout:3334: snap_create 2022-04-28T19:55:17.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: cluster 2022-04-28T19:55:16.020758+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2022-04-28T19:55:17.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.021862+0000 mon.a (mon.0) 1225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:17.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.024580+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:17.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.026114+0000 mon.a (mon.0) 1227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:17.288 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.026968+0000 mon.a (mon.0) 1228 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:17.289 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: cephadm 2022-04-28T19:55:16.027852+0000 mgr.y (mgr.24961) 328 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:17.289 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.028779+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:17.289 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.920019+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T19:55:17.289 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.920581+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:55:17.290 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: audit 2022-04-28T19:55:16.921049+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:55:17.290 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[59745]: cluster 2022-04-28T19:55:16.956175+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2022-04-28T19:55:17.290 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: cluster 2022-04-28T19:55:16.020758+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2022-04-28T19:55:17.290 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.021862+0000 mon.a (mon.0) 1225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:17.291 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.024580+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:17.291 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.026114+0000 mon.a (mon.0) 1227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:17.291 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.026968+0000 mon.a (mon.0) 1228 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:17.291 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: cephadm 2022-04-28T19:55:16.027852+0000 mgr.y (mgr.24961) 328 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.028779+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.920019+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T19:55:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.920581+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:55:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: audit 2022-04-28T19:55:16.921049+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:55:17.292 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[61152]: cluster 2022-04-28T19:55:16.956175+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2022-04-28T19:55:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: cluster 2022-04-28T19:55:16.020758+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2022-04-28T19:55:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.021862+0000 mon.a (mon.0) 1225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.024580+0000 mon.a (mon.0) 1226 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.026114+0000 mon.a (mon.0) 1227 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.026968+0000 mon.a (mon.0) 1228 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: cephadm 2022-04-28T19:55:16.027852+0000 mgr.y (mgr.24961) 328 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.028779+0000 mon.a (mon.0) 1229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.920019+0000 mon.a (mon.0) 1230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T19:55:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.920581+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:55:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: audit 2022-04-28T19:55:16.921049+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:55:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:17 smithi139 bash[43301]: cluster 2022-04-28T19:55:16.956175+0000 mon.a (mon.0) 1233 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2022-04-28T19:55:17.590 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.287Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.287Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.292Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.59 port=9094 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.294Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.347Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.348Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.352Z caller=main.go:518 msg=Listening address=:9093 2022-04-28T19:55:17.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:17 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:17.352Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2022-04-28T19:55:18.041 INFO:tasks.rados.rados.0.smithi059.stdout:3334: done (0 left) 2022-04-28T19:55:18.041 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 324 2022-04-28T19:55:18.041 INFO:tasks.rados.rados.0.smithi059.stdout:3335: seq_num 1045 ranges {9035776=704512} 2022-04-28T19:55:18.096 INFO:tasks.rados.rados.0.smithi059.stdout:3335: writing smithi05955964-31 from 9035776 to 9740288 tid 1 2022-04-28T19:55:18.096 INFO:tasks.rados.rados.0.smithi059.stdout:3336: delete oid 30 current snap is 324 2022-04-28T19:55:18.109 INFO:tasks.rados.rados.0.smithi059.stdout:3335: finishing write tid 1 to smithi05955964-31 2022-04-28T19:55:18.110 INFO:tasks.rados.rados.0.smithi059.stdout:3335: finishing write tid 3 to smithi05955964-31 2022-04-28T19:55:18.111 INFO:tasks.rados.rados.0.smithi059.stdout:3335: finishing write tid 4 to smithi05955964-31 2022-04-28T19:55:18.111 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1365 (ObjNum 1045 snap 324 seq_num 1045) dirty exists 2022-04-28T19:55:18.111 INFO:tasks.rados.rados.0.smithi059.stdout:3335: left oid 31 (ObjNum 1045 snap 324 seq_num 1045) 2022-04-28T19:55:18.111 INFO:tasks.rados.rados.0.smithi059.stdout:3335: done (1 left) 2022-04-28T19:55:18.111 INFO:tasks.rados.rados.0.smithi059.stdout:3336: done (0 left) 2022-04-28T19:55:18.112 INFO:tasks.rados.rados.0.smithi059.stdout:3337: delete oid 48 current snap is 324 2022-04-28T19:55:18.114 INFO:tasks.rados.rados.0.smithi059.stdout:3337: done (0 left) 2022-04-28T19:55:18.114 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 324 2022-04-28T19:55:18.114 INFO:tasks.rados.rados.0.smithi059.stdout:3338: seq_num 1048 ranges {524288=761856} 2022-04-28T19:55:18.121 INFO:tasks.rados.rados.0.smithi059.stdout:3338: writing smithi05955964-44 from 524288 to 1286144 tid 1 2022-04-28T19:55:18.122 INFO:tasks.rados.rados.0.smithi059.stdout:3339: snap_remove snap 305 2022-04-28T19:55:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:18 smithi059 bash[59745]: cluster 2022-04-28T19:55:16.991855+0000 mgr.y (mgr.24961) 329 : cluster [DBG] pgmap v527: 89 pgs: 89 active+clean; 825 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 330 KiB/s wr, 11 op/s 2022-04-28T19:55:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:18 smithi059 bash[59745]: audit 2022-04-28T19:55:17.148991+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:55:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:18 smithi059 bash[59745]: audit 2022-04-28T19:55:17.153795+0000 mon.a (mon.0) 1235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:55:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:18 smithi059 bash[61152]: cluster 2022-04-28T19:55:16.991855+0000 mgr.y (mgr.24961) 329 : cluster [DBG] pgmap v527: 89 pgs: 89 active+clean; 825 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 330 KiB/s wr, 11 op/s 2022-04-28T19:55:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:18 smithi059 bash[61152]: audit 2022-04-28T19:55:17.148991+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:55:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:18 smithi059 bash[61152]: audit 2022-04-28T19:55:17.153795+0000 mon.a (mon.0) 1235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:18 smithi139 bash[43301]: cluster 2022-04-28T19:55:16.991855+0000 mgr.y (mgr.24961) 329 : cluster [DBG] pgmap v527: 89 pgs: 89 active+clean; 825 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 330 KiB/s wr, 11 op/s 2022-04-28T19:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:18 smithi139 bash[43301]: audit 2022-04-28T19:55:17.148991+0000 mon.a (mon.0) 1234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:18 smithi139 bash[43301]: audit 2022-04-28T19:55:17.153795+0000 mon.a (mon.0) 1235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:55:19.051 INFO:tasks.rados.rados.0.smithi059.stdout:3339: done (1 left) 2022-04-28T19:55:19.052 INFO:tasks.rados.rados.0.smithi059.stdout:3340: snap_create 2022-04-28T19:55:19.052 INFO:tasks.rados.rados.0.smithi059.stdout:3338: finishing write tid 1 to smithi05955964-44 2022-04-28T19:55:19.052 INFO:tasks.rados.rados.0.smithi059.stdout:3338: finishing write tid 3 to smithi05955964-44 2022-04-28T19:55:19.052 INFO:tasks.rados.rados.0.smithi059.stdout:3338: finishing write tid 4 to smithi05955964-44 2022-04-28T19:55:19.052 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1193 (ObjNum 1048 snap 324 seq_num 1048) dirty exists 2022-04-28T19:55:19.052 INFO:tasks.rados.rados.0.smithi059.stdout:3338: left oid 44 (ObjNum 1048 snap 324 seq_num 1048) 2022-04-28T19:55:19.295 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:19 smithi059 bash[59745]: cluster 2022-04-28T19:55:18.038273+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2022-04-28T19:55:19.296 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:19 smithi059 bash[61152]: cluster 2022-04-28T19:55:18.038273+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2022-04-28T19:55:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:19 smithi139 bash[43301]: cluster 2022-04-28T19:55:18.038273+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2022-04-28T19:55:19.590 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:19 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:19.295Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000248737s 2022-04-28T19:55:20.053 INFO:tasks.rados.rados.0.smithi059.stdout:3338: done (1 left) 2022-04-28T19:55:20.053 INFO:tasks.rados.rados.0.smithi059.stdout:3340: done (0 left) 2022-04-28T19:55:20.053 INFO:tasks.rados.rados.0.smithi059.stdout:3341: rollback oid 21 current snap is 325 2022-04-28T19:55:20.054 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 308 2022-04-28T19:55:20.054 INFO:tasks.rados.rados.0.smithi059.stdout:3342: rollback oid 38 current snap is 325 2022-04-28T19:55:20.054 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 38 to 319 2022-04-28T19:55:20.054 INFO:tasks.rados.rados.0.smithi059.stdout:3343: snap_create 2022-04-28T19:55:20.061 INFO:tasks.rados.rados.0.smithi059.stdout:3341: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:55:20.061 INFO:tasks.rados.rados.0.smithi059.stdout:3341: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:55:20.062 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1159 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:55:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:3342: finishing rollback tid 0 to smithi05955964-38 2022-04-28T19:55:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:3342: finishing rollback tid 1 to smithi05955964-38 2022-04-28T19:55:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:3342: finishing rollback tid 2 to smithi05955964-38 2022-04-28T19:55:20.063 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1210 (ObjNum 1018 snap 314 seq_num 1018) dirty exists 2022-04-28T19:55:20.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:20 smithi059 bash[61152]: cluster 2022-04-28T19:55:18.992760+0000 mgr.y (mgr.24961) 330 : cluster [DBG] pgmap v529: 89 pgs: 89 active+clean; 826 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 172 KiB/s wr, 1 op/s 2022-04-28T19:55:20.402 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:20 smithi059 bash[61152]: cluster 2022-04-28T19:55:19.049388+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2022-04-28T19:55:20.403 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:20 smithi059 bash[59745]: cluster 2022-04-28T19:55:18.992760+0000 mgr.y (mgr.24961) 330 : cluster [DBG] pgmap v529: 89 pgs: 89 active+clean; 826 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 172 KiB/s wr, 1 op/s 2022-04-28T19:55:20.403 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:20 smithi059 bash[59745]: cluster 2022-04-28T19:55:19.049388+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2022-04-28T19:55:20.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:20 smithi139 bash[43301]: cluster 2022-04-28T19:55:18.992760+0000 mgr.y (mgr.24961) 330 : cluster [DBG] pgmap v529: 89 pgs: 89 active+clean; 826 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 172 KiB/s wr, 1 op/s 2022-04-28T19:55:20.404 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:20 smithi139 bash[43301]: cluster 2022-04-28T19:55:19.049388+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2022-04-28T19:55:21.064 INFO:tasks.rados.rados.0.smithi059.stdout:3341: done (2 left) 2022-04-28T19:55:21.064 INFO:tasks.rados.rados.0.smithi059.stdout:3342: done (1 left) 2022-04-28T19:55:21.064 INFO:tasks.rados.rados.0.smithi059.stdout:3343: done (0 left) 2022-04-28T19:55:21.065 INFO:tasks.rados.rados.0.smithi059.stdout:3344: read oid 9 snap 325 2022-04-28T19:55:21.065 INFO:tasks.rados.rados.0.smithi059.stdout:3344: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:55:21.065 INFO:tasks.rados.rados.0.smithi059.stdout:3345: snap_create 2022-04-28T19:55:21.091 INFO:tasks.rados.rados.0.smithi059.stdout:3344: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:55:21.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:21 smithi059 bash[61152]: cluster 2022-04-28T19:55:20.052152+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2022-04-28T19:55:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:21 smithi059 bash[59745]: cluster 2022-04-28T19:55:20.052152+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2022-04-28T19:55:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:21 smithi139 bash[43301]: cluster 2022-04-28T19:55:20.052152+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2022-04-28T19:55:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:3344: done (1 left) 2022-04-28T19:55:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:3345: done (0 left) 2022-04-28T19:55:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 327 2022-04-28T19:55:21.965 INFO:tasks.rados.rados.0.smithi059.stdout:3346: seq_num 1049 ranges {188416=778240} 2022-04-28T19:55:21.970 INFO:tasks.rados.rados.0.smithi059.stdout:3346: writing smithi05955964-23 from 188416 to 966656 tid 1 2022-04-28T19:55:21.970 INFO:tasks.rados.rados.0.smithi059.stdout:3347: delete oid 13 current snap is 327 2022-04-28T19:55:21.976 INFO:tasks.rados.rados.0.smithi059.stdout:3346: finishing write tid 1 to smithi05955964-23 2022-04-28T19:55:21.977 INFO:tasks.rados.rados.0.smithi059.stdout:3347: done (1 left) 2022-04-28T19:55:21.977 INFO:tasks.rados.rados.0.smithi059.stdout:3348: copy_from oid 14 from oid 37 current snap is 327 2022-04-28T19:55:21.977 INFO:tasks.rados.rados.0.smithi059.stdout:3349: snap_remove snap 294 2022-04-28T19:55:22.333 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: cluster 2022-04-28T19:55:20.993360+0000 mgr.y (mgr.24961) 331 : cluster [DBG] pgmap v532: 89 pgs: 89 active+clean; 827 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 355 KiB/s wr, 5 op/s 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: cluster 2022-04-28T19:55:21.062549+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.122805+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.128745+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.138593+0000 mon.a (mon.0) 1242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.139160+0000 mgr.y (mgr.24961) 332 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.141300+0000 mon.a (mon.0) 1243 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.143820+0000 mon.a (mon.0) 1244 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.145345+0000 mon.a (mon.0) 1245 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.334 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.146499+0000 mon.a (mon.0) 1246 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.148386+0000 mon.a (mon.0) 1247 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.150659+0000 mon.a (mon.0) 1248 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.152029+0000 mon.a (mon.0) 1249 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.153311+0000 mon.a (mon.0) 1250 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.154718+0000 mon.a (mon.0) 1251 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.156014+0000 mon.a (mon.0) 1252 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.157340+0000 mon.a (mon.0) 1253 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.158506+0000 mon.a (mon.0) 1254 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.159686+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: audit 2022-04-28T19:55:21.160858+0000 mon.a (mon.0) 1256 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.335 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:22 smithi139 bash[43301]: cluster 2022-04-28T19:55:21.962921+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: cluster 2022-04-28T19:55:20.993360+0000 mgr.y (mgr.24961) 331 : cluster [DBG] pgmap v532: 89 pgs: 89 active+clean; 827 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 355 KiB/s wr, 5 op/s 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: cluster 2022-04-28T19:55:21.062549+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.122805+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.128745+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.138593+0000 mon.a (mon.0) 1242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.139160+0000 mgr.y (mgr.24961) 332 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.141300+0000 mon.a (mon.0) 1243 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.143820+0000 mon.a (mon.0) 1244 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.145345+0000 mon.a (mon.0) 1245 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.146499+0000 mon.a (mon.0) 1246 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.148386+0000 mon.a (mon.0) 1247 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.150659+0000 mon.a (mon.0) 1248 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.152029+0000 mon.a (mon.0) 1249 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.153311+0000 mon.a (mon.0) 1250 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.154718+0000 mon.a (mon.0) 1251 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.156014+0000 mon.a (mon.0) 1252 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.157340+0000 mon.a (mon.0) 1253 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.158506+0000 mon.a (mon.0) 1254 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.159686+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: audit 2022-04-28T19:55:21.160858+0000 mon.a (mon.0) 1256 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[59745]: cluster 2022-04-28T19:55:21.962921+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: cluster 2022-04-28T19:55:20.993360+0000 mgr.y (mgr.24961) 331 : cluster [DBG] pgmap v532: 89 pgs: 89 active+clean; 827 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 355 KiB/s wr, 5 op/s 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: cluster 2022-04-28T19:55:21.062549+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.122805+0000 mon.a (mon.0) 1240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.128745+0000 mon.a (mon.0) 1241 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.138593+0000 mon.a (mon.0) 1242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.139160+0000 mgr.y (mgr.24961) 332 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.141300+0000 mon.a (mon.0) 1243 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.143820+0000 mon.a (mon.0) 1244 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.145345+0000 mon.a (mon.0) 1245 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.146499+0000 mon.a (mon.0) 1246 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.148386+0000 mon.a (mon.0) 1247 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.150659+0000 mon.a (mon.0) 1248 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.152029+0000 mon.a (mon.0) 1249 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.153311+0000 mon.a (mon.0) 1250 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.154718+0000 mon.a (mon.0) 1251 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.156014+0000 mon.a (mon.0) 1252 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.157340+0000 mon.a (mon.0) 1253 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.158506+0000 mon.a (mon.0) 1254 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.159686+0000 mon.a (mon.0) 1255 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: audit 2022-04-28T19:55:21.160858+0000 mon.a (mon.0) 1256 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:22.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:22 smithi059 bash[61152]: cluster 2022-04-28T19:55:21.962921+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2022-04-28T19:55:22.977 INFO:tasks.rados.rados.0.smithi059.stdout:3349: done (2 left) 2022-04-28T19:55:22.978 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 327 2022-04-28T19:55:22.978 INFO:tasks.rados.rados.0.smithi059.stdout:3350: seq_num 1051 ranges {0=409600,409600=401408} 2022-04-28T19:55:22.979 INFO:tasks.rados.rados.0.smithi059.stdout:3350: writing smithi05955964-19 from 0 to 409600 tid 1 2022-04-28T19:55:22.982 INFO:tasks.rados.rados.0.smithi059.stdout:3350: writing smithi05955964-19 from 409600 to 811008 tid 2 2022-04-28T19:55:22.982 INFO:tasks.rados.rados.0.smithi059.stdout:3351: snap_create 2022-04-28T19:55:22.982 INFO:tasks.rados.rados.0.smithi059.stdout:3346: finishing write tid 3 to smithi05955964-23 2022-04-28T19:55:22.983 INFO:tasks.rados.rados.0.smithi059.stdout:3346: finishing write tid 4 to smithi05955964-23 2022-04-28T19:55:22.983 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1343 (ObjNum 1049 snap 327 seq_num 1049) dirty exists 2022-04-28T19:55:22.983 INFO:tasks.rados.rados.0.smithi059.stdout:3346: left oid 23 (ObjNum 1049 snap 327 seq_num 1049) 2022-04-28T19:55:22.983 INFO:tasks.rados.rados.0.smithi059.stdout:3348: finishing copy_from to smithi05955964-14 2022-04-28T19:55:22.984 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1331 (ObjNum 898 snap 268 seq_num 898) dirty exists 2022-04-28T19:55:22.984 INFO:tasks.rados.rados.0.smithi059.stdout:3348: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:55:22.984 INFO:tasks.rados.rados.0.smithi059.stdout:3350: finishing write tid 1 to smithi05955964-19 2022-04-28T19:55:22.986 INFO:tasks.rados.rados.0.smithi059.stdout:3350: finishing write tid 2 to smithi05955964-19 2022-04-28T19:55:22.986 INFO:tasks.rados.rados.0.smithi059.stdout:3350: finishing write tid 4 to smithi05955964-19 2022-04-28T19:55:22.988 INFO:tasks.rados.rados.0.smithi059.stdout:3350: finishing write tid 5 to smithi05955964-19 2022-04-28T19:55:22.988 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1474 (ObjNum 1051 snap 327 seq_num 1051) dirty exists 2022-04-28T19:55:22.988 INFO:tasks.rados.rados.0.smithi059.stdout:3350: left oid 19 (ObjNum 1051 snap 327 seq_num 1051) 2022-04-28T19:55:23.987 INFO:tasks.rados.rados.0.smithi059.stdout:3346: done (3 left) 2022-04-28T19:55:23.987 INFO:tasks.rados.rados.0.smithi059.stdout:3348: done (2 left) 2022-04-28T19:55:23.987 INFO:tasks.rados.rados.0.smithi059.stdout:3350: done (1 left) 2022-04-28T19:55:23.987 INFO:tasks.rados.rados.0.smithi059.stdout:3351: done (0 left) 2022-04-28T19:55:23.988 INFO:tasks.rados.rados.0.smithi059.stdout:3352: copy_from oid 7 from oid 49 current snap is 328 2022-04-28T19:55:23.988 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 37 current snap is 328 2022-04-28T19:55:23.988 INFO:tasks.rados.rados.0.smithi059.stdout:3353: seq_num 1052 ranges {5283840=425984,5709824=401408} 2022-04-28T19:55:24.021 INFO:tasks.rados.rados.0.smithi059.stdout:3353: writing smithi05955964-37 from 5283840 to 5709824 tid 1 2022-04-28T19:55:24.023 INFO:tasks.rados.rados.0.smithi059.stdout:3353: writing smithi05955964-37 from 5709824 to 6111232 tid 2 2022-04-28T19:55:24.023 INFO:tasks.rados.rados.0.smithi059.stdout:3354: copy_from oid 5 from oid 6 current snap is 328 2022-04-28T19:55:24.023 INFO:tasks.rados.rados.0.smithi059.stdout:3352: finishing copy_from to smithi05955964-7 2022-04-28T19:55:24.023 INFO:tasks.rados.rados.0.smithi059.stdout:3352: got expected ENOENT (src dne) 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:3352: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:3352: done (2 left) 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:3355: rollback oid 46 current snap is 328 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 301 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:3356: rollback oid 33 current snap is 328 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 322 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:3357: copy_from oid 34 from oid 48 current snap is 328 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 328 2022-04-28T19:55:24.024 INFO:tasks.rados.rados.0.smithi059.stdout:3358: seq_num 1053 ranges {3850240=442368,4292608=401408} 2022-04-28T19:55:24.047 INFO:tasks.rados.rados.0.smithi059.stdout:3358: writing smithi05955964-26 from 3850240 to 4292608 tid 1 2022-04-28T19:55:24.050 INFO:tasks.rados.rados.0.smithi059.stdout:3358: writing smithi05955964-26 from 4292608 to 4694016 tid 2 2022-04-28T19:55:24.050 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 328 2022-04-28T19:55:24.050 INFO:tasks.rados.rados.0.smithi059.stdout:3359: seq_num 1054 ranges {0=458752,458752=401408} 2022-04-28T19:55:24.053 INFO:tasks.rados.rados.0.smithi059.stdout:3359: writing smithi05955964-43 from 0 to 458752 tid 1 2022-04-28T19:55:24.055 INFO:tasks.rados.rados.0.smithi059.stdout:3359: writing smithi05955964-43 from 458752 to 860160 tid 2 2022-04-28T19:55:24.056 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 328 2022-04-28T19:55:24.056 INFO:tasks.rados.rados.0.smithi059.stdout:3360: seq_num 1055 ranges {1286144=475136,1761280=401408} 2022-04-28T19:55:24.065 INFO:tasks.rados.rados.0.smithi059.stdout:3360: writing smithi05955964-44 from 1286144 to 1761280 tid 1 2022-04-28T19:55:24.068 INFO:tasks.rados.rados.0.smithi059.stdout:3360: writing smithi05955964-44 from 1761280 to 2162688 tid 2 2022-04-28T19:55:24.068 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 328 2022-04-28T19:55:24.068 INFO:tasks.rados.rados.0.smithi059.stdout:3361: seq_num 1056 ranges {0=491520,491520=401408} 2022-04-28T19:55:24.071 INFO:tasks.rados.rados.0.smithi059.stdout:3361: writing smithi05955964-2 from 0 to 491520 tid 1 2022-04-28T19:55:24.074 INFO:tasks.rados.rados.0.smithi059.stdout:3361: writing smithi05955964-2 from 491520 to 892928 tid 2 2022-04-28T19:55:24.074 INFO:tasks.rados.rados.0.smithi059.stdout:3362: read oid 11 snap -1 2022-04-28T19:55:24.074 INFO:tasks.rados.rados.0.smithi059.stdout:3362: expect (ObjNum 1030 snap 319 seq_num 1030) 2022-04-28T19:55:24.074 INFO:tasks.rados.rados.0.smithi059.stdout:3363: copy_from oid 27 from oid 21 current snap is 328 2022-04-28T19:55:24.074 INFO:tasks.rados.rados.0.smithi059.stdout:3356: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:55:24.074 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1472 (ObjNum 0 snap 0 seq_num 188341408) dirty dne 2022-04-28T19:55:24.075 INFO:tasks.rados.rados.0.smithi059.stdout:3356: done (10 left) 2022-04-28T19:55:24.075 INFO:tasks.rados.rados.0.smithi059.stdout:3364: snap_remove snap 252 2022-04-28T19:55:24.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:23 smithi059 bash[61152]: cluster 2022-04-28T19:55:22.975591+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2022-04-28T19:55:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:23 smithi059 bash[61152]: cluster 2022-04-28T19:55:22.993819+0000 mgr.y (mgr.24961) 333 : cluster [DBG] pgmap v536: 89 pgs: 89 active+clean; 827 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 188 KiB/s wr, 4 op/s 2022-04-28T19:55:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:23 smithi059 bash[61152]: cephadm 2022-04-28T19:55:23.318068+0000 mgr.y (mgr.24961) 334 : cephadm [INF] Upgrade: Updating grafana.a 2022-04-28T19:55:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:23 smithi059 bash[59745]: cluster 2022-04-28T19:55:22.975591+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2022-04-28T19:55:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:23 smithi059 bash[59745]: cluster 2022-04-28T19:55:22.993819+0000 mgr.y (mgr.24961) 333 : cluster [DBG] pgmap v536: 89 pgs: 89 active+clean; 827 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 188 KiB/s wr, 4 op/s 2022-04-28T19:55:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:23 smithi059 bash[59745]: cephadm 2022-04-28T19:55:23.318068+0000 mgr.y (mgr.24961) 334 : cephadm [INF] Upgrade: Updating grafana.a 2022-04-28T19:55:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:23 smithi139 bash[43301]: cluster 2022-04-28T19:55:22.975591+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2022-04-28T19:55:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:23 smithi139 bash[43301]: cluster 2022-04-28T19:55:22.993819+0000 mgr.y (mgr.24961) 333 : cluster [DBG] pgmap v536: 89 pgs: 89 active+clean; 827 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.2 MiB/s rd, 188 KiB/s wr, 4 op/s 2022-04-28T19:55:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:23 smithi139 bash[43301]: cephadm 2022-04-28T19:55:23.318068+0000 mgr.y (mgr.24961) 334 : cephadm [INF] Upgrade: Updating grafana.a 2022-04-28T19:55:25.002 INFO:tasks.rados.rados.0.smithi059.stdout:3364: done (10 left) 2022-04-28T19:55:25.002 INFO:tasks.rados.rados.0.smithi059.stdout:3365: copy_from oid 24 from oid 36 current snap is 328 2022-04-28T19:55:25.002 INFO:tasks.rados.rados.0.smithi059.stdout:3353: finishing write tid 1 to smithi05955964-37 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3353: finishing write tid 2 to smithi05955964-37 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3353: finishing write tid 4 to smithi05955964-37 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3357: finishing copy_from to smithi05955964-34 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3357: got expected ENOENT (src dne) 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3357: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3354: finishing copy_from to smithi05955964-5 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3354: got expected ENOENT (src dne) 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3354: finishing copy_from racing read to smithi05955964-5 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3355: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:55:25.003 INFO:tasks.rados.rados.0.smithi059.stdout:3355: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3353: finishing write tid 5 to smithi05955964-37 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1351 (ObjNum 1052 snap 328 seq_num 1052) dirty exists 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3353: left oid 37 (ObjNum 1052 snap 328 seq_num 1052) 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3355: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1512 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3358: finishing write tid 1 to smithi05955964-26 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3358: finishing write tid 2 to smithi05955964-26 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3358: finishing write tid 4 to smithi05955964-26 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3353: done (10 left) 2022-04-28T19:55:25.004 INFO:tasks.rados.rados.0.smithi059.stdout:3354: done (9 left) 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3355: done (8 left) 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3357: done (7 left) 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3366: rollback oid 29 current snap is 328 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 303 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3358: finishing write tid 5 to smithi05955964-26 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1477 (ObjNum 1053 snap 328 seq_num 1053) dirty exists 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3358: left oid 26 (ObjNum 1053 snap 328 seq_num 1053) 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3358: done (7 left) 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:3367: copy_from oid 8 from oid 41 current snap is 328 2022-04-28T19:55:25.005 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 328 2022-04-28T19:55:25.006 INFO:tasks.rados.rados.0.smithi059.stdout:3368: seq_num 1057 ranges {9224192=507904,9732096=401408} 2022-04-28T19:55:25.056 INFO:tasks.rados.rados.0.smithi059.stdout:3368: writing smithi05955964-47 from 9224192 to 9732096 tid 1 2022-04-28T19:55:25.060 INFO:tasks.rados.rados.0.smithi059.stdout:3368: writing smithi05955964-47 from 9732096 to 10133504 tid 2 2022-04-28T19:55:25.060 INFO:tasks.rados.rados.0.smithi059.stdout:3369: copy_from oid 50 from oid 7 current snap is 328 2022-04-28T19:55:25.061 INFO:tasks.rados.rados.0.smithi059.stdout:3359: finishing write tid 1 to smithi05955964-43 2022-04-28T19:55:25.061 INFO:tasks.rados.rados.0.smithi059.stdout:3359: finishing write tid 2 to smithi05955964-43 2022-04-28T19:55:25.061 INFO:tasks.rados.rados.0.smithi059.stdout:3359: finishing write tid 4 to smithi05955964-43 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:3359: finishing write tid 5 to smithi05955964-43 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1478 (ObjNum 1054 snap 328 seq_num 1054) dirty exists 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:3359: left oid 43 (ObjNum 1054 snap 328 seq_num 1054) 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:3359: done (9 left) 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:3370: rollback oid 17 current snap is 328 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 17 to 300 2022-04-28T19:55:25.062 INFO:tasks.rados.rados.0.smithi059.stdout:3360: finishing write tid 1 to smithi05955964-44 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3360: finishing write tid 2 to smithi05955964-44 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3360: finishing write tid 4 to smithi05955964-44 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3371: snap_create 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3360: finishing write tid 5 to smithi05955964-44 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1196 (ObjNum 1055 snap 328 seq_num 1055) dirty exists 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3360: left oid 44 (ObjNum 1055 snap 328 seq_num 1055) 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3361: finishing write tid 1 to smithi05955964-2 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3361: finishing write tid 2 to smithi05955964-2 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3361: finishing write tid 4 to smithi05955964-2 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:3361: finishing write tid 5 to smithi05955964-2 2022-04-28T19:55:25.063 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1452 (ObjNum 1056 snap 328 seq_num 1056) dirty exists 2022-04-28T19:55:25.064 INFO:tasks.rados.rados.0.smithi059.stdout:3361: left oid 2 (ObjNum 1056 snap 328 seq_num 1056) 2022-04-28T19:55:25.064 INFO:tasks.rados.rados.0.smithi059.stdout:3362: expect (ObjNum 1030 snap 319 seq_num 1030) 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:3363: finishing copy_from to smithi05955964-27 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1159 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:3363: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:3365: finishing copy_from to smithi05955964-24 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:3365: got expected ENOENT (src dne) 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:3365: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:55:25.168 INFO:tasks.rados.rados.0.smithi059.stdout:3366: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3366: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3366: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1516 (ObjNum 998 snap 297 seq_num 998) dirty exists 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3367: finishing copy_from to smithi05955964-8 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1020 (ObjNum 1038 snap 321 seq_num 1038) dirty exists 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3367: finishing copy_from racing read to smithi05955964-8 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3370: finishing rollback tid 1 to smithi05955964-17 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3368: finishing write tid 1 to smithi05955964-47 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3368: finishing write tid 2 to smithi05955964-47 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:3370: finishing rollback tid 2 to smithi05955964-17 2022-04-28T19:55:25.169 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 1074 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:3368: finishing write tid 4 to smithi05955964-47 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:3368: finishing write tid 5 to smithi05955964-47 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1346 (ObjNum 1057 snap 328 seq_num 1057) dirty exists 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:3368: left oid 47 (ObjNum 1057 snap 328 seq_num 1057) 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:3369: finishing copy_from to smithi05955964-50 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1363 (ObjNum 1037 snap 321 seq_num 1037) dirty exists 2022-04-28T19:55:25.170 INFO:tasks.rados.rados.0.smithi059.stdout:3369: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:55:25.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:24 smithi059 bash[59745]: cephadm 2022-04-28T19:55:23.366222+0000 mgr.y (mgr.24961) 335 : cephadm [INF] Deploying daemon grafana.a on smithi139 2022-04-28T19:55:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:24 smithi059 bash[59745]: cluster 2022-04-28T19:55:23.986777+0000 mon.a (mon.0) 1259 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2022-04-28T19:55:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:24 smithi059 bash[61152]: cephadm 2022-04-28T19:55:23.366222+0000 mgr.y (mgr.24961) 335 : cephadm [INF] Deploying daemon grafana.a on smithi139 2022-04-28T19:55:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:24 smithi059 bash[61152]: cluster 2022-04-28T19:55:23.986777+0000 mon.a (mon.0) 1259 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2022-04-28T19:55:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:24 smithi139 bash[43301]: cephadm 2022-04-28T19:55:23.366222+0000 mgr.y (mgr.24961) 335 : cephadm [INF] Deploying daemon grafana.a on smithi139 2022-04-28T19:55:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:24 smithi139 bash[43301]: cluster 2022-04-28T19:55:23.986777+0000 mon.a (mon.0) 1259 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2022-04-28T19:55:26.007 INFO:tasks.rados.rados.0.smithi059.stdout:3360: done (10 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3361: done (9 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3362: done (8 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3363: done (7 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3365: done (6 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3366: done (5 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3367: done (4 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3368: done (3 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3369: done (2 left) 2022-04-28T19:55:26.008 INFO:tasks.rados.rados.0.smithi059.stdout:3370: done (1 left) 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:3371: done (0 left) 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:3372: rollback oid 42 current snap is 329 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 324 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:3373: read oid 27 snap -1 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:3373: expect (ObjNum 916 snap 273 seq_num 916) 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:3374: rollback oid 26 current snap is 329 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 301 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 329 2022-04-28T19:55:26.009 INFO:tasks.rados.rados.0.smithi059.stdout:3375: seq_num 1058 ranges {7888896=524288,8413184=401408} 2022-04-28T19:55:26.055 INFO:tasks.rados.rados.0.smithi059.stdout:3375: writing smithi05955964-38 from 7888896 to 8413184 tid 1 2022-04-28T19:55:26.058 INFO:tasks.rados.rados.0.smithi059.stdout:3375: writing smithi05955964-38 from 8413184 to 8814592 tid 2 2022-04-28T19:55:26.058 INFO:tasks.rados.rados.0.smithi059.stdout:3376: read oid 36 snap -1 2022-04-28T19:55:26.058 INFO:tasks.rados.rados.0.smithi059.stdout:3376: expect deleted 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:3377: setattr oid 14 current snap is 329 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:3372: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1340 (ObjNum -133139536 snap 21893 seq_num 4161921984) dirty dne 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:3374: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:3374: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:3374: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1480 (ObjNum 966 snap 289 seq_num 966) dirty exists 2022-04-28T19:55:26.059 INFO:tasks.rados.rados.0.smithi059.stdout:3373: expect (ObjNum 916 snap 273 seq_num 916) 2022-04-28T19:55:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3375: finishing write tid 1 to smithi05955964-38 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3375: finishing write tid 2 to smithi05955964-38 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3375: finishing write tid 4 to smithi05955964-38 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3375: finishing write tid 5 to smithi05955964-38 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1213 (ObjNum 1058 snap 329 seq_num 1058) dirty exists 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3375: left oid 38 (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3372: done (5 left) 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3373: done (4 left) 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3374: done (3 left) 2022-04-28T19:55:26.090 INFO:tasks.rados.rados.0.smithi059.stdout:3375: done (2 left) 2022-04-28T19:55:26.091 INFO:tasks.rados.rados.0.smithi059.stdout:3376: done (1 left) 2022-04-28T19:55:26.091 INFO:tasks.rados.rados.0.smithi059.stdout:3378: delete oid 33 current snap is 329 2022-04-28T19:55:26.094 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1481 (ObjNum 898 snap 268 seq_num 898) dirty exists 2022-04-28T19:55:26.094 INFO:tasks.rados.rados.0.smithi059.stdout:3377: done (1 left) 2022-04-28T19:55:26.095 INFO:tasks.rados.rados.0.smithi059.stdout:3378: done (0 left) 2022-04-28T19:55:26.095 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 329 2022-04-28T19:55:26.095 INFO:tasks.rados.rados.0.smithi059.stdout:3379: seq_num 1060 ranges {3784704=557056,4341760=401408} 2022-04-28T19:55:26.119 INFO:tasks.rados.rados.0.smithi059.stdout:3379: writing smithi05955964-25 from 3784704 to 4341760 tid 1 2022-04-28T19:55:26.121 INFO:tasks.rados.rados.0.smithi059.stdout:3379: writing smithi05955964-25 from 4341760 to 4743168 tid 2 2022-04-28T19:55:26.122 INFO:tasks.rados.rados.0.smithi059.stdout:3380: delete oid 50 current snap is 329 2022-04-28T19:55:26.126 INFO:tasks.rados.rados.0.smithi059.stdout:3380: done (1 left) 2022-04-28T19:55:26.126 INFO:tasks.rados.rados.0.smithi059.stdout:3381: snap_remove snap 303 2022-04-28T19:55:26.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:26 smithi059 bash[59745]: cluster 2022-04-28T19:55:24.994970+0000 mgr.y (mgr.24961) 336 : cluster [DBG] pgmap v538: 89 pgs: 89 active+clean; 831 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 846 KiB/s wr, 4 op/s 2022-04-28T19:55:26.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:26 smithi059 bash[59745]: cluster 2022-04-28T19:55:24.997329+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2022-04-28T19:55:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:26 smithi059 bash[61152]: cluster 2022-04-28T19:55:24.994970+0000 mgr.y (mgr.24961) 336 : cluster [DBG] pgmap v538: 89 pgs: 89 active+clean; 831 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 846 KiB/s wr, 4 op/s 2022-04-28T19:55:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:26 smithi059 bash[61152]: cluster 2022-04-28T19:55:24.997329+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2022-04-28T19:55:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:25 smithi139 bash[43301]: cluster 2022-04-28T19:55:24.994970+0000 mgr.y (mgr.24961) 336 : cluster [DBG] pgmap v538: 89 pgs: 89 active+clean; 831 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 846 KiB/s wr, 4 op/s 2022-04-28T19:55:26.475 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:25 smithi139 bash[43301]: cluster 2022-04-28T19:55:24.997329+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2022-04-28T19:55:26.967 INFO:tasks.rados.rados.0.smithi059.stdout:3381: done (1 left) 2022-04-28T19:55:26.967 INFO:tasks.rados.rados.0.smithi059.stdout:3382: read oid 18 snap -1 2022-04-28T19:55:26.967 INFO:tasks.rados.rados.0.smithi059.stdout:3382: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:55:26.967 INFO:tasks.rados.rados.0.smithi059.stdout:3383: read oid 2 snap -1 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3383: expect (ObjNum 1056 snap 328 seq_num 1056) 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3379: finishing write tid 1 to smithi05955964-25 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3379: finishing write tid 2 to smithi05955964-25 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3379: finishing write tid 4 to smithi05955964-25 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3379: finishing write tid 5 to smithi05955964-25 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1522 (ObjNum 1060 snap 329 seq_num 1060) dirty exists 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3379: left oid 25 (ObjNum 1060 snap 329 seq_num 1060) 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3379: done (2 left) 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:3384: rollback oid 27 current snap is 329 2022-04-28T19:55:26.968 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 311 2022-04-28T19:55:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 329 2022-04-28T19:55:26.969 INFO:tasks.rados.rados.0.smithi059.stdout:3385: seq_num 1062 ranges {286720=589824,876544=401408} 2022-04-28T19:55:26.972 INFO:tasks.rados.rados.0.smithi059.stdout:3385: writing smithi05955964-10 from 286720 to 876544 tid 1 2022-04-28T19:55:26.975 INFO:tasks.rados.rados.0.smithi059.stdout:3385: writing smithi05955964-10 from 876544 to 1277952 tid 2 2022-04-28T19:55:26.976 INFO:tasks.rados.rados.0.smithi059.stdout:3386: read oid 15 snap 287 2022-04-28T19:55:26.976 INFO:tasks.rados.rados.0.smithi059.stdout:3386: expect (ObjNum 946 snap 277 seq_num 946) 2022-04-28T19:55:26.976 INFO:tasks.rados.rados.0.smithi059.stdout:3387: read oid 3 snap -1 2022-04-28T19:55:26.976 INFO:tasks.rados.rados.0.smithi059.stdout:3387: expect (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:55:26.976 INFO:tasks.rados.rados.0.smithi059.stdout:3388: read oid 41 snap -1 2022-04-28T19:55:26.977 INFO:tasks.rados.rados.0.smithi059.stdout:3388: expect (ObjNum 1038 snap 321 seq_num 1038) 2022-04-28T19:55:26.977 INFO:tasks.rados.rados.0.smithi059.stdout:3389: snap_create 2022-04-28T19:55:26.989 INFO:tasks.rados.rados.0.smithi059.stdout:3384: finishing rollback tid 0 to smithi05955964-27 2022-04-28T19:55:26.990 INFO:tasks.rados.rados.0.smithi059.stdout:3383: expect (ObjNum 1056 snap 328 seq_num 1056) 2022-04-28T19:55:27.014 INFO:tasks.rados.rados.0.smithi059.stdout:3388: expect (ObjNum 1038 snap 321 seq_num 1038) 2022-04-28T19:55:27.074 INFO:tasks.rados.rados.0.smithi059.stdout:3387: expect (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:55:27.207 INFO:tasks.rados.rados.0.smithi059.stdout:3384: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:55:27.207 INFO:tasks.rados.rados.0.smithi059.stdout:3384: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:55:27.208 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1349 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:55:27.208 INFO:tasks.rados.rados.0.smithi059.stdout:3385: finishing write tid 1 to smithi05955964-10 2022-04-28T19:55:27.208 INFO:tasks.rados.rados.0.smithi059.stdout:3385: finishing write tid 2 to smithi05955964-10 2022-04-28T19:55:27.208 INFO:tasks.rados.rados.0.smithi059.stdout:3385: finishing write tid 4 to smithi05955964-10 2022-04-28T19:55:27.208 INFO:tasks.rados.rados.0.smithi059.stdout:3385: finishing write tid 5 to smithi05955964-10 2022-04-28T19:55:27.209 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1352 (ObjNum 1062 snap 329 seq_num 1062) dirty exists 2022-04-28T19:55:27.209 INFO:tasks.rados.rados.0.smithi059.stdout:3385: left oid 10 (ObjNum 1062 snap 329 seq_num 1062) 2022-04-28T19:55:27.209 INFO:tasks.rados.rados.0.smithi059.stdout:3386: expect (ObjNum 946 snap 277 seq_num 946) 2022-04-28T19:55:27.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:27 smithi059 bash[61152]: cluster 2022-04-28T19:55:26.005699+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2022-04-28T19:55:27.299 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:27 smithi059 bash[61152]: cluster 2022-04-28T19:55:26.964681+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2022-04-28T19:55:27.299 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:27 smithi059 bash[59745]: cluster 2022-04-28T19:55:26.005699+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2022-04-28T19:55:27.300 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:27 smithi059 bash[59745]: cluster 2022-04-28T19:55:26.964681+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2022-04-28T19:55:27.309 INFO:tasks.rados.rados.0.smithi059.stdout:3382: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:55:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:27 smithi139 bash[43301]: cluster 2022-04-28T19:55:26.005699+0000 mon.a (mon.0) 1261 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2022-04-28T19:55:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:27 smithi139 bash[43301]: cluster 2022-04-28T19:55:26.964681+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2022-04-28T19:55:27.590 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:27 smithi059 bash[72035]: level=info ts=2022-04-28T19:55:27.299Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004249073s 2022-04-28T19:55:28.144 INFO:tasks.rados.rados.0.smithi059.stdout:3382: done (7 left) 2022-04-28T19:55:28.144 INFO:tasks.rados.rados.0.smithi059.stdout:3383: done (6 left) 2022-04-28T19:55:28.145 INFO:tasks.rados.rados.0.smithi059.stdout:3384: done (5 left) 2022-04-28T19:55:28.145 INFO:tasks.rados.rados.0.smithi059.stdout:3385: done (4 left) 2022-04-28T19:55:28.145 INFO:tasks.rados.rados.0.smithi059.stdout:3386: done (3 left) 2022-04-28T19:55:28.145 INFO:tasks.rados.rados.0.smithi059.stdout:3387: done (2 left) 2022-04-28T19:55:28.145 INFO:tasks.rados.rados.0.smithi059.stdout:3388: done (1 left) 2022-04-28T19:55:28.145 INFO:tasks.rados.rados.0.smithi059.stdout:3389: done (0 left) 2022-04-28T19:55:28.146 INFO:tasks.rados.rados.0.smithi059.stdout:3390: snap_create 2022-04-28T19:55:28.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:28 smithi059 bash[59745]: cluster 2022-04-28T19:55:26.995585+0000 mgr.y (mgr.24961) 337 : cluster [DBG] pgmap v542: 89 pgs: 89 active+clean; 838 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2022-04-28T19:55:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:28 smithi059 bash[59745]: cluster 2022-04-28T19:55:27.985634+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2022-04-28T19:55:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:28 smithi059 bash[61152]: cluster 2022-04-28T19:55:26.995585+0000 mgr.y (mgr.24961) 337 : cluster [DBG] pgmap v542: 89 pgs: 89 active+clean; 838 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2022-04-28T19:55:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:28 smithi059 bash[61152]: cluster 2022-04-28T19:55:27.985634+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2022-04-28T19:55:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:28 smithi139 bash[43301]: cluster 2022-04-28T19:55:26.995585+0000 mgr.y (mgr.24961) 337 : cluster [DBG] pgmap v542: 89 pgs: 89 active+clean; 838 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.7 MiB/s wr, 9 op/s 2022-04-28T19:55:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:28 smithi139 bash[43301]: cluster 2022-04-28T19:55:27.985634+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2022-04-28T19:55:29.020 INFO:tasks.rados.rados.0.smithi059.stdout:3390: done (0 left) 2022-04-28T19:55:29.020 INFO:tasks.rados.rados.0.smithi059.stdout:3391: snap_remove snap 312 2022-04-28T19:55:30.027 INFO:tasks.rados.rados.0.smithi059.stdout:3391: done (0 left) 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:3392: copy_from oid 49 from oid 2 current snap is 331 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:3393: read oid 18 snap -1 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:3393: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:3394: rollback oid 29 current snap is 331 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 322 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 331 2022-04-28T19:55:30.104 INFO:tasks.rados.rados.0.smithi059.stdout:3395: seq_num 1063 ranges {0=606208,606208=401408} 2022-04-28T19:55:30.107 INFO:tasks.rados.rados.0.smithi059.stdout:3395: writing smithi05955964-50 from 0 to 606208 tid 1 2022-04-28T19:55:30.107 INFO:tasks.rados.rados.0.smithi059.stdout:3395: writing smithi05955964-50 from 606208 to 1007616 tid 2 2022-04-28T19:55:30.108 INFO:tasks.rados.rados.0.smithi059.stdout:3396: read oid 22 snap 322 2022-04-28T19:55:30.108 INFO:tasks.rados.rados.0.smithi059.stdout:3396: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:55:30.108 INFO:tasks.rados.rados.0.smithi059.stdout:3397: read oid 14 snap -1 2022-04-28T19:55:30.108 INFO:tasks.rados.rados.0.smithi059.stdout:3397: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:55:30.108 INFO:tasks.rados.rados.0.smithi059.stdout:3398: setattr oid 8 current snap is 331 2022-04-28T19:55:30.109 INFO:tasks.rados.rados.0.smithi059.stdout:3394: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:55:30.109 INFO:tasks.rados.rados.0.smithi059.stdout:3394: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:55:30.109 INFO:tasks.rados.rados.0.smithi059.stdout:3399: read oid 21 snap -1 2022-04-28T19:55:30.109 INFO:tasks.rados.rados.0.smithi059.stdout:3399: expect (ObjNum 916 snap 273 seq_num 916) 2022-04-28T19:55:30.110 INFO:tasks.rados.rados.0.smithi059.stdout:3400: rollback oid 42 current snap is 331 2022-04-28T19:55:30.110 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 309 2022-04-28T19:55:30.110 INFO:tasks.rados.rados.0.smithi059.stdout:3401: read oid 17 snap 327 2022-04-28T19:55:30.110 INFO:tasks.rados.rados.0.smithi059.stdout:3401: expect deleted 2022-04-28T19:55:30.110 INFO:tasks.rados.rados.0.smithi059.stdout:3402: copy_from oid 33 from oid 4 current snap is 331 2022-04-28T19:55:30.110 INFO:tasks.rados.rados.0.smithi059.stdout:3403: snap_create 2022-04-28T19:55:30.111 INFO:tasks.rados.rados.0.smithi059.stdout:3394: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:55:30.111 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1526 (ObjNum 1014 snap 314 seq_num 1014) dirty exists 2022-04-28T19:55:30.111 INFO:tasks.rados.rados.0.smithi059.stdout:3395: finishing write tid 1 to smithi05955964-50 2022-04-28T19:55:30.111 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1528 (ObjNum 1038 snap 321 seq_num 1038) dirty exists 2022-04-28T19:55:30.111 INFO:tasks.rados.rados.0.smithi059.stdout:3395: finishing write tid 2 to smithi05955964-50 2022-04-28T19:55:30.111 INFO:tasks.rados.rados.0.smithi059.stdout:3399: expect (ObjNum 916 snap 273 seq_num 916) 2022-04-28T19:55:30.112 INFO:tasks.rados.rados.0.smithi059.stdout:3396: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:55:30.134 INFO:tasks.rados.rados.0.smithi059.stdout:3392: finishing copy_from to smithi05955964-49 2022-04-28T19:55:30.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1452 (ObjNum 1056 snap 328 seq_num 1056) dirty exists 2022-04-28T19:55:30.134 INFO:tasks.rados.rados.0.smithi059.stdout:3395: finishing write tid 4 to smithi05955964-50 2022-04-28T19:55:30.134 INFO:tasks.rados.rados.0.smithi059.stdout:3392: finishing copy_from racing read to smithi05955964-49 2022-04-28T19:55:30.135 INFO:tasks.rados.rados.0.smithi059.stdout:3395: finishing write tid 5 to smithi05955964-50 2022-04-28T19:55:30.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1486 (ObjNum 1063 snap 331 seq_num 1063) dirty exists 2022-04-28T19:55:30.135 INFO:tasks.rados.rados.0.smithi059.stdout:3395: left oid 50 (ObjNum 1063 snap 331 seq_num 1063) 2022-04-28T19:55:30.135 INFO:tasks.rados.rados.0.smithi059.stdout:3397: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:55:30.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:30 smithi059 bash[61152]: cluster 2022-04-28T19:55:28.996396+0000 mgr.y (mgr.24961) 338 : cluster [DBG] pgmap v544: 89 pgs: 89 active+clean; 849 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.1 MiB/s wr, 7 op/s 2022-04-28T19:55:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:30 smithi059 bash[61152]: cluster 2022-04-28T19:55:29.018205+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2022-04-28T19:55:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:30 smithi059 bash[59745]: cluster 2022-04-28T19:55:28.996396+0000 mgr.y (mgr.24961) 338 : cluster [DBG] pgmap v544: 89 pgs: 89 active+clean; 849 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.1 MiB/s wr, 7 op/s 2022-04-28T19:55:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:30 smithi059 bash[59745]: cluster 2022-04-28T19:55:29.018205+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2022-04-28T19:55:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:30 smithi139 bash[43301]: cluster 2022-04-28T19:55:28.996396+0000 mgr.y (mgr.24961) 338 : cluster [DBG] pgmap v544: 89 pgs: 89 active+clean; 849 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 2.1 MiB/s wr, 7 op/s 2022-04-28T19:55:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:30 smithi139 bash[43301]: cluster 2022-04-28T19:55:29.018205+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2022-04-28T19:55:30.413 INFO:tasks.rados.rados.0.smithi059.stdout:3400: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:55:30.413 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1340 (ObjNum 0 snap 0 seq_num 188341408) dirty dne 2022-04-28T19:55:30.413 INFO:tasks.rados.rados.0.smithi059.stdout:3402: finishing copy_from to smithi05955964-33 2022-04-28T19:55:30.414 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1146 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:55:30.414 INFO:tasks.rados.rados.0.smithi059.stdout:3402: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:55:30.414 INFO:tasks.rados.rados.0.smithi059.stdout:3393: expect (ObjNum 701 snap 203 seq_num 701) 2022-04-28T19:55:31.248 INFO:tasks.rados.rados.0.smithi059.stdout:3392: done (11 left) 2022-04-28T19:55:31.248 INFO:tasks.rados.rados.0.smithi059.stdout:3393: done (10 left) 2022-04-28T19:55:31.248 INFO:tasks.rados.rados.0.smithi059.stdout:3394: done (9 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3395: done (8 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3396: done (7 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3397: done (6 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3398: done (5 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3399: done (4 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3400: done (3 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3401: done (2 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3402: done (1 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3403: done (0 left) 2022-04-28T19:55:31.249 INFO:tasks.rados.rados.0.smithi059.stdout:3404: delete oid 8 current snap is 332 2022-04-28T19:55:31.256 INFO:tasks.rados.rados.0.smithi059.stdout:3404: done (0 left) 2022-04-28T19:55:31.257 INFO:tasks.rados.rados.0.smithi059.stdout:3405: snap_remove snap 246 2022-04-28T19:55:31.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:31 smithi059 bash[59745]: cluster 2022-04-28T19:55:30.025494+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2022-04-28T19:55:31.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:31 smithi059 bash[61152]: cluster 2022-04-28T19:55:30.025494+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2022-04-28T19:55:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:31 smithi139 bash[43301]: cluster 2022-04-28T19:55:30.025494+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2022-04-28T19:55:31.414 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:55:31.746 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:55:31.746 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (15s) 10s ago 14m 13.8M - ba2b418f427c 852576bdb38c 2022-04-28T19:55:31.747 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (14m) 26s ago 14m 26.5M - 6.7.4 557c83e11646 e08850de16f0 2022-04-28T19:55:31.747 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (75s) 10s ago 12m 13.6M - 16.2.7-969-g4fa079ba 863db77b5b5e 0b1d79d85003 2022-04-28T19:55:31.747 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (62s) 26s ago 13m 11.8M - 16.2.7-969-g4fa079ba 863db77b5b5e 01b2328b32c7 2022-04-28T19:55:31.747 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (6m) 26s ago 18m 377M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:55:31.747 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (7m) 10s ago 20m 420M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:55:31.748 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (6m) 10s ago 21m 86.3M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:55:31.748 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (5m) 26s ago 19m 58.9M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:55:31.748 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (6m) 10s ago 19m 70.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:55:31.748 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (50s) 10s ago 15m 12.2M - 1dbe0e931976 c196770fa402 2022-04-28T19:55:31.748 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (41s) 26s ago 15m 8680k - 1dbe0e931976 97aaa15a2dd3 2022-04-28T19:55:31.749 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (4m) 10s ago 18m 343M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:55:31.749 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (4m) 10s ago 17m 259M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:55:31.749 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (3m) 10s ago 17m 329M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:55:31.749 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (3m) 10s ago 17m 239M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:55:31.749 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (2m) 26s ago 16m 295M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:55:31.750 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (2m) 26s ago 16m 362M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 4d55ca87a6a9 2022-04-28T19:55:31.750 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (117s) 26s ago 16m 223M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 2cf53c9fc29e 2022-04-28T19:55:31.750 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (87s) 26s ago 15m 185M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 8655092f1b14 2022-04-28T19:55:31.750 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (31s) 26s ago 15m 32.0M - 514e6a882f6e 1187d983585d 2022-04-28T19:55:31.970 INFO:tasks.rados.rados.0.smithi059.stdout:3405: done (0 left) 2022-04-28T19:55:31.970 INFO:tasks.rados.rados.0.smithi059.stdout:3406: rmattr oid 5 current snap is 332 2022-04-28T19:55:31.970 INFO:tasks.rados.rados.0.smithi059.stdout:3406: done (0 left) 2022-04-28T19:55:31.970 INFO:tasks.rados.rados.0.smithi059.stdout:3407: snap_create 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:55:32.134 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 8 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 15 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:55:32.135 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: cluster 2022-04-28T19:55:30.997303+0000 mgr.y (mgr.24961) 339 : cluster [DBG] pgmap v547: 89 pgs: 89 active+clean; 859 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: cluster 2022-04-28T19:55:31.036685+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.724582+0000 mon.a (mon.0) 1267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.725293+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.726020+0000 mon.a (mon.0) 1269 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.726715+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.727471+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.728299+0000 mon.a (mon.0) 1272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.732723+0000 mon.a (mon.0) 1273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.733388+0000 mon.a (mon.0) 1274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.734036+0000 mon.a (mon.0) 1275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.734713+0000 mon.a (mon.0) 1276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: audit 2022-04-28T19:55:31.735407+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[59745]: cluster 2022-04-28T19:55:31.968836+0000 mon.a (mon.0) 1278 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: cluster 2022-04-28T19:55:30.997303+0000 mgr.y (mgr.24961) 339 : cluster [DBG] pgmap v547: 89 pgs: 89 active+clean; 859 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2022-04-28T19:55:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: cluster 2022-04-28T19:55:31.036685+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.724582+0000 mon.a (mon.0) 1267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.725293+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.726020+0000 mon.a (mon.0) 1269 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.726715+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.727471+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.728299+0000 mon.a (mon.0) 1272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.732723+0000 mon.a (mon.0) 1273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.733388+0000 mon.a (mon.0) 1274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.734036+0000 mon.a (mon.0) 1275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.734713+0000 mon.a (mon.0) 1276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: audit 2022-04-28T19:55:31.735407+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:32 smithi059 bash[61152]: cluster 2022-04-28T19:55:31.968836+0000 mon.a (mon.0) 1278 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2022-04-28T19:55:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: cluster 2022-04-28T19:55:30.997303+0000 mgr.y (mgr.24961) 339 : cluster [DBG] pgmap v547: 89 pgs: 89 active+clean; 859 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.6 MiB/s wr, 22 op/s 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: cluster 2022-04-28T19:55:31.036685+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.724582+0000 mon.a (mon.0) 1267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.725293+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.726020+0000 mon.a (mon.0) 1269 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.726715+0000 mon.a (mon.0) 1270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.727471+0000 mon.a (mon.0) 1271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.728299+0000 mon.a (mon.0) 1272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.732723+0000 mon.a (mon.0) 1273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:55:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.733388+0000 mon.a (mon.0) 1274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.734036+0000 mon.a (mon.0) 1275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.734713+0000 mon.a (mon.0) 1276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: audit 2022-04-28T19:55:31.735407+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:55:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:32 smithi139 bash[43301]: cluster 2022-04-28T19:55:31.968836+0000 mon.a (mon.0) 1278 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2022-04-28T19:55:32.981 INFO:tasks.rados.rados.0.smithi059.stdout:3407: done (0 left) 2022-04-28T19:55:32.981 INFO:tasks.rados.rados.0.smithi059.stdout:3408: snap_remove snap 275 2022-04-28T19:55:33.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[61152]: audit 2022-04-28T19:55:31.400796+0000 mgr.y (mgr.24961) 340 : audit [DBG] from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[61152]: audit 2022-04-28T19:55:31.719152+0000 mgr.y (mgr.24961) 341 : audit [DBG] from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[61152]: audit 2022-04-28T19:55:32.133491+0000 mon.a (mon.0) 1279 : audit [DBG] from='client.? 172.21.15.59:0/3521187850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[61152]: cluster 2022-04-28T19:55:32.980287+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2022-04-28T19:55:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[59745]: audit 2022-04-28T19:55:31.400796+0000 mgr.y (mgr.24961) 340 : audit [DBG] from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[59745]: audit 2022-04-28T19:55:31.719152+0000 mgr.y (mgr.24961) 341 : audit [DBG] from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[59745]: audit 2022-04-28T19:55:32.133491+0000 mon.a (mon.0) 1279 : audit [DBG] from='client.? 172.21.15.59:0/3521187850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:33 smithi059 bash[59745]: cluster 2022-04-28T19:55:32.980287+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2022-04-28T19:55:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:33 smithi139 bash[43301]: audit 2022-04-28T19:55:31.400796+0000 mgr.y (mgr.24961) 340 : audit [DBG] from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:33 smithi139 bash[43301]: audit 2022-04-28T19:55:31.719152+0000 mgr.y (mgr.24961) 341 : audit [DBG] from='client.44409 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:55:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:33 smithi139 bash[43301]: audit 2022-04-28T19:55:32.133491+0000 mon.a (mon.0) 1279 : audit [DBG] from='client.? 172.21.15.59:0/3521187850' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:33 smithi139 bash[43301]: cluster 2022-04-28T19:55:32.980287+0000 mon.a (mon.0) 1280 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2022-04-28T19:55:33.989 INFO:tasks.rados.rados.0.smithi059.stdout:3408: done (0 left) 2022-04-28T19:55:33.989 INFO:tasks.rados.rados.0.smithi059.stdout:3409: snap_remove snap 306 2022-04-28T19:55:34.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:34 smithi059 bash[61152]: cluster 2022-04-28T19:55:32.998045+0000 mgr.y (mgr.24961) 342 : cluster [DBG] pgmap v551: 89 pgs: 89 active+clean; 859 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2022-04-28T19:55:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:34 smithi059 bash[61152]: cluster 2022-04-28T19:55:33.985946+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2022-04-28T19:55:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:34 smithi059 bash[59745]: cluster 2022-04-28T19:55:32.998045+0000 mgr.y (mgr.24961) 342 : cluster [DBG] pgmap v551: 89 pgs: 89 active+clean; 859 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2022-04-28T19:55:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:34 smithi059 bash[59745]: cluster 2022-04-28T19:55:33.985946+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2022-04-28T19:55:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:34 smithi139 bash[43301]: cluster 2022-04-28T19:55:32.998045+0000 mgr.y (mgr.24961) 342 : cluster [DBG] pgmap v551: 89 pgs: 89 active+clean; 859 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2022-04-28T19:55:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:34 smithi139 bash[43301]: cluster 2022-04-28T19:55:33.985946+0000 mon.a (mon.0) 1281 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2022-04-28T19:55:34.995 INFO:tasks.rados.rados.0.smithi059.stdout:3409: done (0 left) 2022-04-28T19:55:34.996 INFO:tasks.rados.rados.0.smithi059.stdout:3410: snap_create 2022-04-28T19:55:36.001 INFO:tasks.rados.rados.0.smithi059.stdout:3410: done (0 left) 2022-04-28T19:55:36.002 INFO:tasks.rados.rados.0.smithi059.stdout:3411: read oid 36 snap 321 2022-04-28T19:55:36.002 INFO:tasks.rados.rados.0.smithi059.stdout:3411: expect deleted 2022-04-28T19:55:36.002 INFO:tasks.rados.rados.0.smithi059.stdout:3412: setattr oid 19 current snap is 334 2022-04-28T19:55:36.002 INFO:tasks.rados.rados.0.smithi059.stdout:3413: copy_from oid 31 from oid 11 current snap is 334 2022-04-28T19:55:36.002 INFO:tasks.rados.rados.0.smithi059.stdout:3414: snap_remove snap 332 2022-04-28T19:55:36.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:35 smithi059 bash[61152]: cluster 2022-04-28T19:55:34.992853+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2022-04-28T19:55:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:35 smithi059 bash[61152]: cluster 2022-04-28T19:55:34.998451+0000 mgr.y (mgr.24961) 343 : cluster [DBG] pgmap v554: 89 pgs: 89 active+clean; 868 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.3 MiB/s wr, 9 op/s 2022-04-28T19:55:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:35 smithi059 bash[59745]: cluster 2022-04-28T19:55:34.992853+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2022-04-28T19:55:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:35 smithi059 bash[59745]: cluster 2022-04-28T19:55:34.998451+0000 mgr.y (mgr.24961) 343 : cluster [DBG] pgmap v554: 89 pgs: 89 active+clean; 868 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.3 MiB/s wr, 9 op/s 2022-04-28T19:55:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:35 smithi139 bash[43301]: cluster 2022-04-28T19:55:34.992853+0000 mon.a (mon.0) 1282 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2022-04-28T19:55:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:35 smithi139 bash[43301]: cluster 2022-04-28T19:55:34.998451+0000 mgr.y (mgr.24961) 343 : cluster [DBG] pgmap v554: 89 pgs: 89 active+clean; 868 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.3 MiB/s wr, 9 op/s 2022-04-28T19:55:36.970 INFO:tasks.rados.rados.0.smithi059.stdout:3414: done (3 left) 2022-04-28T19:55:36.970 INFO:tasks.rados.rados.0.smithi059.stdout:3415: rmattr oid 15 current snap is 334 2022-04-28T19:55:36.970 INFO:tasks.rados.rados.0.smithi059.stdout:3415: done (3 left) 2022-04-28T19:55:36.970 INFO:tasks.rados.rados.0.smithi059.stdout:3416: rollback oid 12 current snap is 334 2022-04-28T19:55:36.970 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 12 to 322 2022-04-28T19:55:36.971 INFO:tasks.rados.rados.0.smithi059.stdout:3417: snap_remove snap 326 2022-04-28T19:55:37.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:36 smithi059 bash[61152]: cluster 2022-04-28T19:55:36.000349+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2022-04-28T19:55:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:36 smithi059 bash[61152]: cluster 2022-04-28T19:55:36.967761+0000 mon.a (mon.0) 1284 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2022-04-28T19:55:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:36 smithi059 bash[59745]: cluster 2022-04-28T19:55:36.000349+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2022-04-28T19:55:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:36 smithi059 bash[59745]: cluster 2022-04-28T19:55:36.967761+0000 mon.a (mon.0) 1284 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2022-04-28T19:55:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:36 smithi139 bash[43301]: cluster 2022-04-28T19:55:36.000349+0000 mon.a (mon.0) 1283 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2022-04-28T19:55:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:36 smithi139 bash[43301]: cluster 2022-04-28T19:55:36.967761+0000 mon.a (mon.0) 1284 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2022-04-28T19:55:37.982 INFO:tasks.rados.rados.0.smithi059.stdout:3417: done (4 left) 2022-04-28T19:55:37.982 INFO:tasks.rados.rados.0.smithi059.stdout:3418: snap_remove snap 325 2022-04-28T19:55:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:38 smithi059 bash[61152]: cluster 2022-04-28T19:55:36.999015+0000 mgr.y (mgr.24961) 344 : cluster [DBG] pgmap v557: 89 pgs: 89 active+clean; 868 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.3 MiB/s wr, 8 op/s 2022-04-28T19:55:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:38 smithi059 bash[61152]: cluster 2022-04-28T19:55:37.979552+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2022-04-28T19:55:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:38 smithi059 bash[59745]: cluster 2022-04-28T19:55:36.999015+0000 mgr.y (mgr.24961) 344 : cluster [DBG] pgmap v557: 89 pgs: 89 active+clean; 868 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.3 MiB/s wr, 8 op/s 2022-04-28T19:55:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:38 smithi059 bash[59745]: cluster 2022-04-28T19:55:37.979552+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2022-04-28T19:55:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:38 smithi139 bash[43301]: cluster 2022-04-28T19:55:36.999015+0000 mgr.y (mgr.24961) 344 : cluster [DBG] pgmap v557: 89 pgs: 89 active+clean; 868 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 2.3 MiB/s wr, 8 op/s 2022-04-28T19:55:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:38 smithi139 bash[43301]: cluster 2022-04-28T19:55:37.979552+0000 mon.a (mon.0) 1285 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2022-04-28T19:55:38.985 INFO:tasks.rados.rados.0.smithi059.stdout:3418: done (4 left) 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:3419: copy_from oid 30 from oid 17 current snap is 334 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1482 (ObjNum 1051 snap 327 seq_num 1051) dirty exists 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:3413: finishing copy_from to smithi05955964-31 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1206 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:3413: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:3416: finishing rollback tid 0 to smithi05955964-12 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:3416: finishing rollback tid 1 to smithi05955964-12 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:3416: finishing rollback tid 2 to smithi05955964-12 2022-04-28T19:55:38.996 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1368 (ObjNum 1031 snap 319 seq_num 1031) dirty exists 2022-04-28T19:55:38.997 INFO:tasks.rados.rados.0.smithi059.stdout:3411: done (4 left) 2022-04-28T19:55:38.997 INFO:tasks.rados.rados.0.smithi059.stdout:3412: done (3 left) 2022-04-28T19:55:38.997 INFO:tasks.rados.rados.0.smithi059.stdout:3413: done (2 left) 2022-04-28T19:55:38.997 INFO:tasks.rados.rados.0.smithi059.stdout:3416: done (1 left) 2022-04-28T19:55:38.997 INFO:tasks.rados.rados.0.smithi059.stdout:3420: copy_from oid 28 from oid 12 current snap is 334 2022-04-28T19:55:38.997 INFO:tasks.rados.rados.0.smithi059.stdout:3421: snap_remove snap 313 2022-04-28T19:55:39.998 INFO:tasks.rados.rados.0.smithi059.stdout:3421: done (2 left) 2022-04-28T19:55:40.069 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 334 2022-04-28T19:55:40.069 INFO:tasks.rados.rados.0.smithi059.stdout:3422: seq_num 1065 ranges {860160=638976,1499136=401408} 2022-04-28T19:55:40.074 INFO:tasks.rados.rados.0.smithi059.stdout:3422: writing smithi05955964-43 from 860160 to 1499136 tid 1 2022-04-28T19:55:40.074 INFO:tasks.rados.rados.0.smithi059.stdout:3422: writing smithi05955964-43 from 1499136 to 1900544 tid 2 2022-04-28T19:55:40.074 INFO:tasks.rados.rados.0.smithi059.stdout:3423: rmattr oid 36 current snap is 334 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3423: done (3 left) 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 334 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3420: finishing copy_from to smithi05955964-28 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1368 (ObjNum 1031 snap 319 seq_num 1031) dirty exists 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3424: seq_num 1066 ranges {188416=663552,851968=401408} 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3424: writing smithi05955964-29 from 188416 to 851968 tid 1 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3424: writing smithi05955964-29 from 851968 to 1253376 tid 2 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3425: setattr oid 26 current snap is 334 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3419: finishing copy_from to smithi05955964-30 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1074 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:55:40.075 INFO:tasks.rados.rados.0.smithi059.stdout:3420: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3419: finishing copy_from racing read to smithi05955964-30 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3422: finishing write tid 1 to smithi05955964-43 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3422: finishing write tid 2 to smithi05955964-43 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3419: done (4 left) 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3420: done (3 left) 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3426: read oid 34 snap -1 2022-04-28T19:55:40.076 INFO:tasks.rados.rados.0.smithi059.stdout:3426: expect deleted 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3427: delete oid 7 current snap is 334 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3422: finishing write tid 4 to smithi05955964-43 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3424: finishing write tid 1 to smithi05955964-29 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3424: finishing write tid 2 to smithi05955964-29 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3424: finishing write tid 4 to smithi05955964-29 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3426: done (4 left) 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3427: done (3 left) 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 334 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3428: seq_num 1068 ranges {1007616=696320,1703936=401408} 2022-04-28T19:55:40.077 INFO:tasks.rados.rados.0.smithi059.stdout:3428: writing smithi05955964-50 from 1007616 to 1703936 tid 1 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:3428: writing smithi05955964-50 from 1703936 to 2105344 tid 2 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:3429: rollback oid 21 current snap is 334 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 299 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:3422: finishing write tid 5 to smithi05955964-43 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1491 (ObjNum 1065 snap 334 seq_num 1065) dirty exists 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:3422: left oid 43 (ObjNum 1065 snap 334 seq_num 1065) 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:3424: finishing write tid 5 to smithi05955964-29 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1534 (ObjNum 1066 snap 334 seq_num 1066) dirty exists 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:3424: left oid 29 (ObjNum 1066 snap 334 seq_num 1066) 2022-04-28T19:55:40.078 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1483 (ObjNum 966 snap 289 seq_num 966) dirty exists 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3422: done (4 left) 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3424: done (3 left) 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3425: done (2 left) 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 334 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3430: seq_num 1069 ranges {1900544=712704,2613248=401408} 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3430: writing smithi05955964-43 from 1900544 to 2613248 tid 1 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3430: writing smithi05955964-43 from 2613248 to 3014656 tid 2 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3431: read oid 9 snap -1 2022-04-28T19:55:40.079 INFO:tasks.rados.rados.0.smithi059.stdout:3431: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3432: copy_from oid 24 from oid 41 current snap is 334 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3428: finishing write tid 1 to smithi05955964-50 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3428: finishing write tid 2 to smithi05955964-50 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3428: finishing write tid 4 to smithi05955964-50 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3429: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3428: finishing write tid 5 to smithi05955964-50 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1495 (ObjNum 1068 snap 334 seq_num 1068) dirty exists 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3428: left oid 50 (ObjNum 1068 snap 334 seq_num 1068) 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3429: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:55:40.080 INFO:tasks.rados.rados.0.smithi059.stdout:3429: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1162 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3433: read oid 31 snap -1 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3433: expect (ObjNum 1030 snap 319 seq_num 1030) 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3428: done (5 left) 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3429: done (4 left) 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3434: read oid 45 snap -1 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3434: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3435: rmattr oid 37 current snap is 334 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 334 2022-04-28T19:55:40.081 INFO:tasks.rados.rados.0.smithi059.stdout:3436: seq_num 1070 ranges {3850240=729088,4579328=401408} 2022-04-28T19:55:40.082 INFO:tasks.rados.rados.0.smithi059.stdout:3436: writing smithi05955964-26 from 3850240 to 4579328 tid 1 2022-04-28T19:55:40.085 INFO:tasks.rados.rados.0.smithi059.stdout:3436: writing smithi05955964-26 from 4579328 to 4980736 tid 2 2022-04-28T19:55:40.085 INFO:tasks.rados.rados.0.smithi059.stdout:3437: setattr oid 15 current snap is 334 2022-04-28T19:55:40.086 INFO:tasks.rados.rados.0.smithi059.stdout:3430: finishing write tid 1 to smithi05955964-43 2022-04-28T19:55:40.087 INFO:tasks.rados.rados.0.smithi059.stdout:3430: finishing write tid 2 to smithi05955964-43 2022-04-28T19:55:40.087 INFO:tasks.rados.rados.0.smithi059.stdout:3430: finishing write tid 4 to smithi05955964-43 2022-04-28T19:55:40.087 INFO:tasks.rados.rados.0.smithi059.stdout:3430: finishing write tid 5 to smithi05955964-43 2022-04-28T19:55:40.087 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1498 (ObjNum 1069 snap 334 seq_num 1069) dirty exists 2022-04-28T19:55:40.087 INFO:tasks.rados.rados.0.smithi059.stdout:3430: left oid 43 (ObjNum 1069 snap 334 seq_num 1069) 2022-04-28T19:55:40.087 INFO:tasks.rados.rados.0.smithi059.stdout:3433: expect (ObjNum 1030 snap 319 seq_num 1030) 2022-04-28T19:55:40.193 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1352 (ObjNum 1052 snap 328 seq_num 1052) dirty exists 2022-04-28T19:55:40.193 INFO:tasks.rados.rados.0.smithi059.stdout:3432: finishing copy_from to smithi05955964-24 2022-04-28T19:55:40.193 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1020 (ObjNum 1038 snap 321 seq_num 1038) dirty exists 2022-04-28T19:55:40.194 INFO:tasks.rados.rados.0.smithi059.stdout:3432: finishing copy_from racing read to smithi05955964-24 2022-04-28T19:55:40.194 INFO:tasks.rados.rados.0.smithi059.stdout:3431: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:55:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:39 smithi059 bash[61152]: cluster 2022-04-28T19:55:38.983742+0000 mon.a (mon.0) 1286 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2022-04-28T19:55:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:39 smithi059 bash[61152]: cluster 2022-04-28T19:55:38.999605+0000 mgr.y (mgr.24961) 345 : cluster [DBG] pgmap v560: 89 pgs: 89 active+clean; 871 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 690 KiB/s wr, 0 op/s 2022-04-28T19:55:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:39 smithi059 bash[59745]: cluster 2022-04-28T19:55:38.983742+0000 mon.a (mon.0) 1286 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2022-04-28T19:55:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:39 smithi059 bash[59745]: cluster 2022-04-28T19:55:38.999605+0000 mgr.y (mgr.24961) 345 : cluster [DBG] pgmap v560: 89 pgs: 89 active+clean; 871 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 690 KiB/s wr, 0 op/s 2022-04-28T19:55:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:39 smithi139 bash[43301]: cluster 2022-04-28T19:55:38.983742+0000 mon.a (mon.0) 1286 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2022-04-28T19:55:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:39 smithi139 bash[43301]: cluster 2022-04-28T19:55:38.999605+0000 mgr.y (mgr.24961) 345 : cluster [DBG] pgmap v560: 89 pgs: 89 active+clean; 871 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 690 KiB/s wr, 0 op/s 2022-04-28T19:55:40.494 INFO:tasks.rados.rados.0.smithi059.stdout:3436: finishing write tid 1 to smithi05955964-26 2022-04-28T19:55:40.494 INFO:tasks.rados.rados.0.smithi059.stdout:3436: finishing write tid 2 to smithi05955964-26 2022-04-28T19:55:40.494 INFO:tasks.rados.rados.0.smithi059.stdout:3436: finishing write tid 4 to smithi05955964-26 2022-04-28T19:55:40.495 INFO:tasks.rados.rados.0.smithi059.stdout:3436: finishing write tid 5 to smithi05955964-26 2022-04-28T19:55:40.495 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1486 (ObjNum 1070 snap 334 seq_num 1070) dirty exists 2022-04-28T19:55:40.495 INFO:tasks.rados.rados.0.smithi059.stdout:3436: left oid 26 (ObjNum 1070 snap 334 seq_num 1070) 2022-04-28T19:55:40.495 INFO:tasks.rados.rados.0.smithi059.stdout:3434: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:55:41.245 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:40 smithi139 bash[43301]: cluster 2022-04-28T19:55:39.992266+0000 mon.a (mon.0) 1287 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2022-04-28T19:55:41.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:40 smithi059 bash[61152]: cluster 2022-04-28T19:55:39.992266+0000 mon.a (mon.0) 1287 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2022-04-28T19:55:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:40 smithi059 bash[59745]: cluster 2022-04-28T19:55:39.992266+0000 mon.a (mon.0) 1287 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2022-04-28T19:55:42.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:42 smithi059 bash[61152]: cluster 2022-04-28T19:55:41.000469+0000 mgr.y (mgr.24961) 346 : cluster [DBG] pgmap v562: 89 pgs: 89 active+clean; 884 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.7 MiB/s wr, 12 op/s 2022-04-28T19:55:42.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:42 smithi059 bash[61152]: cluster 2022-04-28T19:55:41.975375+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2022-04-28T19:55:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:41 smithi059 bash[59745]: cluster 2022-04-28T19:55:41.000469+0000 mgr.y (mgr.24961) 346 : cluster [DBG] pgmap v562: 89 pgs: 89 active+clean; 884 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.7 MiB/s wr, 12 op/s 2022-04-28T19:55:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:41 smithi059 bash[59745]: cluster 2022-04-28T19:55:41.975375+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2022-04-28T19:55:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:41 smithi139 bash[43301]: cluster 2022-04-28T19:55:41.000469+0000 mgr.y (mgr.24961) 346 : cluster [DBG] pgmap v562: 89 pgs: 89 active+clean; 884 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.7 MiB/s wr, 12 op/s 2022-04-28T19:55:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:41 smithi139 bash[43301]: cluster 2022-04-28T19:55:41.975375+0000 mon.a (mon.0) 1288 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2022-04-28T19:55:42.647 INFO:tasks.rados.rados.0.smithi059.stdout:3430: done (7 left) 2022-04-28T19:55:42.647 INFO:tasks.rados.rados.0.smithi059.stdout:3431: done (6 left) 2022-04-28T19:55:42.647 INFO:tasks.rados.rados.0.smithi059.stdout:3432: done (5 left) 2022-04-28T19:55:42.647 INFO:tasks.rados.rados.0.smithi059.stdout:3433: done (4 left) 2022-04-28T19:55:42.647 INFO:tasks.rados.rados.0.smithi059.stdout:3434: done (3 left) 2022-04-28T19:55:42.647 INFO:tasks.rados.rados.0.smithi059.stdout:3435: done (2 left) 2022-04-28T19:55:42.648 INFO:tasks.rados.rados.0.smithi059.stdout:3436: done (1 left) 2022-04-28T19:55:42.648 INFO:tasks.rados.rados.0.smithi059.stdout:3438: delete oid 20 current snap is 334 2022-04-28T19:55:42.650 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1370 (ObjNum 1027 snap 318 seq_num 1027) dirty exists 2022-04-28T19:55:42.652 INFO:tasks.rados.rados.0.smithi059.stdout:3437: done (1 left) 2022-04-28T19:55:42.652 INFO:tasks.rados.rados.0.smithi059.stdout:3438: done (0 left) 2022-04-28T19:55:42.652 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 334 2022-04-28T19:55:42.652 INFO:tasks.rados.rados.0.smithi059.stdout:3439: seq_num 1072 ranges {4743168=761856,5505024=401408} 2022-04-28T19:55:42.683 INFO:tasks.rados.rados.0.smithi059.stdout:3439: writing smithi05955964-25 from 4743168 to 5505024 tid 1 2022-04-28T19:55:42.685 INFO:tasks.rados.rados.0.smithi059.stdout:3439: writing smithi05955964-25 from 5505024 to 5906432 tid 2 2022-04-28T19:55:42.685 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 334 2022-04-28T19:55:42.685 INFO:tasks.rados.rados.0.smithi059.stdout:3440: seq_num 1073 ranges {2162688=778240,2940928=401408} 2022-04-28T19:55:42.702 INFO:tasks.rados.rados.0.smithi059.stdout:3440: writing smithi05955964-44 from 2162688 to 2940928 tid 1 2022-04-28T19:55:42.704 INFO:tasks.rados.rados.0.smithi059.stdout:3440: writing smithi05955964-44 from 2940928 to 3342336 tid 2 2022-04-28T19:55:42.704 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 29 current snap is 334 2022-04-28T19:55:42.704 INFO:tasks.rados.rados.0.smithi059.stdout:3441: seq_num 1074 ranges {1253376=794624,2048000=401408} 2022-04-28T19:55:42.716 INFO:tasks.rados.rados.0.smithi059.stdout:3441: writing smithi05955964-29 from 1253376 to 2048000 tid 1 2022-04-28T19:55:42.718 INFO:tasks.rados.rados.0.smithi059.stdout:3441: writing smithi05955964-29 from 2048000 to 2449408 tid 2 2022-04-28T19:55:42.718 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 334 2022-04-28T19:55:42.719 INFO:tasks.rados.rados.0.smithi059.stdout:3442: seq_num 1075 ranges {11132928=409600,11542528=671744,12214272=131072} 2022-04-28T19:55:42.783 INFO:tasks.rados.rados.0.smithi059.stdout:3442: writing smithi05955964-12 from 11132928 to 11542528 tid 1 2022-04-28T19:55:42.786 INFO:tasks.rados.rados.0.smithi059.stdout:3442: writing smithi05955964-12 from 11542528 to 12214272 tid 2 2022-04-28T19:55:42.788 INFO:tasks.rados.rados.0.smithi059.stdout:3442: writing smithi05955964-12 from 12214272 to 12345344 tid 3 2022-04-28T19:55:42.788 INFO:tasks.rados.rados.0.smithi059.stdout:3443: snap_create 2022-04-28T19:55:42.788 INFO:tasks.rados.rados.0.smithi059.stdout:3439: finishing write tid 1 to smithi05955964-25 2022-04-28T19:55:42.789 INFO:tasks.rados.rados.0.smithi059.stdout:3439: finishing write tid 2 to smithi05955964-25 2022-04-28T19:55:42.789 INFO:tasks.rados.rados.0.smithi059.stdout:3439: finishing write tid 4 to smithi05955964-25 2022-04-28T19:55:42.789 INFO:tasks.rados.rados.0.smithi059.stdout:3439: finishing write tid 5 to smithi05955964-25 2022-04-28T19:55:42.789 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1538 (ObjNum 1072 snap 334 seq_num 1072) dirty exists 2022-04-28T19:55:42.789 INFO:tasks.rados.rados.0.smithi059.stdout:3439: left oid 25 (ObjNum 1072 snap 334 seq_num 1072) 2022-04-28T19:55:42.790 INFO:tasks.rados.rados.0.smithi059.stdout:3440: finishing write tid 1 to smithi05955964-44 2022-04-28T19:55:42.790 INFO:tasks.rados.rados.0.smithi059.stdout:3440: finishing write tid 2 to smithi05955964-44 2022-04-28T19:55:42.790 INFO:tasks.rados.rados.0.smithi059.stdout:3440: finishing write tid 4 to smithi05955964-44 2022-04-28T19:55:42.790 INFO:tasks.rados.rados.0.smithi059.stdout:3440: finishing write tid 5 to smithi05955964-44 2022-04-28T19:55:42.790 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1199 (ObjNum 1073 snap 334 seq_num 1073) dirty exists 2022-04-28T19:55:42.791 INFO:tasks.rados.rados.0.smithi059.stdout:3440: left oid 44 (ObjNum 1073 snap 334 seq_num 1073) 2022-04-28T19:55:42.791 INFO:tasks.rados.rados.0.smithi059.stdout:3441: finishing write tid 1 to smithi05955964-29 2022-04-28T19:55:42.791 INFO:tasks.rados.rados.0.smithi059.stdout:3441: finishing write tid 2 to smithi05955964-29 2022-04-28T19:55:42.791 INFO:tasks.rados.rados.0.smithi059.stdout:3441: finishing write tid 4 to smithi05955964-29 2022-04-28T19:55:42.791 INFO:tasks.rados.rados.0.smithi059.stdout:3441: finishing write tid 5 to smithi05955964-29 2022-04-28T19:55:42.791 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1541 (ObjNum 1074 snap 334 seq_num 1074) dirty exists 2022-04-28T19:55:42.792 INFO:tasks.rados.rados.0.smithi059.stdout:3441: left oid 29 (ObjNum 1074 snap 334 seq_num 1074) 2022-04-28T19:55:42.792 INFO:tasks.rados.rados.0.smithi059.stdout:3442: finishing write tid 1 to smithi05955964-12 2022-04-28T19:55:42.792 INFO:tasks.rados.rados.0.smithi059.stdout:3442: finishing write tid 2 to smithi05955964-12 2022-04-28T19:55:42.795 INFO:tasks.rados.rados.0.smithi059.stdout:3442: finishing write tid 3 to smithi05955964-12 2022-04-28T19:55:42.795 INFO:tasks.rados.rados.0.smithi059.stdout:3442: finishing write tid 5 to smithi05955964-12 2022-04-28T19:55:42.799 INFO:tasks.rados.rados.0.smithi059.stdout:3442: finishing write tid 6 to smithi05955964-12 2022-04-28T19:55:42.799 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1374 (ObjNum 1075 snap 334 seq_num 1075) dirty exists 2022-04-28T19:55:42.799 INFO:tasks.rados.rados.0.smithi059.stdout:3442: left oid 12 (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:55:43.009 INFO:tasks.rados.rados.0.smithi059.stdout:3439: done (4 left) 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3440: done (3 left) 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3441: done (2 left) 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3442: done (1 left) 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3443: done (0 left) 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3444: read oid 22 snap -1 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3444: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 335 2022-04-28T19:55:43.010 INFO:tasks.rados.rados.0.smithi059.stdout:3445: seq_num 1076 ranges {892928=425984,1318912=548864,1867776=253952} 2022-04-28T19:55:43.017 INFO:tasks.rados.rados.0.smithi059.stdout:3445: writing smithi05955964-2 from 892928 to 1318912 tid 1 2022-04-28T19:55:43.021 INFO:tasks.rados.rados.0.smithi059.stdout:3445: writing smithi05955964-2 from 1318912 to 1867776 tid 2 2022-04-28T19:55:43.023 INFO:tasks.rados.rados.0.smithi059.stdout:3445: writing smithi05955964-2 from 1867776 to 2121728 tid 3 2022-04-28T19:55:43.023 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 335 2022-04-28T19:55:43.023 INFO:tasks.rados.rados.0.smithi059.stdout:3446: seq_num 1077 ranges {3342336=442368,3784704=434176,4218880=368640} 2022-04-28T19:55:43.044 INFO:tasks.rados.rados.0.smithi059.stdout:3446: writing smithi05955964-44 from 3342336 to 3784704 tid 1 2022-04-28T19:55:43.046 INFO:tasks.rados.rados.0.smithi059.stdout:3446: writing smithi05955964-44 from 3784704 to 4218880 tid 2 2022-04-28T19:55:43.049 INFO:tasks.rados.rados.0.smithi059.stdout:3446: writing smithi05955964-44 from 4218880 to 4587520 tid 3 2022-04-28T19:55:43.049 INFO:tasks.rados.rados.0.smithi059.stdout:3447: copy_from oid 50 from oid 29 current snap is 335 2022-04-28T19:55:43.049 INFO:tasks.rados.rados.0.smithi059.stdout:3444: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:55:43.100 INFO:tasks.rados.rados.0.smithi059.stdout:3445: finishing write tid 1 to smithi05955964-2 2022-04-28T19:55:43.100 INFO:tasks.rados.rados.0.smithi059.stdout:3445: finishing write tid 2 to smithi05955964-2 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3445: finishing write tid 3 to smithi05955964-2 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3445: finishing write tid 5 to smithi05955964-2 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3445: finishing write tid 6 to smithi05955964-2 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1456 (ObjNum 1076 snap 335 seq_num 1076) dirty exists 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3445: left oid 2 (ObjNum 1076 snap 335 seq_num 1076) 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3446: finishing write tid 1 to smithi05955964-44 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3446: finishing write tid 2 to smithi05955964-44 2022-04-28T19:55:43.101 INFO:tasks.rados.rados.0.smithi059.stdout:3446: finishing write tid 3 to smithi05955964-44 2022-04-28T19:55:43.102 INFO:tasks.rados.rados.0.smithi059.stdout:3444: done (3 left) 2022-04-28T19:55:43.102 INFO:tasks.rados.rados.0.smithi059.stdout:3445: done (2 left) 2022-04-28T19:55:43.102 INFO:tasks.rados.rados.0.smithi059.stdout:3448: read oid 41 snap 307 2022-04-28T19:55:43.102 INFO:tasks.rados.rados.0.smithi059.stdout:3448: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:55:43.102 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 27 current snap is 335 2022-04-28T19:55:43.102 INFO:tasks.rados.rados.0.smithi059.stdout:3449: seq_num 1078 ranges {0=458752,458752=720896,1179648=81920} 2022-04-28T19:55:43.104 INFO:tasks.rados.rados.0.smithi059.stdout:3449: writing smithi05955964-27 from 0 to 458752 tid 1 2022-04-28T19:55:43.110 INFO:tasks.rados.rados.0.smithi059.stdout:3449: writing smithi05955964-27 from 458752 to 1179648 tid 2 2022-04-28T19:55:43.111 INFO:tasks.rados.rados.0.smithi059.stdout:3449: writing smithi05955964-27 from 1179648 to 1261568 tid 3 2022-04-28T19:55:43.111 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 335 2022-04-28T19:55:43.111 INFO:tasks.rados.rados.0.smithi059.stdout:3450: seq_num 1079 ranges {5906432=475136,6381568=606208,6987776=196608} 2022-04-28T19:55:43.147 INFO:tasks.rados.rados.0.smithi059.stdout:3450: writing smithi05955964-25 from 5906432 to 6381568 tid 1 2022-04-28T19:55:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3450: writing smithi05955964-25 from 6381568 to 6987776 tid 2 2022-04-28T19:55:43.151 INFO:tasks.rados.rados.0.smithi059.stdout:3450: writing smithi05955964-25 from 6987776 to 7184384 tid 3 2022-04-28T19:55:43.151 INFO:tasks.rados.rados.0.smithi059.stdout:3451: snap_remove snap 299 2022-04-28T19:55:43.926 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 systemd[1]: Stopping Ceph grafana.a for 20a1061c-c72a-11ec-8c39-001a4aab830c... 2022-04-28T19:55:43.927 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2022-04-28T19:55:43.927 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped Stream Manager" 2022-04-28T19:55:43.927 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped NotificationService" logger=server reason="context canceled" 2022-04-28T19:55:43.927 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped BackendPluginManager" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped InternalMetricsService" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped UserAuthTokenService" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped AlertEngine" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped UsageStatsService" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped RemoteCache" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped provisioningServiceImpl" logger=server reason="context canceled" 2022-04-28T19:55:43.928 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped PluginManager" logger=server reason="context canceled" 2022-04-28T19:55:43.929 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:43 smithi139 bash[35227]: t=2022-04-28T19:55:43+0000 lvl=info msg="Stopped CleanUpService" logger=server reason="context canceled" 2022-04-28T19:55:44.005 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:43 smithi059 bash[72035]: level=error ts=2022-04-28T19:55:43.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:55:44.018 INFO:tasks.rados.rados.0.smithi059.stdout:3451: done (5 left) 2022-04-28T19:55:44.018 INFO:tasks.rados.rados.0.smithi059.stdout:3452: rollback oid 9 current snap is 335 2022-04-28T19:55:44.018 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 330 2022-04-28T19:55:44.018 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 335 2022-04-28T19:55:44.018 INFO:tasks.rados.rados.0.smithi059.stdout:3453: seq_num 1080 ranges {7634944=491520,8126464=540672,8667136=262144} 2022-04-28T19:55:44.067 INFO:tasks.rados.rados.0.smithi059.stdout:3453: writing smithi05955964-33 from 7634944 to 8126464 tid 1 2022-04-28T19:55:44.069 INFO:tasks.rados.rados.0.smithi059.stdout:3453: writing smithi05955964-33 from 8126464 to 8667136 tid 2 2022-04-28T19:55:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:3453: writing smithi05955964-33 from 8667136 to 8929280 tid 3 2022-04-28T19:55:44.071 INFO:tasks.rados.rados.0.smithi059.stdout:3454: snap_remove snap 329 2022-04-28T19:55:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:44 smithi059 bash[61152]: cluster 2022-04-28T19:55:43.001188+0000 mgr.y (mgr.24961) 347 : cluster [DBG] pgmap v564: 89 pgs: 89 active+clean; 884 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.0 MiB/s wr, 9 op/s 2022-04-28T19:55:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:44 smithi059 bash[61152]: cluster 2022-04-28T19:55:43.005942+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2022-04-28T19:55:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:44 smithi059 bash[59745]: cluster 2022-04-28T19:55:43.001188+0000 mgr.y (mgr.24961) 347 : cluster [DBG] pgmap v564: 89 pgs: 89 active+clean; 884 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.0 MiB/s wr, 9 op/s 2022-04-28T19:55:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:44 smithi059 bash[59745]: cluster 2022-04-28T19:55:43.005942+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2022-04-28T19:55:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:44 smithi139 bash[43301]: cluster 2022-04-28T19:55:43.001188+0000 mgr.y (mgr.24961) 347 : cluster [DBG] pgmap v564: 89 pgs: 89 active+clean; 884 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 3.0 MiB/s wr, 9 op/s 2022-04-28T19:55:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:44 smithi139 bash[43301]: cluster 2022-04-28T19:55:43.005942+0000 mon.a (mon.0) 1289 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2022-04-28T19:55:45.016 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:44 smithi139 bash[54594]: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-grafana.a 2022-04-28T19:55:45.016 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:44 smithi139 bash[54594]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-grafana-a 2022-04-28T19:55:45.016 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:44 smithi139 bash[54594]: Error response from daemon: No such container: ceph-20a1061c-c72a-11ec-8c39-001a4aab830c-grafana.a 2022-04-28T19:55:45.016 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:44 smithi139 systemd[1]: Stopped Ceph grafana.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:55:45.016 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:44 smithi139 systemd[1]: Started Ceph grafana.a for 20a1061c-c72a-11ec-8c39-001a4aab830c. 2022-04-28T19:55:45.033 INFO:tasks.rados.rados.0.smithi059.stdout:3454: done (7 left) 2022-04-28T19:55:45.033 INFO:tasks.rados.rados.0.smithi059.stdout:3455: read oid 5 snap -1 2022-04-28T19:55:45.033 INFO:tasks.rados.rados.0.smithi059.stdout:3455: expect deleted 2022-04-28T19:55:45.033 INFO:tasks.rados.rados.0.smithi059.stdout:3456: setattr oid 3 current snap is 335 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:3446: finishing write tid 5 to smithi05955964-44 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:3446: finishing write tid 6 to smithi05955964-44 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1203 (ObjNum 1077 snap 335 seq_num 1077) dirty exists 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:3446: left oid 44 (ObjNum 1077 snap 335 seq_num 1077) 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:3447: finishing copy_from to smithi05955964-50 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1541 (ObjNum 1074 snap 334 seq_num 1074) dirty exists 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:3447: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:55:45.034 INFO:tasks.rados.rados.0.smithi059.stdout:3448: expect (ObjNum 785 snap 234 seq_num 785) 2022-04-28T19:55:45.063 INFO:tasks.rados.rados.0.smithi059.stdout:3449: finishing write tid 1 to smithi05955964-27 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:3449: finishing write tid 2 to smithi05955964-27 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:3449: finishing write tid 3 to smithi05955964-27 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:3449: finishing write tid 5 to smithi05955964-27 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:3449: finishing write tid 6 to smithi05955964-27 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1356 (ObjNum 1078 snap 335 seq_num 1078) dirty exists 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:3449: left oid 27 (ObjNum 1078 snap 335 seq_num 1078) 2022-04-28T19:55:45.064 INFO:tasks.rados.rados.0.smithi059.stdout:3450: finishing write tid 1 to smithi05955964-25 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3450: finishing write tid 2 to smithi05955964-25 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3450: finishing write tid 3 to smithi05955964-25 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3450: finishing write tid 5 to smithi05955964-25 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3450: finishing write tid 6 to smithi05955964-25 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1546 (ObjNum 1079 snap 335 seq_num 1079) dirty exists 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3450: left oid 25 (ObjNum 1079 snap 335 seq_num 1079) 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3452: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3452: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:55:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3452: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:55:45.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1206 (ObjNum 1000 snap 297 seq_num 1000) dirty exists 2022-04-28T19:55:45.066 INFO:tasks.rados.rados.0.smithi059.stdout:3453: finishing write tid 1 to smithi05955964-33 2022-04-28T19:55:45.066 INFO:tasks.rados.rados.0.smithi059.stdout:3453: finishing write tid 2 to smithi05955964-33 2022-04-28T19:55:45.066 INFO:tasks.rados.rados.0.smithi059.stdout:3453: finishing write tid 3 to smithi05955964-33 2022-04-28T19:55:45.066 INFO:tasks.rados.rados.0.smithi059.stdout:3453: finishing write tid 5 to smithi05955964-33 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:3453: finishing write tid 6 to smithi05955964-33 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1545 (ObjNum 1080 snap 335 seq_num 1080) dirty exists 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:3453: left oid 33 (ObjNum 1080 snap 335 seq_num 1080) 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:3446: done (8 left) 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:3447: done (7 left) 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:3448: done (6 left) 2022-04-28T19:55:45.069 INFO:tasks.rados.rados.0.smithi059.stdout:3449: done (5 left) 2022-04-28T19:55:45.070 INFO:tasks.rados.rados.0.smithi059.stdout:3450: done (4 left) 2022-04-28T19:55:45.070 INFO:tasks.rados.rados.0.smithi059.stdout:3452: done (3 left) 2022-04-28T19:55:45.070 INFO:tasks.rados.rados.0.smithi059.stdout:3453: done (2 left) 2022-04-28T19:55:45.070 INFO:tasks.rados.rados.0.smithi059.stdout:3455: done (1 left) 2022-04-28T19:55:45.070 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 335 2022-04-28T19:55:45.070 INFO:tasks.rados.rados.0.smithi059.stdout:3457: seq_num 1081 ranges {2121728=507904,2629632=425984,3055616=376832} 2022-04-28T19:55:45.078 INFO:tasks.rados.rados.0.smithi059.stdout:3457: writing smithi05955964-2 from 2121728 to 2629632 tid 1 2022-04-28T19:55:45.081 INFO:tasks.rados.rados.0.smithi059.stdout:3457: writing smithi05955964-2 from 2629632 to 3055616 tid 2 2022-04-28T19:55:45.084 INFO:tasks.rados.rados.0.smithi059.stdout:3457: writing smithi05955964-2 from 3055616 to 3432448 tid 3 2022-04-28T19:55:45.084 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 1 current snap is 335 2022-04-28T19:55:45.084 INFO:tasks.rados.rados.0.smithi059.stdout:3458: seq_num 1082 ranges {4210688=524288,4734976=704512,5439488=98304} 2022-04-28T19:55:45.111 INFO:tasks.rados.rados.0.smithi059.stdout:3458: writing smithi05955964-1 from 4210688 to 4734976 tid 1 2022-04-28T19:55:45.117 INFO:tasks.rados.rados.0.smithi059.stdout:3458: writing smithi05955964-1 from 4734976 to 5439488 tid 2 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:3458: writing smithi05955964-1 from 5439488 to 5537792 tid 3 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:3459: copy_from oid 8 from oid 11 current snap is 335 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1283 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:3457: finishing write tid 1 to smithi05955964-2 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:3457: finishing write tid 2 to smithi05955964-2 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:3457: finishing write tid 3 to smithi05955964-2 2022-04-28T19:55:45.118 INFO:tasks.rados.rados.0.smithi059.stdout:3457: finishing write tid 5 to smithi05955964-2 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:3457: finishing write tid 6 to smithi05955964-2 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1460 (ObjNum 1081 snap 335 seq_num 1081) dirty exists 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:3457: left oid 2 (ObjNum 1081 snap 335 seq_num 1081) 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:3458: finishing write tid 1 to smithi05955964-1 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:3456: done (3 left) 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:3457: done (2 left) 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 335 2022-04-28T19:55:45.119 INFO:tasks.rados.rados.0.smithi059.stdout:3460: seq_num 1083 ranges {1744896=540672,2285568=589824,2875392=212992} 2022-04-28T19:55:45.128 INFO:tasks.rados.rados.0.smithi059.stdout:3460: writing smithi05955964-24 from 1744896 to 2285568 tid 1 2022-04-28T19:55:45.131 INFO:tasks.rados.rados.0.smithi059.stdout:3460: writing smithi05955964-24 from 2285568 to 2875392 tid 2 2022-04-28T19:55:45.133 INFO:tasks.rados.rados.0.smithi059.stdout:3460: writing smithi05955964-24 from 2875392 to 3088384 tid 3 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3461: copy_from oid 29 from oid 26 current snap is 335 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3458: finishing write tid 2 to smithi05955964-1 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3458: finishing write tid 3 to smithi05955964-1 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3458: finishing write tid 5 to smithi05955964-1 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3458: finishing write tid 6 to smithi05955964-1 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1456 (ObjNum 1082 snap 335 seq_num 1082) dirty exists 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3458: left oid 1 (ObjNum 1082 snap 335 seq_num 1082) 2022-04-28T19:55:45.134 INFO:tasks.rados.rados.0.smithi059.stdout:3459: finishing copy_from to smithi05955964-8 2022-04-28T19:55:45.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1206 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:55:45.135 INFO:tasks.rados.rados.0.smithi059.stdout:3459: finishing copy_from racing read to smithi05955964-8 2022-04-28T19:55:45.135 INFO:tasks.rados.rados.0.smithi059.stdout:3460: finishing write tid 1 to smithi05955964-24 2022-04-28T19:55:45.135 INFO:tasks.rados.rados.0.smithi059.stdout:3458: done (3 left) 2022-04-28T19:55:45.135 INFO:tasks.rados.rados.0.smithi059.stdout:3459: done (2 left) 2022-04-28T19:55:45.135 INFO:tasks.rados.rados.0.smithi059.stdout:3462: snap_remove snap 287 2022-04-28T19:55:45.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[59745]: cluster 2022-04-28T19:55:44.014950+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2022-04-28T19:55:45.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[59745]: audit 2022-04-28T19:55:44.904121+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:45.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[59745]: audit 2022-04-28T19:55:44.905891+0000 mon.a (mon.0) 1292 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:45.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[59745]: audit 2022-04-28T19:55:44.908371+0000 mon.a (mon.0) 1293 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:45.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[59745]: audit 2022-04-28T19:55:44.909939+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:45.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[59745]: audit 2022-04-28T19:55:44.913532+0000 mon.a (mon.0) 1295 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:45.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[61152]: cluster 2022-04-28T19:55:44.014950+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2022-04-28T19:55:45.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[61152]: audit 2022-04-28T19:55:44.904121+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:45.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[61152]: audit 2022-04-28T19:55:44.905891+0000 mon.a (mon.0) 1292 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:45.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[61152]: audit 2022-04-28T19:55:44.908371+0000 mon.a (mon.0) 1293 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:45.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[61152]: audit 2022-04-28T19:55:44.909939+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:45.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:45 smithi059 bash[61152]: audit 2022-04-28T19:55:44.913532+0000 mon.a (mon.0) 1295 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:45.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:45 smithi139 bash[43301]: cluster 2022-04-28T19:55:44.014950+0000 mon.a (mon.0) 1290 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2022-04-28T19:55:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:45 smithi139 bash[43301]: audit 2022-04-28T19:55:44.904121+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:45 smithi139 bash[43301]: audit 2022-04-28T19:55:44.905891+0000 mon.a (mon.0) 1292 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:45 smithi139 bash[43301]: audit 2022-04-28T19:55:44.908371+0000 mon.a (mon.0) 1293 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:45 smithi139 bash[43301]: audit 2022-04-28T19:55:44.909939+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:45 smithi139 bash[43301]: audit 2022-04-28T19:55:44.913532+0000 mon.a (mon.0) 1295 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3462: done (2 left) 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3463: rollback oid 50 current snap is 335 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 50 to 308 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3460: finishing write tid 2 to smithi05955964-24 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3460: finishing write tid 3 to smithi05955964-24 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3460: finishing write tid 5 to smithi05955964-24 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3460: finishing write tid 6 to smithi05955964-24 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1217 (ObjNum 1083 snap 335 seq_num 1083) dirty exists 2022-04-28T19:55:46.037 INFO:tasks.rados.rados.0.smithi059.stdout:3460: left oid 24 (ObjNum 1083 snap 335 seq_num 1083) 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:3461: finishing copy_from to smithi05955964-29 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1486 (ObjNum 1070 snap 334 seq_num 1070) dirty exists 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:3461: finishing copy_from racing read to smithi05955964-29 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:3460: done (2 left) 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:3461: done (1 left) 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 335 2022-04-28T19:55:46.038 INFO:tasks.rados.rados.0.smithi059.stdout:3464: seq_num 1084 ranges {3571712=557056,4128768=475136,4603904=327680} 2022-04-28T19:55:46.060 INFO:tasks.rados.rados.0.smithi059.stdout:3464: writing smithi05955964-32 from 3571712 to 4128768 tid 1 2022-04-28T19:55:46.063 INFO:tasks.rados.rados.0.smithi059.stdout:3464: writing smithi05955964-32 from 4128768 to 4603904 tid 2 2022-04-28T19:55:46.065 INFO:tasks.rados.rados.0.smithi059.stdout:3464: writing smithi05955964-32 from 4603904 to 4931584 tid 3 2022-04-28T19:55:46.065 INFO:tasks.rados.rados.0.smithi059.stdout:3465: read oid 7 snap 319 2022-04-28T19:55:46.065 INFO:tasks.rados.rados.0.smithi059.stdout:3465: expect deleted 2022-04-28T19:55:46.065 INFO:tasks.rados.rados.0.smithi059.stdout:3466: snap_remove snap 333 2022-04-28T19:55:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:46 smithi059 bash[59745]: cephadm 2022-04-28T19:55:44.911307+0000 mgr.y (mgr.24961) 348 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:46 smithi059 bash[59745]: cluster 2022-04-28T19:55:45.002073+0000 mgr.y (mgr.24961) 349 : cluster [DBG] pgmap v567: 89 pgs: 89 active+clean; 910 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2022-04-28T19:55:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:46 smithi059 bash[59745]: cluster 2022-04-28T19:55:45.030598+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2022-04-28T19:55:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:46 smithi059 bash[61152]: cephadm 2022-04-28T19:55:44.911307+0000 mgr.y (mgr.24961) 348 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:46 smithi059 bash[61152]: cluster 2022-04-28T19:55:45.002073+0000 mgr.y (mgr.24961) 349 : cluster [DBG] pgmap v567: 89 pgs: 89 active+clean; 910 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2022-04-28T19:55:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:46 smithi059 bash[61152]: cluster 2022-04-28T19:55:45.030598+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2022-04-28T19:55:46.356 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[43301]: cephadm 2022-04-28T19:55:44.911307+0000 mgr.y (mgr.24961) 348 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:46.356 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[43301]: cluster 2022-04-28T19:55:45.002073+0000 mgr.y (mgr.24961) 349 : cluster [DBG] pgmap v567: 89 pgs: 89 active+clean; 910 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2022-04-28T19:55:46.356 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[43301]: cluster 2022-04-28T19:55:45.030598+0000 mon.a (mon.0) 1296 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2022-04-28T19:55:46.616 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="App mode production" logger=settings 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2022-04-28T19:55:46.617 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Starting DB migrations" logger=migrator 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2022-04-28T19:55:46.618 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2022-04-28T19:55:46.868 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2022-04-28T19:55:46.868 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2022-04-28T19:55:46.868 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2022-04-28T19:55:46.868 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2022-04-28T19:55:46.868 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2022-04-28T19:55:46.869 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2022-04-28T19:55:46.869 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3466: done (3 left) 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3467: copy_from oid 16 from oid 30 current snap is 335 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3463: finishing rollback tid 0 to smithi05955964-50 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3463: finishing rollback tid 1 to smithi05955964-50 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3463: finishing rollback tid 2 to smithi05955964-50 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1548 (ObjNum 1003 snap 299 seq_num 1003) dirty exists 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3464: finishing write tid 1 to smithi05955964-32 2022-04-28T19:55:46.979 INFO:tasks.rados.rados.0.smithi059.stdout:3464: finishing write tid 2 to smithi05955964-32 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3464: finishing write tid 3 to smithi05955964-32 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3464: finishing write tid 5 to smithi05955964-32 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3464: finishing write tid 6 to smithi05955964-32 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1552 (ObjNum 1084 snap 335 seq_num 1084) dirty exists 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3464: left oid 32 (ObjNum 1084 snap 335 seq_num 1084) 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3463: done (3 left) 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3464: done (2 left) 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3465: done (1 left) 2022-04-28T19:55:46.980 INFO:tasks.rados.rados.0.smithi059.stdout:3468: snap_remove snap 323 2022-04-28T19:55:47.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[43301]: cluster 2022-04-28T19:55:46.034766+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2022-04-28T19:55:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[43301]: cluster 2022-04-28T19:55:46.976363+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2022-04-28T19:55:47.122 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2022-04-28T19:55:47.123 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2022-04-28T19:55:47.123 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2022-04-28T19:55:47.123 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:46 smithi139 bash[54700]: t=2022-04-28T19:55:46+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2022-04-28T19:55:47.123 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2022-04-28T19:55:47.123 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2022-04-28T19:55:47.123 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2022-04-28T19:55:47.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:47 smithi059 bash[61152]: cluster 2022-04-28T19:55:46.034766+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2022-04-28T19:55:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:47 smithi059 bash[61152]: cluster 2022-04-28T19:55:46.976363+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2022-04-28T19:55:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:47 smithi059 bash[59745]: cluster 2022-04-28T19:55:46.034766+0000 mon.a (mon.0) 1297 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2022-04-28T19:55:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:47 smithi059 bash[59745]: cluster 2022-04-28T19:55:46.976363+0000 mon.a (mon.0) 1298 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2022-04-28T19:55:47.423 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2022-04-28T19:55:47.424 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2022-04-28T19:55:47.700 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2022-04-28T19:55:47.701 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2022-04-28T19:55:47.953 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2022-04-28T19:55:47.963 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2022-04-28T19:55:47.963 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2022-04-28T19:55:47.963 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2022-04-28T19:55:47.963 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2022-04-28T19:55:47.963 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2022-04-28T19:55:47.964 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2022-04-28T19:55:47.964 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2022-04-28T19:55:47.995 INFO:tasks.rados.rados.0.smithi059.stdout:3468: done (1 left) 2022-04-28T19:55:47.995 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 335 2022-04-28T19:55:47.995 INFO:tasks.rados.rados.0.smithi059.stdout:3469: seq_num 1085 ranges {3014656=581632,3596288=761856,4358144=40960} 2022-04-28T19:55:48.015 INFO:tasks.rados.rados.0.smithi059.stdout:3469: writing smithi05955964-43 from 3014656 to 3596288 tid 1 2022-04-28T19:55:48.020 INFO:tasks.rados.rados.0.smithi059.stdout:3469: writing smithi05955964-43 from 3596288 to 4358144 tid 2 2022-04-28T19:55:48.020 INFO:tasks.rados.rados.0.smithi059.stdout:3469: writing smithi05955964-43 from 4358144 to 4399104 tid 3 2022-04-28T19:55:48.021 INFO:tasks.rados.rados.0.smithi059.stdout:3470: snap_create 2022-04-28T19:55:48.021 INFO:tasks.rados.rados.0.smithi059.stdout:3467: finishing copy_from to smithi05955964-16 2022-04-28T19:55:48.021 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1074 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:55:48.021 INFO:tasks.rados.rados.0.smithi059.stdout:3467: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:55:48.026 INFO:tasks.rados.rados.0.smithi059.stdout:3469: finishing write tid 1 to smithi05955964-43 2022-04-28T19:55:48.027 INFO:tasks.rados.rados.0.smithi059.stdout:3469: finishing write tid 2 to smithi05955964-43 2022-04-28T19:55:48.028 INFO:tasks.rados.rados.0.smithi059.stdout:3469: finishing write tid 3 to smithi05955964-43 2022-04-28T19:55:48.028 INFO:tasks.rados.rados.0.smithi059.stdout:3469: finishing write tid 5 to smithi05955964-43 2022-04-28T19:55:48.029 INFO:tasks.rados.rados.0.smithi059.stdout:3469: finishing write tid 6 to smithi05955964-43 2022-04-28T19:55:48.029 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1556 (ObjNum 1085 snap 335 seq_num 1085) dirty exists 2022-04-28T19:55:48.030 INFO:tasks.rados.rados.0.smithi059.stdout:3469: left oid 43 (ObjNum 1085 snap 335 seq_num 1085) 2022-04-28T19:55:48.206 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[43301]: cluster 2022-04-28T19:55:47.002594+0000 mgr.y (mgr.24961) 350 : cluster [DBG] pgmap v571: 89 pgs: 89 active+clean; 915 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.7 MiB/s wr, 32 op/s 2022-04-28T19:55:48.347 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[43301]: cluster 2022-04-28T19:55:47.993941+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2022-04-28T19:55:48.347 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:47 smithi139 bash[54700]: t=2022-04-28T19:55:47+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2022-04-28T19:55:48.347 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2022-04-28T19:55:48.347 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2022-04-28T19:55:48.347 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2022-04-28T19:55:48.347 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2022-04-28T19:55:48.347 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2022-04-28T19:55:48.348 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2022-04-28T19:55:48.348 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2022-04-28T19:55:48.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:48 smithi059 bash[61152]: cluster 2022-04-28T19:55:47.002594+0000 mgr.y (mgr.24961) 350 : cluster [DBG] pgmap v571: 89 pgs: 89 active+clean; 915 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.7 MiB/s wr, 32 op/s 2022-04-28T19:55:48.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:48 smithi059 bash[61152]: cluster 2022-04-28T19:55:47.993941+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2022-04-28T19:55:48.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:48 smithi059 bash[59745]: cluster 2022-04-28T19:55:47.002594+0000 mgr.y (mgr.24961) 350 : cluster [DBG] pgmap v571: 89 pgs: 89 active+clean; 915 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.7 MiB/s wr, 32 op/s 2022-04-28T19:55:48.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:48 smithi059 bash[59745]: cluster 2022-04-28T19:55:47.993941+0000 mon.a (mon.0) 1299 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2022-04-28T19:55:48.467 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2022-04-28T19:55:48.468 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2022-04-28T19:55:48.468 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2022-04-28T19:55:48.727 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2022-04-28T19:55:48.727 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2022-04-28T19:55:48.727 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2022-04-28T19:55:48.728 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2022-04-28T19:55:48.728 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2022-04-28T19:55:48.728 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2022-04-28T19:55:48.728 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2022-04-28T19:55:48.728 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2022-04-28T19:55:48.990 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2022-04-28T19:55:48.990 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2022-04-28T19:55:48.991 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2022-04-28T19:55:48.991 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2022-04-28T19:55:48.991 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2022-04-28T19:55:48.991 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2022-04-28T19:55:48.991 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:48 smithi139 bash[54700]: t=2022-04-28T19:55:48+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2022-04-28T19:55:49.001 INFO:tasks.rados.rados.0.smithi059.stdout:3467: done (2 left) 2022-04-28T19:55:49.001 INFO:tasks.rados.rados.0.smithi059.stdout:3469: done (1 left) 2022-04-28T19:55:49.001 INFO:tasks.rados.rados.0.smithi059.stdout:3470: done (0 left) 2022-04-28T19:55:49.001 INFO:tasks.rados.rados.0.smithi059.stdout:3471: snap_remove snap 328 2022-04-28T19:55:49.241 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2022-04-28T19:55:49.344 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2022-04-28T19:55:49.344 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2022-04-28T19:55:49.344 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2022-04-28T19:55:49.344 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2022-04-28T19:55:49.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2022-04-28T19:55:49.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2022-04-28T19:55:49.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2022-04-28T19:55:49.510 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2022-04-28T19:55:49.511 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2022-04-28T19:55:49.511 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2022-04-28T19:55:49.511 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2022-04-28T19:55:49.770 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2022-04-28T19:55:49.771 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2022-04-28T19:55:49.771 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2022-04-28T19:55:49.771 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2022-04-28T19:55:49.771 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2022-04-28T19:55:49.771 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2022-04-28T19:55:50.011 INFO:tasks.rados.rados.0.smithi059.stdout:3471: done (0 left) 2022-04-28T19:55:50.011 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 336 2022-04-28T19:55:50.011 INFO:tasks.rados.rados.0.smithi059.stdout:3472: seq_num 1086 ranges {0=598016,598016=647168,1245184=155648} 2022-04-28T19:55:50.014 INFO:tasks.rados.rados.0.smithi059.stdout:3472: writing smithi05955964-20 from 0 to 598016 tid 1 2022-04-28T19:55:50.018 INFO:tasks.rados.rados.0.smithi059.stdout:3472: writing smithi05955964-20 from 598016 to 1245184 tid 2 2022-04-28T19:55:50.020 INFO:tasks.rados.rados.0.smithi059.stdout:3472: writing smithi05955964-20 from 1245184 to 1400832 tid 3 2022-04-28T19:55:50.020 INFO:tasks.rados.rados.0.smithi059.stdout:3473: read oid 45 snap -1 2022-04-28T19:55:50.020 INFO:tasks.rados.rados.0.smithi059.stdout:3473: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:55:50.020 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 44 current snap is 336 2022-04-28T19:55:50.020 INFO:tasks.rados.rados.0.smithi059.stdout:3474: seq_num 1087 ranges {4587520=614400,5201920=532480,5734400=270336} 2022-04-28T19:55:50.031 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[43301]: cluster 2022-04-28T19:55:49.001223+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2022-04-28T19:55:50.031 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[43301]: cluster 2022-04-28T19:55:49.003302+0000 mgr.y (mgr.24961) 351 : cluster [DBG] pgmap v574: 89 pgs: 89 active+clean; 915 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2022-04-28T19:55:50.031 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2022-04-28T19:55:50.031 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2022-04-28T19:55:50.031 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2022-04-28T19:55:50.032 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2022-04-28T19:55:50.032 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2022-04-28T19:55:50.032 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2022-04-28T19:55:50.032 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:49 smithi139 bash[54700]: t=2022-04-28T19:55:49+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2022-04-28T19:55:50.050 INFO:tasks.rados.rados.0.smithi059.stdout:3474: writing smithi05955964-44 from 4587520 to 5201920 tid 1 2022-04-28T19:55:50.054 INFO:tasks.rados.rados.0.smithi059.stdout:3474: writing smithi05955964-44 from 5201920 to 5734400 tid 2 2022-04-28T19:55:50.056 INFO:tasks.rados.rados.0.smithi059.stdout:3474: writing smithi05955964-44 from 5734400 to 6004736 tid 3 2022-04-28T19:55:50.056 INFO:tasks.rados.rados.0.smithi059.stdout:3475: snap_remove snap 300 2022-04-28T19:55:50.301 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2022-04-28T19:55:50.301 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2022-04-28T19:55:50.301 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2022-04-28T19:55:50.301 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2022-04-28T19:55:50.301 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2022-04-28T19:55:50.302 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2022-04-28T19:55:50.302 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2022-04-28T19:55:50.302 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2022-04-28T19:55:50.302 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2022-04-28T19:55:50.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:50 smithi059 bash[61152]: cluster 2022-04-28T19:55:49.001223+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2022-04-28T19:55:50.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:50 smithi059 bash[61152]: cluster 2022-04-28T19:55:49.003302+0000 mgr.y (mgr.24961) 351 : cluster [DBG] pgmap v574: 89 pgs: 89 active+clean; 915 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2022-04-28T19:55:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:50 smithi059 bash[59745]: cluster 2022-04-28T19:55:49.001223+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2022-04-28T19:55:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:50 smithi059 bash[59745]: cluster 2022-04-28T19:55:49.003302+0000 mgr.y (mgr.24961) 351 : cluster [DBG] pgmap v574: 89 pgs: 89 active+clean; 915 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.2 MiB/s wr, 5 op/s 2022-04-28T19:55:50.578 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2022-04-28T19:55:50.579 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2022-04-28T19:55:50.579 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2022-04-28T19:55:50.579 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2022-04-28T19:55:50.579 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2022-04-28T19:55:50.579 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2022-04-28T19:55:50.579 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2022-04-28T19:55:50.838 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2022-04-28T19:55:50.839 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2022-04-28T19:55:50.839 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2022-04-28T19:55:50.839 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2022-04-28T19:55:50.839 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2022-04-28T19:55:50.839 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2022-04-28T19:55:51.013 INFO:tasks.rados.rados.0.smithi059.stdout:3475: done (3 left) 2022-04-28T19:55:51.013 INFO:tasks.rados.rados.0.smithi059.stdout:3476: read oid 23 snap -1 2022-04-28T19:55:51.014 INFO:tasks.rados.rados.0.smithi059.stdout:3476: expect (ObjNum 1049 snap 327 seq_num 1049) 2022-04-28T19:55:51.014 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 336 2022-04-28T19:55:51.014 INFO:tasks.rados.rados.0.smithi059.stdout:3477: seq_num 1088 ranges {2826240=630784,3457024=466944,3923968=335872} 2022-04-28T19:55:51.033 INFO:tasks.rados.rados.0.smithi059.stdout:3477: writing smithi05955964-31 from 2826240 to 3457024 tid 1 2022-04-28T19:55:51.036 INFO:tasks.rados.rados.0.smithi059.stdout:3477: writing smithi05955964-31 from 3457024 to 3923968 tid 2 2022-04-28T19:55:51.038 INFO:tasks.rados.rados.0.smithi059.stdout:3477: writing smithi05955964-31 from 3923968 to 4259840 tid 3 2022-04-28T19:55:51.038 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 35 current snap is 336 2022-04-28T19:55:51.038 INFO:tasks.rados.rados.0.smithi059.stdout:3478: seq_num 1089 ranges {0=647168,647168=745472,1392640=57344} 2022-04-28T19:55:51.041 INFO:tasks.rados.rados.0.smithi059.stdout:3478: writing smithi05955964-35 from 0 to 647168 tid 1 2022-04-28T19:55:51.045 INFO:tasks.rados.rados.0.smithi059.stdout:3478: writing smithi05955964-35 from 647168 to 1392640 tid 2 2022-04-28T19:55:51.046 INFO:tasks.rados.rados.0.smithi059.stdout:3478: writing smithi05955964-35 from 1392640 to 1449984 tid 3 2022-04-28T19:55:51.047 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 336 2022-04-28T19:55:51.047 INFO:tasks.rados.rados.0.smithi059.stdout:3472: finishing write tid 1 to smithi05955964-20 2022-04-28T19:55:51.047 INFO:tasks.rados.rados.0.smithi059.stdout:3479: seq_num 1090 ranges {3432448=663552,4096000=630784,4726784=172032} 2022-04-28T19:55:51.069 INFO:tasks.rados.rados.0.smithi059.stdout:3479: writing smithi05955964-2 from 3432448 to 4096000 tid 1 2022-04-28T19:55:51.073 INFO:tasks.rados.rados.0.smithi059.stdout:3479: writing smithi05955964-2 from 4096000 to 4726784 tid 2 2022-04-28T19:55:51.075 INFO:tasks.rados.rados.0.smithi059.stdout:3479: writing smithi05955964-2 from 4726784 to 4898816 tid 3 2022-04-28T19:55:51.075 INFO:tasks.rados.rados.0.smithi059.stdout:3480: rollback oid 29 current snap is 336 2022-04-28T19:55:51.075 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 310 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3472: finishing write tid 2 to smithi05955964-20 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3472: finishing write tid 3 to smithi05955964-20 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3472: finishing write tid 5 to smithi05955964-20 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3472: finishing write tid 6 to smithi05955964-20 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1287 (ObjNum 1086 snap 336 seq_num 1086) dirty exists 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3472: left oid 20 (ObjNum 1086 snap 336 seq_num 1086) 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3474: finishing write tid 1 to smithi05955964-44 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3474: finishing write tid 2 to smithi05955964-44 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3472: done (7 left) 2022-04-28T19:55:51.076 INFO:tasks.rados.rados.0.smithi059.stdout:3481: copy_from oid 26 from oid 46 current snap is 336 2022-04-28T19:55:51.077 INFO:tasks.rados.rados.0.smithi059.stdout:3474: finishing write tid 3 to smithi05955964-44 2022-04-28T19:55:51.077 INFO:tasks.rados.rados.0.smithi059.stdout:3474: finishing write tid 5 to smithi05955964-44 2022-04-28T19:55:51.077 INFO:tasks.rados.rados.0.smithi059.stdout:3474: finishing write tid 6 to smithi05955964-44 2022-04-28T19:55:51.078 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1210 (ObjNum 1087 snap 336 seq_num 1087) dirty exists 2022-04-28T19:55:51.078 INFO:tasks.rados.rados.0.smithi059.stdout:3474: left oid 44 (ObjNum 1087 snap 336 seq_num 1087) 2022-04-28T19:55:51.078 INFO:tasks.rados.rados.0.smithi059.stdout:3473: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:50 smithi139 bash[54700]: t=2022-04-28T19:55:50+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2022-04-28T19:55:51.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2022-04-28T19:55:51.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: cluster 2022-04-28T19:55:50.006957+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2022-04-28T19:55:51.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.147784+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.428159+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.441312+0000 mon.a (mon.0) 1304 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.443991+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139.front.sepia.ceph.com:3000"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.450581+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.456265+0000 mon.a (mon.0) 1307 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.459668+0000 mon.a (mon.0) 1308 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.461507+0000 mon.a (mon.0) 1309 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.462926+0000 mon.a (mon.0) 1310 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.465018+0000 mon.a (mon.0) 1311 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.467625+0000 mon.a (mon.0) 1312 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.469020+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.470707+0000 mon.a (mon.0) 1314 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.472513+0000 mon.a (mon.0) 1315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.474141+0000 mon.a (mon.0) 1316 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.475684+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.477518+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.479251+0000 mon.a (mon.0) 1319 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.481041+0000 mon.a (mon.0) 1320 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.482677+0000 mon.a (mon.0) 1321 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.490196+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.491701+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.498009+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.499537+0000 mon.a (mon.0) 1325 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.505189+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.506775+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.511765+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.513209+0000 mon.a (mon.0) 1329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.519816+0000 mon.a (mon.0) 1330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.521115+0000 mon.a (mon.0) 1331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2022-04-28T19:55:51.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.526963+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.528296+0000 mon.a (mon.0) 1333 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.534216+0000 mon.a (mon.0) 1334 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.535397+0000 mon.a (mon.0) 1335 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.541772+0000 mon.a (mon.0) 1336 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.543040+0000 mon.a (mon.0) 1337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.544203+0000 mon.a (mon.0) 1338 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.548897+0000 mon.a (mon.0) 1339 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.550079+0000 mon.a (mon.0) 1340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2022-04-28T19:55:51.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.555914+0000 mon.a (mon.0) 1341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.557143+0000 mon.a (mon.0) 1342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.558249+0000 mon.a (mon.0) 1343 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.559322+0000 mon.a (mon.0) 1344 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.560406+0000 mon.a (mon.0) 1345 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.561439+0000 mon.a (mon.0) 1346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.566912+0000 mon.a (mon.0) 1347 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.568774+0000 mon.a (mon.0) 1348 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.573455+0000 mon.a (mon.0) 1349 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.574311+0000 mon.a (mon.0) 1350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.576207+0000 mon.a (mon.0) 1351 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:51.126 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.577463+0000 mon.a (mon.0) 1352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:51.126 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.584517+0000 mon.a (mon.0) 1353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.126 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.591658+0000 mon.a (mon.0) 1354 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.126 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.593192+0000 mon.a (mon.0) 1355 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:51.126 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.594343+0000 mon.a (mon.0) 1356 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:51.126 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[43301]: audit 2022-04-28T19:55:50.602154+0000 mon.a (mon.0) 1357 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: cluster 2022-04-28T19:55:50.006957+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2022-04-28T19:55:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.147784+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.428159+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.441312+0000 mon.a (mon.0) 1304 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.443991+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139.front.sepia.ceph.com:3000"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.450581+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.456265+0000 mon.a (mon.0) 1307 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.459668+0000 mon.a (mon.0) 1308 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.461507+0000 mon.a (mon.0) 1309 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.462926+0000 mon.a (mon.0) 1310 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.465018+0000 mon.a (mon.0) 1311 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.467625+0000 mon.a (mon.0) 1312 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.469020+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.470707+0000 mon.a (mon.0) 1314 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.472513+0000 mon.a (mon.0) 1315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.474141+0000 mon.a (mon.0) 1316 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.475684+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.477518+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.479251+0000 mon.a (mon.0) 1319 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.481041+0000 mon.a (mon.0) 1320 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: cluster 2022-04-28T19:55:50.006957+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.147784+0000 mon.a (mon.0) 1302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.428159+0000 mon.a (mon.0) 1303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.441312+0000 mon.a (mon.0) 1304 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.443991+0000 mon.a (mon.0) 1305 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139.front.sepia.ceph.com:3000"}]: dispatch 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.450581+0000 mon.a (mon.0) 1306 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.456265+0000 mon.a (mon.0) 1307 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.459668+0000 mon.a (mon.0) 1308 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.461507+0000 mon.a (mon.0) 1309 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.462926+0000 mon.a (mon.0) 1310 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.465018+0000 mon.a (mon.0) 1311 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.467625+0000 mon.a (mon.0) 1312 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.469020+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.470707+0000 mon.a (mon.0) 1314 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.472513+0000 mon.a (mon.0) 1315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.474141+0000 mon.a (mon.0) 1316 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.475684+0000 mon.a (mon.0) 1317 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.477518+0000 mon.a (mon.0) 1318 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.479251+0000 mon.a (mon.0) 1319 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.481041+0000 mon.a (mon.0) 1320 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.482677+0000 mon.a (mon.0) 1321 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.490196+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.491701+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.498009+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.499537+0000 mon.a (mon.0) 1325 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.505189+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.506775+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.511765+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.513209+0000 mon.a (mon.0) 1329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.519816+0000 mon.a (mon.0) 1330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.521115+0000 mon.a (mon.0) 1331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2022-04-28T19:55:51.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.526963+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.528296+0000 mon.a (mon.0) 1333 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.534216+0000 mon.a (mon.0) 1334 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.535397+0000 mon.a (mon.0) 1335 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.541772+0000 mon.a (mon.0) 1336 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.482677+0000 mon.a (mon.0) 1321 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.490196+0000 mon.a (mon.0) 1322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.491701+0000 mon.a (mon.0) 1323 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2022-04-28T19:55:51.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.498009+0000 mon.a (mon.0) 1324 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.499537+0000 mon.a (mon.0) 1325 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.505189+0000 mon.a (mon.0) 1326 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.506775+0000 mon.a (mon.0) 1327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.511765+0000 mon.a (mon.0) 1328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.513209+0000 mon.a (mon.0) 1329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.519816+0000 mon.a (mon.0) 1330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.521115+0000 mon.a (mon.0) 1331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2022-04-28T19:55:51.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.526963+0000 mon.a (mon.0) 1332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2022-04-28T19:55:51.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.528296+0000 mon.a (mon.0) 1333 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2022-04-28T19:55:51.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.534216+0000 mon.a (mon.0) 1334 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2022-04-28T19:55:51.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.535397+0000 mon.a (mon.0) 1335 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2022-04-28T19:55:51.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.541772+0000 mon.a (mon.0) 1336 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.543040+0000 mon.a (mon.0) 1337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.544203+0000 mon.a (mon.0) 1338 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.548897+0000 mon.a (mon.0) 1339 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.550079+0000 mon.a (mon.0) 1340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.555914+0000 mon.a (mon.0) 1341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.557143+0000 mon.a (mon.0) 1342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.558249+0000 mon.a (mon.0) 1343 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.559322+0000 mon.a (mon.0) 1344 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.560406+0000 mon.a (mon.0) 1345 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.561439+0000 mon.a (mon.0) 1346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.566912+0000 mon.a (mon.0) 1347 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.568774+0000 mon.a (mon.0) 1348 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.573455+0000 mon.a (mon.0) 1349 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.574311+0000 mon.a (mon.0) 1350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.576207+0000 mon.a (mon.0) 1351 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:51.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.577463+0000 mon.a (mon.0) 1352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.584517+0000 mon.a (mon.0) 1353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.591658+0000 mon.a (mon.0) 1354 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.593192+0000 mon.a (mon.0) 1355 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.594343+0000 mon.a (mon.0) 1356 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[59745]: audit 2022-04-28T19:55:50.602154+0000 mon.a (mon.0) 1357 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.543040+0000 mon.a (mon.0) 1337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.544203+0000 mon.a (mon.0) 1338 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.548897+0000 mon.a (mon.0) 1339 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.550079+0000 mon.a (mon.0) 1340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.555914+0000 mon.a (mon.0) 1341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.557143+0000 mon.a (mon.0) 1342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.558249+0000 mon.a (mon.0) 1343 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.559322+0000 mon.a (mon.0) 1344 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.560406+0000 mon.a (mon.0) 1345 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.561439+0000 mon.a (mon.0) 1346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.566912+0000 mon.a (mon.0) 1347 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2022-04-28T19:55:51.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.568774+0000 mon.a (mon.0) 1348 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.573455+0000 mon.a (mon.0) 1349 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.574311+0000 mon.a (mon.0) 1350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.576207+0000 mon.a (mon.0) 1351 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.577463+0000 mon.a (mon.0) 1352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.584517+0000 mon.a (mon.0) 1353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.591658+0000 mon.a (mon.0) 1354 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.593192+0000 mon.a (mon.0) 1355 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.594343+0000 mon.a (mon.0) 1356 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:55:51.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:51 smithi059 bash[61152]: audit 2022-04-28T19:55:50.602154+0000 mon.a (mon.0) 1357 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2022-04-28T19:55:51.377 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2022-04-28T19:55:51.378 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2022-04-28T19:55:51.637 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2022-04-28T19:55:51.638 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2022-04-28T19:55:51.898 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2022-04-28T19:55:51.899 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2022-04-28T19:55:52.184 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2022-04-28T19:55:52.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2022-04-28T19:55:52.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:51 smithi139 bash[54700]: t=2022-04-28T19:55:51+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2022-04-28T19:55:52.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2022-04-28T19:55:52.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2022-04-28T19:55:52.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2022-04-28T19:55:52.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2022-04-28T19:55:52.185 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: audit 2022-04-28T19:55:50.441953+0000 mgr.y (mgr.24961) 352 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: audit 2022-04-28T19:55:50.444742+0000 mgr.y (mgr.24961) 353 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139.front.sepia.ceph.com:3000"}]: dispatch 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cephadm 2022-04-28T19:55:50.483925+0000 mgr.y (mgr.24961) 354 : cephadm [INF] Upgrade: Finalizing container_image settings 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cephadm 2022-04-28T19:55:50.567996+0000 mgr.y (mgr.24961) 355 : cephadm [INF] Upgrade: Complete! 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cephadm 2022-04-28T19:55:50.578786+0000 mgr.y (mgr.24961) 356 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cephadm 2022-04-28T19:55:50.595374+0000 mgr.y (mgr.24961) 357 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cluster 2022-04-28T19:55:51.004453+0000 mgr.y (mgr.24961) 358 : cluster [DBG] pgmap v576: 89 pgs: 89 active+clean; 936 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 761 B/s rd, 5.3 MiB/s wr, 10 op/s 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cluster 2022-04-28T19:55:51.010141+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2022-04-28T19:55:52.186 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[43301]: cluster 2022-04-28T19:55:51.977225+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2022-04-28T19:55:52.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: audit 2022-04-28T19:55:50.441953+0000 mgr.y (mgr.24961) 352 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: audit 2022-04-28T19:55:50.444742+0000 mgr.y (mgr.24961) 353 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139.front.sepia.ceph.com:3000"}]: dispatch 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cephadm 2022-04-28T19:55:50.483925+0000 mgr.y (mgr.24961) 354 : cephadm [INF] Upgrade: Finalizing container_image settings 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cephadm 2022-04-28T19:55:50.567996+0000 mgr.y (mgr.24961) 355 : cephadm [INF] Upgrade: Complete! 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cephadm 2022-04-28T19:55:50.578786+0000 mgr.y (mgr.24961) 356 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cephadm 2022-04-28T19:55:50.595374+0000 mgr.y (mgr.24961) 357 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cluster 2022-04-28T19:55:51.004453+0000 mgr.y (mgr.24961) 358 : cluster [DBG] pgmap v576: 89 pgs: 89 active+clean; 936 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 761 B/s rd, 5.3 MiB/s wr, 10 op/s 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cluster 2022-04-28T19:55:51.010141+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2022-04-28T19:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[61152]: cluster 2022-04-28T19:55:51.977225+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2022-04-28T19:55:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: audit 2022-04-28T19:55:50.441953+0000 mgr.y (mgr.24961) 352 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2022-04-28T19:55:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: audit 2022-04-28T19:55:50.444742+0000 mgr.y (mgr.24961) 353 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi139.front.sepia.ceph.com:3000"}]: dispatch 2022-04-28T19:55:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cephadm 2022-04-28T19:55:50.483925+0000 mgr.y (mgr.24961) 354 : cephadm [INF] Upgrade: Finalizing container_image settings 2022-04-28T19:55:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cephadm 2022-04-28T19:55:50.567996+0000 mgr.y (mgr.24961) 355 : cephadm [INF] Upgrade: Complete! 2022-04-28T19:55:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cephadm 2022-04-28T19:55:50.578786+0000 mgr.y (mgr.24961) 356 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cephadm 2022-04-28T19:55:50.595374+0000 mgr.y (mgr.24961) 357 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:55:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cluster 2022-04-28T19:55:51.004453+0000 mgr.y (mgr.24961) 358 : cluster [DBG] pgmap v576: 89 pgs: 89 active+clean; 936 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 761 B/s rd, 5.3 MiB/s wr, 10 op/s 2022-04-28T19:55:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cluster 2022-04-28T19:55:51.010141+0000 mon.a (mon.0) 1358 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2022-04-28T19:55:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:52 smithi059 bash[59745]: cluster 2022-04-28T19:55:51.977225+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2022-04-28T19:55:52.445 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2022-04-28T19:55:52.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2022-04-28T19:55:52.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2022-04-28T19:55:52.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2022-04-28T19:55:52.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2022-04-28T19:55:52.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2022-04-28T19:55:52.697 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2022-04-28T19:55:52.698 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2022-04-28T19:55:52.698 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2022-04-28T19:55:52.975 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2022-04-28T19:55:52.976 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2022-04-28T19:55:53.240 INFO:tasks.rados.rados.0.smithi059.stdout:3476: expect (ObjNum 1049 snap 327 seq_num 1049) 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:52 smithi139 bash[54700]: t=2022-04-28T19:55:52+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2022-04-28T19:55:53.244 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2022-04-28T19:55:53.245 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2022-04-28T19:55:53.245 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2022-04-28T19:55:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:3477: finishing write tid 1 to smithi05955964-31 2022-04-28T19:55:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:3477: finishing write tid 2 to smithi05955964-31 2022-04-28T19:55:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:3477: finishing write tid 3 to smithi05955964-31 2022-04-28T19:55:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:3477: finishing write tid 5 to smithi05955964-31 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3477: finishing write tid 6 to smithi05955964-31 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1378 (ObjNum 1088 snap 336 seq_num 1088) dirty exists 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3477: left oid 31 (ObjNum 1088 snap 336 seq_num 1088) 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3478: finishing write tid 1 to smithi05955964-35 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3478: finishing write tid 2 to smithi05955964-35 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3478: finishing write tid 3 to smithi05955964-35 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3478: finishing write tid 5 to smithi05955964-35 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3478: finishing write tid 6 to smithi05955964-35 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1340 (ObjNum 1089 snap 336 seq_num 1089) dirty exists 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3478: left oid 35 (ObjNum 1089 snap 336 seq_num 1089) 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3480: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:55:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3480: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:3479: finishing write tid 1 to smithi05955964-2 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:3480: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1553 (ObjNum 998 snap 297 seq_num 998) dirty exists 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:3479: finishing write tid 2 to smithi05955964-2 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:3473: done (7 left) 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:3474: done (6 left) 2022-04-28T19:55:53.266 INFO:tasks.rados.rados.0.smithi059.stdout:3476: done (5 left) 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3477: done (4 left) 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3478: done (3 left) 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3480: done (2 left) 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3482: snap_create 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3479: finishing write tid 3 to smithi05955964-2 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3479: finishing write tid 5 to smithi05955964-2 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3479: finishing write tid 6 to smithi05955964-2 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1464 (ObjNum 1090 snap 336 seq_num 1090) dirty exists 2022-04-28T19:55:53.267 INFO:tasks.rados.rados.0.smithi059.stdout:3479: left oid 2 (ObjNum 1090 snap 336 seq_num 1090) 2022-04-28T19:55:53.268 INFO:tasks.rados.rados.0.smithi059.stdout:3481: finishing copy_from to smithi05955964-26 2022-04-28T19:55:53.268 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1512 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:55:53.268 INFO:tasks.rados.rados.0.smithi059.stdout:3481: finishing copy_from racing read to smithi05955964-26 2022-04-28T19:55:53.513 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2022-04-28T19:55:53.513 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2022-04-28T19:55:53.513 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2022-04-28T19:55:53.513 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2022-04-28T19:55:53.513 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2022-04-28T19:55:53.513 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2022-04-28T19:55:53.514 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2022-04-28T19:55:53.514 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2022-04-28T19:55:53.540 INFO:tasks.rados.rados.0.smithi059.stdout:3479: done (2 left) 2022-04-28T19:55:53.541 INFO:tasks.rados.rados.0.smithi059.stdout:3481: done (1 left) 2022-04-28T19:55:53.541 INFO:tasks.rados.rados.0.smithi059.stdout:3482: done (0 left) 2022-04-28T19:55:53.541 INFO:tasks.rados.rados.0.smithi059.stdout:3483: rmattr oid 4 current snap is 337 2022-04-28T19:55:53.541 INFO:tasks.rados.rados.0.smithi059.stdout:3483: done (0 left) 2022-04-28T19:55:53.541 INFO:tasks.rados.rados.0.smithi059.stdout:3484: read oid 4 snap -1 2022-04-28T19:55:53.541 INFO:tasks.rados.rados.0.smithi059.stdout:3484: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3485: copy_from oid 16 from oid 24 current snap is 337 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3486: read oid 44 snap -1 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3486: expect (ObjNum 1087 snap 336 seq_num 1087) 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3487: rollback oid 29 current snap is 337 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 29 to 319 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3488: rmattr oid 13 current snap is 337 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3488: done (4 left) 2022-04-28T19:55:53.542 INFO:tasks.rados.rados.0.smithi059.stdout:3489: setattr oid 11 current snap is 337 2022-04-28T19:55:53.543 INFO:tasks.rados.rados.0.smithi059.stdout:3490: read oid 17 snap -1 2022-04-28T19:55:53.543 INFO:tasks.rados.rados.0.smithi059.stdout:3490: expect (ObjNum 960 snap 285 seq_num 960) 2022-04-28T19:55:53.543 INFO:tasks.rados.rados.0.smithi059.stdout:3491: read oid 10 snap 295 2022-04-28T19:55:53.543 INFO:tasks.rados.rados.0.smithi059.stdout:3491: expect (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:55:53.544 INFO:tasks.rados.rados.0.smithi059.stdout:3492: snap_create 2022-04-28T19:55:53.579 INFO:tasks.rados.rados.0.smithi059.stdout:3491: expect (ObjNum 917 snap 273 seq_num 917) 2022-04-28T19:55:53.781 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2022-04-28T19:55:53.781 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2022-04-28T19:55:53.781 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2022-04-28T19:55:53.782 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2022-04-28T19:55:53.782 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[43301]: audit 2022-04-28T19:55:52.525415+0000 mon.a (mon.0) 1360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:53.782 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[43301]: cluster 2022-04-28T19:55:53.005214+0000 mgr.y (mgr.24961) 359 : cluster [DBG] pgmap v579: 89 pgs: 89 active+clean; 936 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 766 B/s rd, 5.4 MiB/s wr, 10 op/s 2022-04-28T19:55:53.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:53 smithi059 bash[59745]: audit 2022-04-28T19:55:52.525415+0000 mon.a (mon.0) 1360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:53.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:53 smithi059 bash[59745]: cluster 2022-04-28T19:55:53.005214+0000 mgr.y (mgr.24961) 359 : cluster [DBG] pgmap v579: 89 pgs: 89 active+clean; 936 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 766 B/s rd, 5.4 MiB/s wr, 10 op/s 2022-04-28T19:55:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:53 smithi059 bash[61152]: audit 2022-04-28T19:55:52.525415+0000 mon.a (mon.0) 1360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:55:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:53 smithi059 bash[61152]: cluster 2022-04-28T19:55:53.005214+0000 mgr.y (mgr.24961) 359 : cluster [DBG] pgmap v579: 89 pgs: 89 active+clean; 936 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 766 B/s rd, 5.4 MiB/s wr, 10 op/s 2022-04-28T19:55:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:55:53 smithi059 bash[72035]: level=error ts=2022-04-28T19:55:53.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:55:53.843 INFO:tasks.rados.rados.0.smithi059.stdout:3487: finishing rollback tid 0 to smithi05955964-29 2022-04-28T19:55:53.844 INFO:tasks.rados.rados.0.smithi059.stdout:3487: finishing rollback tid 1 to smithi05955964-29 2022-04-28T19:55:53.844 INFO:tasks.rados.rados.0.smithi059.stdout:3487: finishing rollback tid 2 to smithi05955964-29 2022-04-28T19:55:53.844 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1557 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:55:53.844 INFO:tasks.rados.rados.0.smithi059.stdout:3486: expect (ObjNum 1087 snap 336 seq_num 1087) 2022-04-28T19:55:54.058 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2022-04-28T19:55:54.059 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2022-04-28T19:55:54.059 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2022-04-28T19:55:54.059 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2022-04-28T19:55:54.059 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2022-04-28T19:55:54.059 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:53 smithi139 bash[54700]: t=2022-04-28T19:55:53+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2022-04-28T19:55:54.059 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2022-04-28T19:55:54.179 INFO:tasks.rados.rados.0.smithi059.stdout:3484: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:55:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2022-04-28T19:55:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2022-04-28T19:55:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2022-04-28T19:55:54.310 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2022-04-28T19:55:54.311 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2022-04-28T19:55:54.311 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2022-04-28T19:55:54.587 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[43301]: cluster 2022-04-28T19:55:53.538511+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2022-04-28T19:55:54.587 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2022-04-28T19:55:54.587 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2022-04-28T19:55:54.588 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2022-04-28T19:55:54.588 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2022-04-28T19:55:54.588 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2022-04-28T19:55:54.588 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2022-04-28T19:55:54.588 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2022-04-28T19:55:54.588 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2022-04-28T19:55:54.781 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1218 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:55:54.781 INFO:tasks.rados.rados.0.smithi059.stdout:3490: expect (ObjNum 960 snap 285 seq_num 960) 2022-04-28T19:55:54.840 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2022-04-28T19:55:54.840 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2022-04-28T19:55:54.840 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2022-04-28T19:55:54.840 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2022-04-28T19:55:54.840 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2022-04-28T19:55:54.840 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2022-04-28T19:55:54.841 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2022-04-28T19:55:54.841 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2022-04-28T19:55:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:54 smithi059 bash[61152]: cluster 2022-04-28T19:55:53.538511+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2022-04-28T19:55:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:54 smithi059 bash[59745]: cluster 2022-04-28T19:55:53.538511+0000 mon.a (mon.0) 1361 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2022-04-28T19:55:55.108 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2022-04-28T19:55:55.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2022-04-28T19:55:55.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2022-04-28T19:55:55.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2022-04-28T19:55:55.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:54 smithi139 bash[54700]: t=2022-04-28T19:55:54+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2022-04-28T19:55:55.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2022-04-28T19:55:55.109 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2022-04-28T19:55:55.375 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2022-04-28T19:55:55.663 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[43301]: cluster 2022-04-28T19:55:54.547092+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2022-04-28T19:55:55.663 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[43301]: cluster 2022-04-28T19:55:55.006104+0000 mgr.y (mgr.24961) 360 : cluster [DBG] pgmap v582: 89 pgs: 89 active+clean; 951 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2022-04-28T19:55:55.663 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2022-04-28T19:55:55.664 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2022-04-28T19:55:55.697 INFO:tasks.rados.rados.0.smithi059.stdout:3485: finishing copy_from to smithi05955964-16 2022-04-28T19:55:55.697 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1217 (ObjNum 1083 snap 335 seq_num 1083) dirty exists 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3485: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3484: done (7 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3485: done (6 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3486: done (5 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3487: done (4 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3489: done (3 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3490: done (2 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3491: done (1 left) 2022-04-28T19:55:55.698 INFO:tasks.rados.rados.0.smithi059.stdout:3492: done (0 left) 2022-04-28T19:55:55.699 INFO:tasks.rados.rados.0.smithi059.stdout:3493: read oid 23 snap -1 2022-04-28T19:55:55.699 INFO:tasks.rados.rados.0.smithi059.stdout:3493: expect (ObjNum 1049 snap 327 seq_num 1049) 2022-04-28T19:55:55.699 INFO:tasks.rados.rados.0.smithi059.stdout:3494: read oid 45 snap -1 2022-04-28T19:55:55.699 INFO:tasks.rados.rados.0.smithi059.stdout:3494: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:55:55.699 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 338 2022-04-28T19:55:55.699 INFO:tasks.rados.rados.0.smithi059.stdout:3495: seq_num 1091 ranges {1605632=679936,2285568=516096,2801664=286720} 2022-04-28T19:55:55.717 INFO:tasks.rados.rados.0.smithi059.stdout:3495: writing smithi05955964-26 from 1605632 to 2285568 tid 1 2022-04-28T19:55:55.720 INFO:tasks.rados.rados.0.smithi059.stdout:3495: writing smithi05955964-26 from 2285568 to 2801664 tid 2 2022-04-28T19:55:55.722 INFO:tasks.rados.rados.0.smithi059.stdout:3495: writing smithi05955964-26 from 2801664 to 3088384 tid 3 2022-04-28T19:55:55.723 INFO:tasks.rados.rados.0.smithi059.stdout:3496: read oid 12 snap -1 2022-04-28T19:55:55.723 INFO:tasks.rados.rados.0.smithi059.stdout:3496: expect (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:55:55.723 INFO:tasks.rados.rados.0.smithi059.stdout:3497: snap_remove snap 321 2022-04-28T19:55:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:55 smithi059 bash[61152]: cluster 2022-04-28T19:55:54.547092+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2022-04-28T19:55:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:55 smithi059 bash[61152]: cluster 2022-04-28T19:55:55.006104+0000 mgr.y (mgr.24961) 360 : cluster [DBG] pgmap v582: 89 pgs: 89 active+clean; 951 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2022-04-28T19:55:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:55 smithi059 bash[59745]: cluster 2022-04-28T19:55:54.547092+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2022-04-28T19:55:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:55 smithi059 bash[59745]: cluster 2022-04-28T19:55:55.006104+0000 mgr.y (mgr.24961) 360 : cluster [DBG] pgmap v582: 89 pgs: 89 active+clean; 951 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 2.1 MiB/s wr, 11 op/s 2022-04-28T19:55:55.923 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2022-04-28T19:55:55.924 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2022-04-28T19:55:55.924 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2022-04-28T19:55:55.924 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2022-04-28T19:55:55.924 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2022-04-28T19:55:55.924 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2022-04-28T19:55:55.924 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2022-04-28T19:55:56.184 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:55 smithi139 bash[54700]: t=2022-04-28T19:55:55+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2022-04-28T19:55:56.185 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2022-04-28T19:55:56.438 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2022-04-28T19:55:56.438 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2022-04-28T19:55:56.438 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2022-04-28T19:55:56.438 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2022-04-28T19:55:56.439 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2022-04-28T19:55:56.439 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2022-04-28T19:55:56.439 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2022-04-28T19:55:56.439 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2022-04-28T19:55:56.439 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2022-04-28T19:55:56.439 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2022-04-28T19:55:56.565 INFO:tasks.rados.rados.0.smithi059.stdout:3497: done (4 left) 2022-04-28T19:55:56.565 INFO:tasks.rados.rados.0.smithi059.stdout:3498: snap_create 2022-04-28T19:55:56.565 INFO:tasks.rados.rados.0.smithi059.stdout:3493: expect (ObjNum 1049 snap 327 seq_num 1049) 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:3495: finishing write tid 1 to smithi05955964-26 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:3495: finishing write tid 2 to smithi05955964-26 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:3495: finishing write tid 3 to smithi05955964-26 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:3495: finishing write tid 5 to smithi05955964-26 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:3495: finishing write tid 6 to smithi05955964-26 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1516 (ObjNum 1091 snap 338 seq_num 1091) dirty exists 2022-04-28T19:55:56.590 INFO:tasks.rados.rados.0.smithi059.stdout:3495: left oid 26 (ObjNum 1091 snap 338 seq_num 1091) 2022-04-28T19:55:56.591 INFO:tasks.rados.rados.0.smithi059.stdout:3494: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:55:56.731 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2022-04-28T19:55:56.731 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2022-04-28T19:55:56.731 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2022-04-28T19:55:56.731 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2022-04-28T19:55:56.731 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2022-04-28T19:55:56.731 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2022-04-28T19:55:56.732 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2022-04-28T19:55:57.016 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2022-04-28T19:55:57.017 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2022-04-28T19:55:57.017 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2022-04-28T19:55:57.017 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2022-04-28T19:55:57.017 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2022-04-28T19:55:57.017 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:56 smithi139 bash[54700]: t=2022-04-28T19:55:56+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2022-04-28T19:55:57.277 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2022-04-28T19:55:57.278 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2022-04-28T19:55:57.538 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2022-04-28T19:55:57.815 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[43301]: cluster 2022-04-28T19:55:56.564306+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2022-04-28T19:55:57.815 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[43301]: cluster 2022-04-28T19:55:56.983682+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2022-04-28T19:55:57.815 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[43301]: cluster 2022-04-28T19:55:57.006636+0000 mgr.y (mgr.24961) 361 : cluster [DBG] pgmap v585: 89 pgs: 89 active+clean; 954 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 2.8 MiB/s wr, 19 op/s 2022-04-28T19:55:57.816 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2022-04-28T19:55:57.816 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2022-04-28T19:55:57.816 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2022-04-28T19:55:57.816 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2022-04-28T19:55:57.817 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2022-04-28T19:55:57.817 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2022-04-28T19:55:57.817 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2022-04-28T19:55:57.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:57 smithi059 bash[61152]: cluster 2022-04-28T19:55:56.564306+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2022-04-28T19:55:57.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:57 smithi059 bash[61152]: cluster 2022-04-28T19:55:56.983682+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2022-04-28T19:55:57.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:55:57 smithi059 bash[61152]: cluster 2022-04-28T19:55:57.006636+0000 mgr.y (mgr.24961) 361 : cluster [DBG] pgmap v585: 89 pgs: 89 active+clean; 954 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 2.8 MiB/s wr, 19 op/s 2022-04-28T19:55:57.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:57 smithi059 bash[59745]: cluster 2022-04-28T19:55:56.564306+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2022-04-28T19:55:57.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:57 smithi059 bash[59745]: cluster 2022-04-28T19:55:56.983682+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2022-04-28T19:55:57.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:55:57 smithi059 bash[59745]: cluster 2022-04-28T19:55:57.006636+0000 mgr.y (mgr.24961) 361 : cluster [DBG] pgmap v585: 89 pgs: 89 active+clean; 954 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 2.8 MiB/s wr, 19 op/s 2022-04-28T19:55:58.100 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2022-04-28T19:55:58.100 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2022-04-28T19:55:58.100 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2022-04-28T19:55:58.101 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:57 smithi139 bash[54700]: t=2022-04-28T19:55:57+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2022-04-28T19:55:58.101 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2022-04-28T19:55:58.101 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2022-04-28T19:55:58.101 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2022-04-28T19:55:58.368 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2022-04-28T19:55:58.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2022-04-28T19:55:58.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2022-04-28T19:55:58.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2022-04-28T19:55:58.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2022-04-28T19:55:58.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2022-04-28T19:55:58.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2022-04-28T19:55:58.629 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2022-04-28T19:55:58.630 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2022-04-28T19:55:58.631 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2022-04-28T19:55:58.742 INFO:tasks.rados.rados.0.smithi059.stdout:3496: expect (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:55:58.891 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2022-04-28T19:55:58.891 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2022-04-28T19:55:58.891 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2022-04-28T19:55:58.892 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2022-04-28T19:55:58.892 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2022-04-28T19:55:58.892 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2022-04-28T19:55:58.892 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2022-04-28T19:55:58.892 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2022-04-28T19:55:58.893 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2022-04-28T19:55:59.159 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2022-04-28T19:55:59.160 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2022-04-28T19:55:59.160 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:58 smithi139 bash[54700]: t=2022-04-28T19:55:58+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2022-04-28T19:55:59.160 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2022-04-28T19:55:59.160 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2022-04-28T19:55:59.160 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2022-04-28T19:55:59.160 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2022-04-28T19:55:59.428 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2022-04-28T19:55:59.428 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2022-04-28T19:55:59.428 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2022-04-28T19:55:59.428 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2022-04-28T19:55:59.429 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2022-04-28T19:55:59.429 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2022-04-28T19:55:59.429 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2022-04-28T19:55:59.688 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=13.30560298s 2022-04-28T19:55:59.689 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Created default organization" logger=sqlstore 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2022-04-28T19:55:59.955 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2022-04-28T19:55:59.956 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="warming cache for startup" logger=ngalert 2022-04-28T19:55:59.956 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2022-04-28T19:56:00.059 INFO:tasks.rados.rados.0.smithi059.stdout:3493: done (4 left) 2022-04-28T19:56:00.059 INFO:tasks.rados.rados.0.smithi059.stdout:3494: done (3 left) 2022-04-28T19:56:00.059 INFO:tasks.rados.rados.0.smithi059.stdout:3495: done (2 left) 2022-04-28T19:56:00.059 INFO:tasks.rados.rados.0.smithi059.stdout:3496: done (1 left) 2022-04-28T19:56:00.059 INFO:tasks.rados.rados.0.smithi059.stdout:3498: done (0 left) 2022-04-28T19:56:00.059 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 339 2022-04-28T19:56:00.060 INFO:tasks.rados.rados.0.smithi059.stdout:3499: seq_num 1092 ranges {0=696320,696320=802816} 2022-04-28T19:56:00.068 INFO:tasks.rados.rados.0.smithi059.stdout:3499: writing smithi05955964-34 from 0 to 696320 tid 1 2022-04-28T19:56:00.073 INFO:tasks.rados.rados.0.smithi059.stdout:3499: writing smithi05955964-34 from 696320 to 1499136 tid 2 2022-04-28T19:56:00.073 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 339 2022-04-28T19:56:00.073 INFO:tasks.rados.rados.0.smithi059.stdout:3500: seq_num 1093 ranges {950272=712704,1662976=688128,2351104=114688} 2022-04-28T19:56:00.083 INFO:tasks.rados.rados.0.smithi059.stdout:3500: writing smithi05955964-21 from 950272 to 1662976 tid 1 2022-04-28T19:56:00.087 INFO:tasks.rados.rados.0.smithi059.stdout:3500: writing smithi05955964-21 from 1662976 to 2351104 tid 2 2022-04-28T19:56:00.089 INFO:tasks.rados.rados.0.smithi059.stdout:3500: writing smithi05955964-21 from 2351104 to 2465792 tid 3 2022-04-28T19:56:00.089 INFO:tasks.rados.rados.0.smithi059.stdout:3501: read oid 36 snap 288 2022-04-28T19:56:00.089 INFO:tasks.rados.rados.0.smithi059.stdout:3501: expect (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:56:00.089 INFO:tasks.rados.rados.0.smithi059.stdout:3502: rollback oid 20 current snap is 339 2022-04-28T19:56:00.089 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 337 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3499: finishing write tid 1 to smithi05955964-34 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3499: finishing write tid 2 to smithi05955964-34 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3499: finishing write tid 4 to smithi05955964-34 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3499: finishing write tid 5 to smithi05955964-34 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1355 (ObjNum 1092 snap 339 seq_num 1092) dirty exists 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3499: left oid 34 (ObjNum 1092 snap 339 seq_num 1092) 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3500: finishing write tid 1 to smithi05955964-21 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3499: done (3 left) 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3503: copy_from oid 1 from oid 38 current snap is 339 2022-04-28T19:56:00.090 INFO:tasks.rados.rados.0.smithi059.stdout:3504: read oid 14 snap 288 2022-04-28T19:56:00.091 INFO:tasks.rados.rados.0.smithi059.stdout:3504: expect (ObjNum 800 snap 235 seq_num 800) 2022-04-28T19:56:00.091 INFO:tasks.rados.rados.0.smithi059.stdout:3505: delete oid 32 current snap is 339 2022-04-28T19:56:00.093 INFO:tasks.rados.rados.0.smithi059.stdout:3505: done (5 left) 2022-04-28T19:56:00.093 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 339 2022-04-28T19:56:00.093 INFO:tasks.rados.rados.0.smithi059.stdout:3506: seq_num 1095 ranges {0=745472,745472=499712,1245184=303104} 2022-04-28T19:56:00.098 INFO:tasks.rados.rados.0.smithi059.stdout:3506: writing smithi05955964-32 from 0 to 745472 tid 1 2022-04-28T19:56:00.101 INFO:tasks.rados.rados.0.smithi059.stdout:3506: writing smithi05955964-32 from 745472 to 1245184 tid 2 2022-04-28T19:56:00.104 INFO:tasks.rados.rados.0.smithi059.stdout:3506: writing smithi05955964-32 from 1245184 to 1548288 tid 3 2022-04-28T19:56:00.104 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 28 current snap is 339 2022-04-28T19:56:00.106 INFO:tasks.rados.rados.0.smithi059.stdout:3507: seq_num 1096 ranges {11132928=761856,11894784=786432,12681216=16384} 2022-04-28T19:56:00.171 INFO:tasks.rados.rados.0.smithi059.stdout:3507: writing smithi05955964-28 from 11132928 to 11894784 tid 1 2022-04-28T19:56:00.176 INFO:tasks.rados.rados.0.smithi059.stdout:3507: writing smithi05955964-28 from 11894784 to 12681216 tid 2 2022-04-28T19:56:00.176 INFO:tasks.rados.rados.0.smithi059.stdout:3507: writing smithi05955964-28 from 12681216 to 12697600 tid 3 2022-04-28T19:56:00.176 INFO:tasks.rados.rados.0.smithi059.stdout:3508: rollback oid 6 current snap is 339 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 6 to 307 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3509: read oid 15 snap -1 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3509: expect (ObjNum 1027 snap 318 seq_num 1027) 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3510: delete oid 16 current snap is 339 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3500: finishing write tid 2 to smithi05955964-21 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3500: finishing write tid 3 to smithi05955964-21 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3500: finishing write tid 5 to smithi05955964-21 2022-04-28T19:56:00.177 INFO:tasks.rados.rados.0.smithi059.stdout:3501: expect (ObjNum 678 snap 197 seq_num 678) 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:3502: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:3502: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:3502: finishing rollback tid 2 to smithi05955964-20 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1290 (ObjNum 1086 snap 336 seq_num 1086) dirty exists 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:3500: finishing write tid 6 to smithi05955964-21 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1166 (ObjNum 1093 snap 339 seq_num 1093) dirty exists 2022-04-28T19:56:00.225 INFO:tasks.rados.rados.0.smithi059.stdout:3500: left oid 21 (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:3506: finishing write tid 1 to smithi05955964-32 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:3500: done (9 left) 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:3501: done (8 left) 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:3502: done (7 left) 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:3510: done (6 left) 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 339 2022-04-28T19:56:00.226 INFO:tasks.rados.rados.0.smithi059.stdout:3511: seq_num 1098 ranges {0=794624,794624=557056,1351680=245760} 2022-04-28T19:56:00.231 INFO:tasks.rados.rados.0.smithi059.stdout:3511: writing smithi05955964-16 from 0 to 794624 tid 1 2022-04-28T19:56:00.236 INFO:tasks.rados.rados.0.smithi059.stdout:3511: writing smithi05955964-16 from 794624 to 1351680 tid 2 2022-04-28T19:56:00.240 INFO:tasks.rados.rados.0.smithi059.stdout:3511: writing smithi05955964-16 from 1351680 to 1597440 tid 3 2022-04-28T19:56:00.240 INFO:tasks.rados.rados.0.smithi059.stdout:3512: read oid 46 snap -1 2022-04-28T19:56:00.240 INFO:tasks.rados.rados.0.smithi059.stdout:3512: expect (ObjNum 955 snap 283 seq_num 955) 2022-04-28T19:56:00.241 INFO:tasks.rados.rados.0.smithi059.stdout:3513: rollback oid 26 current snap is 339 2022-04-28T19:56:00.241 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 334 2022-04-28T19:56:00.241 INFO:tasks.rados.rados.0.smithi059.stdout:3506: finishing write tid 2 to smithi05955964-32 2022-04-28T19:56:00.241 INFO:tasks.rados.rados.0.smithi059.stdout:3506: finishing write tid 3 to smithi05955964-32 2022-04-28T19:56:00.241 INFO:tasks.rados.rados.0.smithi059.stdout:3506: finishing write tid 5 to smithi05955964-32 2022-04-28T19:56:00.242 INFO:tasks.rados.rados.0.smithi059.stdout:3504: expect (ObjNum 800 snap 235 seq_num 800) 2022-04-28T19:56:00.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:00 smithi059 bash[61152]: cluster 2022-04-28T19:55:59.007171+0000 mgr.y (mgr.24961) 362 : cluster [DBG] pgmap v586: 89 pgs: 89 active+clean; 954 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2022-04-28T19:56:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:00 smithi059 bash[59745]: cluster 2022-04-28T19:55:59.007171+0000 mgr.y (mgr.24961) 362 : cluster [DBG] pgmap v586: 89 pgs: 89 active+clean; 954 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2022-04-28T19:56:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:00 smithi139 bash[43301]: cluster 2022-04-28T19:55:59.007171+0000 mgr.y (mgr.24961) 362 : cluster [DBG] pgmap v586: 89 pgs: 89 active+clean; 954 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2022-04-28T19:56:00.369 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 19:55:59 smithi139 bash[54700]: t=2022-04-28T19:55:59+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2022-04-28T19:56:00.408 INFO:tasks.rados.rados.0.smithi059.stdout:3506: finishing write tid 6 to smithi05955964-32 2022-04-28T19:56:00.408 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1561 (ObjNum 1095 snap 339 seq_num 1095) dirty exists 2022-04-28T19:56:00.409 INFO:tasks.rados.rados.0.smithi059.stdout:3506: left oid 32 (ObjNum 1095 snap 339 seq_num 1095) 2022-04-28T19:56:00.409 INFO:tasks.rados.rados.0.smithi059.stdout:3503: finishing copy_from to smithi05955964-1 2022-04-28T19:56:00.409 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1213 (ObjNum 1058 snap 329 seq_num 1058) dirty exists 2022-04-28T19:56:00.409 INFO:tasks.rados.rados.0.smithi059.stdout:3503: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:56:00.409 INFO:tasks.rados.rados.0.smithi059.stdout:3508: finishing rollback tid 1 to smithi05955964-6 2022-04-28T19:56:00.410 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1241 (ObjNum 0 snap 0 seq_num 188341408) dirty dne 2022-04-28T19:56:00.410 INFO:tasks.rados.rados.0.smithi059.stdout:3503: done (8 left) 2022-04-28T19:56:00.410 INFO:tasks.rados.rados.0.smithi059.stdout:3504: done (7 left) 2022-04-28T19:56:00.410 INFO:tasks.rados.rados.0.smithi059.stdout:3506: done (6 left) 2022-04-28T19:56:00.410 INFO:tasks.rados.rados.0.smithi059.stdout:3508: done (5 left) 2022-04-28T19:56:00.411 INFO:tasks.rados.rados.0.smithi059.stdout:3514: read oid 1 snap -1 2022-04-28T19:56:00.411 INFO:tasks.rados.rados.0.smithi059.stdout:3514: expect (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:56:00.411 INFO:tasks.rados.rados.0.smithi059.stdout:3515: read oid 20 snap -1 2022-04-28T19:56:00.411 INFO:tasks.rados.rados.0.smithi059.stdout:3515: expect (ObjNum 1086 snap 336 seq_num 1086) 2022-04-28T19:56:00.411 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 339 2022-04-28T19:56:00.412 INFO:tasks.rados.rados.0.smithi059.stdout:3516: seq_num 1099 ranges {3088384=409600,3497984=442368,3940352=720896,4661248=40960} 2022-04-28T19:56:00.436 INFO:tasks.rados.rados.0.smithi059.stdout:3516: writing smithi05955964-24 from 3088384 to 3497984 tid 1 2022-04-28T19:56:00.448 INFO:tasks.rados.rados.0.smithi059.stdout:3516: writing smithi05955964-24 from 3497984 to 3940352 tid 2 2022-04-28T19:56:00.449 INFO:tasks.rados.rados.0.smithi059.stdout:3516: writing smithi05955964-24 from 3940352 to 4661248 tid 3 2022-04-28T19:56:00.449 INFO:tasks.rados.rados.0.smithi059.stdout:3516: writing smithi05955964-24 from 4661248 to 4702208 tid 4 2022-04-28T19:56:00.449 INFO:tasks.rados.rados.0.smithi059.stdout:3517: snap_create 2022-04-28T19:56:00.449 INFO:tasks.rados.rados.0.smithi059.stdout:3509: expect (ObjNum 1027 snap 318 seq_num 1027) 2022-04-28T19:56:00.456 INFO:tasks.rados.rados.0.smithi059.stdout:3507: finishing write tid 1 to smithi05955964-28 2022-04-28T19:56:00.456 INFO:tasks.rados.rados.0.smithi059.stdout:3507: finishing write tid 2 to smithi05955964-28 2022-04-28T19:56:00.456 INFO:tasks.rados.rados.0.smithi059.stdout:3507: finishing write tid 3 to smithi05955964-28 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3507: finishing write tid 5 to smithi05955964-28 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3507: finishing write tid 6 to smithi05955964-28 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1451 (ObjNum 1096 snap 339 seq_num 1096) dirty exists 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3507: left oid 28 (ObjNum 1096 snap 339 seq_num 1096) 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3511: finishing write tid 1 to smithi05955964-16 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3511: finishing write tid 2 to smithi05955964-16 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3511: finishing write tid 3 to smithi05955964-16 2022-04-28T19:56:00.457 INFO:tasks.rados.rados.0.smithi059.stdout:3513: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:3511: finishing write tid 5 to smithi05955964-16 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:3513: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:3513: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1519 (ObjNum 1070 snap 334 seq_num 1070) dirty exists 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:3511: finishing write tid 6 to smithi05955964-16 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1456 (ObjNum 1098 snap 339 seq_num 1098) dirty exists 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:3511: left oid 16 (ObjNum 1098 snap 339 seq_num 1098) 2022-04-28T19:56:00.458 INFO:tasks.rados.rados.0.smithi059.stdout:3512: expect (ObjNum 955 snap 283 seq_num 955) 2022-04-28T19:56:00.493 INFO:tasks.rados.rados.0.smithi059.stdout:3515: expect (ObjNum 1086 snap 336 seq_num 1086) 2022-04-28T19:56:00.525 INFO:tasks.rados.rados.0.smithi059.stdout:3514: expect (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:56:01.418 INFO:tasks.rados.rados.0.smithi059.stdout:3516: finishing write tid 1 to smithi05955964-24 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3516: finishing write tid 2 to smithi05955964-24 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3516: finishing write tid 3 to smithi05955964-24 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3516: finishing write tid 4 to smithi05955964-24 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3516: finishing write tid 6 to smithi05955964-24 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3516: finishing write tid 7 to smithi05955964-24 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1223 (ObjNum 1099 snap 339 seq_num 1099) dirty exists 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3516: left oid 24 (ObjNum 1099 snap 339 seq_num 1099) 2022-04-28T19:56:01.419 INFO:tasks.rados.rados.0.smithi059.stdout:3507: done (8 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3509: done (7 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3511: done (6 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3512: done (5 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3513: done (4 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3514: done (3 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3515: done (2 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3516: done (1 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3517: done (0 left) 2022-04-28T19:56:01.420 INFO:tasks.rados.rados.0.smithi059.stdout:3518: delete oid 24 current snap is 340 2022-04-28T19:56:01.424 INFO:tasks.rados.rados.0.smithi059.stdout:3518: done (0 left) 2022-04-28T19:56:01.424 INFO:tasks.rados.rados.0.smithi059.stdout:3519: rmattr oid 49 current snap is 340 2022-04-28T19:56:01.424 INFO:tasks.rados.rados.0.smithi059.stdout:3519: done (0 left) 2022-04-28T19:56:01.424 INFO:tasks.rados.rados.0.smithi059.stdout:3520: snap_remove snap 315 2022-04-28T19:56:01.984 INFO:tasks.rados.rados.0.smithi059.stdout:3520: done (0 left) 2022-04-28T19:56:02.071 INFO:tasks.rados.rados.0.smithi059.stdout:3521: copy_from oid 41 from oid 10 current snap is 340 2022-04-28T19:56:02.071 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 340 2022-04-28T19:56:02.071 INFO:tasks.rados.rados.0.smithi059.stdout:3522: seq_num 1101 ranges {10133504=442368,10575872=606208,11182080=598016} 2022-04-28T19:56:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:3522: writing smithi05955964-47 from 10133504 to 10575872 tid 1 2022-04-28T19:56:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:3522: writing smithi05955964-47 from 10575872 to 11182080 tid 2 2022-04-28T19:56:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:3522: writing smithi05955964-47 from 11182080 to 11780096 tid 3 2022-04-28T19:56:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:3523: copy_from oid 48 from oid 42 current snap is 340 2022-04-28T19:56:02.072 INFO:tasks.rados.rados.0.smithi059.stdout:3521: finishing copy_from to smithi05955964-41 2022-04-28T19:56:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1352 (ObjNum 1062 snap 329 seq_num 1062) dirty exists 2022-04-28T19:56:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:3521: finishing copy_from racing read to smithi05955964-41 2022-04-28T19:56:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:3522: finishing write tid 1 to smithi05955964-47 2022-04-28T19:56:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:3521: done (2 left) 2022-04-28T19:56:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 340 2022-04-28T19:56:02.073 INFO:tasks.rados.rados.0.smithi059.stdout:3524: seq_num 1102 ranges {8814592=458752,9273344=491520,9764864=491520,10256384=221184} 2022-04-28T19:56:02.105 INFO:tasks.rados.rados.0.smithi059.stdout:3524: writing smithi05955964-38 from 8814592 to 9273344 tid 1 2022-04-28T19:56:02.214 INFO:tasks.rados.rados.0.smithi059.stdout:3524: writing smithi05955964-38 from 9273344 to 9764864 tid 2 2022-04-28T19:56:02.214 INFO:tasks.rados.rados.0.smithi059.stdout:3524: writing smithi05955964-38 from 9764864 to 10256384 tid 3 2022-04-28T19:56:02.215 INFO:tasks.rados.rados.0.smithi059.stdout:3524: writing smithi05955964-38 from 10256384 to 10477568 tid 4 2022-04-28T19:56:02.215 INFO:tasks.rados.rados.0.smithi059.stdout:3525: snap_remove snap 335 2022-04-28T19:56:02.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:02 smithi059 bash[61152]: cluster 2022-04-28T19:56:01.008211+0000 mgr.y (mgr.24961) 363 : cluster [DBG] pgmap v587: 89 pgs: 89 active+clean; 958 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2022-04-28T19:56:02.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:02 smithi059 bash[61152]: cluster 2022-04-28T19:56:01.075857+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2022-04-28T19:56:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:02 smithi059 bash[61152]: cluster 2022-04-28T19:56:01.979595+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2022-04-28T19:56:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:02 smithi059 bash[59745]: cluster 2022-04-28T19:56:01.008211+0000 mgr.y (mgr.24961) 363 : cluster [DBG] pgmap v587: 89 pgs: 89 active+clean; 958 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2022-04-28T19:56:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:02 smithi059 bash[59745]: cluster 2022-04-28T19:56:01.075857+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2022-04-28T19:56:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:02 smithi059 bash[59745]: cluster 2022-04-28T19:56:01.979595+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2022-04-28T19:56:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:02 smithi139 bash[43301]: cluster 2022-04-28T19:56:01.008211+0000 mgr.y (mgr.24961) 363 : cluster [DBG] pgmap v587: 89 pgs: 89 active+clean; 958 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 2.0 MiB/s wr, 13 op/s 2022-04-28T19:56:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:02 smithi139 bash[43301]: cluster 2022-04-28T19:56:01.075857+0000 mon.a (mon.0) 1365 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2022-04-28T19:56:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:02 smithi139 bash[43301]: cluster 2022-04-28T19:56:01.979595+0000 mon.a (mon.0) 1366 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2022-04-28T19:56:03.090 INFO:tasks.rados.rados.0.smithi059.stdout:3525: done (3 left) 2022-04-28T19:56:03.090 INFO:tasks.rados.rados.0.smithi059.stdout:3526: delete oid 43 current snap is 340 2022-04-28T19:56:03.090 INFO:tasks.rados.rados.0.smithi059.stdout:3523: finishing copy_from to smithi05955964-48 2022-04-28T19:56:03.090 INFO:tasks.rados.rados.0.smithi059.stdout:3523: got expected ENOENT (src dne) 2022-04-28T19:56:03.090 INFO:tasks.rados.rados.0.smithi059.stdout:3523: finishing copy_from racing read to smithi05955964-48 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3522: finishing write tid 2 to smithi05955964-47 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3522: finishing write tid 3 to smithi05955964-47 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3522: finishing write tid 5 to smithi05955964-47 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3522: finishing write tid 6 to smithi05955964-47 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1360 (ObjNum 1101 snap 340 seq_num 1101) dirty exists 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3522: left oid 47 (ObjNum 1101 snap 340 seq_num 1101) 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3524: finishing write tid 1 to smithi05955964-38 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3524: finishing write tid 2 to smithi05955964-38 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3524: finishing write tid 3 to smithi05955964-38 2022-04-28T19:56:03.091 INFO:tasks.rados.rados.0.smithi059.stdout:3524: finishing write tid 4 to smithi05955964-38 2022-04-28T19:56:03.092 INFO:tasks.rados.rados.0.smithi059.stdout:3524: finishing write tid 6 to smithi05955964-38 2022-04-28T19:56:03.092 INFO:tasks.rados.rados.0.smithi059.stdout:3524: finishing write tid 7 to smithi05955964-38 2022-04-28T19:56:03.092 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1229 (ObjNum 1102 snap 340 seq_num 1102) dirty exists 2022-04-28T19:56:03.092 INFO:tasks.rados.rados.0.smithi059.stdout:3524: left oid 38 (ObjNum 1102 snap 340 seq_num 1102) 2022-04-28T19:56:03.094 INFO:tasks.rados.rados.0.smithi059.stdout:3522: done (3 left) 2022-04-28T19:56:03.094 INFO:tasks.rados.rados.0.smithi059.stdout:3523: done (2 left) 2022-04-28T19:56:03.095 INFO:tasks.rados.rados.0.smithi059.stdout:3524: done (1 left) 2022-04-28T19:56:03.095 INFO:tasks.rados.rados.0.smithi059.stdout:3526: done (0 left) 2022-04-28T19:56:03.095 INFO:tasks.rados.rados.0.smithi059.stdout:3527: rollback oid 47 current snap is 340 2022-04-28T19:56:03.095 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 47 to 319 2022-04-28T19:56:03.095 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 340 2022-04-28T19:56:03.095 INFO:tasks.rados.rados.0.smithi059.stdout:3528: seq_num 1104 ranges {7184384=499712,7684096=712704,8396800=491520} 2022-04-28T19:56:03.138 INFO:tasks.rados.rados.0.smithi059.stdout:3528: writing smithi05955964-25 from 7184384 to 7684096 tid 1 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3528: writing smithi05955964-25 from 7684096 to 8396800 tid 2 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3528: writing smithi05955964-25 from 8396800 to 8888320 tid 3 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3529: read oid 6 snap -1 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3529: expect deleted 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3530: delete oid 40 current snap is 340 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3527: finishing rollback tid 0 to smithi05955964-47 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3527: finishing rollback tid 1 to smithi05955964-47 2022-04-28T19:56:03.259 INFO:tasks.rados.rados.0.smithi059.stdout:3527: finishing rollback tid 2 to smithi05955964-47 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1363 (ObjNum 969 snap 289 seq_num 969) dirty exists 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3528: finishing write tid 1 to smithi05955964-25 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3527: done (3 left) 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3529: done (2 left) 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3530: done (1 left) 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3531: setattr oid 13 current snap is 340 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3528: finishing write tid 2 to smithi05955964-25 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3528: finishing write tid 3 to smithi05955964-25 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3532: snap_create 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3528: finishing write tid 5 to smithi05955964-25 2022-04-28T19:56:03.260 INFO:tasks.rados.rados.0.smithi059.stdout:3528: finishing write tid 6 to smithi05955964-25 2022-04-28T19:56:03.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1562 (ObjNum 1104 snap 340 seq_num 1104) dirty exists 2022-04-28T19:56:03.261 INFO:tasks.rados.rados.0.smithi059.stdout:3528: left oid 25 (ObjNum 1104 snap 340 seq_num 1104) 2022-04-28T19:56:03.261 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1075 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty exists 2022-04-28T19:56:03.405 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -e sha1=4fa079ba14503defa8dc257d7c2d506ebefcfe6d -- bash -c 'ceph orch ps' 2022-04-28T19:56:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:03 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:03.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:04.098 INFO:tasks.rados.rados.0.smithi059.stdout:3528: done (2 left) 2022-04-28T19:56:04.098 INFO:tasks.rados.rados.0.smithi059.stdout:3531: done (1 left) 2022-04-28T19:56:04.098 INFO:tasks.rados.rados.0.smithi059.stdout:3532: done (0 left) 2022-04-28T19:56:04.098 INFO:tasks.rados.rados.0.smithi059.stdout:3533: copy_from oid 15 from oid 17 current snap is 341 2022-04-28T19:56:04.098 INFO:tasks.rados.rados.0.smithi059.stdout:3534: snap_remove snap 308 2022-04-28T19:56:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:04 smithi059 bash[59745]: audit 2022-04-28T19:56:02.455614+0000 mgr.y (mgr.24961) 364 : audit [DBG] from='client.44421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:56:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:04 smithi059 bash[59745]: cluster 2022-04-28T19:56:03.008834+0000 mgr.y (mgr.24961) 365 : cluster [DBG] pgmap v590: 89 pgs: 89 active+clean; 958 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 747 KiB/s wr, 4 op/s 2022-04-28T19:56:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:04 smithi059 bash[59745]: cluster 2022-04-28T19:56:03.085864+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2022-04-28T19:56:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:04 smithi059 bash[61152]: audit 2022-04-28T19:56:02.455614+0000 mgr.y (mgr.24961) 364 : audit [DBG] from='client.44421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:56:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:04 smithi059 bash[61152]: cluster 2022-04-28T19:56:03.008834+0000 mgr.y (mgr.24961) 365 : cluster [DBG] pgmap v590: 89 pgs: 89 active+clean; 958 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 747 KiB/s wr, 4 op/s 2022-04-28T19:56:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:04 smithi059 bash[61152]: cluster 2022-04-28T19:56:03.085864+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2022-04-28T19:56:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:04 smithi139 bash[43301]: audit 2022-04-28T19:56:02.455614+0000 mgr.y (mgr.24961) 364 : audit [DBG] from='client.44421 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:56:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:04 smithi139 bash[43301]: cluster 2022-04-28T19:56:03.008834+0000 mgr.y (mgr.24961) 365 : cluster [DBG] pgmap v590: 89 pgs: 89 active+clean; 958 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 747 KiB/s wr, 4 op/s 2022-04-28T19:56:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:04 smithi139 bash[43301]: cluster 2022-04-28T19:56:03.085864+0000 mon.a (mon.0) 1367 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2022-04-28T19:56:05.109 INFO:tasks.rados.rados.0.smithi059.stdout:3534: done (1 left) 2022-04-28T19:56:05.110 INFO:tasks.rados.rados.0.smithi059.stdout:3535: copy_from oid 18 from oid 46 current snap is 341 2022-04-28T19:56:05.110 INFO:tasks.rados.rados.0.smithi059.stdout:3533: finishing copy_from to smithi05955964-15 2022-04-28T19:56:05.110 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1074 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:56:05.110 INFO:tasks.rados.rados.0.smithi059.stdout:3533: finishing copy_from racing read to smithi05955964-15 2022-04-28T19:56:05.110 INFO:tasks.rados.rados.0.smithi059.stdout:3533: done (1 left) 2022-04-28T19:56:05.110 INFO:tasks.rados.rados.0.smithi059.stdout:3536: snap_create 2022-04-28T19:56:05.126 INFO:tasks.rados.rados.0.smithi059.stdout:3535: finishing copy_from to smithi05955964-18 2022-04-28T19:56:05.126 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1512 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:56:05.127 INFO:tasks.rados.rados.0.smithi059.stdout:3535: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:56:05.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:05 smithi139 bash[43301]: cluster 2022-04-28T19:56:04.096418+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2022-04-28T19:56:05.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:05 smithi059 bash[61152]: cluster 2022-04-28T19:56:04.096418+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2022-04-28T19:56:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:05 smithi059 bash[59745]: cluster 2022-04-28T19:56:04.096418+0000 mon.a (mon.0) 1368 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2022-04-28T19:56:06.113 INFO:tasks.rados.rados.0.smithi059.stdout:3535: done (1 left) 2022-04-28T19:56:06.113 INFO:tasks.rados.rados.0.smithi059.stdout:3536: done (0 left) 2022-04-28T19:56:06.113 INFO:tasks.rados.rados.0.smithi059.stdout:3537: read oid 18 snap -1 2022-04-28T19:56:06.113 INFO:tasks.rados.rados.0.smithi059.stdout:3537: expect (ObjNum 955 snap 283 seq_num 955) 2022-04-28T19:56:06.113 INFO:tasks.rados.rados.0.smithi059.stdout:3538: read oid 24 snap 331 2022-04-28T19:56:06.113 INFO:tasks.rados.rados.0.smithi059.stdout:3538: expect deleted 2022-04-28T19:56:06.114 INFO:tasks.rados.rados.0.smithi059.stdout:3539: rollback oid 45 current snap is 342 2022-04-28T19:56:06.114 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 337 2022-04-28T19:56:06.114 INFO:tasks.rados.rados.0.smithi059.stdout:3540: snap_remove snap 322 2022-04-28T19:56:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:06 smithi139 bash[43301]: cluster 2022-04-28T19:56:05.009716+0000 mgr.y (mgr.24961) 366 : cluster [DBG] pgmap v593: 89 pgs: 89 active+clean; 976 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2022-04-28T19:56:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:06 smithi139 bash[43301]: cluster 2022-04-28T19:56:05.107621+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2022-04-28T19:56:06.415 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:06 smithi059 bash[61152]: cluster 2022-04-28T19:56:05.009716+0000 mgr.y (mgr.24961) 366 : cluster [DBG] pgmap v593: 89 pgs: 89 active+clean; 976 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2022-04-28T19:56:06.416 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:06 smithi059 bash[61152]: cluster 2022-04-28T19:56:05.107621+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2022-04-28T19:56:06.416 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:06 smithi059 bash[59745]: cluster 2022-04-28T19:56:05.009716+0000 mgr.y (mgr.24961) 366 : cluster [DBG] pgmap v593: 89 pgs: 89 active+clean; 976 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 4.7 MiB/s wr, 20 op/s 2022-04-28T19:56:06.416 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:06 smithi059 bash[59745]: cluster 2022-04-28T19:56:05.107621+0000 mon.a (mon.0) 1369 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2022-04-28T19:56:06.763 INFO:teuthology.orchestra.run.smithi059.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:alertmanager.a smithi059 running (50s) 45s ago 15m 13.8M - ba2b418f427c 852576bdb38c 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:grafana.a smithi139 running (21s) 16s ago 15m 18.1M - 8.3.5 dad864ee21e9 7d5aafd7dd7c 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi059.tnizcj smithi059 running (110s) 45s ago 13m 13.6M - 16.2.7-969-g4fa079ba 863db77b5b5e 0b1d79d85003 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mds.foo.smithi139.oefiza smithi139 running (97s) 16s ago 13m 12.0M - 16.2.7-969-g4fa079ba 863db77b5b5e 01b2328b32c7 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.x smithi139 *:8443 running (7m) 16s ago 19m 378M - 16.2.7-969-g4fa079ba 863db77b5b5e 3adf17d3b8d8 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mgr.y smithi059 *:8443 running (7m) 45s ago 21m 420M - 16.2.7-969-g4fa079ba 863db77b5b5e 6b4aac48ab6d 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mon.a smithi059 running (7m) 45s ago 21m 86.3M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 5d2294b146d1 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mon.b smithi139 running (6m) 16s ago 19m 65.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 7d874bff0464 2022-04-28T19:56:06.764 INFO:teuthology.orchestra.run.smithi059.stdout:mon.c smithi059 running (6m) 45s ago 19m 70.0M 2048M 16.2.7-969-g4fa079ba 863db77b5b5e 6c2184f7a17b 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.a smithi059 running (85s) 45s ago 15m 12.2M - 1dbe0e931976 c196770fa402 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:node-exporter.b smithi139 running (76s) 16s ago 15m 12.3M - 1dbe0e931976 97aaa15a2dd3 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.0 smithi059 running (5m) 45s ago 18m 343M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 919517c0694a 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.1 smithi059 running (5m) 45s ago 18m 259M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 39b737637d02 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.2 smithi059 running (4m) 45s ago 18m 329M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e c42d4b089b24 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.3 smithi059 running (4m) 45s ago 17m 239M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e bfb464fa31e4 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.4 smithi139 running (3m) 16s ago 17m 300M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e e1a7833befb1 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.5 smithi139 running (3m) 16s ago 17m 367M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 4d55ca87a6a9 2022-04-28T19:56:06.765 INFO:teuthology.orchestra.run.smithi059.stdout:osd.6 smithi139 running (2m) 16s ago 16m 248M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 2cf53c9fc29e 2022-04-28T19:56:06.766 INFO:teuthology.orchestra.run.smithi059.stdout:osd.7 smithi139 running (2m) 16s ago 16m 193M 4096M 16.2.7-969-g4fa079ba 863db77b5b5e 8655092f1b14 2022-04-28T19:56:06.766 INFO:teuthology.orchestra.run.smithi059.stdout:prometheus.a smithi139 running (66s) 16s ago 15m 42.7M - 514e6a882f6e 1187d983585d 2022-04-28T19:56:07.139 INFO:tasks.rados.rados.0.smithi059.stdout:3540: done (3 left) 2022-04-28T19:56:07.139 INFO:tasks.rados.rados.0.smithi059.stdout:3541: rmattr oid 29 current snap is 342 2022-04-28T19:56:07.139 INFO:tasks.rados.rados.0.smithi059.stdout:3539: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:56:07.139 INFO:tasks.rados.rados.0.smithi059.stdout:3539: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:56:07.140 INFO:tasks.rados.rados.0.smithi059.stdout:3539: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:56:07.140 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1293 (ObjNum 1011 snap 311 seq_num 1011) dirty exists 2022-04-28T19:56:07.140 INFO:tasks.rados.rados.0.smithi059.stdout:3537: expect (ObjNum 955 snap 283 seq_num 955) 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 29 v 1564 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:3537: done (3 left) 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:3538: done (2 left) 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:3539: done (1 left) 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:3541: done (0 left) 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:3542: copy_from oid 49 from oid 7 current snap is 342 2022-04-28T19:56:07.178 INFO:tasks.rados.rados.0.smithi059.stdout:3543: setattr oid 5 current snap is 342 2022-04-28T19:56:07.179 INFO:tasks.rados.rados.0.smithi059.stdout:3544: delete oid 11 current snap is 342 2022-04-28T19:56:07.182 INFO:tasks.rados.rados.0.smithi059.stdout:3542: finishing copy_from to smithi05955964-49 2022-04-28T19:56:07.182 INFO:tasks.rados.rados.0.smithi059.stdout:3542: got expected ENOENT (src dne) 2022-04-28T19:56:07.182 INFO:tasks.rados.rados.0.smithi059.stdout:3542: finishing copy_from racing read to smithi05955964-49 2022-04-28T19:56:07.183 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1565 (ObjNum 0 snap 32762 seq_num 4162284096) dirty exists 2022-04-28T19:56:07.186 INFO:tasks.rados.rados.0.smithi059.stdout:3542: done (2 left) 2022-04-28T19:56:07.186 INFO:tasks.rados.rados.0.smithi059.stdout:3543: done (1 left) 2022-04-28T19:56:07.186 INFO:tasks.rados.rados.0.smithi059.stdout:3544: done (0 left) 2022-04-28T19:56:07.186 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 15 current snap is 342 2022-04-28T19:56:07.186 INFO:tasks.rados.rados.0.smithi059.stdout:3545: seq_num 1107 ranges {10657792=548864,11206656=770048,11976704=434176} 2022-04-28T19:56:07.252 INFO:tasks.rados.rados.0.smithi059.stdout:3545: writing smithi05955964-15 from 10657792 to 11206656 tid 1 2022-04-28T19:56:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3545: writing smithi05955964-15 from 11206656 to 11976704 tid 2 2022-04-28T19:56:07.259 INFO:tasks.rados.rados.0.smithi059.stdout:3545: writing smithi05955964-15 from 11976704 to 12410880 tid 3 2022-04-28T19:56:07.259 INFO:tasks.rados.rados.0.smithi059.stdout:3546: copy_from oid 35 from oid 38 current snap is 342 2022-04-28T19:56:07.260 INFO:tasks.rados.rados.0.smithi059.stdout:3547: copy_from oid 47 from oid 11 current snap is 342 2022-04-28T19:56:07.260 INFO:tasks.rados.rados.0.smithi059.stdout:3548: rollback oid 9 current snap is 342 2022-04-28T19:56:07.260 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 311 2022-04-28T19:56:07.260 INFO:tasks.rados.rados.0.smithi059.stdout:3549: snap_remove snap 307 2022-04-28T19:56:07.412 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: cluster 2022-04-28T19:56:06.109851+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2022-04-28T19:56:07.412 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.742358+0000 mon.a (mon.0) 1371 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.412 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.743196+0000 mon.a (mon.0) 1372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.412 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.744024+0000 mon.a (mon.0) 1373 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.412 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.744814+0000 mon.a (mon.0) 1374 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.745603+0000 mon.a (mon.0) 1375 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.746360+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.749720+0000 mon.a (mon.0) 1377 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.750528+0000 mon.a (mon.0) 1378 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.751344+0000 mon.a (mon.0) 1379 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.752203+0000 mon.a (mon.0) 1380 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[59745]: audit 2022-04-28T19:56:06.753037+0000 mon.a (mon.0) 1381 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.413 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: cluster 2022-04-28T19:56:06.109851+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.742358+0000 mon.a (mon.0) 1371 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.743196+0000 mon.a (mon.0) 1372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.744024+0000 mon.a (mon.0) 1373 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.744814+0000 mon.a (mon.0) 1374 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.745603+0000 mon.a (mon.0) 1375 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.746360+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.749720+0000 mon.a (mon.0) 1377 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.750528+0000 mon.a (mon.0) 1378 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.414 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.751344+0000 mon.a (mon.0) 1379 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.415 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.752203+0000 mon.a (mon.0) 1380 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.415 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:07 smithi059 bash[61152]: audit 2022-04-28T19:56:06.753037+0000 mon.a (mon.0) 1381 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.416 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -e sha1=4fa079ba14503defa8dc257d7c2d506ebefcfe6d -- bash -c 'ceph versions' 2022-04-28T19:56:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: cluster 2022-04-28T19:56:06.109851+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2022-04-28T19:56:07.717 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.742358+0000 mon.a (mon.0) 1371 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.a","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.717 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.743196+0000 mon.a (mon.0) 1372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.c","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.717 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.744024+0000 mon.a (mon.0) 1373 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.717 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.744814+0000 mon.a (mon.0) 1374 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.718 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.745603+0000 mon.a (mon.0) 1375 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.718 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.746360+0000 mon.a (mon.0) 1376 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.718 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.749720+0000 mon.a (mon.0) 1377 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon.b","key": "mon_memory_target"}]: dispatch 2022-04-28T19:56:07.718 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.750528+0000 mon.a (mon.0) 1378 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.719 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.751344+0000 mon.a (mon.0) 1379 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.719 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.752203+0000 mon.a (mon.0) 1380 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:07.719 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:07 smithi139 bash[43301]: audit 2022-04-28T19:56:06.753037+0000 mon.a (mon.0) 1381 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:56:08.148 INFO:tasks.rados.rados.0.smithi059.stdout:3549: done (4 left) 2022-04-28T19:56:08.148 INFO:tasks.rados.rados.0.smithi059.stdout:3550: snap_create 2022-04-28T19:56:08.148 INFO:tasks.rados.rados.0.smithi059.stdout:3547: finishing copy_from to smithi05955964-47 2022-04-28T19:56:08.148 INFO:tasks.rados.rados.0.smithi059.stdout:3547: got expected ENOENT (src dne) 2022-04-28T19:56:08.148 INFO:tasks.rados.rados.0.smithi059.stdout:3547: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3545: finishing write tid 1 to smithi05955964-15 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3545: finishing write tid 2 to smithi05955964-15 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3548: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3548: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3545: finishing write tid 3 to smithi05955964-15 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3548: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1213 (ObjNum 1000 snap 297 seq_num 1000) dirty exists 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3545: finishing write tid 5 to smithi05955964-15 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:3545: finishing write tid 6 to smithi05955964-15 2022-04-28T19:56:08.149 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1382 (ObjNum 1107 snap 342 seq_num 1107) dirty exists 2022-04-28T19:56:08.150 INFO:tasks.rados.rados.0.smithi059.stdout:3545: left oid 15 (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:56:08.150 INFO:tasks.rados.rados.0.smithi059.stdout:3546: finishing copy_from to smithi05955964-35 2022-04-28T19:56:08.150 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1229 (ObjNum 1102 snap 340 seq_num 1102) dirty exists 2022-04-28T19:56:08.150 INFO:tasks.rados.rados.0.smithi059.stdout:3546: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:56:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:08 smithi059 bash[61152]: audit 2022-04-28T19:56:06.739800+0000 mgr.y (mgr.24961) 367 : audit [DBG] from='client.44427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:56:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:08 smithi059 bash[61152]: cluster 2022-04-28T19:56:07.010436+0000 mgr.y (mgr.24961) 368 : cluster [DBG] pgmap v596: 89 pgs: 89 active+clean; 982 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 24 op/s 2022-04-28T19:56:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:08 smithi059 bash[61152]: cluster 2022-04-28T19:56:07.136454+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2022-04-28T19:56:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:08 smithi059 bash[59745]: audit 2022-04-28T19:56:06.739800+0000 mgr.y (mgr.24961) 367 : audit [DBG] from='client.44427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:56:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:08 smithi059 bash[59745]: cluster 2022-04-28T19:56:07.010436+0000 mgr.y (mgr.24961) 368 : cluster [DBG] pgmap v596: 89 pgs: 89 active+clean; 982 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 24 op/s 2022-04-28T19:56:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:08 smithi059 bash[59745]: cluster 2022-04-28T19:56:07.136454+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2022-04-28T19:56:08.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:08 smithi139 bash[43301]: audit 2022-04-28T19:56:06.739800+0000 mgr.y (mgr.24961) 367 : audit [DBG] from='client.44427 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2022-04-28T19:56:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:08 smithi139 bash[43301]: cluster 2022-04-28T19:56:07.010436+0000 mgr.y (mgr.24961) 368 : cluster [DBG] pgmap v596: 89 pgs: 89 active+clean; 982 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.1 MiB/s wr, 24 op/s 2022-04-28T19:56:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:08 smithi139 bash[43301]: cluster 2022-04-28T19:56:07.136454+0000 mon.a (mon.0) 1382 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2022-04-28T19:56:09.155 INFO:tasks.rados.rados.0.smithi059.stdout:3545: done (4 left) 2022-04-28T19:56:09.155 INFO:tasks.rados.rados.0.smithi059.stdout:3546: done (3 left) 2022-04-28T19:56:09.155 INFO:tasks.rados.rados.0.smithi059.stdout:3547: done (2 left) 2022-04-28T19:56:09.155 INFO:tasks.rados.rados.0.smithi059.stdout:3548: done (1 left) 2022-04-28T19:56:09.155 INFO:tasks.rados.rados.0.smithi059.stdout:3550: done (0 left) 2022-04-28T19:56:09.155 INFO:tasks.rados.rados.0.smithi059.stdout:3551: setattr oid 5 current snap is 343 2022-04-28T19:56:09.156 INFO:tasks.rados.rados.0.smithi059.stdout:3552: snap_remove snap 316 2022-04-28T19:56:09.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:09 smithi059 bash[59745]: cluster 2022-04-28T19:56:08.146500+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2022-04-28T19:56:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:09 smithi059 bash[61152]: cluster 2022-04-28T19:56:08.146500+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2022-04-28T19:56:09.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:09 smithi139 bash[43301]: cluster 2022-04-28T19:56:08.146500+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2022-04-28T19:56:10.167 INFO:tasks.rados.rados.0.smithi059.stdout:3552: done (1 left) 2022-04-28T19:56:10.167 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 343 2022-04-28T19:56:10.167 INFO:tasks.rados.rados.0.smithi059.stdout:3553: seq_num 1108 ranges {0=565248,565248=647168,1212416=483328,1695744=73728} 2022-04-28T19:56:10.167 INFO:tasks.rados.rados.0.smithi059.stdout:3553: writing smithi05955964-36 from 0 to 565248 tid 1 2022-04-28T19:56:10.171 INFO:tasks.rados.rados.0.smithi059.stdout:3553: writing smithi05955964-36 from 565248 to 1212416 tid 2 2022-04-28T19:56:10.175 INFO:tasks.rados.rados.0.smithi059.stdout:3553: writing smithi05955964-36 from 1212416 to 1695744 tid 3 2022-04-28T19:56:10.175 INFO:tasks.rados.rados.0.smithi059.stdout:3553: writing smithi05955964-36 from 1695744 to 1769472 tid 4 2022-04-28T19:56:10.176 INFO:tasks.rados.rados.0.smithi059.stdout:3554: setattr oid 31 current snap is 343 2022-04-28T19:56:10.176 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1567 (ObjNum 1 snap 32762 seq_num 4162130400) dirty exists 2022-04-28T19:56:10.176 INFO:tasks.rados.rados.0.smithi059.stdout:3553: finishing write tid 1 to smithi05955964-36 2022-04-28T19:56:10.177 INFO:tasks.rados.rados.0.smithi059.stdout:3553: finishing write tid 2 to smithi05955964-36 2022-04-28T19:56:10.178 INFO:tasks.rados.rados.0.smithi059.stdout:3551: done (2 left) 2022-04-28T19:56:10.178 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 26 current snap is 343 2022-04-28T19:56:10.178 INFO:tasks.rados.rados.0.smithi059.stdout:3555: seq_num 1109 ranges {4980736=581632,5562368=532480,6094848=647168,6742016=24576} 2022-04-28T19:56:10.210 INFO:tasks.rados.rados.0.smithi059.stdout:3555: writing smithi05955964-26 from 4980736 to 5562368 tid 1 2022-04-28T19:56:10.212 INFO:tasks.rados.rados.0.smithi059.stdout:3555: writing smithi05955964-26 from 5562368 to 6094848 tid 2 2022-04-28T19:56:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:3555: writing smithi05955964-26 from 6094848 to 6742016 tid 3 2022-04-28T19:56:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:3555: writing smithi05955964-26 from 6742016 to 6766592 tid 4 2022-04-28T19:56:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:3556: snap_create 2022-04-28T19:56:10.217 INFO:tasks.rados.rados.0.smithi059.stdout:3553: finishing write tid 3 to smithi05955964-36 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:3553: finishing write tid 4 to smithi05955964-36 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:3553: finishing write tid 6 to smithi05955964-36 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:3553: finishing write tid 7 to smithi05955964-36 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1462 (ObjNum 1108 snap 343 seq_num 1108) dirty exists 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:3553: left oid 36 (ObjNum 1108 snap 343 seq_num 1108) 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1383 (ObjNum 1088 snap 336 seq_num 1088) dirty exists 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:3555: finishing write tid 1 to smithi05955964-26 2022-04-28T19:56:10.218 INFO:tasks.rados.rados.0.smithi059.stdout:3555: finishing write tid 2 to smithi05955964-26 2022-04-28T19:56:10.221 INFO:tasks.rados.rados.0.smithi059.stdout:3555: finishing write tid 3 to smithi05955964-26 2022-04-28T19:56:10.221 INFO:tasks.rados.rados.0.smithi059.stdout:3555: finishing write tid 4 to smithi05955964-26 2022-04-28T19:56:10.221 INFO:tasks.rados.rados.0.smithi059.stdout:3555: finishing write tid 6 to smithi05955964-26 2022-04-28T19:56:10.223 INFO:tasks.rados.rados.0.smithi059.stdout:3555: finishing write tid 7 to smithi05955964-26 2022-04-28T19:56:10.223 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1524 (ObjNum 1109 snap 343 seq_num 1109) dirty exists 2022-04-28T19:56:10.223 INFO:tasks.rados.rados.0.smithi059.stdout:3555: left oid 26 (ObjNum 1109 snap 343 seq_num 1109) 2022-04-28T19:56:10.437 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:10 smithi059 bash[61152]: cluster 2022-04-28T19:56:09.011304+0000 mgr.y (mgr.24961) 369 : cluster [DBG] pgmap v599: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 6.2 MiB/s wr, 7 op/s 2022-04-28T19:56:10.437 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:10 smithi059 bash[61152]: cluster 2022-04-28T19:56:09.153703+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2022-04-28T19:56:10.437 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:10 smithi059 bash[59745]: cluster 2022-04-28T19:56:09.011304+0000 mgr.y (mgr.24961) 369 : cluster [DBG] pgmap v599: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 6.2 MiB/s wr, 7 op/s 2022-04-28T19:56:10.438 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:10 smithi059 bash[59745]: cluster 2022-04-28T19:56:09.153703+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2022-04-28T19:56:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:10 smithi139 bash[43301]: cluster 2022-04-28T19:56:09.011304+0000 mgr.y (mgr.24961) 369 : cluster [DBG] pgmap v599: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 6.2 MiB/s wr, 7 op/s 2022-04-28T19:56:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:10 smithi139 bash[43301]: cluster 2022-04-28T19:56:09.153703+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2022-04-28T19:56:10.901 INFO:teuthology.orchestra.run.smithi059.stdout:{ 2022-04-28T19:56:10.901 INFO:teuthology.orchestra.run.smithi059.stdout: "mon": { 2022-04-28T19:56:10.901 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 3 2022-04-28T19:56:10.902 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:56:10.902 INFO:teuthology.orchestra.run.smithi059.stdout: "mgr": { 2022-04-28T19:56:10.902 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:56:10.902 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: "osd": { 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 8 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: "mds": { 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 2 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: }, 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: "overall": { 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)": 15 2022-04-28T19:56:10.903 INFO:teuthology.orchestra.run.smithi059.stdout: } 2022-04-28T19:56:10.904 INFO:teuthology.orchestra.run.smithi059.stdout:} 2022-04-28T19:56:11.179 INFO:tasks.rados.rados.0.smithi059.stdout:3553: done (3 left) 2022-04-28T19:56:11.229 INFO:tasks.rados.rados.0.smithi059.stdout:3554: done (2 left) 2022-04-28T19:56:11.229 INFO:tasks.rados.rados.0.smithi059.stdout:3555: done (1 left) 2022-04-28T19:56:11.230 INFO:tasks.rados.rados.0.smithi059.stdout:3556: done (0 left) 2022-04-28T19:56:11.230 INFO:tasks.rados.rados.0.smithi059.stdout:3557: rollback oid 46 current snap is 344 2022-04-28T19:56:11.230 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 343 2022-04-28T19:56:11.230 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 344 2022-04-28T19:56:11.231 INFO:tasks.rados.rados.0.smithi059.stdout:3558: seq_num 1110 ranges {0=598016,598016=466944,1064960=450560,1515520=286720} 2022-04-28T19:56:11.237 INFO:tasks.rados.rados.0.smithi059.stdout:3558: writing smithi05955964-48 from 0 to 598016 tid 1 2022-04-28T19:56:11.237 INFO:tasks.rados.rados.0.smithi059.stdout:3558: writing smithi05955964-48 from 598016 to 1064960 tid 2 2022-04-28T19:56:11.237 INFO:tasks.rados.rados.0.smithi059.stdout:3558: writing smithi05955964-48 from 1064960 to 1515520 tid 3 2022-04-28T19:56:11.237 INFO:tasks.rados.rados.0.smithi059.stdout:3558: writing smithi05955964-48 from 1515520 to 1802240 tid 4 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3559: read oid 33 snap -1 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3559: expect (ObjNum 1080 snap 335 seq_num 1080) 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3557: finishing rollback tid 0 to smithi05955964-46 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3560: setattr oid 14 current snap is 344 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3557: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3557: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1571 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:56:11.238 INFO:tasks.rados.rados.0.smithi059.stdout:3558: finishing write tid 1 to smithi05955964-48 2022-04-28T19:56:11.239 INFO:tasks.rados.rados.0.smithi059.stdout:3558: finishing write tid 2 to smithi05955964-48 2022-04-28T19:56:11.239 INFO:tasks.rados.rados.0.smithi059.stdout:3558: finishing write tid 3 to smithi05955964-48 2022-04-28T19:56:11.239 INFO:tasks.rados.rados.0.smithi059.stdout:3557: done (3 left) 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3561: read oid 3 snap -1 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3561: expect (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3562: read oid 12 snap -1 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3562: expect (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3563: copy_from oid 11 from oid 2 current snap is 344 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3564: delete oid 7 current snap is 344 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3558: finishing write tid 4 to smithi05955964-48 2022-04-28T19:56:11.240 INFO:tasks.rados.rados.0.smithi059.stdout:3558: finishing write tid 6 to smithi05955964-48 2022-04-28T19:56:11.241 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1530 (ObjNum 898 snap 268 seq_num 898) dirty exists 2022-04-28T19:56:11.241 INFO:tasks.rados.rados.0.smithi059.stdout:3558: finishing write tid 7 to smithi05955964-48 2022-04-28T19:56:11.241 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1529 (ObjNum 1110 snap 344 seq_num 1110) dirty exists 2022-04-28T19:56:11.241 INFO:tasks.rados.rados.0.smithi059.stdout:3558: left oid 48 (ObjNum 1110 snap 344 seq_num 1110) 2022-04-28T19:56:11.241 INFO:tasks.rados.rados.0.smithi059.stdout:3561: expect (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:56:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:11 smithi059 bash[59745]: cluster 2022-04-28T19:56:10.163304+0000 mon.a (mon.0) 1385 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2022-04-28T19:56:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:11 smithi059 bash[59745]: audit 2022-04-28T19:56:10.900861+0000 mon.a (mon.0) 1386 : audit [DBG] from='client.? 172.21.15.59:0/2215700702' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:11 smithi059 bash[61152]: cluster 2022-04-28T19:56:10.163304+0000 mon.a (mon.0) 1385 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2022-04-28T19:56:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:11 smithi059 bash[61152]: audit 2022-04-28T19:56:10.900861+0000 mon.a (mon.0) 1386 : audit [DBG] from='client.? 172.21.15.59:0/2215700702' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:11.350 INFO:tasks.rados.rados.0.smithi059.stdout:3563: finishing copy_from to smithi05955964-11 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1464 (ObjNum 1090 snap 336 seq_num 1090) dirty exists 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:3563: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:3558: done (6 left) 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:3560: done (5 left) 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:3561: done (4 left) 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:3563: done (3 left) 2022-04-28T19:56:11.351 INFO:tasks.rados.rados.0.smithi059.stdout:3564: done (2 left) 2022-04-28T19:56:11.352 INFO:tasks.rados.rados.0.smithi059.stdout:3565: snap_remove snap 290 2022-04-28T19:56:11.562 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -e sha1=4fa079ba14503defa8dc257d7c2d506ebefcfe6d -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2022-04-28T19:56:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:11 smithi139 bash[43301]: cluster 2022-04-28T19:56:10.163304+0000 mon.a (mon.0) 1385 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2022-04-28T19:56:11.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:11 smithi139 bash[43301]: audit 2022-04-28T19:56:10.900861+0000 mon.a (mon.0) 1386 : audit [DBG] from='client.? 172.21.15.59:0/2215700702' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:11.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:11 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:11.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:11.991 INFO:tasks.rados.rados.0.smithi059.stdout:3565: done (2 left) 2022-04-28T19:56:11.991 INFO:tasks.rados.rados.0.smithi059.stdout:3566: read oid 1 snap -1 2022-04-28T19:56:11.991 INFO:tasks.rados.rados.0.smithi059.stdout:3566: expect (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:56:11.991 INFO:tasks.rados.rados.0.smithi059.stdout:3567: snap_create 2022-04-28T19:56:11.992 INFO:tasks.rados.rados.0.smithi059.stdout:3559: expect (ObjNum 1080 snap 335 seq_num 1080) 2022-04-28T19:56:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:12 smithi059 bash[61152]: cluster 2022-04-28T19:56:11.012200+0000 mgr.y (mgr.24961) 370 : cluster [DBG] pgmap v602: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.8 MiB/s wr, 11 op/s 2022-04-28T19:56:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:12 smithi059 bash[61152]: cluster 2022-04-28T19:56:11.177289+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2022-04-28T19:56:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:12 smithi059 bash[61152]: cluster 2022-04-28T19:56:11.989263+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2022-04-28T19:56:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:12 smithi059 bash[59745]: cluster 2022-04-28T19:56:11.012200+0000 mgr.y (mgr.24961) 370 : cluster [DBG] pgmap v602: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.8 MiB/s wr, 11 op/s 2022-04-28T19:56:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:12 smithi059 bash[59745]: cluster 2022-04-28T19:56:11.177289+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2022-04-28T19:56:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:12 smithi059 bash[59745]: cluster 2022-04-28T19:56:11.989263+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2022-04-28T19:56:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:12 smithi139 bash[43301]: cluster 2022-04-28T19:56:11.012200+0000 mgr.y (mgr.24961) 370 : cluster [DBG] pgmap v602: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 5.8 MiB/s wr, 11 op/s 2022-04-28T19:56:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:12 smithi139 bash[43301]: cluster 2022-04-28T19:56:11.177289+0000 mon.a (mon.0) 1387 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2022-04-28T19:56:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:12 smithi139 bash[43301]: cluster 2022-04-28T19:56:11.989263+0000 mon.a (mon.0) 1388 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2022-04-28T19:56:12.761 INFO:tasks.rados.rados.0.smithi059.stdout:3562: expect (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:56:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:13 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:13.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:14.082 INFO:tasks.rados.rados.0.smithi059.stdout:3566: expect (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:56:14.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:14 smithi059 bash[61152]: cluster 2022-04-28T19:56:12.997908+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2022-04-28T19:56:14.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:14 smithi059 bash[61152]: cluster 2022-04-28T19:56:13.012732+0000 mgr.y (mgr.24961) 371 : cluster [DBG] pgmap v606: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2022-04-28T19:56:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:14 smithi059 bash[59745]: cluster 2022-04-28T19:56:12.997908+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2022-04-28T19:56:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:14 smithi059 bash[59745]: cluster 2022-04-28T19:56:13.012732+0000 mgr.y (mgr.24961) 371 : cluster [DBG] pgmap v606: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2022-04-28T19:56:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:14 smithi139 bash[43301]: cluster 2022-04-28T19:56:12.997908+0000 mon.a (mon.0) 1389 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2022-04-28T19:56:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:14 smithi139 bash[43301]: cluster 2022-04-28T19:56:13.012732+0000 mgr.y (mgr.24961) 371 : cluster [DBG] pgmap v606: 89 pgs: 89 active+clean; 1005 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2022-04-28T19:56:14.972 INFO:tasks.rados.rados.0.smithi059.stdout:3559: done (3 left) 2022-04-28T19:56:14.972 INFO:tasks.rados.rados.0.smithi059.stdout:3562: done (2 left) 2022-04-28T19:56:14.972 INFO:tasks.rados.rados.0.smithi059.stdout:3566: done (1 left) 2022-04-28T19:56:14.973 INFO:tasks.rados.rados.0.smithi059.stdout:3567: done (0 left) 2022-04-28T19:56:14.973 INFO:tasks.rados.rados.0.smithi059.stdout:3568: snap_create 2022-04-28T19:56:14.980 INFO:teuthology.orchestra.run.smithi059.stdout:true 2022-04-28T19:56:15.019 INFO:tasks.rados.rados.0.smithi059.stdout:3568: done (0 left) 2022-04-28T19:56:15.019 INFO:tasks.rados.rados.0.smithi059.stdout:3569: read oid 39 snap 345 2022-04-28T19:56:15.020 INFO:tasks.rados.rados.0.smithi059.stdout:3569: expect (ObjNum 894 snap 267 seq_num 894) 2022-04-28T19:56:15.020 INFO:tasks.rados.rados.0.smithi059.stdout:3570: rollback oid 44 current snap is 346 2022-04-28T19:56:15.020 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 44 to 337 2022-04-28T19:56:15.020 INFO:tasks.rados.rados.0.smithi059.stdout:3571: snap_remove snap 342 2022-04-28T19:56:15.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:15 smithi059 bash[59745]: audit 2022-04-28T19:56:14.969260+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.59:0/1429337315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:15 smithi059 bash[61152]: audit 2022-04-28T19:56:14.969260+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.59:0/1429337315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:15 smithi139 bash[43301]: audit 2022-04-28T19:56:14.969260+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.59:0/1429337315' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:15.610 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -e sha1=4fa079ba14503defa8dc257d7c2d506ebefcfe6d -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2022-04-28T19:56:16.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:16 smithi059 bash[59745]: cluster 2022-04-28T19:56:15.013394+0000 mgr.y (mgr.24961) 372 : cluster [DBG] pgmap v607: 89 pgs: 89 active+clean; 1007 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 356 KiB/s wr, 13 op/s 2022-04-28T19:56:16.024 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:16 smithi059 bash[59745]: cluster 2022-04-28T19:56:15.018493+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2022-04-28T19:56:16.031 INFO:tasks.rados.rados.0.smithi059.stdout:3571: done (2 left) 2022-04-28T19:56:16.031 INFO:tasks.rados.rados.0.smithi059.stdout:3572: read oid 10 snap -1 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:3572: expect (ObjNum 1062 snap 329 seq_num 1062) 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:3573: delete oid 19 current snap is 346 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:3570: finishing rollback tid 0 to smithi05955964-44 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:3570: finishing rollback tid 1 to smithi05955964-44 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:3570: finishing rollback tid 2 to smithi05955964-44 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1216 (ObjNum 1087 snap 336 seq_num 1087) dirty exists 2022-04-28T19:56:16.032 INFO:tasks.rados.rados.0.smithi059.stdout:3569: expect (ObjNum 894 snap 267 seq_num 894) 2022-04-28T19:56:16.048 INFO:tasks.rados.rados.0.smithi059.stdout:3572: expect (ObjNum 1062 snap 329 seq_num 1062) 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:3569: done (3 left) 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:3570: done (2 left) 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:3572: done (1 left) 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:3573: done (0 left) 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:3574: rollback oid 11 current snap is 346 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 11 to 340 2022-04-28T19:56:16.082 INFO:tasks.rados.rados.0.smithi059.stdout:3575: delete oid 24 current snap is 346 2022-04-28T19:56:16.088 INFO:tasks.rados.rados.0.smithi059.stdout:3574: finishing rollback tid 0 to smithi05955964-11 2022-04-28T19:56:16.089 INFO:tasks.rados.rados.0.smithi059.stdout:3574: finishing rollback tid 1 to smithi05955964-11 2022-04-28T19:56:16.089 INFO:tasks.rados.rados.0.smithi059.stdout:3574: finishing rollback tid 2 to smithi05955964-11 2022-04-28T19:56:16.089 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1467 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:56:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:3574: done (1 left) 2022-04-28T19:56:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:3575: done (0 left) 2022-04-28T19:56:16.090 INFO:tasks.rados.rados.0.smithi059.stdout:3576: snap_remove snap 301 2022-04-28T19:56:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:16 smithi059 bash[61152]: cluster 2022-04-28T19:56:15.013394+0000 mgr.y (mgr.24961) 372 : cluster [DBG] pgmap v607: 89 pgs: 89 active+clean; 1007 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 356 KiB/s wr, 13 op/s 2022-04-28T19:56:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:16 smithi059 bash[61152]: cluster 2022-04-28T19:56:15.018493+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2022-04-28T19:56:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:16 smithi139 bash[43301]: cluster 2022-04-28T19:56:15.013394+0000 mgr.y (mgr.24961) 372 : cluster [DBG] pgmap v607: 89 pgs: 89 active+clean; 1007 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 356 KiB/s wr, 13 op/s 2022-04-28T19:56:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:16 smithi139 bash[43301]: cluster 2022-04-28T19:56:15.018493+0000 mon.a (mon.0) 1390 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2022-04-28T19:56:17.038 INFO:tasks.rados.rados.0.smithi059.stdout:3576: done (0 left) 2022-04-28T19:56:17.121 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 346 2022-04-28T19:56:17.121 INFO:tasks.rados.rados.0.smithi059.stdout:3577: seq_num 1114 ranges {1605632=663552,2269184=802816,3072000=401408} 2022-04-28T19:56:17.124 INFO:tasks.rados.rados.0.smithi059.stdout:3577: writing smithi05955964-46 from 1605632 to 2269184 tid 1 2022-04-28T19:56:17.124 INFO:tasks.rados.rados.0.smithi059.stdout:3577: writing smithi05955964-46 from 2269184 to 3072000 tid 2 2022-04-28T19:56:17.124 INFO:tasks.rados.rados.0.smithi059.stdout:3577: writing smithi05955964-46 from 3072000 to 3473408 tid 3 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3578: rmattr oid 27 current snap is 346 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3579: delete oid 11 current snap is 346 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1364 (ObjNum 1078 snap 335 seq_num 1078) dirty exists 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3577: finishing write tid 1 to smithi05955964-46 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3578: done (2 left) 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3579: done (1 left) 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 346 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3580: seq_num 1116 ranges {1277952=696320,1974272=573440,2547712=450560,2998272=180224} 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3580: writing smithi05955964-10 from 1277952 to 1974272 tid 1 2022-04-28T19:56:17.125 INFO:tasks.rados.rados.0.smithi059.stdout:3580: writing smithi05955964-10 from 1974272 to 2547712 tid 2 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3580: writing smithi05955964-10 from 2547712 to 2998272 tid 3 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3580: writing smithi05955964-10 from 2998272 to 3178496 tid 4 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3581: read oid 23 snap 310 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3581: expect deleted 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3577: finishing write tid 2 to smithi05955964-46 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3582: read oid 22 snap -1 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3582: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3583: read oid 49 snap -1 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3583: expect (ObjNum 1056 snap 328 seq_num 1056) 2022-04-28T19:56:17.126 INFO:tasks.rados.rados.0.smithi059.stdout:3584: read oid 7 snap -1 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3584: expect deleted 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3585: setattr oid 16 current snap is 346 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3577: finishing write tid 3 to smithi05955964-46 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3577: finishing write tid 5 to smithi05955964-46 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3577: finishing write tid 6 to smithi05955964-46 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1576 (ObjNum 1114 snap 346 seq_num 1114) dirty exists 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3577: left oid 46 (ObjNum 1114 snap 346 seq_num 1114) 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3580: finishing write tid 1 to smithi05955964-10 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3580: finishing write tid 2 to smithi05955964-10 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3577: done (6 left) 2022-04-28T19:56:17.127 INFO:tasks.rados.rados.0.smithi059.stdout:3586: read oid 13 snap -1 2022-04-28T19:56:17.128 INFO:tasks.rados.rados.0.smithi059.stdout:3586: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:56:17.128 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 346 2022-04-28T19:56:17.128 INFO:tasks.rados.rados.0.smithi059.stdout:3587: seq_num 1117 ranges {12345344=712704,13058048=458752,13516800=655360,14172160=90112} 2022-04-28T19:56:17.198 INFO:tasks.rados.rados.0.smithi059.stdout:3587: writing smithi05955964-12 from 12345344 to 13058048 tid 1 2022-04-28T19:56:17.200 INFO:tasks.rados.rados.0.smithi059.stdout:3587: writing smithi05955964-12 from 13058048 to 13516800 tid 2 2022-04-28T19:56:17.204 INFO:tasks.rados.rados.0.smithi059.stdout:3587: writing smithi05955964-12 from 13516800 to 14172160 tid 3 2022-04-28T19:56:17.205 INFO:tasks.rados.rados.0.smithi059.stdout:3587: writing smithi05955964-12 from 14172160 to 14262272 tid 4 2022-04-28T19:56:17.205 INFO:tasks.rados.rados.0.smithi059.stdout:3588: delete oid 30 current snap is 346 2022-04-28T19:56:17.205 INFO:tasks.rados.rados.0.smithi059.stdout:3580: finishing write tid 3 to smithi05955964-10 2022-04-28T19:56:17.205 INFO:tasks.rados.rados.0.smithi059.stdout:3586: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:56:17.208 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1457 (ObjNum 1098 snap 339 seq_num 1098) dirty exists 2022-04-28T19:56:17.208 INFO:tasks.rados.rados.0.smithi059.stdout:3583: expect (ObjNum 1056 snap 328 seq_num 1056) 2022-04-28T19:56:17.217 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[59745]: cluster 2022-04-28T19:56:16.028150+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2022-04-28T19:56:17.217 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[59745]: audit 2022-04-28T19:56:16.928379+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:56:17.217 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[59745]: audit 2022-04-28T19:56:16.929338+0000 mon.a (mon.0) 1393 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T19:56:17.217 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[59745]: audit 2022-04-28T19:56:16.929785+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:56:17.218 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[61152]: cluster 2022-04-28T19:56:16.028150+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2022-04-28T19:56:17.218 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[61152]: audit 2022-04-28T19:56:16.928379+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:56:17.218 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[61152]: audit 2022-04-28T19:56:16.929338+0000 mon.a (mon.0) 1393 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T19:56:17.218 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:17 smithi059 bash[61152]: audit 2022-04-28T19:56:16.929785+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:56:17.235 INFO:tasks.rados.rados.0.smithi059.stdout:3580: finishing write tid 4 to smithi05955964-10 2022-04-28T19:56:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:3580: finishing write tid 6 to smithi05955964-10 2022-04-28T19:56:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:3580: finishing write tid 7 to smithi05955964-10 2022-04-28T19:56:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1369 (ObjNum 1116 snap 346 seq_num 1116) dirty exists 2022-04-28T19:56:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:3580: left oid 10 (ObjNum 1116 snap 346 seq_num 1116) 2022-04-28T19:56:17.236 INFO:tasks.rados.rados.0.smithi059.stdout:3582: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:56:17.286 INFO:tasks.rados.rados.0.smithi059.stdout:3587: finishing write tid 1 to smithi05955964-12 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3587: finishing write tid 2 to smithi05955964-12 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3587: finishing write tid 3 to smithi05955964-12 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3587: finishing write tid 4 to smithi05955964-12 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3587: finishing write tid 6 to smithi05955964-12 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3587: finishing write tid 7 to smithi05955964-12 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1388 (ObjNum 1117 snap 346 seq_num 1117) dirty exists 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3587: left oid 12 (ObjNum 1117 snap 346 seq_num 1117) 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3580: done (8 left) 2022-04-28T19:56:17.287 INFO:tasks.rados.rados.0.smithi059.stdout:3581: done (7 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3582: done (6 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3583: done (5 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3584: done (4 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3585: done (3 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3586: done (2 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3587: done (1 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3588: done (0 left) 2022-04-28T19:56:17.288 INFO:tasks.rados.rados.0.smithi059.stdout:3589: delete oid 10 current snap is 346 2022-04-28T19:56:17.289 INFO:tasks.rados.rados.0.smithi059.stdout:3589: done (0 left) 2022-04-28T19:56:17.289 INFO:tasks.rados.rados.0.smithi059.stdout:3590: delete oid 14 current snap is 346 2022-04-28T19:56:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:3590: done (0 left) 2022-04-28T19:56:17.294 INFO:tasks.rados.rados.0.smithi059.stdout:3591: snap_create 2022-04-28T19:56:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:17 smithi139 bash[43301]: cluster 2022-04-28T19:56:16.028150+0000 mon.a (mon.0) 1391 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2022-04-28T19:56:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:17 smithi139 bash[43301]: audit 2022-04-28T19:56:16.928379+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:56:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:17 smithi139 bash[43301]: audit 2022-04-28T19:56:16.929338+0000 mon.a (mon.0) 1393 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T19:56:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:17 smithi139 bash[43301]: audit 2022-04-28T19:56:16.929785+0000 mon.a (mon.0) 1394 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T19:56:18.046 INFO:tasks.rados.rados.0.smithi059.stdout:3591: done (0 left) 2022-04-28T19:56:18.046 INFO:tasks.rados.rados.0.smithi059.stdout:3592: copy_from oid 1 from oid 28 current snap is 347 2022-04-28T19:56:18.046 INFO:tasks.rados.rados.0.smithi059.stdout:3593: delete oid 34 current snap is 347 2022-04-28T19:56:18.054 INFO:tasks.rados.rados.0.smithi059.stdout:3593: done (1 left) 2022-04-28T19:56:18.054 INFO:tasks.rados.rados.0.smithi059.stdout:3594: snap_create 2022-04-28T19:56:18.121 INFO:tasks.rados.rados.0.smithi059.stdout:3592: finishing copy_from to smithi05955964-1 2022-04-28T19:56:18.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1451 (ObjNum 1096 snap 339 seq_num 1096) dirty exists 2022-04-28T19:56:18.122 INFO:tasks.rados.rados.0.smithi059.stdout:3592: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:56:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[59745]: cluster 2022-04-28T19:56:17.013973+0000 mgr.y (mgr.24961) 373 : cluster [DBG] pgmap v610: 89 pgs: 89 active+clean; 1015 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2022-04-28T19:56:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[59745]: cluster 2022-04-28T19:56:17.034056+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2022-04-28T19:56:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[59745]: audit 2022-04-28T19:56:17.153431+0000 mon.a (mon.0) 1396 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:56:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[59745]: audit 2022-04-28T19:56:17.157201+0000 mon.a (mon.0) 1397 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:56:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[61152]: cluster 2022-04-28T19:56:17.013973+0000 mgr.y (mgr.24961) 373 : cluster [DBG] pgmap v610: 89 pgs: 89 active+clean; 1015 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2022-04-28T19:56:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[61152]: cluster 2022-04-28T19:56:17.034056+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2022-04-28T19:56:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[61152]: audit 2022-04-28T19:56:17.153431+0000 mon.a (mon.0) 1396 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:56:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:18 smithi059 bash[61152]: audit 2022-04-28T19:56:17.157201+0000 mon.a (mon.0) 1397 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:56:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:18 smithi139 bash[43301]: cluster 2022-04-28T19:56:17.013973+0000 mgr.y (mgr.24961) 373 : cluster [DBG] pgmap v610: 89 pgs: 89 active+clean; 1015 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2022-04-28T19:56:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:18 smithi139 bash[43301]: cluster 2022-04-28T19:56:17.034056+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2022-04-28T19:56:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:18 smithi139 bash[43301]: audit 2022-04-28T19:56:17.153431+0000 mon.a (mon.0) 1396 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:56:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:18 smithi139 bash[43301]: audit 2022-04-28T19:56:17.157201+0000 mon.a (mon.0) 1397 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:56:19.055 INFO:tasks.rados.rados.0.smithi059.stdout:3592: done (1 left) 2022-04-28T19:56:19.055 INFO:tasks.rados.rados.0.smithi059.stdout:3594: done (0 left) 2022-04-28T19:56:19.056 INFO:tasks.rados.rados.0.smithi059.stdout:3595: copy_from oid 44 from oid 13 current snap is 348 2022-04-28T19:56:19.056 INFO:tasks.rados.rados.0.smithi059.stdout:3596: snap_remove snap 330 2022-04-28T19:56:19.259 INFO:teuthology.orchestra.run.smithi059.stdout: "ceph version 16.2.7-969-g4fa079ba (4fa079ba14503defa8dc257d7c2d506ebefcfe6d) pacific (stable)" 2022-04-28T19:56:19.317 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:19 smithi059 bash[61152]: cluster 2022-04-28T19:56:18.045748+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2022-04-28T19:56:19.317 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:19 smithi059 bash[59745]: cluster 2022-04-28T19:56:18.045748+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2022-04-28T19:56:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:19 smithi139 bash[43301]: cluster 2022-04-28T19:56:18.045748+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2022-04-28T19:56:19.939 INFO:teuthology.task.sequential:In sequential, running task print... 2022-04-28T19:56:19.939 INFO:teuthology.task.print:**** done end upgrade, wait... 2022-04-28T19:56:19.940 DEBUG:teuthology.parallel:result is None 2022-04-28T19:56:20.066 INFO:tasks.rados.rados.0.smithi059.stdout:3596: done (1 left) 2022-04-28T19:56:20.066 INFO:tasks.rados.rados.0.smithi059.stdout:3597: copy_from oid 12 from oid 17 current snap is 348 2022-04-28T19:56:20.066 INFO:tasks.rados.rados.0.smithi059.stdout:3598: delete oid 3 current snap is 348 2022-04-28T19:56:20.066 INFO:tasks.rados.rados.0.smithi059.stdout:3595: finishing copy_from to smithi05955964-44 2022-04-28T19:56:20.067 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 44 v 1075 (ObjNum 5 snap 4 seq_num 6) dirty exists 2022-04-28T19:56:20.067 INFO:tasks.rados.rados.0.smithi059.stdout:3595: finishing copy_from racing read to smithi05955964-44 2022-04-28T19:56:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:3595: done (2 left) 2022-04-28T19:56:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:3598: done (1 left) 2022-04-28T19:56:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:3599: read oid 47 snap -1 2022-04-28T19:56:20.070 INFO:tasks.rados.rados.0.smithi059.stdout:3599: expect (ObjNum 969 snap 289 seq_num 969) 2022-04-28T19:56:20.071 INFO:tasks.rados.rados.0.smithi059.stdout:3600: snap_create 2022-04-28T19:56:20.105 INFO:tasks.rados.rados.0.smithi059.stdout:3599: expect (ObjNum 969 snap 289 seq_num 969) 2022-04-28T19:56:20.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:20 smithi059 bash[59745]: cluster 2022-04-28T19:56:19.014772+0000 mgr.y (mgr.24961) 374 : cluster [DBG] pgmap v613: 89 pgs: 89 active+clean; 1017 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.5 MiB/s wr, 11 op/s 2022-04-28T19:56:20.380 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:20 smithi059 bash[59745]: cluster 2022-04-28T19:56:19.053694+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2022-04-28T19:56:20.380 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:20 smithi059 bash[59745]: audit 2022-04-28T19:56:19.246398+0000 mon.a (mon.0) 1400 : audit [DBG] from='client.? 172.21.15.59:0/3506212181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:20.380 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:20 smithi059 bash[61152]: cluster 2022-04-28T19:56:19.014772+0000 mgr.y (mgr.24961) 374 : cluster [DBG] pgmap v613: 89 pgs: 89 active+clean; 1017 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.5 MiB/s wr, 11 op/s 2022-04-28T19:56:20.381 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:20 smithi059 bash[61152]: cluster 2022-04-28T19:56:19.053694+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2022-04-28T19:56:20.381 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:20 smithi059 bash[61152]: audit 2022-04-28T19:56:19.246398+0000 mon.a (mon.0) 1400 : audit [DBG] from='client.? 172.21.15.59:0/3506212181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:20.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:20 smithi139 bash[43301]: cluster 2022-04-28T19:56:19.014772+0000 mgr.y (mgr.24961) 374 : cluster [DBG] pgmap v613: 89 pgs: 89 active+clean; 1017 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.5 MiB/s wr, 11 op/s 2022-04-28T19:56:20.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:20 smithi139 bash[43301]: cluster 2022-04-28T19:56:19.053694+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2022-04-28T19:56:20.382 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:20 smithi139 bash[43301]: audit 2022-04-28T19:56:19.246398+0000 mon.a (mon.0) 1400 : audit [DBG] from='client.? 172.21.15.59:0/3506212181' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T19:56:20.789 INFO:tasks.rados.rados.0.smithi059.stdout:3597: finishing copy_from to smithi05955964-12 2022-04-28T19:56:20.789 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1074 (ObjNum 960 snap 285 seq_num 960) dirty exists 2022-04-28T19:56:20.789 INFO:tasks.rados.rados.0.smithi059.stdout:3597: finishing copy_from racing read to smithi05955964-12 2022-04-28T19:56:21.069 INFO:tasks.rados.rados.0.smithi059.stdout:3597: done (2 left) 2022-04-28T19:56:21.118 INFO:tasks.rados.rados.0.smithi059.stdout:3599: done (1 left) 2022-04-28T19:56:21.119 INFO:tasks.rados.rados.0.smithi059.stdout:3600: done (0 left) 2022-04-28T19:56:21.119 INFO:tasks.rados.rados.0.smithi059.stdout:3601: copy_from oid 30 from oid 21 current snap is 349 2022-04-28T19:56:21.119 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 349 2022-04-28T19:56:21.119 INFO:tasks.rados.rados.0.smithi059.stdout:3602: seq_num 1123 ranges {0=409600,409600=614400,1024000=598016,1622016=393216} 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3602: writing smithi05955964-7 from 0 to 409600 tid 1 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3602: writing smithi05955964-7 from 409600 to 1024000 tid 2 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3602: writing smithi05955964-7 from 1024000 to 1622016 tid 3 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3602: writing smithi05955964-7 from 1622016 to 2015232 tid 4 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3603: delete oid 47 current snap is 349 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3602: finishing write tid 1 to smithi05955964-7 2022-04-28T19:56:21.121 INFO:tasks.rados.rados.0.smithi059.stdout:3602: finishing write tid 2 to smithi05955964-7 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3602: finishing write tid 3 to smithi05955964-7 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3602: finishing write tid 4 to smithi05955964-7 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3603: done (2 left) 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3604: snap_create 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3602: finishing write tid 6 to smithi05955964-7 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3602: finishing write tid 7 to smithi05955964-7 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 1393 (ObjNum 1123 snap 349 seq_num 1123) dirty exists 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3602: left oid 7 (ObjNum 1123 snap 349 seq_num 1123) 2022-04-28T19:56:21.122 INFO:tasks.rados.rados.0.smithi059.stdout:3601: finishing copy_from to smithi05955964-30 2022-04-28T19:56:21.123 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 30 v 1166 (ObjNum 1093 snap 339 seq_num 1093) dirty exists 2022-04-28T19:56:21.123 INFO:tasks.rados.rados.0.smithi059.stdout:3601: finishing copy_from racing read to smithi05955964-30 2022-04-28T19:56:21.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:21 smithi059 bash[61152]: cluster 2022-04-28T19:56:20.062163+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2022-04-28T19:56:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:21 smithi059 bash[59745]: cluster 2022-04-28T19:56:20.062163+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2022-04-28T19:56:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:21 smithi139 bash[43301]: cluster 2022-04-28T19:56:20.062163+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2022-04-28T19:56:22.003 INFO:tasks.rados.rados.0.smithi059.stdout:3601: done (2 left) 2022-04-28T19:56:22.003 INFO:tasks.rados.rados.0.smithi059.stdout:3602: done (1 left) 2022-04-28T19:56:22.003 INFO:tasks.rados.rados.0.smithi059.stdout:3604: done (0 left) 2022-04-28T19:56:22.003 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 350 2022-04-28T19:56:22.003 INFO:tasks.rados.rados.0.smithi059.stdout:3605: seq_num 1125 ranges {0=450560,450560=786432,1236992=614400,1851392=204800} 2022-04-28T19:56:22.005 INFO:tasks.rados.rados.0.smithi059.stdout:3605: writing smithi05955964-47 from 0 to 450560 tid 1 2022-04-28T19:56:22.010 INFO:tasks.rados.rados.0.smithi059.stdout:3605: writing smithi05955964-47 from 450560 to 1236992 tid 2 2022-04-28T19:56:22.014 INFO:tasks.rados.rados.0.smithi059.stdout:3605: writing smithi05955964-47 from 1236992 to 1851392 tid 3 2022-04-28T19:56:22.015 INFO:tasks.rados.rados.0.smithi059.stdout:3605: writing smithi05955964-47 from 1851392 to 2056192 tid 4 2022-04-28T19:56:22.015 INFO:tasks.rados.rados.0.smithi059.stdout:3606: rmattr oid 41 current snap is 350 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:3606: done (1 left) 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:3607: rollback oid 26 current snap is 350 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 26 to 340 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:3605: finishing write tid 1 to smithi05955964-47 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:3605: finishing write tid 2 to smithi05955964-47 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 350 2022-04-28T19:56:22.016 INFO:tasks.rados.rados.0.smithi059.stdout:3608: seq_num 1126 ranges {0=466944,466944=720896,1187840=778240,1966080=106496} 2022-04-28T19:56:22.018 INFO:tasks.rados.rados.0.smithi059.stdout:3608: writing smithi05955964-40 from 0 to 466944 tid 1 2022-04-28T19:56:22.023 INFO:tasks.rados.rados.0.smithi059.stdout:3608: writing smithi05955964-40 from 466944 to 1187840 tid 2 2022-04-28T19:56:22.028 INFO:tasks.rados.rados.0.smithi059.stdout:3608: writing smithi05955964-40 from 1187840 to 1966080 tid 3 2022-04-28T19:56:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:3608: writing smithi05955964-40 from 1966080 to 2072576 tid 4 2022-04-28T19:56:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:3609: rollback oid 23 current snap is 350 2022-04-28T19:56:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 327 2022-04-28T19:56:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:3605: finishing write tid 3 to smithi05955964-47 2022-04-28T19:56:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:3605: finishing write tid 4 to smithi05955964-47 2022-04-28T19:56:22.029 INFO:tasks.rados.rados.0.smithi059.stdout:3605: finishing write tid 6 to smithi05955964-47 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3607: finishing rollback tid 0 to smithi05955964-26 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3607: finishing rollback tid 1 to smithi05955964-26 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3605: finishing write tid 7 to smithi05955964-47 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1376 (ObjNum 1125 snap 350 seq_num 1125) dirty exists 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3605: left oid 47 (ObjNum 1125 snap 350 seq_num 1125) 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3607: finishing rollback tid 2 to smithi05955964-26 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 26 v 1535 (ObjNum 1070 snap 334 seq_num 1070) dirty exists 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3608: finishing write tid 1 to smithi05955964-40 2022-04-28T19:56:22.030 INFO:tasks.rados.rados.0.smithi059.stdout:3605: done (3 left) 2022-04-28T19:56:22.031 INFO:tasks.rados.rados.0.smithi059.stdout:3607: done (2 left) 2022-04-28T19:56:22.031 INFO:tasks.rados.rados.0.smithi059.stdout:3610: snap_remove snap 309 2022-04-28T19:56:22.074 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:21 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:21.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:22.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:22 smithi059 bash[59745]: cluster 2022-04-28T19:56:21.015742+0000 mgr.y (mgr.24961) 375 : cluster [DBG] pgmap v616: 89 pgs: 89 active+clean; 1.0 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 4.4 MiB/s wr, 19 op/s 2022-04-28T19:56:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:22 smithi059 bash[59745]: cluster 2022-04-28T19:56:21.068014+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2022-04-28T19:56:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:22 smithi059 bash[59745]: cluster 2022-04-28T19:56:22.002096+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2022-04-28T19:56:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:22 smithi059 bash[61152]: cluster 2022-04-28T19:56:21.015742+0000 mgr.y (mgr.24961) 375 : cluster [DBG] pgmap v616: 89 pgs: 89 active+clean; 1.0 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 4.4 MiB/s wr, 19 op/s 2022-04-28T19:56:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:22 smithi059 bash[61152]: cluster 2022-04-28T19:56:21.068014+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2022-04-28T19:56:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:22 smithi059 bash[61152]: cluster 2022-04-28T19:56:22.002096+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2022-04-28T19:56:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:22 smithi139 bash[43301]: cluster 2022-04-28T19:56:21.015742+0000 mgr.y (mgr.24961) 375 : cluster [DBG] pgmap v616: 89 pgs: 89 active+clean; 1.0 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 4.4 MiB/s wr, 19 op/s 2022-04-28T19:56:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:22 smithi139 bash[43301]: cluster 2022-04-28T19:56:21.068014+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2022-04-28T19:56:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:22 smithi139 bash[43301]: cluster 2022-04-28T19:56:22.002096+0000 mon.a (mon.0) 1403 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2022-04-28T19:56:23.014 INFO:tasks.rados.rados.0.smithi059.stdout:3610: done (2 left) 2022-04-28T19:56:23.014 INFO:tasks.rados.rados.0.smithi059.stdout:3611: delete oid 24 current snap is 350 2022-04-28T19:56:23.014 INFO:tasks.rados.rados.0.smithi059.stdout:3608: finishing write tid 2 to smithi05955964-40 2022-04-28T19:56:23.014 INFO:tasks.rados.rados.0.smithi059.stdout:3608: finishing write tid 3 to smithi05955964-40 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3609: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3608: finishing write tid 4 to smithi05955964-40 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3608: finishing write tid 6 to smithi05955964-40 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3609: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3609: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1379 (ObjNum 1049 snap 327 seq_num 1049) dirty exists 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3608: finishing write tid 7 to smithi05955964-40 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1467 (ObjNum 1126 snap 350 seq_num 1126) dirty exists 2022-04-28T19:56:23.015 INFO:tasks.rados.rados.0.smithi059.stdout:3608: left oid 40 (ObjNum 1126 snap 350 seq_num 1126) 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:3608: done (2 left) 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:3609: done (1 left) 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:3611: done (0 left) 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:3612: copy_from oid 31 from oid 37 current snap is 350 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:3613: rollback oid 27 current snap is 350 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 338 2022-04-28T19:56:23.021 INFO:tasks.rados.rados.0.smithi059.stdout:3614: delete oid 41 current snap is 350 2022-04-28T19:56:23.024 INFO:tasks.rados.rados.0.smithi059.stdout:3614: done (2 left) 2022-04-28T19:56:23.024 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 350 2022-04-28T19:56:23.024 INFO:tasks.rados.rados.0.smithi059.stdout:3615: seq_num 1129 ranges {0=516096,516096=770048,1286144=598016,1884160=237568} 2022-04-28T19:56:23.027 INFO:tasks.rados.rados.0.smithi059.stdout:3615: writing smithi05955964-24 from 0 to 516096 tid 1 2022-04-28T19:56:23.031 INFO:tasks.rados.rados.0.smithi059.stdout:3615: writing smithi05955964-24 from 516096 to 1286144 tid 2 2022-04-28T19:56:23.035 INFO:tasks.rados.rados.0.smithi059.stdout:3615: writing smithi05955964-24 from 1286144 to 1884160 tid 3 2022-04-28T19:56:23.037 INFO:tasks.rados.rados.0.smithi059.stdout:3615: writing smithi05955964-24 from 1884160 to 2121728 tid 4 2022-04-28T19:56:23.037 INFO:tasks.rados.rados.0.smithi059.stdout:3616: snap_create 2022-04-28T19:56:23.037 INFO:tasks.rados.rados.0.smithi059.stdout:3613: finishing rollback tid 0 to smithi05955964-27 2022-04-28T19:56:23.037 INFO:tasks.rados.rados.0.smithi059.stdout:3613: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:56:23.037 INFO:tasks.rados.rados.0.smithi059.stdout:3613: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:56:23.037 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1382 (ObjNum 1078 snap 335 seq_num 1078) dirty exists 2022-04-28T19:56:23.038 INFO:tasks.rados.rados.0.smithi059.stdout:3615: finishing write tid 1 to smithi05955964-24 2022-04-28T19:56:23.038 INFO:tasks.rados.rados.0.smithi059.stdout:3615: finishing write tid 2 to smithi05955964-24 2022-04-28T19:56:23.039 INFO:tasks.rados.rados.0.smithi059.stdout:3615: finishing write tid 3 to smithi05955964-24 2022-04-28T19:56:23.039 INFO:tasks.rados.rados.0.smithi059.stdout:3615: finishing write tid 4 to smithi05955964-24 2022-04-28T19:56:23.039 INFO:tasks.rados.rados.0.smithi059.stdout:3615: finishing write tid 6 to smithi05955964-24 2022-04-28T19:56:23.041 INFO:tasks.rados.rados.0.smithi059.stdout:3615: finishing write tid 7 to smithi05955964-24 2022-04-28T19:56:23.041 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1473 (ObjNum 1129 snap 350 seq_num 1129) dirty exists 2022-04-28T19:56:23.041 INFO:tasks.rados.rados.0.smithi059.stdout:3615: left oid 24 (ObjNum 1129 snap 350 seq_num 1129) 2022-04-28T19:56:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:3612: finishing copy_from to smithi05955964-31 2022-04-28T19:56:23.054 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1352 (ObjNum 1052 snap 328 seq_num 1052) dirty exists 2022-04-28T19:56:23.055 INFO:tasks.rados.rados.0.smithi059.stdout:3612: finishing copy_from racing read to smithi05955964-31 2022-04-28T19:56:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:23 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:23.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:24.026 INFO:tasks.rados.rados.0.smithi059.stdout:3612: done (3 left) 2022-04-28T19:56:24.026 INFO:tasks.rados.rados.0.smithi059.stdout:3613: done (2 left) 2022-04-28T19:56:24.026 INFO:tasks.rados.rados.0.smithi059.stdout:3615: done (1 left) 2022-04-28T19:56:24.027 INFO:tasks.rados.rados.0.smithi059.stdout:3616: done (0 left) 2022-04-28T19:56:24.027 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 351 2022-04-28T19:56:24.027 INFO:tasks.rados.rados.0.smithi059.stdout:3617: seq_num 1130 ranges {0=532480,532480=655360,1187840=753664,1941504=196608} 2022-04-28T19:56:24.029 INFO:tasks.rados.rados.0.smithi059.stdout:3617: writing smithi05955964-42 from 0 to 532480 tid 1 2022-04-28T19:56:24.032 INFO:tasks.rados.rados.0.smithi059.stdout:3617: writing smithi05955964-42 from 532480 to 1187840 tid 2 2022-04-28T19:56:24.037 INFO:tasks.rados.rados.0.smithi059.stdout:3617: writing smithi05955964-42 from 1187840 to 1941504 tid 3 2022-04-28T19:56:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:3617: writing smithi05955964-42 from 1941504 to 2138112 tid 4 2022-04-28T19:56:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:3618: read oid 41 snap -1 2022-04-28T19:56:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:3618: expect deleted 2022-04-28T19:56:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:3619: snap_create 2022-04-28T19:56:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:3617: finishing write tid 1 to smithi05955964-42 2022-04-28T19:56:24.039 INFO:tasks.rados.rados.0.smithi059.stdout:3617: finishing write tid 2 to smithi05955964-42 2022-04-28T19:56:24.043 INFO:tasks.rados.rados.0.smithi059.stdout:3617: finishing write tid 3 to smithi05955964-42 2022-04-28T19:56:24.044 INFO:tasks.rados.rados.0.smithi059.stdout:3617: finishing write tid 4 to smithi05955964-42 2022-04-28T19:56:24.044 INFO:tasks.rados.rados.0.smithi059.stdout:3617: finishing write tid 6 to smithi05955964-42 2022-04-28T19:56:24.046 INFO:tasks.rados.rados.0.smithi059.stdout:3617: finishing write tid 7 to smithi05955964-42 2022-04-28T19:56:24.046 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1519 (ObjNum 1130 snap 351 seq_num 1130) dirty exists 2022-04-28T19:56:24.046 INFO:tasks.rados.rados.0.smithi059.stdout:3617: left oid 42 (ObjNum 1130 snap 351 seq_num 1130) 2022-04-28T19:56:24.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:24 smithi059 bash[61152]: cluster 2022-04-28T19:56:23.011503+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2022-04-28T19:56:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:24 smithi059 bash[61152]: cluster 2022-04-28T19:56:23.016231+0000 mgr.y (mgr.24961) 376 : cluster [DBG] pgmap v620: 89 pgs: 89 active+clean; 1.0 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.0 MiB/s wr, 15 op/s 2022-04-28T19:56:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:24 smithi059 bash[59745]: cluster 2022-04-28T19:56:23.011503+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2022-04-28T19:56:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:24 smithi059 bash[59745]: cluster 2022-04-28T19:56:23.016231+0000 mgr.y (mgr.24961) 376 : cluster [DBG] pgmap v620: 89 pgs: 89 active+clean; 1.0 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.0 MiB/s wr, 15 op/s 2022-04-28T19:56:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:24 smithi139 bash[43301]: cluster 2022-04-28T19:56:23.011503+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2022-04-28T19:56:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:24 smithi139 bash[43301]: cluster 2022-04-28T19:56:23.016231+0000 mgr.y (mgr.24961) 376 : cluster [DBG] pgmap v620: 89 pgs: 89 active+clean; 1.0 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 4.0 MiB/s wr, 15 op/s 2022-04-28T19:56:25.027 INFO:tasks.rados.rados.0.smithi059.stdout:3617: done (2 left) 2022-04-28T19:56:25.027 INFO:tasks.rados.rados.0.smithi059.stdout:3618: done (1 left) 2022-04-28T19:56:25.027 INFO:tasks.rados.rados.0.smithi059.stdout:3619: done (0 left) 2022-04-28T19:56:25.028 INFO:tasks.rados.rados.0.smithi059.stdout:3620: copy_from oid 39 from oid 30 current snap is 352 2022-04-28T19:56:25.028 INFO:tasks.rados.rados.0.smithi059.stdout:3621: read oid 40 snap -1 2022-04-28T19:56:25.028 INFO:tasks.rados.rados.0.smithi059.stdout:3621: expect (ObjNum 1126 snap 350 seq_num 1126) 2022-04-28T19:56:25.028 INFO:tasks.rados.rados.0.smithi059.stdout:3622: read oid 43 snap -1 2022-04-28T19:56:25.028 INFO:tasks.rados.rados.0.smithi059.stdout:3622: expect deleted 2022-04-28T19:56:25.028 INFO:tasks.rados.rados.0.smithi059.stdout:3623: delete oid 25 current snap is 352 2022-04-28T19:56:25.035 INFO:tasks.rados.rados.0.smithi059.stdout:3622: done (3 left) 2022-04-28T19:56:25.035 INFO:tasks.rados.rados.0.smithi059.stdout:3623: done (2 left) 2022-04-28T19:56:25.035 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 13 current snap is 352 2022-04-28T19:56:25.035 INFO:tasks.rados.rados.0.smithi059.stdout:3624: seq_num 1132 ranges {0=565248,565248=425984,991232=770048,1761280=409600} 2022-04-28T19:56:25.038 INFO:tasks.rados.rados.0.smithi059.stdout:3624: writing smithi05955964-13 from 0 to 565248 tid 1 2022-04-28T19:56:25.041 INFO:tasks.rados.rados.0.smithi059.stdout:3624: writing smithi05955964-13 from 565248 to 991232 tid 2 2022-04-28T19:56:25.045 INFO:tasks.rados.rados.0.smithi059.stdout:3624: writing smithi05955964-13 from 991232 to 1761280 tid 3 2022-04-28T19:56:25.047 INFO:tasks.rados.rados.0.smithi059.stdout:3624: writing smithi05955964-13 from 1761280 to 2170880 tid 4 2022-04-28T19:56:25.048 INFO:tasks.rados.rados.0.smithi059.stdout:3625: setattr oid 5 current snap is 352 2022-04-28T19:56:25.048 INFO:tasks.rados.rados.0.smithi059.stdout:3621: expect (ObjNum 1126 snap 350 seq_num 1126) 2022-04-28T19:56:25.096 INFO:tasks.rados.rados.0.smithi059.stdout:3624: finishing write tid 1 to smithi05955964-13 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3624: finishing write tid 2 to smithi05955964-13 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3624: finishing write tid 3 to smithi05955964-13 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3624: finishing write tid 4 to smithi05955964-13 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3624: finishing write tid 6 to smithi05955964-13 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3620: finishing copy_from to smithi05955964-39 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1166 (ObjNum 1093 snap 339 seq_num 1093) dirty exists 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3620: finishing copy_from racing read to smithi05955964-39 2022-04-28T19:56:25.097 INFO:tasks.rados.rados.0.smithi059.stdout:3624: finishing write tid 7 to smithi05955964-13 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1080 (ObjNum 1132 snap 352 seq_num 1132) dirty exists 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:3624: left oid 13 (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:3620: done (3 left) 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:3621: done (2 left) 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:3624: done (1 left) 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 352 2022-04-28T19:56:25.098 INFO:tasks.rados.rados.0.smithi059.stdout:3626: seq_num 1133 ranges {1802240=581632,2383872=753664,3137536=573440,3710976=278528} 2022-04-28T19:56:25.118 INFO:tasks.rados.rados.0.smithi059.stdout:3626: writing smithi05955964-48 from 1802240 to 2383872 tid 1 2022-04-28T19:56:25.122 INFO:tasks.rados.rados.0.smithi059.stdout:3626: writing smithi05955964-48 from 2383872 to 3137536 tid 2 2022-04-28T19:56:25.126 INFO:tasks.rados.rados.0.smithi059.stdout:3626: writing smithi05955964-48 from 3137536 to 3710976 tid 3 2022-04-28T19:56:25.128 INFO:tasks.rados.rados.0.smithi059.stdout:3626: writing smithi05955964-48 from 3710976 to 3989504 tid 4 2022-04-28T19:56:25.128 INFO:tasks.rados.rados.0.smithi059.stdout:3627: read oid 47 snap -1 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:3627: expect (ObjNum 1125 snap 350 seq_num 1125) 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:3628: read oid 2 snap -1 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:3628: expect (ObjNum 1090 snap 336 seq_num 1090) 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:3629: delete oid 46 current snap is 352 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1580 (ObjNum 347164504 snap 32762 seq_num 1) dirty exists 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:3626: finishing write tid 1 to smithi05955964-48 2022-04-28T19:56:25.129 INFO:tasks.rados.rados.0.smithi059.stdout:3626: finishing write tid 2 to smithi05955964-48 2022-04-28T19:56:25.130 INFO:tasks.rados.rados.0.smithi059.stdout:3626: finishing write tid 3 to smithi05955964-48 2022-04-28T19:56:25.131 INFO:tasks.rados.rados.0.smithi059.stdout:3625: done (4 left) 2022-04-28T19:56:25.131 INFO:tasks.rados.rados.0.smithi059.stdout:3629: done (3 left) 2022-04-28T19:56:25.131 INFO:tasks.rados.rados.0.smithi059.stdout:3630: snap_create 2022-04-28T19:56:25.134 INFO:tasks.rados.rados.0.smithi059.stdout:3626: finishing write tid 4 to smithi05955964-48 2022-04-28T19:56:25.135 INFO:tasks.rados.rados.0.smithi059.stdout:3626: finishing write tid 6 to smithi05955964-48 2022-04-28T19:56:25.136 INFO:tasks.rados.rados.0.smithi059.stdout:3626: finishing write tid 7 to smithi05955964-48 2022-04-28T19:56:25.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1540 (ObjNum 1133 snap 352 seq_num 1133) dirty exists 2022-04-28T19:56:25.137 INFO:tasks.rados.rados.0.smithi059.stdout:3626: left oid 48 (ObjNum 1133 snap 352 seq_num 1133) 2022-04-28T19:56:25.138 INFO:tasks.rados.rados.0.smithi059.stdout:3627: expect (ObjNum 1125 snap 350 seq_num 1125) 2022-04-28T19:56:25.188 INFO:tasks.rados.rados.0.smithi059.stdout:3628: expect (ObjNum 1090 snap 336 seq_num 1090) 2022-04-28T19:56:25.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:25 smithi059 bash[59745]: cluster 2022-04-28T19:56:24.026095+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2022-04-28T19:56:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:25 smithi059 bash[61152]: cluster 2022-04-28T19:56:24.026095+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2022-04-28T19:56:25.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:25 smithi139 bash[43301]: cluster 2022-04-28T19:56:24.026095+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2022-04-28T19:56:26.084 INFO:tasks.rados.rados.0.smithi059.stdout:3626: done (3 left) 2022-04-28T19:56:26.085 INFO:tasks.rados.rados.0.smithi059.stdout:3627: done (2 left) 2022-04-28T19:56:26.085 INFO:tasks.rados.rados.0.smithi059.stdout:3628: done (1 left) 2022-04-28T19:56:26.085 INFO:tasks.rados.rados.0.smithi059.stdout:3630: done (0 left) 2022-04-28T19:56:26.085 INFO:tasks.rados.rados.0.smithi059.stdout:3631: rmattr oid 9 current snap is 353 2022-04-28T19:56:26.085 INFO:tasks.rados.rados.0.smithi059.stdout:3632: read oid 45 snap -1 2022-04-28T19:56:26.085 INFO:tasks.rados.rados.0.smithi059.stdout:3632: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:56:26.086 INFO:tasks.rados.rados.0.smithi059.stdout:3633: delete oid 17 current snap is 353 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3633: done (2 left) 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3634: rollback oid 36 current snap is 353 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 36 to 324 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3635: read oid 12 snap -1 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3635: expect (ObjNum 960 snap 285 seq_num 960) 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3636: read oid 41 snap -1 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3636: expect deleted 2022-04-28T19:56:26.089 INFO:tasks.rados.rados.0.smithi059.stdout:3637: delete oid 7 current snap is 353 2022-04-28T19:56:26.106 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1217 (ObjNum 1000 snap 297 seq_num 1000) dirty exists 2022-04-28T19:56:26.108 INFO:tasks.rados.rados.0.smithi059.stdout:3634: finishing rollback tid 0 to smithi05955964-36 2022-04-28T19:56:26.108 INFO:tasks.rados.rados.0.smithi059.stdout:3634: finishing rollback tid 1 to smithi05955964-36 2022-04-28T19:56:26.109 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1469 (ObjNum 1432967772 snap 1684426576 seq_num 1415797876) dirty dne 2022-04-28T19:56:26.124 INFO:tasks.rados.rados.0.smithi059.stdout:3631: done (5 left) 2022-04-28T19:56:26.124 INFO:tasks.rados.rados.0.smithi059.stdout:3634: done (4 left) 2022-04-28T19:56:26.124 INFO:tasks.rados.rados.0.smithi059.stdout:3636: done (3 left) 2022-04-28T19:56:26.125 INFO:tasks.rados.rados.0.smithi059.stdout:3637: done (2 left) 2022-04-28T19:56:26.125 INFO:tasks.rados.rados.0.smithi059.stdout:3638: copy_from oid 18 from oid 50 current snap is 353 2022-04-28T19:56:26.125 INFO:tasks.rados.rados.0.smithi059.stdout:3639: snap_create 2022-04-28T19:56:26.147 INFO:tasks.rados.rados.0.smithi059.stdout:3635: expect (ObjNum 960 snap 285 seq_num 960) 2022-04-28T19:56:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:26 smithi059 bash[61152]: cluster 2022-04-28T19:56:25.017080+0000 mgr.y (mgr.24961) 377 : cluster [DBG] pgmap v622: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.0 MiB/s wr, 8 op/s 2022-04-28T19:56:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:26 smithi059 bash[61152]: cluster 2022-04-28T19:56:25.024555+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2022-04-28T19:56:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:26 smithi059 bash[59745]: cluster 2022-04-28T19:56:25.017080+0000 mgr.y (mgr.24961) 377 : cluster [DBG] pgmap v622: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.0 MiB/s wr, 8 op/s 2022-04-28T19:56:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:26 smithi059 bash[59745]: cluster 2022-04-28T19:56:25.024555+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2022-04-28T19:56:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:26 smithi139 bash[43301]: cluster 2022-04-28T19:56:25.017080+0000 mgr.y (mgr.24961) 377 : cluster [DBG] pgmap v622: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.0 MiB/s wr, 8 op/s 2022-04-28T19:56:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:26 smithi139 bash[43301]: cluster 2022-04-28T19:56:25.024555+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2022-04-28T19:56:27.066 INFO:tasks.rados.rados.0.smithi059.stdout:3638: finishing copy_from to smithi05955964-18 2022-04-28T19:56:27.066 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1548 (ObjNum 1003 snap 299 seq_num 1003) dirty exists 2022-04-28T19:56:27.066 INFO:tasks.rados.rados.0.smithi059.stdout:3638: finishing copy_from racing read to smithi05955964-18 2022-04-28T19:56:27.066 INFO:tasks.rados.rados.0.smithi059.stdout:3632: expect (ObjNum 1011 snap 311 seq_num 1011) 2022-04-28T19:56:27.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:27 smithi139 bash[43301]: cluster 2022-04-28T19:56:26.083603+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2022-04-28T19:56:27.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:27 smithi059 bash[61152]: cluster 2022-04-28T19:56:26.083603+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2022-04-28T19:56:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:27 smithi059 bash[59745]: cluster 2022-04-28T19:56:26.083603+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2022-04-28T19:56:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:28 smithi059 bash[61152]: cluster 2022-04-28T19:56:27.017587+0000 mgr.y (mgr.24961) 378 : cluster [DBG] pgmap v625: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 6.5 MiB/s wr, 11 op/s 2022-04-28T19:56:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:28 smithi059 bash[61152]: cluster 2022-04-28T19:56:27.125442+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2022-04-28T19:56:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:28 smithi059 bash[59745]: cluster 2022-04-28T19:56:27.017587+0000 mgr.y (mgr.24961) 378 : cluster [DBG] pgmap v625: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 6.5 MiB/s wr, 11 op/s 2022-04-28T19:56:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:28 smithi059 bash[59745]: cluster 2022-04-28T19:56:27.125442+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2022-04-28T19:56:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:28 smithi139 bash[43301]: cluster 2022-04-28T19:56:27.017587+0000 mgr.y (mgr.24961) 378 : cluster [DBG] pgmap v625: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 6.5 MiB/s wr, 11 op/s 2022-04-28T19:56:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:28 smithi139 bash[43301]: cluster 2022-04-28T19:56:27.125442+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2022-04-28T19:56:29.223 INFO:tasks.rados.rados.0.smithi059.stdout:3632: done (3 left) 2022-04-28T19:56:29.224 INFO:tasks.rados.rados.0.smithi059.stdout:3635: done (2 left) 2022-04-28T19:56:29.224 INFO:tasks.rados.rados.0.smithi059.stdout:3638: done (1 left) 2022-04-28T19:56:29.224 INFO:tasks.rados.rados.0.smithi059.stdout:3639: done (0 left) 2022-04-28T19:56:29.224 INFO:tasks.rados.rados.0.smithi059.stdout:3640: rollback oid 35 current snap is 354 2022-04-28T19:56:29.225 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 35 to 349 2022-04-28T19:56:29.225 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 45 current snap is 354 2022-04-28T19:56:29.225 INFO:tasks.rados.rados.0.smithi059.stdout:3641: seq_num 1137 ranges {17801216=647168,18448384=696320,19144704=557056,19701760=352256} 2022-04-28T19:56:29.326 INFO:tasks.rados.rados.0.smithi059.stdout:3641: writing smithi05955964-45 from 17801216 to 18448384 tid 1 2022-04-28T19:56:29.330 INFO:tasks.rados.rados.0.smithi059.stdout:3641: writing smithi05955964-45 from 18448384 to 19144704 tid 2 2022-04-28T19:56:29.333 INFO:tasks.rados.rados.0.smithi059.stdout:3641: writing smithi05955964-45 from 19144704 to 19701760 tid 3 2022-04-28T19:56:29.336 INFO:tasks.rados.rados.0.smithi059.stdout:3641: writing smithi05955964-45 from 19701760 to 20054016 tid 4 2022-04-28T19:56:29.336 INFO:tasks.rados.rados.0.smithi059.stdout:3642: snap_remove snap 319 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3642: done (2 left) 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3643: snap_create 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3640: finishing rollback tid 0 to smithi05955964-35 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3640: finishing rollback tid 1 to smithi05955964-35 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3640: finishing rollback tid 2 to smithi05955964-35 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1343 (ObjNum 1102 snap 340 seq_num 1102) dirty exists 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3641: finishing write tid 1 to smithi05955964-45 2022-04-28T19:56:30.137 INFO:tasks.rados.rados.0.smithi059.stdout:3641: finishing write tid 2 to smithi05955964-45 2022-04-28T19:56:30.138 INFO:tasks.rados.rados.0.smithi059.stdout:3641: finishing write tid 3 to smithi05955964-45 2022-04-28T19:56:30.138 INFO:tasks.rados.rados.0.smithi059.stdout:3641: finishing write tid 4 to smithi05955964-45 2022-04-28T19:56:30.138 INFO:tasks.rados.rados.0.smithi059.stdout:3641: finishing write tid 6 to smithi05955964-45 2022-04-28T19:56:30.138 INFO:tasks.rados.rados.0.smithi059.stdout:3641: finishing write tid 7 to smithi05955964-45 2022-04-28T19:56:30.138 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1299 (ObjNum 1137 snap 354 seq_num 1137) dirty exists 2022-04-28T19:56:30.138 INFO:tasks.rados.rados.0.smithi059.stdout:3641: left oid 45 (ObjNum 1137 snap 354 seq_num 1137) 2022-04-28T19:56:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:30 smithi059 bash[61152]: cluster 2022-04-28T19:56:29.018378+0000 mgr.y (mgr.24961) 379 : cluster [DBG] pgmap v627: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 11 op/s 2022-04-28T19:56:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:30 smithi059 bash[59745]: cluster 2022-04-28T19:56:29.018378+0000 mgr.y (mgr.24961) 379 : cluster [DBG] pgmap v627: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 11 op/s 2022-04-28T19:56:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:30 smithi139 bash[43301]: cluster 2022-04-28T19:56:29.018378+0000 mgr.y (mgr.24961) 379 : cluster [DBG] pgmap v627: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 11 op/s 2022-04-28T19:56:31.149 INFO:tasks.rados.rados.0.smithi059.stdout:3640: done (2 left) 2022-04-28T19:56:31.149 INFO:tasks.rados.rados.0.smithi059.stdout:3641: done (1 left) 2022-04-28T19:56:31.149 INFO:tasks.rados.rados.0.smithi059.stdout:3643: done (0 left) 2022-04-28T19:56:31.149 INFO:tasks.rados.rados.0.smithi059.stdout:3644: snap_remove snap 334 2022-04-28T19:56:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:31 smithi059 bash[61152]: cluster 2022-04-28T19:56:30.135125+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2022-04-28T19:56:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:31 smithi059 bash[59745]: cluster 2022-04-28T19:56:30.135125+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2022-04-28T19:56:31.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:31 smithi139 bash[43301]: cluster 2022-04-28T19:56:30.135125+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2022-04-28T19:56:32.015 INFO:tasks.rados.rados.0.smithi059.stdout:3644: done (0 left) 2022-04-28T19:56:32.015 INFO:tasks.rados.rados.0.smithi059.stdout:3645: rollback oid 37 current snap is 355 2022-04-28T19:56:32.016 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 37 to 341 2022-04-28T19:56:32.016 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 36 current snap is 355 2022-04-28T19:56:32.016 INFO:tasks.rados.rados.0.smithi059.stdout:3646: seq_num 1138 ranges {0=663552,663552=581632,1245184=712704,1957888=311296} 2022-04-28T19:56:32.019 INFO:tasks.rados.rados.0.smithi059.stdout:3646: writing smithi05955964-36 from 0 to 663552 tid 1 2022-04-28T19:56:32.023 INFO:tasks.rados.rados.0.smithi059.stdout:3646: writing smithi05955964-36 from 663552 to 1245184 tid 2 2022-04-28T19:56:32.028 INFO:tasks.rados.rados.0.smithi059.stdout:3646: writing smithi05955964-36 from 1245184 to 1957888 tid 3 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3646: writing smithi05955964-36 from 1957888 to 2269184 tid 4 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3645: finishing rollback tid 0 to smithi05955964-37 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3647: read oid 21 snap -1 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3647: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3648: copy_from oid 6 from oid 43 current snap is 355 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3645: finishing rollback tid 1 to smithi05955964-37 2022-04-28T19:56:32.031 INFO:tasks.rados.rados.0.smithi059.stdout:3645: finishing rollback tid 2 to smithi05955964-37 2022-04-28T19:56:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1551 (ObjNum 1052 snap 328 seq_num 1052) dirty exists 2022-04-28T19:56:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:3645: done (3 left) 2022-04-28T19:56:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:3649: snap_create 2022-04-28T19:56:32.032 INFO:tasks.rados.rados.0.smithi059.stdout:3646: finishing write tid 1 to smithi05955964-36 2022-04-28T19:56:32.039 INFO:tasks.rados.rados.0.smithi059.stdout:3646: finishing write tid 2 to smithi05955964-36 2022-04-28T19:56:32.039 INFO:tasks.rados.rados.0.smithi059.stdout:3648: finishing copy_from to smithi05955964-6 2022-04-28T19:56:32.039 INFO:tasks.rados.rados.0.smithi059.stdout:3648: got expected ENOENT (src dne) 2022-04-28T19:56:32.039 INFO:tasks.rados.rados.0.smithi059.stdout:3648: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:56:32.039 INFO:tasks.rados.rados.0.smithi059.stdout:3646: finishing write tid 3 to smithi05955964-36 2022-04-28T19:56:32.040 INFO:tasks.rados.rados.0.smithi059.stdout:3646: finishing write tid 4 to smithi05955964-36 2022-04-28T19:56:32.042 INFO:tasks.rados.rados.0.smithi059.stdout:3646: finishing write tid 6 to smithi05955964-36 2022-04-28T19:56:32.043 INFO:tasks.rados.rados.0.smithi059.stdout:3647: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:56:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:31 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:31.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:32.116 INFO:tasks.rados.rados.0.smithi059.stdout:3646: finishing write tid 7 to smithi05955964-36 2022-04-28T19:56:32.117 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 36 v 1474 (ObjNum 1138 snap 355 seq_num 1138) dirty exists 2022-04-28T19:56:32.117 INFO:tasks.rados.rados.0.smithi059.stdout:3646: left oid 36 (ObjNum 1138 snap 355 seq_num 1138) 2022-04-28T19:56:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:32 smithi059 bash[61152]: cluster 2022-04-28T19:56:31.019318+0000 mgr.y (mgr.24961) 380 : cluster [DBG] pgmap v629: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2022-04-28T19:56:32.666 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:32 smithi059 bash[61152]: cluster 2022-04-28T19:56:31.148126+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2022-04-28T19:56:32.666 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:32 smithi059 bash[61152]: cluster 2022-04-28T19:56:32.013159+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2022-04-28T19:56:32.667 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:32 smithi059 bash[59745]: cluster 2022-04-28T19:56:31.019318+0000 mgr.y (mgr.24961) 380 : cluster [DBG] pgmap v629: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2022-04-28T19:56:32.667 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:32 smithi059 bash[59745]: cluster 2022-04-28T19:56:31.148126+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2022-04-28T19:56:32.667 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:32 smithi059 bash[59745]: cluster 2022-04-28T19:56:32.013159+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2022-04-28T19:56:32.667 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:32 smithi139 bash[43301]: cluster 2022-04-28T19:56:31.019318+0000 mgr.y (mgr.24961) 380 : cluster [DBG] pgmap v629: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 14 op/s 2022-04-28T19:56:32.668 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:32 smithi139 bash[43301]: cluster 2022-04-28T19:56:31.148126+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2022-04-28T19:56:32.668 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:32 smithi139 bash[43301]: cluster 2022-04-28T19:56:32.013159+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2022-04-28T19:56:33.029 INFO:tasks.rados.rados.0.smithi059.stdout:3646: done (3 left) 2022-04-28T19:56:33.030 INFO:tasks.rados.rados.0.smithi059.stdout:3647: done (2 left) 2022-04-28T19:56:33.030 INFO:tasks.rados.rados.0.smithi059.stdout:3648: done (1 left) 2022-04-28T19:56:33.030 INFO:tasks.rados.rados.0.smithi059.stdout:3649: done (0 left) 2022-04-28T19:56:33.030 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 356 2022-04-28T19:56:33.030 INFO:tasks.rados.rados.0.smithi059.stdout:3650: seq_num 1139 ranges {1400832=679936,2080768=458752,2539520=524288,3063808=540672,3604480=81920} 2022-04-28T19:56:33.043 INFO:tasks.rados.rados.0.smithi059.stdout:3650: writing smithi05955964-20 from 1400832 to 2080768 tid 1 2022-04-28T19:56:33.045 INFO:tasks.rados.rados.0.smithi059.stdout:3650: writing smithi05955964-20 from 2080768 to 2539520 tid 2 2022-04-28T19:56:33.049 INFO:tasks.rados.rados.0.smithi059.stdout:3650: writing smithi05955964-20 from 2539520 to 3063808 tid 3 2022-04-28T19:56:33.052 INFO:tasks.rados.rados.0.smithi059.stdout:3650: writing smithi05955964-20 from 3063808 to 3604480 tid 4 2022-04-28T19:56:33.053 INFO:tasks.rados.rados.0.smithi059.stdout:3650: writing smithi05955964-20 from 3604480 to 3686400 tid 5 2022-04-28T19:56:33.053 INFO:tasks.rados.rados.0.smithi059.stdout:3651: read oid 30 snap -1 2022-04-28T19:56:33.053 INFO:tasks.rados.rados.0.smithi059.stdout:3651: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:56:33.054 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 14 current snap is 356 2022-04-28T19:56:33.054 INFO:tasks.rados.rados.0.smithi059.stdout:3652: seq_num 1140 ranges {0=696320,696320=745472,1441792=729088,2170880=131072} 2022-04-28T19:56:33.057 INFO:tasks.rados.rados.0.smithi059.stdout:3652: writing smithi05955964-14 from 0 to 696320 tid 1 2022-04-28T19:56:33.062 INFO:tasks.rados.rados.0.smithi059.stdout:3652: writing smithi05955964-14 from 696320 to 1441792 tid 2 2022-04-28T19:56:33.067 INFO:tasks.rados.rados.0.smithi059.stdout:3652: writing smithi05955964-14 from 1441792 to 2170880 tid 3 2022-04-28T19:56:33.068 INFO:tasks.rados.rados.0.smithi059.stdout:3652: writing smithi05955964-14 from 2170880 to 2301952 tid 4 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3653: rollback oid 22 current snap is 356 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 22 to 351 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 1 to smithi05955964-20 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 2 to smithi05955964-20 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 3 to smithi05955964-20 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 4 to smithi05955964-20 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 5 to smithi05955964-20 2022-04-28T19:56:33.069 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 7 to smithi05955964-20 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:3650: finishing write tid 8 to smithi05955964-20 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1305 (ObjNum 1139 snap 356 seq_num 1139) dirty exists 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:3650: left oid 20 (ObjNum 1139 snap 356 seq_num 1139) 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:3652: finishing write tid 1 to smithi05955964-14 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:3652: finishing write tid 2 to smithi05955964-14 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:3650: done (3 left) 2022-04-28T19:56:33.070 INFO:tasks.rados.rados.0.smithi059.stdout:3654: setattr oid 49 current snap is 356 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3655: copy_from oid 27 from oid 43 current snap is 356 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3656: copy_from oid 13 from oid 19 current snap is 356 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3657: read oid 4 snap -1 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3657: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3658: copy_from oid 38 from oid 5 current snap is 356 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3659: delete oid 10 current snap is 356 2022-04-28T19:56:33.072 INFO:tasks.rados.rados.0.smithi059.stdout:3651: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:56:33.151 INFO:tasks.rados.rados.0.smithi059.stdout:3655: finishing copy_from to smithi05955964-27 2022-04-28T19:56:33.151 INFO:tasks.rados.rados.0.smithi059.stdout:3655: got expected ENOENT (src dne) 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3655: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3652: finishing write tid 3 to smithi05955964-14 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3656: finishing copy_from to smithi05955964-13 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3656: got expected ENOENT (src dne) 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3656: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3651: done (8 left) 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3655: done (7 left) 2022-04-28T19:56:33.152 INFO:tasks.rados.rados.0.smithi059.stdout:3656: done (6 left) 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:3659: done (5 left) 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:3660: rollback oid 42 current snap is 356 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 331 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:3652: finishing write tid 4 to smithi05955964-14 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:3652: finishing write tid 6 to smithi05955964-14 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1475 (ObjNum 1056 snap 328 seq_num 1056) dirty exists 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:3653: finishing rollback tid 0 to smithi05955964-22 2022-04-28T19:56:33.153 INFO:tasks.rados.rados.0.smithi059.stdout:3653: finishing rollback tid 1 to smithi05955964-22 2022-04-28T19:56:33.154 INFO:tasks.rados.rados.0.smithi059.stdout:3653: finishing rollback tid 2 to smithi05955964-22 2022-04-28T19:56:33.154 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1548 (ObjNum 988 snap 293 seq_num 988) dirty exists 2022-04-28T19:56:33.154 INFO:tasks.rados.rados.0.smithi059.stdout:3652: finishing write tid 7 to smithi05955964-14 2022-04-28T19:56:33.154 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1545 (ObjNum 1140 snap 356 seq_num 1140) dirty exists 2022-04-28T19:56:33.154 INFO:tasks.rados.rados.0.smithi059.stdout:3652: left oid 14 (ObjNum 1140 snap 356 seq_num 1140) 2022-04-28T19:56:33.154 INFO:tasks.rados.rados.0.smithi059.stdout:3657: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:3658: finishing copy_from to smithi05955964-38 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1580 (ObjNum -132434528 snap 21893 seq_num 4161565888) dirty exists 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:3658: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:3660: finishing rollback tid 0 to smithi05955964-42 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:3660: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1553 (ObjNum -156545328 snap 21893 seq_num 1744896) dirty dne 2022-04-28T19:56:33.758 INFO:tasks.rados.rados.0.smithi059.stdout:3652: done (5 left) 2022-04-28T19:56:33.759 INFO:tasks.rados.rados.0.smithi059.stdout:3653: done (4 left) 2022-04-28T19:56:33.759 INFO:tasks.rados.rados.0.smithi059.stdout:3654: done (3 left) 2022-04-28T19:56:33.759 INFO:tasks.rados.rados.0.smithi059.stdout:3657: done (2 left) 2022-04-28T19:56:33.759 INFO:tasks.rados.rados.0.smithi059.stdout:3658: done (1 left) 2022-04-28T19:56:33.759 INFO:tasks.rados.rados.0.smithi059.stdout:3660: done (0 left) 2022-04-28T19:56:33.759 INFO:tasks.rados.rados.0.smithi059.stdout:3661: snap_create 2022-04-28T19:56:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:33 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:33.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:34.164 INFO:tasks.rados.rados.0.smithi059.stdout:3661: done (0 left) 2022-04-28T19:56:34.164 INFO:tasks.rados.rados.0.smithi059.stdout:3662: setattr oid 49 current snap is 357 2022-04-28T19:56:34.165 INFO:tasks.rados.rados.0.smithi059.stdout:3663: rollback oid 23 current snap is 357 2022-04-28T19:56:34.165 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 352 2022-04-28T19:56:34.165 INFO:tasks.rados.rados.0.smithi059.stdout:3664: snap_remove snap 348 2022-04-28T19:56:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:34 smithi059 bash[59745]: cluster 2022-04-28T19:56:33.020002+0000 mgr.y (mgr.24961) 381 : cluster [DBG] pgmap v632: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2022-04-28T19:56:34.444 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:34 smithi059 bash[59745]: cluster 2022-04-28T19:56:33.028687+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2022-04-28T19:56:34.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:34 smithi059 bash[61152]: cluster 2022-04-28T19:56:33.020002+0000 mgr.y (mgr.24961) 381 : cluster [DBG] pgmap v632: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2022-04-28T19:56:34.445 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:34 smithi059 bash[61152]: cluster 2022-04-28T19:56:33.028687+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2022-04-28T19:56:34.445 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:34 smithi139 bash[43301]: cluster 2022-04-28T19:56:33.020002+0000 mgr.y (mgr.24961) 381 : cluster [DBG] pgmap v632: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2022-04-28T19:56:34.445 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:34 smithi139 bash[43301]: cluster 2022-04-28T19:56:33.028687+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2022-04-28T19:56:35.173 INFO:tasks.rados.rados.0.smithi059.stdout:3664: done (2 left) 2022-04-28T19:56:35.174 INFO:tasks.rados.rados.0.smithi059.stdout:3665: read oid 22 snap 288 2022-04-28T19:56:35.174 INFO:tasks.rados.rados.0.smithi059.stdout:3665: expect (ObjNum 958 snap 284 seq_num 958) 2022-04-28T19:56:35.174 INFO:tasks.rados.rados.0.smithi059.stdout:3666: snap_remove snap 340 2022-04-28T19:56:35.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:35 smithi059 bash[61152]: cluster 2022-04-28T19:56:34.162712+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2022-04-28T19:56:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:35 smithi059 bash[59745]: cluster 2022-04-28T19:56:34.162712+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2022-04-28T19:56:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:35 smithi139 bash[43301]: cluster 2022-04-28T19:56:34.162712+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2022-04-28T19:56:36.177 INFO:tasks.rados.rados.0.smithi059.stdout:3666: done (3 left) 2022-04-28T19:56:36.177 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 357 2022-04-28T19:56:36.178 INFO:tasks.rados.rados.0.smithi059.stdout:3667: seq_num 1142 ranges {2408448=729088,3137536=565248,3702784=696320,4399104=344064} 2022-04-28T19:56:36.195 INFO:tasks.rados.rados.0.smithi059.stdout:3667: writing smithi05955964-50 from 2408448 to 3137536 tid 1 2022-04-28T19:56:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3667: writing smithi05955964-50 from 3137536 to 3702784 tid 2 2022-04-28T19:56:36.202 INFO:tasks.rados.rados.0.smithi059.stdout:3667: writing smithi05955964-50 from 3702784 to 4399104 tid 3 2022-04-28T19:56:36.205 INFO:tasks.rados.rados.0.smithi059.stdout:3667: writing smithi05955964-50 from 4399104 to 4743168 tid 4 2022-04-28T19:56:36.205 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 17 current snap is 357 2022-04-28T19:56:36.205 INFO:tasks.rados.rados.0.smithi059.stdout:3668: seq_num 1143 ranges {0=753664,753664=450560,1204224=507904,1712128=589824,2301952=57344} 2022-04-28T19:56:36.208 INFO:tasks.rados.rados.0.smithi059.stdout:3668: writing smithi05955964-17 from 0 to 753664 tid 1 2022-04-28T19:56:36.211 INFO:tasks.rados.rados.0.smithi059.stdout:3668: writing smithi05955964-17 from 753664 to 1204224 tid 2 2022-04-28T19:56:36.215 INFO:tasks.rados.rados.0.smithi059.stdout:3668: writing smithi05955964-17 from 1204224 to 1712128 tid 3 2022-04-28T19:56:36.218 INFO:tasks.rados.rados.0.smithi059.stdout:3668: writing smithi05955964-17 from 1712128 to 2301952 tid 4 2022-04-28T19:56:36.220 INFO:tasks.rados.rados.0.smithi059.stdout:3668: writing smithi05955964-17 from 2301952 to 2359296 tid 5 2022-04-28T19:56:36.220 INFO:tasks.rados.rados.0.smithi059.stdout:3669: copy_from oid 38 from oid 28 current snap is 357 2022-04-28T19:56:36.220 INFO:tasks.rados.rados.0.smithi059.stdout:3663: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:56:36.221 INFO:tasks.rados.rados.0.smithi059.stdout:3663: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:56:36.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1476 (ObjNum 1056 snap 328 seq_num 1056) dirty exists 2022-04-28T19:56:36.221 INFO:tasks.rados.rados.0.smithi059.stdout:3663: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:56:36.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1385 (ObjNum 1049 snap 327 seq_num 1049) dirty exists 2022-04-28T19:56:36.221 INFO:tasks.rados.rados.0.smithi059.stdout:3665: expect (ObjNum 958 snap 284 seq_num 958) 2022-04-28T19:56:36.265 INFO:tasks.rados.rados.0.smithi059.stdout:3667: finishing write tid 1 to smithi05955964-50 2022-04-28T19:56:36.265 INFO:tasks.rados.rados.0.smithi059.stdout:3667: finishing write tid 2 to smithi05955964-50 2022-04-28T19:56:36.265 INFO:tasks.rados.rados.0.smithi059.stdout:3667: finishing write tid 3 to smithi05955964-50 2022-04-28T19:56:36.265 INFO:tasks.rados.rados.0.smithi059.stdout:3667: finishing write tid 4 to smithi05955964-50 2022-04-28T19:56:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:3667: finishing write tid 6 to smithi05955964-50 2022-04-28T19:56:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:3667: finishing write tid 7 to smithi05955964-50 2022-04-28T19:56:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1567 (ObjNum 1142 snap 357 seq_num 1142) dirty exists 2022-04-28T19:56:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:3667: left oid 50 (ObjNum 1142 snap 357 seq_num 1142) 2022-04-28T19:56:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 1 to smithi05955964-17 2022-04-28T19:56:36.266 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 2 to smithi05955964-17 2022-04-28T19:56:36.267 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 3 to smithi05955964-17 2022-04-28T19:56:36.267 INFO:tasks.rados.rados.0.smithi059.stdout:3662: done (5 left) 2022-04-28T19:56:36.267 INFO:tasks.rados.rados.0.smithi059.stdout:3663: done (4 left) 2022-04-28T19:56:36.267 INFO:tasks.rados.rados.0.smithi059.stdout:3665: done (3 left) 2022-04-28T19:56:36.268 INFO:tasks.rados.rados.0.smithi059.stdout:3667: done (2 left) 2022-04-28T19:56:36.268 INFO:tasks.rados.rados.0.smithi059.stdout:3670: delete oid 25 current snap is 357 2022-04-28T19:56:36.268 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 4 to smithi05955964-17 2022-04-28T19:56:36.268 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 5 to smithi05955964-17 2022-04-28T19:56:36.268 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 7 to smithi05955964-17 2022-04-28T19:56:36.269 INFO:tasks.rados.rados.0.smithi059.stdout:3668: finishing write tid 8 to smithi05955964-17 2022-04-28T19:56:36.269 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 17 v 1087 (ObjNum 1143 snap 357 seq_num 1143) dirty exists 2022-04-28T19:56:36.269 INFO:tasks.rados.rados.0.smithi059.stdout:3668: left oid 17 (ObjNum 1143 snap 357 seq_num 1143) 2022-04-28T19:56:36.270 INFO:tasks.rados.rados.0.smithi059.stdout:3668: done (2 left) 2022-04-28T19:56:36.270 INFO:tasks.rados.rados.0.smithi059.stdout:3670: done (1 left) 2022-04-28T19:56:36.270 INFO:tasks.rados.rados.0.smithi059.stdout:3671: snap_remove snap 352 2022-04-28T19:56:36.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:36 smithi059 bash[59745]: cluster 2022-04-28T19:56:35.020797+0000 mgr.y (mgr.24961) 382 : cluster [DBG] pgmap v635: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2022-04-28T19:56:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:36 smithi059 bash[59745]: cluster 2022-04-28T19:56:35.171127+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2022-04-28T19:56:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:36 smithi059 bash[61152]: cluster 2022-04-28T19:56:35.020797+0000 mgr.y (mgr.24961) 382 : cluster [DBG] pgmap v635: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2022-04-28T19:56:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:36 smithi059 bash[61152]: cluster 2022-04-28T19:56:35.171127+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2022-04-28T19:56:36.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:36 smithi139 bash[43301]: cluster 2022-04-28T19:56:35.020797+0000 mgr.y (mgr.24961) 382 : cluster [DBG] pgmap v635: 89 pgs: 89 active+clean; 1.0 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.1 MiB/s wr, 16 op/s 2022-04-28T19:56:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:36 smithi139 bash[43301]: cluster 2022-04-28T19:56:35.171127+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2022-04-28T19:56:37.195 INFO:tasks.rados.rados.0.smithi059.stdout:3671: done (1 left) 2022-04-28T19:56:37.195 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 357 2022-04-28T19:56:37.196 INFO:tasks.rados.rados.0.smithi059.stdout:3672: seq_num 1145 ranges {2072576=786432,2859008=622592,3481600=516096,3997696=417792,4415488=49152} 2022-04-28T19:56:37.211 INFO:tasks.rados.rados.0.smithi059.stdout:3672: writing smithi05955964-40 from 2072576 to 2859008 tid 1 2022-04-28T19:56:37.215 INFO:tasks.rados.rados.0.smithi059.stdout:3672: writing smithi05955964-40 from 2859008 to 3481600 tid 2 2022-04-28T19:56:37.218 INFO:tasks.rados.rados.0.smithi059.stdout:3672: writing smithi05955964-40 from 3481600 to 3997696 tid 3 2022-04-28T19:56:37.221 INFO:tasks.rados.rados.0.smithi059.stdout:3672: writing smithi05955964-40 from 3997696 to 4415488 tid 4 2022-04-28T19:56:37.222 INFO:tasks.rados.rados.0.smithi059.stdout:3672: writing smithi05955964-40 from 4415488 to 4464640 tid 5 2022-04-28T19:56:37.222 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 357 2022-04-28T19:56:37.222 INFO:tasks.rados.rados.0.smithi059.stdout:3673: seq_num 1146 ranges {892928=802816,1695744=499712,2195456=679936,2875392=425984} 2022-04-28T19:56:37.231 INFO:tasks.rados.rados.0.smithi059.stdout:3673: writing smithi05955964-49 from 892928 to 1695744 tid 1 2022-04-28T19:56:37.234 INFO:tasks.rados.rados.0.smithi059.stdout:3673: writing smithi05955964-49 from 1695744 to 2195456 tid 2 2022-04-28T19:56:37.238 INFO:tasks.rados.rados.0.smithi059.stdout:3673: writing smithi05955964-49 from 2195456 to 2875392 tid 3 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3673: writing smithi05955964-49 from 2875392 to 3301376 tid 4 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3674: read oid 42 snap 324 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3674: expect deleted 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3675: delete oid 48 current snap is 357 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3669: finishing copy_from to smithi05955964-38 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1451 (ObjNum 1096 snap 339 seq_num 1096) dirty exists 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3669: finishing copy_from racing read to smithi05955964-38 2022-04-28T19:56:37.242 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 1 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 2 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 3 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 4 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 5 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 7 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: finishing write tid 8 to smithi05955964-40 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1482 (ObjNum 1145 snap 357 seq_num 1145) dirty exists 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3672: left oid 40 (ObjNum 1145 snap 357 seq_num 1145) 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3673: finishing write tid 1 to smithi05955964-49 2022-04-28T19:56:37.243 INFO:tasks.rados.rados.0.smithi059.stdout:3673: finishing write tid 2 to smithi05955964-49 2022-04-28T19:56:37.244 INFO:tasks.rados.rados.0.smithi059.stdout:3673: finishing write tid 3 to smithi05955964-49 2022-04-28T19:56:37.244 INFO:tasks.rados.rados.0.smithi059.stdout:3673: finishing write tid 4 to smithi05955964-49 2022-04-28T19:56:37.244 INFO:tasks.rados.rados.0.smithi059.stdout:3673: finishing write tid 6 to smithi05955964-49 2022-04-28T19:56:37.245 INFO:tasks.rados.rados.0.smithi059.stdout:3673: finishing write tid 7 to smithi05955964-49 2022-04-28T19:56:37.246 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1487 (ObjNum 1146 snap 357 seq_num 1146) dirty exists 2022-04-28T19:56:37.246 INFO:tasks.rados.rados.0.smithi059.stdout:3673: left oid 49 (ObjNum 1146 snap 357 seq_num 1146) 2022-04-28T19:56:37.247 INFO:tasks.rados.rados.0.smithi059.stdout:3669: done (4 left) 2022-04-28T19:56:37.247 INFO:tasks.rados.rados.0.smithi059.stdout:3672: done (3 left) 2022-04-28T19:56:37.248 INFO:tasks.rados.rados.0.smithi059.stdout:3673: done (2 left) 2022-04-28T19:56:37.248 INFO:tasks.rados.rados.0.smithi059.stdout:3674: done (1 left) 2022-04-28T19:56:37.248 INFO:tasks.rados.rados.0.smithi059.stdout:3675: done (0 left) 2022-04-28T19:56:37.248 INFO:tasks.rados.rados.0.smithi059.stdout:3676: snap_remove snap 354 2022-04-28T19:56:37.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:37 smithi059 bash[59745]: cluster 2022-04-28T19:56:36.176807+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2022-04-28T19:56:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:37 smithi059 bash[61152]: cluster 2022-04-28T19:56:36.176807+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2022-04-28T19:56:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:37 smithi139 bash[43301]: cluster 2022-04-28T19:56:36.176807+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:3676: done (0 left) 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:3677: rmattr oid 2 current snap is 357 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:3677: done (0 left) 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 3 current snap is 357 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:3678: seq_num 1148 ranges {0=32768} 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:3678: writing smithi05955964-3 from 0 to 32768 tid 1 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:3679: rollback oid 43 current snap is 357 2022-04-28T19:56:38.200 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 43 to 347 2022-04-28T19:56:38.201 INFO:tasks.rados.rados.0.smithi059.stdout:3680: rollback oid 18 current snap is 357 2022-04-28T19:56:38.201 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 18 to 347 2022-04-28T19:56:38.201 INFO:tasks.rados.rados.0.smithi059.stdout:3681: snap_create 2022-04-28T19:56:38.206 INFO:tasks.rados.rados.0.smithi059.stdout:3679: finishing rollback tid 1 to smithi05955964-43 2022-04-28T19:56:38.206 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1562 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty dne 2022-04-28T19:56:38.207 INFO:tasks.rados.rados.0.smithi059.stdout:3678: finishing write tid 1 to smithi05955964-3 2022-04-28T19:56:38.207 INFO:tasks.rados.rados.0.smithi059.stdout:3678: finishing write tid 3 to smithi05955964-3 2022-04-28T19:56:38.209 INFO:tasks.rados.rados.0.smithi059.stdout:3680: finishing rollback tid 0 to smithi05955964-18 2022-04-28T19:56:38.209 INFO:tasks.rados.rados.0.smithi059.stdout:3680: finishing rollback tid 1 to smithi05955964-18 2022-04-28T19:56:38.210 INFO:tasks.rados.rados.0.smithi059.stdout:3678: finishing write tid 4 to smithi05955964-3 2022-04-28T19:56:38.210 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1307 (ObjNum 1148 snap 357 seq_num 1148) dirty exists 2022-04-28T19:56:38.211 INFO:tasks.rados.rados.0.smithi059.stdout:3678: left oid 3 (ObjNum 1148 snap 357 seq_num 1148) 2022-04-28T19:56:38.211 INFO:tasks.rados.rados.0.smithi059.stdout:3680: finishing rollback tid 2 to smithi05955964-18 2022-04-28T19:56:38.211 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1556 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:56:38.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:38 smithi059 bash[59745]: cluster 2022-04-28T19:56:37.021483+0000 mgr.y (mgr.24961) 383 : cluster [DBG] pgmap v638: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.6 MiB/s wr, 21 op/s 2022-04-28T19:56:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:38 smithi059 bash[59745]: cluster 2022-04-28T19:56:37.194080+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2022-04-28T19:56:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:38 smithi059 bash[61152]: cluster 2022-04-28T19:56:37.021483+0000 mgr.y (mgr.24961) 383 : cluster [DBG] pgmap v638: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.6 MiB/s wr, 21 op/s 2022-04-28T19:56:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:38 smithi059 bash[61152]: cluster 2022-04-28T19:56:37.194080+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2022-04-28T19:56:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:38 smithi139 bash[43301]: cluster 2022-04-28T19:56:37.021483+0000 mgr.y (mgr.24961) 383 : cluster [DBG] pgmap v638: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.6 MiB/s wr, 21 op/s 2022-04-28T19:56:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:38 smithi139 bash[43301]: cluster 2022-04-28T19:56:37.194080+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2022-04-28T19:56:39.211 INFO:tasks.rados.rados.0.smithi059.stdout:3678: done (3 left) 2022-04-28T19:56:39.212 INFO:tasks.rados.rados.0.smithi059.stdout:3679: done (2 left) 2022-04-28T19:56:39.212 INFO:tasks.rados.rados.0.smithi059.stdout:3680: done (1 left) 2022-04-28T19:56:39.212 INFO:tasks.rados.rados.0.smithi059.stdout:3681: done (0 left) 2022-04-28T19:56:39.212 INFO:tasks.rados.rados.0.smithi059.stdout:3682: copy_from oid 1 from oid 29 current snap is 358 2022-04-28T19:56:39.212 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 358 2022-04-28T19:56:39.212 INFO:tasks.rados.rados.0.smithi059.stdout:3683: seq_num 1149 ranges {3301376=49152} 2022-04-28T19:56:39.234 INFO:tasks.rados.rados.0.smithi059.stdout:3683: writing smithi05955964-49 from 3301376 to 3350528 tid 1 2022-04-28T19:56:39.289 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 358 2022-04-28T19:56:39.289 INFO:tasks.rados.rados.0.smithi059.stdout:3684: seq_num 1150 ranges {966656=65536} 2022-04-28T19:56:39.291 INFO:tasks.rados.rados.0.smithi059.stdout:3684: writing smithi05955964-23 from 966656 to 1032192 tid 1 2022-04-28T19:56:39.291 INFO:tasks.rados.rados.0.smithi059.stdout:3685: rmattr oid 37 current snap is 358 2022-04-28T19:56:39.291 INFO:tasks.rados.rados.0.smithi059.stdout:3682: finishing copy_from to smithi05955964-1 2022-04-28T19:56:39.291 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1564 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3686: snap_create 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3682: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3683: finishing write tid 1 to smithi05955964-49 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3683: finishing write tid 3 to smithi05955964-49 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3683: finishing write tid 4 to smithi05955964-49 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1566 (ObjNum 1149 snap 358 seq_num 1149) dirty exists 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3683: left oid 49 (ObjNum 1149 snap 358 seq_num 1149) 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1557 (ObjNum 1052 snap 328 seq_num 1052) dirty exists 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3684: finishing write tid 1 to smithi05955964-23 2022-04-28T19:56:39.292 INFO:tasks.rados.rados.0.smithi059.stdout:3684: finishing write tid 3 to smithi05955964-23 2022-04-28T19:56:39.293 INFO:tasks.rados.rados.0.smithi059.stdout:3684: finishing write tid 4 to smithi05955964-23 2022-04-28T19:56:39.293 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1387 (ObjNum 1150 snap 358 seq_num 1150) dirty exists 2022-04-28T19:56:39.293 INFO:tasks.rados.rados.0.smithi059.stdout:3684: left oid 23 (ObjNum 1150 snap 358 seq_num 1150) 2022-04-28T19:56:39.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:39 smithi059 bash[59745]: cluster 2022-04-28T19:56:38.198628+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2022-04-28T19:56:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:39 smithi059 bash[61152]: cluster 2022-04-28T19:56:38.198628+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2022-04-28T19:56:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:39 smithi139 bash[43301]: cluster 2022-04-28T19:56:38.198628+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2022-04-28T19:56:40.223 INFO:tasks.rados.rados.0.smithi059.stdout:3682: done (4 left) 2022-04-28T19:56:40.223 INFO:tasks.rados.rados.0.smithi059.stdout:3683: done (3 left) 2022-04-28T19:56:40.223 INFO:tasks.rados.rados.0.smithi059.stdout:3684: done (2 left) 2022-04-28T19:56:40.223 INFO:tasks.rados.rados.0.smithi059.stdout:3685: done (1 left) 2022-04-28T19:56:40.223 INFO:tasks.rados.rados.0.smithi059.stdout:3686: done (0 left) 2022-04-28T19:56:40.223 INFO:tasks.rados.rados.0.smithi059.stdout:3687: snap_create 2022-04-28T19:56:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:40 smithi059 bash[61152]: cluster 2022-04-28T19:56:39.022298+0000 mgr.y (mgr.24961) 384 : cluster [DBG] pgmap v641: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2022-04-28T19:56:40.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:40 smithi059 bash[61152]: cluster 2022-04-28T19:56:39.208791+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2022-04-28T19:56:40.602 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:40 smithi059 bash[59745]: cluster 2022-04-28T19:56:39.022298+0000 mgr.y (mgr.24961) 384 : cluster [DBG] pgmap v641: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2022-04-28T19:56:40.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:40 smithi059 bash[59745]: cluster 2022-04-28T19:56:39.208791+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2022-04-28T19:56:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:40 smithi139 bash[43301]: cluster 2022-04-28T19:56:39.022298+0000 mgr.y (mgr.24961) 384 : cluster [DBG] pgmap v641: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.1 MiB/s wr, 6 op/s 2022-04-28T19:56:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:40 smithi139 bash[43301]: cluster 2022-04-28T19:56:39.208791+0000 mon.a (mon.0) 1418 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2022-04-28T19:56:41.237 INFO:tasks.rados.rados.0.smithi059.stdout:3687: done (0 left) 2022-04-28T19:56:41.237 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 48 current snap is 360 2022-04-28T19:56:41.237 INFO:tasks.rados.rados.0.smithi059.stdout:3688: seq_num 1151 ranges {0=81920} 2022-04-28T19:56:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:3688: writing smithi05955964-48 from 0 to 81920 tid 1 2022-04-28T19:56:41.238 INFO:tasks.rados.rados.0.smithi059.stdout:3689: snap_create 2022-04-28T19:56:41.242 INFO:tasks.rados.rados.0.smithi059.stdout:3688: finishing write tid 1 to smithi05955964-48 2022-04-28T19:56:41.242 INFO:tasks.rados.rados.0.smithi059.stdout:3688: finishing write tid 3 to smithi05955964-48 2022-04-28T19:56:41.243 INFO:tasks.rados.rados.0.smithi059.stdout:3688: finishing write tid 4 to smithi05955964-48 2022-04-28T19:56:41.244 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1551 (ObjNum 1151 snap 360 seq_num 1151) dirty exists 2022-04-28T19:56:41.244 INFO:tasks.rados.rados.0.smithi059.stdout:3688: left oid 48 (ObjNum 1151 snap 360 seq_num 1151) 2022-04-28T19:56:41.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:41 smithi059 bash[61152]: cluster 2022-04-28T19:56:40.221769+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2022-04-28T19:56:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:41 smithi059 bash[59745]: cluster 2022-04-28T19:56:40.221769+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2022-04-28T19:56:41.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:41 smithi139 bash[43301]: cluster 2022-04-28T19:56:40.221769+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2022-04-28T19:56:42.021 INFO:tasks.rados.rados.0.smithi059.stdout:3688: done (1 left) 2022-04-28T19:56:42.021 INFO:tasks.rados.rados.0.smithi059.stdout:3689: done (0 left) 2022-04-28T19:56:42.021 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 31 current snap is 361 2022-04-28T19:56:42.021 INFO:tasks.rados.rados.0.smithi059.stdout:3690: seq_num 1152 ranges {6111232=98304} 2022-04-28T19:56:42.057 INFO:tasks.rados.rados.0.smithi059.stdout:3690: writing smithi05955964-31 from 6111232 to 6209536 tid 1 2022-04-28T19:56:42.057 INFO:tasks.rados.rados.0.smithi059.stdout:3691: copy_from oid 50 from oid 15 current snap is 361 2022-04-28T19:56:42.057 INFO:tasks.rados.rados.0.smithi059.stdout:3692: snap_create 2022-04-28T19:56:42.066 INFO:tasks.rados.rados.0.smithi059.stdout:3690: finishing write tid 1 to smithi05955964-31 2022-04-28T19:56:42.067 INFO:tasks.rados.rados.0.smithi059.stdout:3690: finishing write tid 3 to smithi05955964-31 2022-04-28T19:56:42.075 INFO:tasks.rados.rados.0.smithi059.stdout:3690: finishing write tid 4 to smithi05955964-31 2022-04-28T19:56:42.075 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1396 (ObjNum 1152 snap 361 seq_num 1152) dirty exists 2022-04-28T19:56:42.076 INFO:tasks.rados.rados.0.smithi059.stdout:3690: left oid 31 (ObjNum 1152 snap 361 seq_num 1152) 2022-04-28T19:56:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:41 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:41.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:42.122 INFO:tasks.rados.rados.0.smithi059.stdout:3691: finishing copy_from to smithi05955964-50 2022-04-28T19:56:42.122 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1382 (ObjNum 1107 snap 342 seq_num 1107) dirty exists 2022-04-28T19:56:42.122 INFO:tasks.rados.rados.0.smithi059.stdout:3691: finishing copy_from racing read to smithi05955964-50 2022-04-28T19:56:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:42 smithi059 bash[61152]: cluster 2022-04-28T19:56:41.023216+0000 mgr.y (mgr.24961) 385 : cluster [DBG] pgmap v644: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.8 MiB/s wr, 11 op/s 2022-04-28T19:56:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:42 smithi059 bash[61152]: cluster 2022-04-28T19:56:41.234357+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2022-04-28T19:56:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:42 smithi059 bash[61152]: cluster 2022-04-28T19:56:42.020746+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2022-04-28T19:56:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:42 smithi059 bash[59745]: cluster 2022-04-28T19:56:41.023216+0000 mgr.y (mgr.24961) 385 : cluster [DBG] pgmap v644: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.8 MiB/s wr, 11 op/s 2022-04-28T19:56:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:42 smithi059 bash[59745]: cluster 2022-04-28T19:56:41.234357+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2022-04-28T19:56:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:42 smithi059 bash[59745]: cluster 2022-04-28T19:56:42.020746+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2022-04-28T19:56:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:42 smithi139 bash[43301]: cluster 2022-04-28T19:56:41.023216+0000 mgr.y (mgr.24961) 385 : cluster [DBG] pgmap v644: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.8 MiB/s wr, 11 op/s 2022-04-28T19:56:42.659 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:42 smithi139 bash[43301]: cluster 2022-04-28T19:56:41.234357+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2022-04-28T19:56:42.659 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:42 smithi139 bash[43301]: cluster 2022-04-28T19:56:42.020746+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2022-04-28T19:56:43.039 INFO:tasks.rados.rados.0.smithi059.stdout:3690: done (2 left) 2022-04-28T19:56:43.039 INFO:tasks.rados.rados.0.smithi059.stdout:3691: done (1 left) 2022-04-28T19:56:43.039 INFO:tasks.rados.rados.0.smithi059.stdout:3692: done (0 left) 2022-04-28T19:56:43.039 INFO:tasks.rados.rados.0.smithi059.stdout:3693: rollback oid 31 current snap is 362 2022-04-28T19:56:43.039 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 331 2022-04-28T19:56:43.040 INFO:tasks.rados.rados.0.smithi059.stdout:3694: read oid 29 snap -1 2022-04-28T19:56:43.040 INFO:tasks.rados.rados.0.smithi059.stdout:3694: expect (ObjNum 949 snap 279 seq_num 949) 2022-04-28T19:56:43.040 INFO:tasks.rados.rados.0.smithi059.stdout:3695: rollback oid 3 current snap is 362 2022-04-28T19:56:43.041 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 344 2022-04-28T19:56:43.041 INFO:tasks.rados.rados.0.smithi059.stdout:3696: setattr oid 50 current snap is 362 2022-04-28T19:56:43.041 INFO:tasks.rados.rados.0.smithi059.stdout:3697: rollback oid 8 current snap is 362 2022-04-28T19:56:43.041 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 8 to 345 2022-04-28T19:56:43.042 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 362 2022-04-28T19:56:43.043 INFO:tasks.rados.rados.0.smithi059.stdout:3698: seq_num 1153 ranges {4464640=114688} 2022-04-28T19:56:43.065 INFO:tasks.rados.rados.0.smithi059.stdout:3698: writing smithi05955964-40 from 4464640 to 4579328 tid 1 2022-04-28T19:56:43.065 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 362 2022-04-28T19:56:43.066 INFO:tasks.rados.rados.0.smithi059.stdout:3699: seq_num 1154 ranges {0=131072} 2022-04-28T19:56:43.066 INFO:tasks.rados.rados.0.smithi059.stdout:3699: writing smithi05955964-7 from 0 to 131072 tid 1 2022-04-28T19:56:43.066 INFO:tasks.rados.rados.0.smithi059.stdout:3700: rmattr oid 1 current snap is 362 2022-04-28T19:56:43.066 INFO:tasks.rados.rados.0.smithi059.stdout:3693: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:56:43.066 INFO:tasks.rados.rados.0.smithi059.stdout:3693: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:3693: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1399 (ObjNum 1045 snap 324 seq_num 1045) dirty exists 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:3695: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:3695: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:3695: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1310 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2022-04-28T19:56:43.067 INFO:tasks.rados.rados.0.smithi059.stdout:3694: expect (ObjNum 949 snap 279 seq_num 949) 2022-04-28T19:56:43.130 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1568 (ObjNum 1107 snap 342 seq_num 1107) dirty exists 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:3697: finishing rollback tid 0 to smithi05955964-8 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:3697: finishing rollback tid 1 to smithi05955964-8 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:3697: finishing rollback tid 2 to smithi05955964-8 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1587 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:3698: finishing write tid 1 to smithi05955964-40 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:3698: finishing write tid 3 to smithi05955964-40 2022-04-28T19:56:43.131 INFO:tasks.rados.rados.0.smithi059.stdout:3699: finishing write tid 1 to smithi05955964-7 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:3699: finishing write tid 3 to smithi05955964-7 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:3698: finishing write tid 4 to smithi05955964-40 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1568 (ObjNum 1153 snap 362 seq_num 1153) dirty exists 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:3698: left oid 40 (ObjNum 1153 snap 362 seq_num 1153) 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1569 (ObjNum 949 snap 279 seq_num 949) dirty exists 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:3699: finishing write tid 4 to smithi05955964-7 2022-04-28T19:56:43.132 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 1401 (ObjNum 1154 snap 362 seq_num 1154) dirty exists 2022-04-28T19:56:43.133 INFO:tasks.rados.rados.0.smithi059.stdout:3699: left oid 7 (ObjNum 1154 snap 362 seq_num 1154) 2022-04-28T19:56:43.133 INFO:tasks.rados.rados.0.smithi059.stdout:3693: done (7 left) 2022-04-28T19:56:43.133 INFO:tasks.rados.rados.0.smithi059.stdout:3694: done (6 left) 2022-04-28T19:56:43.133 INFO:tasks.rados.rados.0.smithi059.stdout:3695: done (5 left) 2022-04-28T19:56:43.133 INFO:tasks.rados.rados.0.smithi059.stdout:3696: done (4 left) 2022-04-28T19:56:43.133 INFO:tasks.rados.rados.0.smithi059.stdout:3697: done (3 left) 2022-04-28T19:56:43.134 INFO:tasks.rados.rados.0.smithi059.stdout:3698: done (2 left) 2022-04-28T19:56:43.134 INFO:tasks.rados.rados.0.smithi059.stdout:3699: done (1 left) 2022-04-28T19:56:43.134 INFO:tasks.rados.rados.0.smithi059.stdout:3700: done (0 left) 2022-04-28T19:56:43.134 INFO:tasks.rados.rados.0.smithi059.stdout:3701: copy_from oid 3 from oid 25 current snap is 362 2022-04-28T19:56:43.134 INFO:tasks.rados.rados.0.smithi059.stdout:3702: setattr oid 49 current snap is 362 2022-04-28T19:56:43.136 INFO:tasks.rados.rados.0.smithi059.stdout:3701: finishing copy_from to smithi05955964-3 2022-04-28T19:56:43.136 INFO:tasks.rados.rados.0.smithi059.stdout:3701: got expected ENOENT (src dne) 2022-04-28T19:56:43.136 INFO:tasks.rados.rados.0.smithi059.stdout:3701: finishing copy_from racing read to smithi05955964-3 2022-04-28T19:56:43.136 INFO:tasks.rados.rados.0.smithi059.stdout:3701: done (1 left) 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:3703: rollback oid 1 current snap is 362 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 311 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:3704: read oid 36 snap -1 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:3704: expect (ObjNum 1138 snap 355 seq_num 1138) 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:3705: rollback oid 2 current snap is 362 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 2 to 338 2022-04-28T19:56:43.137 INFO:tasks.rados.rados.0.smithi059.stdout:3706: rollback oid 33 current snap is 362 2022-04-28T19:56:43.138 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 33 to 351 2022-04-28T19:56:43.138 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 362 2022-04-28T19:56:43.138 INFO:tasks.rados.rados.0.smithi059.stdout:3707: seq_num 1155 ranges {2056192=147456} 2022-04-28T19:56:43.149 INFO:tasks.rados.rados.0.smithi059.stdout:3707: writing smithi05955964-47 from 2056192 to 2203648 tid 1 2022-04-28T19:56:43.149 INFO:tasks.rados.rados.0.smithi059.stdout:3708: snap_create 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3705: finishing rollback tid 0 to smithi05955964-2 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3705: finishing rollback tid 1 to smithi05955964-2 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3705: finishing rollback tid 2 to smithi05955964-2 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1467 (ObjNum 1090 snap 336 seq_num 1090) dirty exists 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3706: finishing rollback tid 0 to smithi05955964-33 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3706: finishing rollback tid 1 to smithi05955964-33 2022-04-28T19:56:43.150 INFO:tasks.rados.rados.0.smithi059.stdout:3706: finishing rollback tid 2 to smithi05955964-33 2022-04-28T19:56:43.152 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1571 (ObjNum 1080 snap 335 seq_num 1080) dirty exists 2022-04-28T19:56:43.152 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1570 (ObjNum 1149 snap 358 seq_num 1149) dirty exists 2022-04-28T19:56:43.152 INFO:tasks.rados.rados.0.smithi059.stdout:3703: finishing rollback tid 0 to smithi05955964-1 2022-04-28T19:56:43.152 INFO:tasks.rados.rados.0.smithi059.stdout:3703: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:56:43.152 INFO:tasks.rados.rados.0.smithi059.stdout:3703: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:56:43.152 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1573 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:56:43.153 INFO:tasks.rados.rados.0.smithi059.stdout:3704: expect (ObjNum 1138 snap 355 seq_num 1138) 2022-04-28T19:56:43.207 INFO:tasks.rados.rados.0.smithi059.stdout:3707: finishing write tid 1 to smithi05955964-47 2022-04-28T19:56:43.207 INFO:tasks.rados.rados.0.smithi059.stdout:3707: finishing write tid 3 to smithi05955964-47 2022-04-28T19:56:43.207 INFO:tasks.rados.rados.0.smithi059.stdout:3707: finishing write tid 4 to smithi05955964-47 2022-04-28T19:56:43.207 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1389 (ObjNum 1155 snap 362 seq_num 1155) dirty exists 2022-04-28T19:56:43.208 INFO:tasks.rados.rados.0.smithi059.stdout:3707: left oid 47 (ObjNum 1155 snap 362 seq_num 1155) 2022-04-28T19:56:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:43 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:43.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:44.036 INFO:tasks.rados.rados.0.smithi059.stdout:3702: done (6 left) 2022-04-28T19:56:44.036 INFO:tasks.rados.rados.0.smithi059.stdout:3703: done (5 left) 2022-04-28T19:56:44.037 INFO:tasks.rados.rados.0.smithi059.stdout:3704: done (4 left) 2022-04-28T19:56:44.037 INFO:tasks.rados.rados.0.smithi059.stdout:3705: done (3 left) 2022-04-28T19:56:44.037 INFO:tasks.rados.rados.0.smithi059.stdout:3706: done (2 left) 2022-04-28T19:56:44.037 INFO:tasks.rados.rados.0.smithi059.stdout:3707: done (1 left) 2022-04-28T19:56:44.037 INFO:tasks.rados.rados.0.smithi059.stdout:3708: done (0 left) 2022-04-28T19:56:44.037 INFO:tasks.rados.rados.0.smithi059.stdout:3709: snap_create 2022-04-28T19:56:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:44 smithi059 bash[61152]: cluster 2022-04-28T19:56:43.023925+0000 mgr.y (mgr.24961) 386 : cluster [DBG] pgmap v647: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.3 MiB/s wr, 9 op/s 2022-04-28T19:56:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:44 smithi059 bash[61152]: cluster 2022-04-28T19:56:43.035114+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2022-04-28T19:56:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:44 smithi059 bash[59745]: cluster 2022-04-28T19:56:43.023925+0000 mgr.y (mgr.24961) 386 : cluster [DBG] pgmap v647: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.3 MiB/s wr, 9 op/s 2022-04-28T19:56:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:44 smithi059 bash[59745]: cluster 2022-04-28T19:56:43.035114+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2022-04-28T19:56:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:44 smithi139 bash[43301]: cluster 2022-04-28T19:56:43.023925+0000 mgr.y (mgr.24961) 386 : cluster [DBG] pgmap v647: 89 pgs: 89 active+clean; 1.1 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.3 MiB/s wr, 9 op/s 2022-04-28T19:56:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:44 smithi139 bash[43301]: cluster 2022-04-28T19:56:43.035114+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2022-04-28T19:56:45.046 INFO:tasks.rados.rados.0.smithi059.stdout:3709: done (0 left) 2022-04-28T19:56:45.046 INFO:tasks.rados.rados.0.smithi059.stdout:3710: copy_from oid 12 from oid 13 current snap is 364 2022-04-28T19:56:45.046 INFO:tasks.rados.rados.0.smithi059.stdout:3711: snap_create 2022-04-28T19:56:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:3710: finishing copy_from to smithi05955964-12 2022-04-28T19:56:45.065 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1080 (ObjNum 1132 snap 352 seq_num 1132) dirty exists 2022-04-28T19:56:45.066 INFO:tasks.rados.rados.0.smithi059.stdout:3710: finishing copy_from racing read to smithi05955964-12 2022-04-28T19:56:45.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:45 smithi059 bash[61152]: cluster 2022-04-28T19:56:44.032924+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2022-04-28T19:56:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:45 smithi059 bash[59745]: cluster 2022-04-28T19:56:44.032924+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2022-04-28T19:56:45.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:45 smithi139 bash[43301]: cluster 2022-04-28T19:56:44.032924+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2022-04-28T19:56:46.058 INFO:tasks.rados.rados.0.smithi059.stdout:3710: done (1 left) 2022-04-28T19:56:46.058 INFO:tasks.rados.rados.0.smithi059.stdout:3711: done (0 left) 2022-04-28T19:56:46.058 INFO:tasks.rados.rados.0.smithi059.stdout:3712: rollback oid 8 current snap is 365 2022-04-28T19:56:46.059 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 8 to 356 2022-04-28T19:56:46.059 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 365 2022-04-28T19:56:46.059 INFO:tasks.rados.rados.0.smithi059.stdout:3713: seq_num 1156 ranges {1548288=163840} 2022-04-28T19:56:46.070 INFO:tasks.rados.rados.0.smithi059.stdout:3713: writing smithi05955964-32 from 1548288 to 1712128 tid 1 2022-04-28T19:56:46.070 INFO:tasks.rados.rados.0.smithi059.stdout:3714: snap_create 2022-04-28T19:56:46.071 INFO:tasks.rados.rados.0.smithi059.stdout:3712: finishing rollback tid 0 to smithi05955964-8 2022-04-28T19:56:46.071 INFO:tasks.rados.rados.0.smithi059.stdout:3712: finishing rollback tid 1 to smithi05955964-8 2022-04-28T19:56:46.071 INFO:tasks.rados.rados.0.smithi059.stdout:3712: finishing rollback tid 2 to smithi05955964-8 2022-04-28T19:56:46.071 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1591 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:56:46.074 INFO:tasks.rados.rados.0.smithi059.stdout:3713: finishing write tid 1 to smithi05955964-32 2022-04-28T19:56:46.074 INFO:tasks.rados.rados.0.smithi059.stdout:3713: finishing write tid 3 to smithi05955964-32 2022-04-28T19:56:46.075 INFO:tasks.rados.rados.0.smithi059.stdout:3713: finishing write tid 4 to smithi05955964-32 2022-04-28T19:56:46.076 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1573 (ObjNum 1156 snap 365 seq_num 1156) dirty exists 2022-04-28T19:56:46.076 INFO:tasks.rados.rados.0.smithi059.stdout:3713: left oid 32 (ObjNum 1156 snap 365 seq_num 1156) 2022-04-28T19:56:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:46 smithi059 bash[59745]: cluster 2022-04-28T19:56:45.024781+0000 mgr.y (mgr.24961) 387 : cluster [DBG] pgmap v650: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2022-04-28T19:56:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:46 smithi059 bash[59745]: cluster 2022-04-28T19:56:45.045496+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2022-04-28T19:56:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:46 smithi059 bash[61152]: cluster 2022-04-28T19:56:45.024781+0000 mgr.y (mgr.24961) 387 : cluster [DBG] pgmap v650: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2022-04-28T19:56:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:46 smithi059 bash[61152]: cluster 2022-04-28T19:56:45.045496+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2022-04-28T19:56:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:46 smithi139 bash[43301]: cluster 2022-04-28T19:56:45.024781+0000 mgr.y (mgr.24961) 387 : cluster [DBG] pgmap v650: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2022-04-28T19:56:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:46 smithi139 bash[43301]: cluster 2022-04-28T19:56:45.045496+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2022-04-28T19:56:47.073 INFO:tasks.rados.rados.0.smithi059.stdout:3712: done (2 left) 2022-04-28T19:56:47.073 INFO:tasks.rados.rados.0.smithi059.stdout:3713: done (1 left) 2022-04-28T19:56:47.073 INFO:tasks.rados.rados.0.smithi059.stdout:3714: done (0 left) 2022-04-28T19:56:47.074 INFO:tasks.rados.rados.0.smithi059.stdout:3715: rmattr oid 22 current snap is 366 2022-04-28T19:56:47.074 INFO:tasks.rados.rados.0.smithi059.stdout:3715: done (0 left) 2022-04-28T19:56:47.074 INFO:tasks.rados.rados.0.smithi059.stdout:3716: read oid 10 snap -1 2022-04-28T19:56:47.074 INFO:tasks.rados.rados.0.smithi059.stdout:3716: expect deleted 2022-04-28T19:56:47.074 INFO:tasks.rados.rados.0.smithi059.stdout:3717: delete oid 6 current snap is 366 2022-04-28T19:56:47.075 INFO:tasks.rados.rados.0.smithi059.stdout:3716: done (1 left) 2022-04-28T19:56:47.075 INFO:tasks.rados.rados.0.smithi059.stdout:3717: done (0 left) 2022-04-28T19:56:47.075 INFO:tasks.rados.rados.0.smithi059.stdout:3718: snap_remove snap 349 2022-04-28T19:56:47.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:47 smithi059 bash[61152]: cluster 2022-04-28T19:56:46.056501+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2022-04-28T19:56:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:47 smithi059 bash[59745]: cluster 2022-04-28T19:56:46.056501+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2022-04-28T19:56:47.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:47 smithi139 bash[43301]: cluster 2022-04-28T19:56:46.056501+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2022-04-28T19:56:48.086 INFO:tasks.rados.rados.0.smithi059.stdout:3718: done (0 left) 2022-04-28T19:56:48.086 INFO:tasks.rados.rados.0.smithi059.stdout:3719: snap_remove snap 360 2022-04-28T19:56:48.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:48 smithi059 bash[59745]: cluster 2022-04-28T19:56:47.025432+0000 mgr.y (mgr.24961) 388 : cluster [DBG] pgmap v653: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2022-04-28T19:56:48.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:48 smithi059 bash[59745]: cluster 2022-04-28T19:56:47.071915+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2022-04-28T19:56:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:48 smithi059 bash[61152]: cluster 2022-04-28T19:56:47.025432+0000 mgr.y (mgr.24961) 388 : cluster [DBG] pgmap v653: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2022-04-28T19:56:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:48 smithi059 bash[61152]: cluster 2022-04-28T19:56:47.071915+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2022-04-28T19:56:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:48 smithi139 bash[43301]: cluster 2022-04-28T19:56:47.025432+0000 mgr.y (mgr.24961) 388 : cluster [DBG] pgmap v653: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2022-04-28T19:56:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:48 smithi139 bash[43301]: cluster 2022-04-28T19:56:47.071915+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2022-04-28T19:56:49.090 INFO:tasks.rados.rados.0.smithi059.stdout:3719: done (0 left) 2022-04-28T19:56:49.094 INFO:tasks.rados.rados.0.smithi059.stdout:3720: read oid 22 snap -1 2022-04-28T19:56:49.095 INFO:tasks.rados.rados.0.smithi059.stdout:3720: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:56:49.095 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 6 current snap is 366 2022-04-28T19:56:49.095 INFO:tasks.rados.rados.0.smithi059.stdout:3721: seq_num 1158 ranges {0=196608} 2022-04-28T19:56:49.096 INFO:tasks.rados.rados.0.smithi059.stdout:3721: writing smithi05955964-6 from 0 to 196608 tid 1 2022-04-28T19:56:49.096 INFO:tasks.rados.rados.0.smithi059.stdout:3722: snap_create 2022-04-28T19:56:49.098 INFO:tasks.rados.rados.0.smithi059.stdout:3721: finishing write tid 1 to smithi05955964-6 2022-04-28T19:56:49.098 INFO:tasks.rados.rados.0.smithi059.stdout:3721: finishing write tid 3 to smithi05955964-6 2022-04-28T19:56:49.099 INFO:tasks.rados.rados.0.smithi059.stdout:3721: finishing write tid 4 to smithi05955964-6 2022-04-28T19:56:49.100 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1312 (ObjNum 1158 snap 366 seq_num 1158) dirty exists 2022-04-28T19:56:49.100 INFO:tasks.rados.rados.0.smithi059.stdout:3721: left oid 6 (ObjNum 1158 snap 366 seq_num 1158) 2022-04-28T19:56:49.107 INFO:tasks.rados.rados.0.smithi059.stdout:3720: expect (ObjNum 988 snap 293 seq_num 988) 2022-04-28T19:56:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:49 smithi139 bash[43301]: cluster 2022-04-28T19:56:48.084121+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2022-04-28T19:56:49.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:49 smithi059 bash[59745]: cluster 2022-04-28T19:56:48.084121+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2022-04-28T19:56:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:49 smithi059 bash[61152]: cluster 2022-04-28T19:56:48.084121+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2022-04-28T19:56:50.103 INFO:tasks.rados.rados.0.smithi059.stdout:3720: done (2 left) 2022-04-28T19:56:50.103 INFO:tasks.rados.rados.0.smithi059.stdout:3721: done (1 left) 2022-04-28T19:56:50.103 INFO:tasks.rados.rados.0.smithi059.stdout:3722: done (0 left) 2022-04-28T19:56:50.103 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 367 2022-04-28T19:56:50.103 INFO:tasks.rados.rados.0.smithi059.stdout:3723: seq_num 1159 ranges {0=212992} 2022-04-28T19:56:50.104 INFO:tasks.rados.rados.0.smithi059.stdout:3723: writing smithi05955964-25 from 0 to 212992 tid 1 2022-04-28T19:56:50.104 INFO:tasks.rados.rados.0.smithi059.stdout:3724: read oid 17 snap -1 2022-04-28T19:56:50.104 INFO:tasks.rados.rados.0.smithi059.stdout:3724: expect (ObjNum 1143 snap 357 seq_num 1143) 2022-04-28T19:56:50.104 INFO:tasks.rados.rados.0.smithi059.stdout:3725: snap_remove snap 324 2022-04-28T19:56:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:50 smithi139 bash[43301]: cluster 2022-04-28T19:56:49.026300+0000 mgr.y (mgr.24961) 389 : cluster [DBG] pgmap v656: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 550 KiB/s wr, 1 op/s 2022-04-28T19:56:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:50 smithi139 bash[43301]: cluster 2022-04-28T19:56:49.086488+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2022-04-28T19:56:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:50 smithi059 bash[59745]: cluster 2022-04-28T19:56:49.026300+0000 mgr.y (mgr.24961) 389 : cluster [DBG] pgmap v656: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 550 KiB/s wr, 1 op/s 2022-04-28T19:56:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:50 smithi059 bash[59745]: cluster 2022-04-28T19:56:49.086488+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2022-04-28T19:56:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:50 smithi059 bash[61152]: cluster 2022-04-28T19:56:49.026300+0000 mgr.y (mgr.24961) 389 : cluster [DBG] pgmap v656: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 550 KiB/s wr, 1 op/s 2022-04-28T19:56:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:50 smithi059 bash[61152]: cluster 2022-04-28T19:56:49.086488+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2022-04-28T19:56:51.121 INFO:tasks.rados.rados.0.smithi059.stdout:3725: done (2 left) 2022-04-28T19:56:51.181 INFO:tasks.rados.rados.0.smithi059.stdout:3726: read oid 44 snap -1 2022-04-28T19:56:51.181 INFO:tasks.rados.rados.0.smithi059.stdout:3726: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:56:51.181 INFO:tasks.rados.rados.0.smithi059.stdout:3727: read oid 13 snap -1 2022-04-28T19:56:51.181 INFO:tasks.rados.rados.0.smithi059.stdout:3727: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:51.182 INFO:tasks.rados.rados.0.smithi059.stdout:3728: copy_from oid 14 from oid 2 current snap is 367 2022-04-28T19:56:51.182 INFO:tasks.rados.rados.0.smithi059.stdout:3729: snap_remove snap 361 2022-04-28T19:56:51.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:51 smithi139 bash[43301]: cluster 2022-04-28T19:56:50.101581+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2022-04-28T19:56:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:51 smithi139 bash[43301]: audit 2022-04-28T19:56:50.608682+0000 mon.a (mon.0) 1430 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:56:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:51 smithi139 bash[43301]: audit 2022-04-28T19:56:50.611551+0000 mon.a (mon.0) 1431 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:56:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:51 smithi139 bash[43301]: audit 2022-04-28T19:56:50.613387+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:56:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:51 smithi139 bash[43301]: audit 2022-04-28T19:56:50.939952+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:56:51.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[59745]: cluster 2022-04-28T19:56:50.101581+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[59745]: audit 2022-04-28T19:56:50.608682+0000 mon.a (mon.0) 1430 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[59745]: audit 2022-04-28T19:56:50.611551+0000 mon.a (mon.0) 1431 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[59745]: audit 2022-04-28T19:56:50.613387+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[59745]: audit 2022-04-28T19:56:50.939952+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[61152]: cluster 2022-04-28T19:56:50.101581+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[61152]: audit 2022-04-28T19:56:50.608682+0000 mon.a (mon.0) 1430 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:56:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[61152]: audit 2022-04-28T19:56:50.611551+0000 mon.a (mon.0) 1431 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:56:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[61152]: audit 2022-04-28T19:56:50.613387+0000 mon.a (mon.0) 1432 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:56:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[61152]: audit 2022-04-28T19:56:50.939952+0000 mon.a (mon.0) 1433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:56:52.025 INFO:tasks.rados.rados.0.smithi059.stdout:3729: done (5 left) 2022-04-28T19:56:52.025 INFO:tasks.rados.rados.0.smithi059.stdout:3730: copy_from oid 12 from oid 41 current snap is 367 2022-04-28T19:56:52.025 INFO:tasks.rados.rados.0.smithi059.stdout:3731: copy_from oid 33 from oid 15 current snap is 367 2022-04-28T19:56:52.025 INFO:tasks.rados.rados.0.smithi059.stdout:3732: snap_remove snap 347 2022-04-28T19:56:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:51 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:51.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:52 smithi139 bash[43301]: cephadm 2022-04-28T19:56:50.614995+0000 mgr.y (mgr.24961) 390 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:56:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:52 smithi139 bash[43301]: cluster 2022-04-28T19:56:51.027181+0000 mgr.y (mgr.24961) 391 : cluster [DBG] pgmap v659: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 618 KiB/s wr, 3 op/s 2022-04-28T19:56:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:52 smithi139 bash[43301]: cluster 2022-04-28T19:56:51.120250+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2022-04-28T19:56:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:52 smithi139 bash[43301]: cluster 2022-04-28T19:56:52.023815+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2022-04-28T19:56:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[59745]: cephadm 2022-04-28T19:56:50.614995+0000 mgr.y (mgr.24961) 390 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:56:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[59745]: cluster 2022-04-28T19:56:51.027181+0000 mgr.y (mgr.24961) 391 : cluster [DBG] pgmap v659: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 618 KiB/s wr, 3 op/s 2022-04-28T19:56:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[59745]: cluster 2022-04-28T19:56:51.120250+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2022-04-28T19:56:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[59745]: cluster 2022-04-28T19:56:52.023815+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2022-04-28T19:56:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[61152]: cephadm 2022-04-28T19:56:50.614995+0000 mgr.y (mgr.24961) 390 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:56:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[61152]: cluster 2022-04-28T19:56:51.027181+0000 mgr.y (mgr.24961) 391 : cluster [DBG] pgmap v659: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 618 KiB/s wr, 3 op/s 2022-04-28T19:56:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[61152]: cluster 2022-04-28T19:56:51.120250+0000 mon.a (mon.0) 1434 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2022-04-28T19:56:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:52 smithi059 bash[61152]: cluster 2022-04-28T19:56:52.023815+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3732: done (7 left) 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3733: read oid 43 snap -1 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3733: expect deleted 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3734: read oid 5 snap 331 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3734: expect deleted 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3735: read oid 6 snap -1 2022-04-28T19:56:53.028 INFO:tasks.rados.rados.0.smithi059.stdout:3735: expect (ObjNum 1158 snap 366 seq_num 1158) 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:3736: read oid 31 snap -1 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:3736: expect (ObjNum 1045 snap 324 seq_num 1045) 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:3737: copy_from oid 27 from oid 1 current snap is 367 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:3738: read oid 23 snap -1 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:3738: expect (ObjNum 1150 snap 358 seq_num 1150) 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 367 2022-04-28T19:56:53.029 INFO:tasks.rados.rados.0.smithi059.stdout:3739: seq_num 1160 ranges {12697600=229376} 2022-04-28T19:56:53.102 INFO:tasks.rados.rados.0.smithi059.stdout:3739: writing smithi05955964-38 from 12697600 to 12926976 tid 1 2022-04-28T19:56:53.102 INFO:tasks.rados.rados.0.smithi059.stdout:3740: snap_create 2022-04-28T19:56:53.102 INFO:tasks.rados.rados.0.smithi059.stdout:3723: finishing write tid 1 to smithi05955964-25 2022-04-28T19:56:53.102 INFO:tasks.rados.rados.0.smithi059.stdout:3723: finishing write tid 3 to smithi05955964-25 2022-04-28T19:56:53.102 INFO:tasks.rados.rados.0.smithi059.stdout:3723: finishing write tid 4 to smithi05955964-25 2022-04-28T19:56:53.103 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1593 (ObjNum 1159 snap 367 seq_num 1159) dirty exists 2022-04-28T19:56:53.103 INFO:tasks.rados.rados.0.smithi059.stdout:3723: left oid 25 (ObjNum 1159 snap 367 seq_num 1159) 2022-04-28T19:56:53.103 INFO:tasks.rados.rados.0.smithi059.stdout:3724: expect (ObjNum 1143 snap 357 seq_num 1143) 2022-04-28T19:56:53.164 INFO:tasks.rados.rados.0.smithi059.stdout:3726: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:56:53.164 INFO:tasks.rados.rados.0.smithi059.stdout:3727: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:53.216 INFO:tasks.rados.rados.0.smithi059.stdout:3728: finishing copy_from to smithi05955964-14 2022-04-28T19:56:53.216 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1467 (ObjNum 1090 snap 336 seq_num 1090) dirty exists 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:3728: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:3730: finishing copy_from to smithi05955964-12 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:3730: got expected ENOENT (src dne) 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:3730: finishing copy_from racing read to smithi05955964-12 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:3731: finishing copy_from to smithi05955964-33 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1382 (ObjNum 1107 snap 342 seq_num 1107) dirty exists 2022-04-28T19:56:53.217 INFO:tasks.rados.rados.0.smithi059.stdout:3731: finishing copy_from racing read to smithi05955964-33 2022-04-28T19:56:53.218 INFO:tasks.rados.rados.0.smithi059.stdout:3738: expect (ObjNum 1150 snap 358 seq_num 1150) 2022-04-28T19:56:53.260 INFO:tasks.rados.rados.0.smithi059.stdout:3735: expect (ObjNum 1158 snap 366 seq_num 1158) 2022-04-28T19:56:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:3737: finishing copy_from to smithi05955964-27 2022-04-28T19:56:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1573 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:56:53.264 INFO:tasks.rados.rados.0.smithi059.stdout:3737: finishing copy_from racing read to smithi05955964-27 2022-04-28T19:56:53.265 INFO:tasks.rados.rados.0.smithi059.stdout:3736: expect (ObjNum 1045 snap 324 seq_num 1045) 2022-04-28T19:56:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:56:53 smithi059 bash[72035]: level=error ts=2022-04-28T19:56:53.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:56:54.130 INFO:tasks.rados.rados.0.smithi059.stdout:3739: finishing write tid 1 to smithi05955964-38 2022-04-28T19:56:54.130 INFO:tasks.rados.rados.0.smithi059.stdout:3739: finishing write tid 3 to smithi05955964-38 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3739: finishing write tid 4 to smithi05955964-38 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1582 (ObjNum 1160 snap 367 seq_num 1160) dirty exists 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3739: left oid 38 (ObjNum 1160 snap 367 seq_num 1160) 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3723: done (14 left) 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3724: done (13 left) 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3726: done (12 left) 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3727: done (11 left) 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3728: done (10 left) 2022-04-28T19:56:54.131 INFO:tasks.rados.rados.0.smithi059.stdout:3730: done (9 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3731: done (8 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3733: done (7 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3734: done (6 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3735: done (5 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3736: done (4 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3737: done (3 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3738: done (2 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3739: done (1 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3740: done (0 left) 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3741: read oid 13 snap -1 2022-04-28T19:56:54.132 INFO:tasks.rados.rados.0.smithi059.stdout:3741: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:54.133 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 368 2022-04-28T19:56:54.133 INFO:tasks.rados.rados.0.smithi059.stdout:3742: seq_num 1161 ranges {0=245760} 2022-04-28T19:56:54.243 INFO:tasks.rados.rados.0.smithi059.stdout:3742: writing smithi05955964-10 from 0 to 245760 tid 1 2022-04-28T19:56:54.243 INFO:tasks.rados.rados.0.smithi059.stdout:3743: delete oid 2 current snap is 368 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3743: done (2 left) 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3744: read oid 25 snap -1 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3744: expect (ObjNum 1159 snap 367 seq_num 1159) 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3745: delete oid 20 current snap is 368 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3742: finishing write tid 1 to smithi05955964-10 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3742: finishing write tid 3 to smithi05955964-10 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3745: done (3 left) 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:3746: rollback oid 15 current snap is 368 2022-04-28T19:56:54.244 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 15 to 362 2022-04-28T19:56:54.248 INFO:tasks.rados.rados.0.smithi059.stdout:3747: snap_remove snap 364 2022-04-28T19:56:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:54 smithi059 bash[61152]: cluster 2022-04-28T19:56:53.026375+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2022-04-28T19:56:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:54 smithi059 bash[61152]: cluster 2022-04-28T19:56:53.027805+0000 mgr.y (mgr.24961) 392 : cluster [DBG] pgmap v663: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 89 KiB/s wr, 3 op/s 2022-04-28T19:56:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:54 smithi059 bash[59745]: cluster 2022-04-28T19:56:53.026375+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2022-04-28T19:56:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:54 smithi059 bash[59745]: cluster 2022-04-28T19:56:53.027805+0000 mgr.y (mgr.24961) 392 : cluster [DBG] pgmap v663: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 89 KiB/s wr, 3 op/s 2022-04-28T19:56:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:54 smithi139 bash[43301]: cluster 2022-04-28T19:56:53.026375+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2022-04-28T19:56:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:54 smithi139 bash[43301]: cluster 2022-04-28T19:56:53.027805+0000 mgr.y (mgr.24961) 392 : cluster [DBG] pgmap v663: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 89 KiB/s wr, 3 op/s 2022-04-28T19:56:55.044 INFO:tasks.rados.rados.0.smithi059.stdout:3747: done (4 left) 2022-04-28T19:56:55.044 INFO:tasks.rados.rados.0.smithi059.stdout:3748: copy_from oid 43 from oid 8 current snap is 368 2022-04-28T19:56:55.044 INFO:tasks.rados.rados.0.smithi059.stdout:3749: copy_from oid 14 from oid 47 current snap is 368 2022-04-28T19:56:55.044 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 19 current snap is 368 2022-04-28T19:56:55.044 INFO:tasks.rados.rados.0.smithi059.stdout:3750: seq_num 1164 ranges {0=303104} 2022-04-28T19:56:55.045 INFO:tasks.rados.rados.0.smithi059.stdout:3750: writing smithi05955964-19 from 0 to 303104 tid 1 2022-04-28T19:56:55.045 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 368 2022-04-28T19:56:55.045 INFO:tasks.rados.rados.0.smithi059.stdout:3751: seq_num 1165 ranges {12926976=319488} 2022-04-28T19:56:55.121 INFO:tasks.rados.rados.0.smithi059.stdout:3751: writing smithi05955964-38 from 12926976 to 13246464 tid 1 2022-04-28T19:56:55.121 INFO:tasks.rados.rados.0.smithi059.stdout:3752: snap_create 2022-04-28T19:56:55.121 INFO:tasks.rados.rados.0.smithi059.stdout:3742: finishing write tid 4 to smithi05955964-10 2022-04-28T19:56:55.121 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1575 (ObjNum 1161 snap 368 seq_num 1161) dirty exists 2022-04-28T19:56:55.121 INFO:tasks.rados.rados.0.smithi059.stdout:3742: left oid 10 (ObjNum 1161 snap 368 seq_num 1161) 2022-04-28T19:56:55.121 INFO:tasks.rados.rados.0.smithi059.stdout:3741: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:55.178 INFO:tasks.rados.rados.0.smithi059.stdout:3744: expect (ObjNum 1159 snap 367 seq_num 1159) 2022-04-28T19:56:55.183 INFO:tasks.rados.rados.0.smithi059.stdout:3746: finishing rollback tid 0 to smithi05955964-15 2022-04-28T19:56:55.184 INFO:tasks.rados.rados.0.smithi059.stdout:3746: finishing rollback tid 1 to smithi05955964-15 2022-04-28T19:56:55.184 INFO:tasks.rados.rados.0.smithi059.stdout:3746: finishing rollback tid 2 to smithi05955964-15 2022-04-28T19:56:55.184 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 15 v 1404 (ObjNum 1107 snap 342 seq_num 1107) dirty exists 2022-04-28T19:56:55.184 INFO:tasks.rados.rados.0.smithi059.stdout:3750: finishing write tid 1 to smithi05955964-19 2022-04-28T19:56:55.184 INFO:tasks.rados.rados.0.smithi059.stdout:3750: finishing write tid 3 to smithi05955964-19 2022-04-28T19:56:55.184 INFO:tasks.rados.rados.0.smithi059.stdout:3750: finishing write tid 4 to smithi05955964-19 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 19 v 1553 (ObjNum 1164 snap 368 seq_num 1164) dirty exists 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:3750: left oid 19 (ObjNum 1164 snap 368 seq_num 1164) 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:3749: finishing copy_from to smithi05955964-14 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1389 (ObjNum 1155 snap 362 seq_num 1155) dirty exists 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:3748: finishing copy_from to smithi05955964-43 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1591 (ObjNum 1030 snap 319 seq_num 1030) dirty exists 2022-04-28T19:56:55.185 INFO:tasks.rados.rados.0.smithi059.stdout:3749: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:56:55.186 INFO:tasks.rados.rados.0.smithi059.stdout:3748: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:56:55.186 INFO:tasks.rados.rados.0.smithi059.stdout:3751: finishing write tid 1 to smithi05955964-38 2022-04-28T19:56:55.186 INFO:tasks.rados.rados.0.smithi059.stdout:3751: finishing write tid 3 to smithi05955964-38 2022-04-28T19:56:55.186 INFO:tasks.rados.rados.0.smithi059.stdout:3751: finishing write tid 4 to smithi05955964-38 2022-04-28T19:56:55.186 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1584 (ObjNum 1165 snap 368 seq_num 1165) dirty exists 2022-04-28T19:56:55.186 INFO:tasks.rados.rados.0.smithi059.stdout:3751: left oid 38 (ObjNum 1165 snap 368 seq_num 1165) 2022-04-28T19:56:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:55 smithi059 bash[61152]: cluster 2022-04-28T19:56:54.034887+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2022-04-28T19:56:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:55 smithi059 bash[59745]: cluster 2022-04-28T19:56:54.034887+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2022-04-28T19:56:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:55 smithi139 bash[43301]: cluster 2022-04-28T19:56:54.034887+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2022-04-28T19:56:56.092 INFO:tasks.rados.rados.0.smithi059.stdout:3741: done (8 left) 2022-04-28T19:56:56.092 INFO:tasks.rados.rados.0.smithi059.stdout:3742: done (7 left) 2022-04-28T19:56:56.092 INFO:tasks.rados.rados.0.smithi059.stdout:3744: done (6 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3746: done (5 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3748: done (4 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3749: done (3 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3750: done (2 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3751: done (1 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3752: done (0 left) 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3753: copy_from oid 35 from oid 38 current snap is 369 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 8 current snap is 369 2022-04-28T19:56:56.093 INFO:tasks.rados.rados.0.smithi059.stdout:3754: seq_num 1166 ranges {2826240=335872} 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3754: writing smithi05955964-8 from 2826240 to 3162112 tid 1 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3755: rmattr oid 28 current snap is 369 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3756: read oid 47 snap -1 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3756: expect (ObjNum 1155 snap 362 seq_num 1155) 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3757: rmattr oid 41 current snap is 369 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3757: done (4 left) 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3758: read oid 24 snap 336 2022-04-28T19:56:56.111 INFO:tasks.rados.rados.0.smithi059.stdout:3758: expect (ObjNum 1083 snap 335 seq_num 1083) 2022-04-28T19:56:56.112 INFO:tasks.rados.rados.0.smithi059.stdout:3759: read oid 48 snap 339 2022-04-28T19:56:56.112 INFO:tasks.rados.rados.0.smithi059.stdout:3759: expect deleted 2022-04-28T19:56:56.112 INFO:tasks.rados.rados.0.smithi059.stdout:3760: snap_create 2022-04-28T19:56:56.118 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1458 (ObjNum 1096 snap 339 seq_num 1096) dirty exists 2022-04-28T19:56:56.124 INFO:tasks.rados.rados.0.smithi059.stdout:3754: finishing write tid 1 to smithi05955964-8 2022-04-28T19:56:56.125 INFO:tasks.rados.rados.0.smithi059.stdout:3754: finishing write tid 3 to smithi05955964-8 2022-04-28T19:56:56.126 INFO:tasks.rados.rados.0.smithi059.stdout:3754: finishing write tid 4 to smithi05955964-8 2022-04-28T19:56:56.126 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 8 v 1596 (ObjNum 1166 snap 369 seq_num 1166) dirty exists 2022-04-28T19:56:56.126 INFO:tasks.rados.rados.0.smithi059.stdout:3754: left oid 8 (ObjNum 1166 snap 369 seq_num 1166) 2022-04-28T19:56:56.126 INFO:tasks.rados.rados.0.smithi059.stdout:3758: expect (ObjNum 1083 snap 335 seq_num 1083) 2022-04-28T19:56:56.249 INFO:tasks.rados.rados.0.smithi059.stdout:3756: expect (ObjNum 1155 snap 362 seq_num 1155) 2022-04-28T19:56:56.333 INFO:tasks.rados.rados.0.smithi059.stdout:3753: finishing copy_from to smithi05955964-35 2022-04-28T19:56:56.333 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1584 (ObjNum 1165 snap 368 seq_num 1165) dirty exists 2022-04-28T19:56:56.333 INFO:tasks.rados.rados.0.smithi059.stdout:3753: finishing copy_from racing read to smithi05955964-35 2022-04-28T19:56:56.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:56 smithi059 bash[59745]: cluster 2022-04-28T19:56:55.028671+0000 mgr.y (mgr.24961) 393 : cluster [DBG] pgmap v665: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 9 op/s 2022-04-28T19:56:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:56 smithi059 bash[59745]: cluster 2022-04-28T19:56:55.041897+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2022-04-28T19:56:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:56 smithi059 bash[61152]: cluster 2022-04-28T19:56:55.028671+0000 mgr.y (mgr.24961) 393 : cluster [DBG] pgmap v665: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 9 op/s 2022-04-28T19:56:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:56 smithi059 bash[61152]: cluster 2022-04-28T19:56:55.041897+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2022-04-28T19:56:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:56 smithi139 bash[43301]: cluster 2022-04-28T19:56:55.028671+0000 mgr.y (mgr.24961) 393 : cluster [DBG] pgmap v665: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 9 op/s 2022-04-28T19:56:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:56 smithi139 bash[43301]: cluster 2022-04-28T19:56:55.041897+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2022-04-28T19:56:57.027 INFO:tasks.rados.rados.0.smithi059.stdout:3753: done (6 left) 2022-04-28T19:56:57.027 INFO:tasks.rados.rados.0.smithi059.stdout:3754: done (5 left) 2022-04-28T19:56:57.027 INFO:tasks.rados.rados.0.smithi059.stdout:3755: done (4 left) 2022-04-28T19:56:57.027 INFO:tasks.rados.rados.0.smithi059.stdout:3756: done (3 left) 2022-04-28T19:56:57.027 INFO:tasks.rados.rados.0.smithi059.stdout:3758: done (2 left) 2022-04-28T19:56:57.028 INFO:tasks.rados.rados.0.smithi059.stdout:3759: done (1 left) 2022-04-28T19:56:57.028 INFO:tasks.rados.rados.0.smithi059.stdout:3760: done (0 left) 2022-04-28T19:56:57.028 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 370 2022-04-28T19:56:57.028 INFO:tasks.rados.rados.0.smithi059.stdout:3761: seq_num 1167 ranges {2121728=352256} 2022-04-28T19:56:57.041 INFO:tasks.rados.rados.0.smithi059.stdout:3761: writing smithi05955964-24 from 2121728 to 2473984 tid 1 2022-04-28T19:56:57.041 INFO:tasks.rados.rados.0.smithi059.stdout:3762: rollback oid 9 current snap is 370 2022-04-28T19:56:57.041 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 9 to 346 2022-04-28T19:56:57.041 INFO:tasks.rados.rados.0.smithi059.stdout:3763: snap_remove snap 359 2022-04-28T19:56:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:57 smithi059 bash[61152]: cluster 2022-04-28T19:56:56.091651+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2022-04-28T19:56:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:57 smithi059 bash[61152]: cluster 2022-04-28T19:56:57.026246+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2022-04-28T19:56:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:57 smithi059 bash[59745]: cluster 2022-04-28T19:56:56.091651+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2022-04-28T19:56:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:57 smithi059 bash[59745]: cluster 2022-04-28T19:56:57.026246+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2022-04-28T19:56:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:57 smithi139 bash[43301]: cluster 2022-04-28T19:56:56.091651+0000 mon.a (mon.0) 1439 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2022-04-28T19:56:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:57 smithi139 bash[43301]: cluster 2022-04-28T19:56:57.026246+0000 mon.a (mon.0) 1440 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:3763: done (2 left) 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:3764: delete oid 22 current snap is 370 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:3762: finishing rollback tid 0 to smithi05955964-9 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:3762: finishing rollback tid 1 to smithi05955964-9 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:3762: finishing rollback tid 2 to smithi05955964-9 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1220 (ObjNum 1000 snap 297 seq_num 1000) dirty exists 2022-04-28T19:56:58.050 INFO:tasks.rados.rados.0.smithi059.stdout:3761: finishing write tid 1 to smithi05955964-24 2022-04-28T19:56:58.051 INFO:tasks.rados.rados.0.smithi059.stdout:3761: finishing write tid 3 to smithi05955964-24 2022-04-28T19:56:58.051 INFO:tasks.rados.rados.0.smithi059.stdout:3761: finishing write tid 4 to smithi05955964-24 2022-04-28T19:56:58.051 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1586 (ObjNum 1167 snap 370 seq_num 1167) dirty exists 2022-04-28T19:56:58.051 INFO:tasks.rados.rados.0.smithi059.stdout:3761: left oid 24 (ObjNum 1167 snap 370 seq_num 1167) 2022-04-28T19:56:58.053 INFO:tasks.rados.rados.0.smithi059.stdout:3761: done (2 left) 2022-04-28T19:56:58.053 INFO:tasks.rados.rados.0.smithi059.stdout:3762: done (1 left) 2022-04-28T19:56:58.053 INFO:tasks.rados.rados.0.smithi059.stdout:3764: done (0 left) 2022-04-28T19:56:58.054 INFO:tasks.rados.rados.0.smithi059.stdout:3765: read oid 12 snap -1 2022-04-28T19:56:58.054 INFO:tasks.rados.rados.0.smithi059.stdout:3765: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:58.054 INFO:tasks.rados.rados.0.smithi059.stdout:3766: rmattr oid 17 current snap is 370 2022-04-28T19:56:58.054 INFO:tasks.rados.rados.0.smithi059.stdout:3766: done (1 left) 2022-04-28T19:56:58.054 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 39 current snap is 370 2022-04-28T19:56:58.054 INFO:tasks.rados.rados.0.smithi059.stdout:3767: seq_num 1169 ranges {2465792=385024} 2022-04-28T19:56:58.070 INFO:tasks.rados.rados.0.smithi059.stdout:3767: writing smithi05955964-39 from 2465792 to 2850816 tid 1 2022-04-28T19:56:58.070 INFO:tasks.rados.rados.0.smithi059.stdout:3768: copy_from oid 11 from oid 27 current snap is 370 2022-04-28T19:56:58.070 INFO:tasks.rados.rados.0.smithi059.stdout:3765: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:56:58.127 INFO:tasks.rados.rados.0.smithi059.stdout:3767: finishing write tid 1 to smithi05955964-39 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3767: finishing write tid 3 to smithi05955964-39 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3767: finishing write tid 4 to smithi05955964-39 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1593 (ObjNum 1169 snap 370 seq_num 1169) dirty exists 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3767: left oid 39 (ObjNum 1169 snap 370 seq_num 1169) 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3768: finishing copy_from to smithi05955964-11 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1573 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3768: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3765: done (2 left) 2022-04-28T19:56:58.128 INFO:tasks.rados.rados.0.smithi059.stdout:3767: done (1 left) 2022-04-28T19:56:58.129 INFO:tasks.rados.rados.0.smithi059.stdout:3768: done (0 left) 2022-04-28T19:56:58.129 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 370 2022-04-28T19:56:58.129 INFO:tasks.rados.rados.0.smithi059.stdout:3769: seq_num 1170 ranges {2203648=401408} 2022-04-28T19:56:58.147 INFO:tasks.rados.rados.0.smithi059.stdout:3769: writing smithi05955964-47 from 2203648 to 2605056 tid 1 2022-04-28T19:56:58.148 INFO:tasks.rados.rados.0.smithi059.stdout:3770: read oid 22 snap -1 2022-04-28T19:56:58.148 INFO:tasks.rados.rados.0.smithi059.stdout:3770: expect deleted 2022-04-28T19:56:58.148 INFO:tasks.rados.rados.0.smithi059.stdout:3771: snap_create 2022-04-28T19:56:58.152 INFO:tasks.rados.rados.0.smithi059.stdout:3769: finishing write tid 1 to smithi05955964-47 2022-04-28T19:56:58.152 INFO:tasks.rados.rados.0.smithi059.stdout:3769: finishing write tid 3 to smithi05955964-47 2022-04-28T19:56:58.154 INFO:tasks.rados.rados.0.smithi059.stdout:3769: finishing write tid 4 to smithi05955964-47 2022-04-28T19:56:58.154 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1577 (ObjNum 1170 snap 370 seq_num 1170) dirty exists 2022-04-28T19:56:58.154 INFO:tasks.rados.rados.0.smithi059.stdout:3769: left oid 47 (ObjNum 1170 snap 370 seq_num 1170) 2022-04-28T19:56:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:58 smithi139 bash[43301]: cluster 2022-04-28T19:56:57.029165+0000 mgr.y (mgr.24961) 394 : cluster [DBG] pgmap v669: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 6.0 MiB/s wr, 12 op/s 2022-04-28T19:56:58.497 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:56:58 smithi139 bash[43301]: cluster 2022-04-28T19:56:58.049081+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2022-04-28T19:56:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:58 smithi059 bash[61152]: cluster 2022-04-28T19:56:57.029165+0000 mgr.y (mgr.24961) 394 : cluster [DBG] pgmap v669: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 6.0 MiB/s wr, 12 op/s 2022-04-28T19:56:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:56:58 smithi059 bash[61152]: cluster 2022-04-28T19:56:58.049081+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2022-04-28T19:56:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:58 smithi059 bash[59745]: cluster 2022-04-28T19:56:57.029165+0000 mgr.y (mgr.24961) 394 : cluster [DBG] pgmap v669: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 6.0 MiB/s wr, 12 op/s 2022-04-28T19:56:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:56:58 smithi059 bash[59745]: cluster 2022-04-28T19:56:58.049081+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2022-04-28T19:56:59.105 INFO:tasks.rados.rados.0.smithi059.stdout:3769: done (2 left) 2022-04-28T19:56:59.105 INFO:tasks.rados.rados.0.smithi059.stdout:3770: done (1 left) 2022-04-28T19:56:59.105 INFO:tasks.rados.rados.0.smithi059.stdout:3771: done (0 left) 2022-04-28T19:56:59.106 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 23 current snap is 371 2022-04-28T19:56:59.106 INFO:tasks.rados.rados.0.smithi059.stdout:3772: seq_num 1171 ranges {1032192=417792} 2022-04-28T19:56:59.113 INFO:tasks.rados.rados.0.smithi059.stdout:3772: writing smithi05955964-23 from 1032192 to 1449984 tid 1 2022-04-28T19:56:59.113 INFO:tasks.rados.rados.0.smithi059.stdout:3773: delete oid 37 current snap is 371 2022-04-28T19:56:59.119 INFO:tasks.rados.rados.0.smithi059.stdout:3772: finishing write tid 1 to smithi05955964-23 2022-04-28T19:56:59.120 INFO:tasks.rados.rados.0.smithi059.stdout:3772: finishing write tid 3 to smithi05955964-23 2022-04-28T19:56:59.123 INFO:tasks.rados.rados.0.smithi059.stdout:3773: done (1 left) 2022-04-28T19:56:59.123 INFO:tasks.rados.rados.0.smithi059.stdout:3774: read oid 19 snap -1 2022-04-28T19:56:59.123 INFO:tasks.rados.rados.0.smithi059.stdout:3774: expect (ObjNum 1164 snap 368 seq_num 1164) 2022-04-28T19:56:59.123 INFO:tasks.rados.rados.0.smithi059.stdout:3775: rollback oid 4 current snap is 371 2022-04-28T19:56:59.123 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 356 2022-04-28T19:56:59.123 INFO:tasks.rados.rados.0.smithi059.stdout:3776: snap_remove snap 369 2022-04-28T19:57:00.114 INFO:tasks.rados.rados.0.smithi059.stdout:3776: done (3 left) 2022-04-28T19:57:00.114 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 371 2022-04-28T19:57:00.114 INFO:tasks.rados.rados.0.smithi059.stdout:3777: seq_num 1173 ranges {1712128=450560} 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:3777: writing smithi05955964-32 from 1712128 to 2162688 tid 1 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:3778: rollback oid 24 current snap is 371 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 24 to 310 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:3772: finishing write tid 4 to smithi05955964-23 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1579 (ObjNum 1171 snap 371 seq_num 1171) dirty exists 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:3772: left oid 23 (ObjNum 1171 snap 371 seq_num 1171) 2022-04-28T19:57:00.127 INFO:tasks.rados.rados.0.smithi059.stdout:3774: expect (ObjNum 1164 snap 368 seq_num 1164) 2022-04-28T19:57:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:3775: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:57:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:3775: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:57:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:3775: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:57:00.135 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1589 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:57:00.136 INFO:tasks.rados.rados.0.smithi059.stdout:3772: done (4 left) 2022-04-28T19:57:00.136 INFO:tasks.rados.rados.0.smithi059.stdout:3774: done (3 left) 2022-04-28T19:57:00.136 INFO:tasks.rados.rados.0.smithi059.stdout:3775: done (2 left) 2022-04-28T19:57:00.136 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 371 2022-04-28T19:57:00.136 INFO:tasks.rados.rados.0.smithi059.stdout:3779: seq_num 1174 ranges {2826240=466944} 2022-04-28T19:57:00.154 INFO:tasks.rados.rados.0.smithi059.stdout:3779: writing smithi05955964-43 from 2826240 to 3293184 tid 1 2022-04-28T19:57:00.154 INFO:tasks.rados.rados.0.smithi059.stdout:3780: snap_remove snap 288 2022-04-28T19:57:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:00 smithi139 bash[43301]: cluster 2022-04-28T19:56:59.030018+0000 mgr.y (mgr.24961) 395 : cluster [DBG] pgmap v671: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 5.1 MiB/s wr, 6 op/s 2022-04-28T19:57:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:00 smithi139 bash[43301]: cluster 2022-04-28T19:56:59.104807+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2022-04-28T19:57:00.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:00 smithi059 bash[59745]: cluster 2022-04-28T19:56:59.030018+0000 mgr.y (mgr.24961) 395 : cluster [DBG] pgmap v671: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 5.1 MiB/s wr, 6 op/s 2022-04-28T19:57:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:00 smithi059 bash[59745]: cluster 2022-04-28T19:56:59.104807+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2022-04-28T19:57:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:00 smithi059 bash[61152]: cluster 2022-04-28T19:56:59.030018+0000 mgr.y (mgr.24961) 395 : cluster [DBG] pgmap v671: 89 pgs: 89 active+clean; 1.1 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 5.1 MiB/s wr, 6 op/s 2022-04-28T19:57:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:00 smithi059 bash[61152]: cluster 2022-04-28T19:56:59.104807+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2022-04-28T19:57:01.128 INFO:tasks.rados.rados.0.smithi059.stdout:3780: done (3 left) 2022-04-28T19:57:01.128 INFO:tasks.rados.rados.0.smithi059.stdout:3781: read oid 12 snap -1 2022-04-28T19:57:01.128 INFO:tasks.rados.rados.0.smithi059.stdout:3781: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:57:01.128 INFO:tasks.rados.rados.0.smithi059.stdout:3782: snap_create 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3777: finishing write tid 1 to smithi05955964-32 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3777: finishing write tid 3 to smithi05955964-32 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3778: finishing rollback tid 0 to smithi05955964-24 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3777: finishing write tid 4 to smithi05955964-32 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1595 (ObjNum 1173 snap 371 seq_num 1173) dirty exists 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3777: left oid 32 (ObjNum 1173 snap 371 seq_num 1173) 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3778: finishing rollback tid 1 to smithi05955964-24 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:3778: finishing rollback tid 2 to smithi05955964-24 2022-04-28T19:57:01.129 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1592 (ObjNum 636 snap 183 seq_num 636) dirty exists 2022-04-28T19:57:01.130 INFO:tasks.rados.rados.0.smithi059.stdout:3779: finishing write tid 1 to smithi05955964-43 2022-04-28T19:57:01.130 INFO:tasks.rados.rados.0.smithi059.stdout:3779: finishing write tid 3 to smithi05955964-43 2022-04-28T19:57:01.130 INFO:tasks.rados.rados.0.smithi059.stdout:3779: finishing write tid 4 to smithi05955964-43 2022-04-28T19:57:01.130 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1597 (ObjNum 1174 snap 371 seq_num 1174) dirty exists 2022-04-28T19:57:01.130 INFO:tasks.rados.rados.0.smithi059.stdout:3779: left oid 43 (ObjNum 1174 snap 371 seq_num 1174) 2022-04-28T19:57:01.141 INFO:tasks.rados.rados.0.smithi059.stdout:3781: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:57:01.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:01 smithi059 bash[59745]: cluster 2022-04-28T19:57:00.112853+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2022-04-28T19:57:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:01 smithi059 bash[61152]: cluster 2022-04-28T19:57:00.112853+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2022-04-28T19:57:01.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:01 smithi139 bash[43301]: cluster 2022-04-28T19:57:00.112853+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2022-04-28T19:57:02.029 INFO:tasks.rados.rados.0.smithi059.stdout:3777: done (4 left) 2022-04-28T19:57:02.060 INFO:tasks.rados.rados.0.smithi059.stdout:3778: done (3 left) 2022-04-28T19:57:02.061 INFO:tasks.rados.rados.0.smithi059.stdout:3779: done (2 left) 2022-04-28T19:57:02.061 INFO:tasks.rados.rados.0.smithi059.stdout:3781: done (1 left) 2022-04-28T19:57:02.061 INFO:tasks.rados.rados.0.smithi059.stdout:3782: done (0 left) 2022-04-28T19:57:02.061 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 50 current snap is 372 2022-04-28T19:57:02.061 INFO:tasks.rados.rados.0.smithi059.stdout:3783: seq_num 1175 ranges {12410880=483328} 2022-04-28T19:57:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:01 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:01.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:02.105 INFO:tasks.rados.rados.0.smithi059.stdout:3783: writing smithi05955964-50 from 12410880 to 12894208 tid 1 2022-04-28T19:57:02.105 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 372 2022-04-28T19:57:02.105 INFO:tasks.rados.rados.0.smithi059.stdout:3784: seq_num 1176 ranges {131072=499712} 2022-04-28T19:57:02.111 INFO:tasks.rados.rados.0.smithi059.stdout:3784: writing smithi05955964-7 from 131072 to 630784 tid 1 2022-04-28T19:57:02.111 INFO:tasks.rados.rados.0.smithi059.stdout:3785: rollback oid 43 current snap is 372 2022-04-28T19:57:02.111 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 43 to 337 2022-04-28T19:57:02.111 INFO:tasks.rados.rados.0.smithi059.stdout:3786: delete oid 23 current snap is 372 2022-04-28T19:57:02.112 INFO:tasks.rados.rados.0.smithi059.stdout:3786: done (3 left) 2022-04-28T19:57:02.112 INFO:tasks.rados.rados.0.smithi059.stdout:3787: read oid 1 snap 336 2022-04-28T19:57:02.112 INFO:tasks.rados.rados.0.smithi059.stdout:3787: expect (ObjNum 1082 snap 335 seq_num 1082) 2022-04-28T19:57:02.112 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 372 2022-04-28T19:57:02.112 INFO:tasks.rados.rados.0.smithi059.stdout:3788: seq_num 1178 ranges {3350528=532480} 2022-04-28T19:57:02.135 INFO:tasks.rados.rados.0.smithi059.stdout:3788: writing smithi05955964-49 from 3350528 to 3883008 tid 1 2022-04-28T19:57:02.135 INFO:tasks.rados.rados.0.smithi059.stdout:3789: delete oid 23 current snap is 372 2022-04-28T19:57:02.135 INFO:tasks.rados.rados.0.smithi059.stdout:3784: finishing write tid 1 to smithi05955964-7 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3784: finishing write tid 3 to smithi05955964-7 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3783: finishing write tid 1 to smithi05955964-50 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3784: finishing write tid 4 to smithi05955964-7 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 1406 (ObjNum 1176 snap 372 seq_num 1176) dirty exists 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3784: left oid 7 (ObjNum 1176 snap 372 seq_num 1176) 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3783: finishing write tid 3 to smithi05955964-50 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3785: finishing rollback tid 0 to smithi05955964-43 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3783: finishing write tid 4 to smithi05955964-50 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 50 v 1599 (ObjNum 1175 snap 372 seq_num 1175) dirty exists 2022-04-28T19:57:02.136 INFO:tasks.rados.rados.0.smithi059.stdout:3783: left oid 50 (ObjNum 1175 snap 372 seq_num 1175) 2022-04-28T19:57:02.137 INFO:tasks.rados.rados.0.smithi059.stdout:3785: finishing rollback tid 1 to smithi05955964-43 2022-04-28T19:57:02.137 INFO:tasks.rados.rados.0.smithi059.stdout:3785: finishing rollback tid 2 to smithi05955964-43 2022-04-28T19:57:02.137 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1602 (ObjNum 1085 snap 335 seq_num 1085) dirty exists 2022-04-28T19:57:02.137 INFO:tasks.rados.rados.0.smithi059.stdout:3787: expect (ObjNum 1082 snap 335 seq_num 1082) 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:3788: finishing write tid 1 to smithi05955964-49 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:3788: finishing write tid 3 to smithi05955964-49 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:3788: finishing write tid 4 to smithi05955964-49 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1575 (ObjNum 1178 snap 372 seq_num 1178) dirty exists 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:3788: left oid 49 (ObjNum 1178 snap 372 seq_num 1178) 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:3783: done (5 left) 2022-04-28T19:57:02.434 INFO:tasks.rados.rados.0.smithi059.stdout:3784: done (4 left) 2022-04-28T19:57:02.435 INFO:tasks.rados.rados.0.smithi059.stdout:3785: done (3 left) 2022-04-28T19:57:02.435 INFO:tasks.rados.rados.0.smithi059.stdout:3787: done (2 left) 2022-04-28T19:57:02.435 INFO:tasks.rados.rados.0.smithi059.stdout:3788: done (1 left) 2022-04-28T19:57:02.435 INFO:tasks.rados.rados.0.smithi059.stdout:3789: done (0 left) 2022-04-28T19:57:02.435 INFO:tasks.rados.rados.0.smithi059.stdout:3790: delete oid 14 current snap is 372 2022-04-28T19:57:02.438 INFO:tasks.rados.rados.0.smithi059.stdout:3790: done (0 left) 2022-04-28T19:57:02.438 INFO:tasks.rados.rados.0.smithi059.stdout:3791: copy_from oid 14 from oid 39 current snap is 372 2022-04-28T19:57:02.438 INFO:tasks.rados.rados.0.smithi059.stdout:3792: read oid 25 snap -1 2022-04-28T19:57:02.438 INFO:tasks.rados.rados.0.smithi059.stdout:3792: expect (ObjNum 1159 snap 367 seq_num 1159) 2022-04-28T19:57:02.439 INFO:tasks.rados.rados.0.smithi059.stdout:3793: snap_create 2022-04-28T19:57:02.442 INFO:tasks.rados.rados.0.smithi059.stdout:3792: expect (ObjNum 1159 snap 367 seq_num 1159) 2022-04-28T19:57:02.456 INFO:tasks.rados.rados.0.smithi059.stdout:3791: finishing copy_from to smithi05955964-14 2022-04-28T19:57:02.456 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1593 (ObjNum 1169 snap 370 seq_num 1169) dirty exists 2022-04-28T19:57:02.457 INFO:tasks.rados.rados.0.smithi059.stdout:3791: finishing copy_from racing read to smithi05955964-14 2022-04-28T19:57:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:02 smithi059 bash[61152]: cluster 2022-04-28T19:57:01.030921+0000 mgr.y (mgr.24961) 396 : cluster [DBG] pgmap v674: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 5.3 MiB/s wr, 17 op/s 2022-04-28T19:57:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:02 smithi059 bash[61152]: cluster 2022-04-28T19:57:01.126927+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2022-04-28T19:57:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:02 smithi059 bash[61152]: cluster 2022-04-28T19:57:02.026875+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2022-04-28T19:57:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:02 smithi059 bash[59745]: cluster 2022-04-28T19:57:01.030921+0000 mgr.y (mgr.24961) 396 : cluster [DBG] pgmap v674: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 5.3 MiB/s wr, 17 op/s 2022-04-28T19:57:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:02 smithi059 bash[59745]: cluster 2022-04-28T19:57:01.126927+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2022-04-28T19:57:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:02 smithi059 bash[59745]: cluster 2022-04-28T19:57:02.026875+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2022-04-28T19:57:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:02 smithi139 bash[43301]: cluster 2022-04-28T19:57:01.030921+0000 mgr.y (mgr.24961) 396 : cluster [DBG] pgmap v674: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 5.3 MiB/s wr, 17 op/s 2022-04-28T19:57:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:02 smithi139 bash[43301]: cluster 2022-04-28T19:57:01.126927+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2022-04-28T19:57:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:02 smithi139 bash[43301]: cluster 2022-04-28T19:57:02.026875+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2022-04-28T19:57:03.131 INFO:tasks.rados.rados.0.smithi059.stdout:3791: done (2 left) 2022-04-28T19:57:03.245 INFO:tasks.rados.rados.0.smithi059.stdout:3792: done (1 left) 2022-04-28T19:57:03.245 INFO:tasks.rados.rados.0.smithi059.stdout:3793: done (0 left) 2022-04-28T19:57:03.245 INFO:tasks.rados.rados.0.smithi059.stdout:3794: snap_create 2022-04-28T19:57:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:03 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:03.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:04.133 INFO:tasks.rados.rados.0.smithi059.stdout:3794: done (0 left) 2022-04-28T19:57:04.133 INFO:tasks.rados.rados.0.smithi059.stdout:3795: snap_create 2022-04-28T19:57:04.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:04 smithi059 bash[61152]: cluster 2022-04-28T19:57:03.031650+0000 mgr.y (mgr.24961) 397 : cluster [DBG] pgmap v677: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2022-04-28T19:57:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:04 smithi059 bash[61152]: cluster 2022-04-28T19:57:03.130822+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2022-04-28T19:57:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:04 smithi059 bash[59745]: cluster 2022-04-28T19:57:03.031650+0000 mgr.y (mgr.24961) 397 : cluster [DBG] pgmap v677: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2022-04-28T19:57:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:04 smithi059 bash[59745]: cluster 2022-04-28T19:57:03.130822+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2022-04-28T19:57:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:04 smithi139 bash[43301]: cluster 2022-04-28T19:57:03.031650+0000 mgr.y (mgr.24961) 397 : cluster [DBG] pgmap v677: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2022-04-28T19:57:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:04 smithi139 bash[43301]: cluster 2022-04-28T19:57:03.130822+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2022-04-28T19:57:05.145 INFO:tasks.rados.rados.0.smithi059.stdout:3795: done (0 left) 2022-04-28T19:57:05.145 INFO:tasks.rados.rados.0.smithi059.stdout:3796: delete oid 12 current snap is 375 2022-04-28T19:57:05.150 INFO:tasks.rados.rados.0.smithi059.stdout:3796: done (0 left) 2022-04-28T19:57:05.150 INFO:tasks.rados.rados.0.smithi059.stdout:3797: rmattr oid 39 current snap is 375 2022-04-28T19:57:05.150 INFO:tasks.rados.rados.0.smithi059.stdout:3798: rollback oid 42 current snap is 375 2022-04-28T19:57:05.150 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 42 to 358 2022-04-28T19:57:05.150 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 38 current snap is 375 2022-04-28T19:57:05.150 INFO:tasks.rados.rados.0.smithi059.stdout:3799: seq_num 1182 ranges {13246464=606208} 2022-04-28T19:57:05.229 INFO:tasks.rados.rados.0.smithi059.stdout:3799: writing smithi05955964-38 from 13246464 to 13852672 tid 1 2022-04-28T19:57:05.229 INFO:tasks.rados.rados.0.smithi059.stdout:3800: snap_create 2022-04-28T19:57:05.230 INFO:tasks.rados.rados.0.smithi059.stdout:3798: finishing rollback tid 1 to smithi05955964-42 2022-04-28T19:57:05.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1553 (ObjNum -133166544 snap 21893 seq_num 2256) dirty dne 2022-04-28T19:57:05.230 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 39 v 1603 (ObjNum 1169 snap 370 seq_num 1169) dirty exists 2022-04-28T19:57:05.238 INFO:tasks.rados.rados.0.smithi059.stdout:3799: finishing write tid 1 to smithi05955964-38 2022-04-28T19:57:05.239 INFO:tasks.rados.rados.0.smithi059.stdout:3799: finishing write tid 3 to smithi05955964-38 2022-04-28T19:57:05.240 INFO:tasks.rados.rados.0.smithi059.stdout:3799: finishing write tid 4 to smithi05955964-38 2022-04-28T19:57:05.240 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1594 (ObjNum 1182 snap 375 seq_num 1182) dirty exists 2022-04-28T19:57:05.240 INFO:tasks.rados.rados.0.smithi059.stdout:3799: left oid 38 (ObjNum 1182 snap 375 seq_num 1182) 2022-04-28T19:57:05.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:05 smithi059 bash[61152]: cluster 2022-04-28T19:57:04.131887+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2022-04-28T19:57:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:05 smithi059 bash[59745]: cluster 2022-04-28T19:57:04.131887+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2022-04-28T19:57:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:05 smithi139 bash[43301]: cluster 2022-04-28T19:57:04.131887+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2022-04-28T19:57:06.158 INFO:tasks.rados.rados.0.smithi059.stdout:3797: done (3 left) 2022-04-28T19:57:06.158 INFO:tasks.rados.rados.0.smithi059.stdout:3798: done (2 left) 2022-04-28T19:57:06.158 INFO:tasks.rados.rados.0.smithi059.stdout:3799: done (1 left) 2022-04-28T19:57:06.158 INFO:tasks.rados.rados.0.smithi059.stdout:3800: done (0 left) 2022-04-28T19:57:06.158 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 376 2022-04-28T19:57:06.158 INFO:tasks.rados.rados.0.smithi059.stdout:3801: seq_num 1183 ranges {4399104=622592} 2022-04-28T19:57:06.187 INFO:tasks.rados.rados.0.smithi059.stdout:3801: writing smithi05955964-43 from 4399104 to 5021696 tid 1 2022-04-28T19:57:06.187 INFO:tasks.rados.rados.0.smithi059.stdout:3802: delete oid 39 current snap is 376 2022-04-28T19:57:06.198 INFO:tasks.rados.rados.0.smithi059.stdout:3801: finishing write tid 1 to smithi05955964-43 2022-04-28T19:57:06.198 INFO:tasks.rados.rados.0.smithi059.stdout:3801: finishing write tid 3 to smithi05955964-43 2022-04-28T19:57:06.199 INFO:tasks.rados.rados.0.smithi059.stdout:3802: done (1 left) 2022-04-28T19:57:06.199 INFO:tasks.rados.rados.0.smithi059.stdout:3803: snap_remove snap 363 2022-04-28T19:57:06.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:06 smithi059 bash[61152]: cluster 2022-04-28T19:57:05.032487+0000 mgr.y (mgr.24961) 398 : cluster [DBG] pgmap v680: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 594 KiB/s wr, 7 op/s 2022-04-28T19:57:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:06 smithi059 bash[61152]: cluster 2022-04-28T19:57:05.144700+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2022-04-28T19:57:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:06 smithi059 bash[59745]: cluster 2022-04-28T19:57:05.032487+0000 mgr.y (mgr.24961) 398 : cluster [DBG] pgmap v680: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 594 KiB/s wr, 7 op/s 2022-04-28T19:57:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:06 smithi059 bash[59745]: cluster 2022-04-28T19:57:05.144700+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2022-04-28T19:57:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:06 smithi139 bash[43301]: cluster 2022-04-28T19:57:05.032487+0000 mgr.y (mgr.24961) 398 : cluster [DBG] pgmap v680: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 594 KiB/s wr, 7 op/s 2022-04-28T19:57:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:06 smithi139 bash[43301]: cluster 2022-04-28T19:57:05.144700+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2022-04-28T19:57:07.174 INFO:tasks.rados.rados.0.smithi059.stdout:3803: done (1 left) 2022-04-28T19:57:07.252 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 16 current snap is 376 2022-04-28T19:57:07.253 INFO:tasks.rados.rados.0.smithi059.stdout:3804: seq_num 1185 ranges {1597440=655360} 2022-04-28T19:57:07.255 INFO:tasks.rados.rados.0.smithi059.stdout:3804: writing smithi05955964-16 from 1597440 to 2252800 tid 1 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 2 current snap is 376 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3805: seq_num 1186 ranges {0=671744} 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3805: writing smithi05955964-2 from 0 to 671744 tid 1 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3806: delete oid 29 current snap is 376 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3801: finishing write tid 4 to smithi05955964-43 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1605 (ObjNum 1183 snap 376 seq_num 1183) dirty exists 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3801: left oid 43 (ObjNum 1183 snap 376 seq_num 1183) 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3804: finishing write tid 1 to smithi05955964-16 2022-04-28T19:57:07.256 INFO:tasks.rados.rados.0.smithi059.stdout:3805: finishing write tid 1 to smithi05955964-2 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3804: finishing write tid 3 to smithi05955964-16 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3805: finishing write tid 3 to smithi05955964-2 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3805: finishing write tid 4 to smithi05955964-2 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1470 (ObjNum 1186 snap 376 seq_num 1186) dirty exists 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3805: left oid 2 (ObjNum 1186 snap 376 seq_num 1186) 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3801: done (3 left) 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3805: done (2 left) 2022-04-28T19:57:07.257 INFO:tasks.rados.rados.0.smithi059.stdout:3806: done (1 left) 2022-04-28T19:57:07.258 INFO:tasks.rados.rados.0.smithi059.stdout:3807: snap_create 2022-04-28T19:57:07.258 INFO:tasks.rados.rados.0.smithi059.stdout:3804: finishing write tid 4 to smithi05955964-16 2022-04-28T19:57:07.258 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1460 (ObjNum 1185 snap 376 seq_num 1185) dirty exists 2022-04-28T19:57:07.258 INFO:tasks.rados.rados.0.smithi059.stdout:3804: left oid 16 (ObjNum 1185 snap 376 seq_num 1185) 2022-04-28T19:57:07.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:07 smithi059 bash[61152]: cluster 2022-04-28T19:57:06.156474+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2022-04-28T19:57:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:07 smithi059 bash[59745]: cluster 2022-04-28T19:57:06.156474+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2022-04-28T19:57:07.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:07 smithi139 bash[43301]: cluster 2022-04-28T19:57:06.156474+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2022-04-28T19:57:08.180 INFO:tasks.rados.rados.0.smithi059.stdout:3804: done (1 left) 2022-04-28T19:57:08.180 INFO:tasks.rados.rados.0.smithi059.stdout:3807: done (0 left) 2022-04-28T19:57:08.180 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 42 current snap is 377 2022-04-28T19:57:08.180 INFO:tasks.rados.rados.0.smithi059.stdout:3808: seq_num 1188 ranges {0=704512} 2022-04-28T19:57:08.188 INFO:tasks.rados.rados.0.smithi059.stdout:3808: writing smithi05955964-42 from 0 to 704512 tid 1 2022-04-28T19:57:08.188 INFO:tasks.rados.rados.0.smithi059.stdout:3809: rmattr oid 22 current snap is 377 2022-04-28T19:57:08.189 INFO:tasks.rados.rados.0.smithi059.stdout:3809: done (1 left) 2022-04-28T19:57:08.189 INFO:tasks.rados.rados.0.smithi059.stdout:3810: read oid 49 snap 362 2022-04-28T19:57:08.189 INFO:tasks.rados.rados.0.smithi059.stdout:3810: expect (ObjNum 1149 snap 358 seq_num 1149) 2022-04-28T19:57:08.189 INFO:tasks.rados.rados.0.smithi059.stdout:3811: snap_create 2022-04-28T19:57:08.197 INFO:tasks.rados.rados.0.smithi059.stdout:3808: finishing write tid 1 to smithi05955964-42 2022-04-28T19:57:08.197 INFO:tasks.rados.rados.0.smithi059.stdout:3808: finishing write tid 3 to smithi05955964-42 2022-04-28T19:57:08.199 INFO:tasks.rados.rados.0.smithi059.stdout:3808: finishing write tid 4 to smithi05955964-42 2022-04-28T19:57:08.199 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 42 v 1560 (ObjNum 1188 snap 377 seq_num 1188) dirty exists 2022-04-28T19:57:08.199 INFO:tasks.rados.rados.0.smithi059.stdout:3808: left oid 42 (ObjNum 1188 snap 377 seq_num 1188) 2022-04-28T19:57:08.202 INFO:tasks.rados.rados.0.smithi059.stdout:3810: expect (ObjNum 1149 snap 358 seq_num 1149) 2022-04-28T19:57:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:08 smithi059 bash[61152]: cluster 2022-04-28T19:57:07.033183+0000 mgr.y (mgr.24961) 399 : cluster [DBG] pgmap v683: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2022-04-28T19:57:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:08 smithi059 bash[61152]: cluster 2022-04-28T19:57:07.173392+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2022-04-28T19:57:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:08 smithi059 bash[59745]: cluster 2022-04-28T19:57:07.033183+0000 mgr.y (mgr.24961) 399 : cluster [DBG] pgmap v683: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2022-04-28T19:57:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:08 smithi059 bash[59745]: cluster 2022-04-28T19:57:07.173392+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2022-04-28T19:57:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:08 smithi139 bash[43301]: cluster 2022-04-28T19:57:07.033183+0000 mgr.y (mgr.24961) 399 : cluster [DBG] pgmap v683: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.4 MiB/s wr, 9 op/s 2022-04-28T19:57:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:08 smithi139 bash[43301]: cluster 2022-04-28T19:57:07.173392+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2022-04-28T19:57:09.201 INFO:tasks.rados.rados.0.smithi059.stdout:3808: done (2 left) 2022-04-28T19:57:09.202 INFO:tasks.rados.rados.0.smithi059.stdout:3810: done (1 left) 2022-04-28T19:57:09.202 INFO:tasks.rados.rados.0.smithi059.stdout:3811: done (0 left) 2022-04-28T19:57:09.202 INFO:tasks.rados.rados.0.smithi059.stdout:3812: read oid 21 snap 353 2022-04-28T19:57:09.202 INFO:tasks.rados.rados.0.smithi059.stdout:3812: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:57:09.203 INFO:tasks.rados.rados.0.smithi059.stdout:3813: read oid 1 snap -1 2022-04-28T19:57:09.203 INFO:tasks.rados.rados.0.smithi059.stdout:3813: expect (ObjNum 933 snap 276 seq_num 933) 2022-04-28T19:57:09.203 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 378 2022-04-28T19:57:09.203 INFO:tasks.rados.rados.0.smithi059.stdout:3814: seq_num 1189 ranges {212992=720896} 2022-04-28T19:57:09.211 INFO:tasks.rados.rados.0.smithi059.stdout:3814: writing smithi05955964-25 from 212992 to 933888 tid 1 2022-04-28T19:57:09.211 INFO:tasks.rados.rados.0.smithi059.stdout:3815: copy_from oid 7 from oid 29 current snap is 378 2022-04-28T19:57:09.211 INFO:tasks.rados.rados.0.smithi059.stdout:3816: setattr oid 41 current snap is 378 2022-04-28T19:57:09.212 INFO:tasks.rados.rados.0.smithi059.stdout:3817: delete oid 27 current snap is 378 2022-04-28T19:57:09.218 INFO:tasks.rados.rados.0.smithi059.stdout:3817: done (5 left) 2022-04-28T19:57:09.218 INFO:tasks.rados.rados.0.smithi059.stdout:3818: delete oid 24 current snap is 378 2022-04-28T19:57:09.223 INFO:tasks.rados.rados.0.smithi059.stdout:3813: expect (ObjNum 933 snap 276 seq_num 933) 2022-04-28T19:57:09.417 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1354 (ObjNum 188341840 snap 32762 seq_num 188341840) dirty exists 2022-04-28T19:57:09.418 INFO:tasks.rados.rados.0.smithi059.stdout:3815: finishing copy_from to smithi05955964-7 2022-04-28T19:57:09.418 INFO:tasks.rados.rados.0.smithi059.stdout:3815: got expected ENOENT (src dne) 2022-04-28T19:57:09.418 INFO:tasks.rados.rados.0.smithi059.stdout:3815: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:57:09.418 INFO:tasks.rados.rados.0.smithi059.stdout:3814: finishing write tid 1 to smithi05955964-25 2022-04-28T19:57:09.418 INFO:tasks.rados.rados.0.smithi059.stdout:3814: finishing write tid 3 to smithi05955964-25 2022-04-28T19:57:09.419 INFO:tasks.rados.rados.0.smithi059.stdout:3812: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:57:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:3814: finishing write tid 4 to smithi05955964-25 2022-04-28T19:57:09.492 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1601 (ObjNum 1189 snap 378 seq_num 1189) dirty exists 2022-04-28T19:57:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:3814: left oid 25 (ObjNum 1189 snap 378 seq_num 1189) 2022-04-28T19:57:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:3812: done (5 left) 2022-04-28T19:57:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:3813: done (4 left) 2022-04-28T19:57:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:3814: done (3 left) 2022-04-28T19:57:09.493 INFO:tasks.rados.rados.0.smithi059.stdout:3815: done (2 left) 2022-04-28T19:57:09.494 INFO:tasks.rados.rados.0.smithi059.stdout:3816: done (1 left) 2022-04-28T19:57:09.494 INFO:tasks.rados.rados.0.smithi059.stdout:3818: done (0 left) 2022-04-28T19:57:09.494 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 10 current snap is 378 2022-04-28T19:57:09.494 INFO:tasks.rados.rados.0.smithi059.stdout:3819: seq_num 1192 ranges {245760=770048} 2022-04-28T19:57:09.507 INFO:tasks.rados.rados.0.smithi059.stdout:3819: writing smithi05955964-10 from 245760 to 1015808 tid 1 2022-04-28T19:57:09.507 INFO:tasks.rados.rados.0.smithi059.stdout:3820: snap_create 2022-04-28T19:57:09.512 INFO:tasks.rados.rados.0.smithi059.stdout:3819: finishing write tid 1 to smithi05955964-10 2022-04-28T19:57:09.512 INFO:tasks.rados.rados.0.smithi059.stdout:3819: finishing write tid 3 to smithi05955964-10 2022-04-28T19:57:09.514 INFO:tasks.rados.rados.0.smithi059.stdout:3819: finishing write tid 4 to smithi05955964-10 2022-04-28T19:57:09.514 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1583 (ObjNum 1192 snap 378 seq_num 1192) dirty exists 2022-04-28T19:57:09.514 INFO:tasks.rados.rados.0.smithi059.stdout:3819: left oid 10 (ObjNum 1192 snap 378 seq_num 1192) 2022-04-28T19:57:09.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:09 smithi059 bash[61152]: cluster 2022-04-28T19:57:08.179772+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2022-04-28T19:57:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:09 smithi059 bash[59745]: cluster 2022-04-28T19:57:08.179772+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2022-04-28T19:57:09.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:09 smithi139 bash[43301]: cluster 2022-04-28T19:57:08.179772+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2022-04-28T19:57:10.208 INFO:tasks.rados.rados.0.smithi059.stdout:3819: done (1 left) 2022-04-28T19:57:10.208 INFO:tasks.rados.rados.0.smithi059.stdout:3820: done (0 left) 2022-04-28T19:57:10.208 INFO:tasks.rados.rados.0.smithi059.stdout:3821: read oid 1 snap 344 2022-04-28T19:57:10.208 INFO:tasks.rados.rados.0.smithi059.stdout:3821: expect (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:57:10.208 INFO:tasks.rados.rados.0.smithi059.stdout:3822: read oid 44 snap -1 2022-04-28T19:57:10.208 INFO:tasks.rados.rados.0.smithi059.stdout:3822: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:57:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:3823: rmattr oid 35 current snap is 379 2022-04-28T19:57:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:3824: rollback oid 14 current snap is 379 2022-04-28T19:57:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 14 to 331 2022-04-28T19:57:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:3825: read oid 21 snap -1 2022-04-28T19:57:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:3825: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:57:10.209 INFO:tasks.rados.rados.0.smithi059.stdout:3826: snap_remove snap 343 2022-04-28T19:57:10.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:10 smithi059 bash[59745]: cluster 2022-04-28T19:57:09.034085+0000 mgr.y (mgr.24961) 400 : cluster [DBG] pgmap v686: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1016 KiB/s wr, 2 op/s 2022-04-28T19:57:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:10 smithi059 bash[59745]: cluster 2022-04-28T19:57:09.199878+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2022-04-28T19:57:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:10 smithi059 bash[61152]: cluster 2022-04-28T19:57:09.034085+0000 mgr.y (mgr.24961) 400 : cluster [DBG] pgmap v686: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1016 KiB/s wr, 2 op/s 2022-04-28T19:57:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:10 smithi059 bash[61152]: cluster 2022-04-28T19:57:09.199878+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2022-04-28T19:57:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:10 smithi139 bash[43301]: cluster 2022-04-28T19:57:09.034085+0000 mgr.y (mgr.24961) 400 : cluster [DBG] pgmap v686: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1016 KiB/s wr, 2 op/s 2022-04-28T19:57:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:10 smithi139 bash[43301]: cluster 2022-04-28T19:57:09.199878+0000 mon.a (mon.0) 1452 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2022-04-28T19:57:11.225 INFO:tasks.rados.rados.0.smithi059.stdout:3826: done (5 left) 2022-04-28T19:57:11.225 INFO:tasks.rados.rados.0.smithi059.stdout:3827: snap_create 2022-04-28T19:57:11.225 INFO:tasks.rados.rados.0.smithi059.stdout:3822: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:57:11.225 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 35 v 1585 (ObjNum 1165 snap 368 seq_num 1165) dirty exists 2022-04-28T19:57:11.226 INFO:tasks.rados.rados.0.smithi059.stdout:3825: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:57:11.300 INFO:tasks.rados.rados.0.smithi059.stdout:3824: finishing rollback tid 0 to smithi05955964-14 2022-04-28T19:57:11.300 INFO:tasks.rados.rados.0.smithi059.stdout:3824: finishing rollback tid 1 to smithi05955964-14 2022-04-28T19:57:11.300 INFO:tasks.rados.rados.0.smithi059.stdout:3824: finishing rollback tid 2 to smithi05955964-14 2022-04-28T19:57:11.301 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 14 v 1596 (ObjNum 898 snap 268 seq_num 898) dirty exists 2022-04-28T19:57:11.301 INFO:tasks.rados.rados.0.smithi059.stdout:3821: expect (ObjNum 1058 snap 329 seq_num 1058) 2022-04-28T19:57:11.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:11 smithi059 bash[59745]: cluster 2022-04-28T19:57:10.201696+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2022-04-28T19:57:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:11 smithi059 bash[61152]: cluster 2022-04-28T19:57:10.201696+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2022-04-28T19:57:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:11 smithi139 bash[43301]: cluster 2022-04-28T19:57:10.201696+0000 mon.a (mon.0) 1453 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2022-04-28T19:57:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:11 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:11.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:12.190 INFO:tasks.rados.rados.0.smithi059.stdout:3821: done (5 left) 2022-04-28T19:57:12.190 INFO:tasks.rados.rados.0.smithi059.stdout:3822: done (4 left) 2022-04-28T19:57:12.190 INFO:tasks.rados.rados.0.smithi059.stdout:3823: done (3 left) 2022-04-28T19:57:12.190 INFO:tasks.rados.rados.0.smithi059.stdout:3824: done (2 left) 2022-04-28T19:57:12.191 INFO:tasks.rados.rados.0.smithi059.stdout:3825: done (1 left) 2022-04-28T19:57:12.191 INFO:tasks.rados.rados.0.smithi059.stdout:3827: done (0 left) 2022-04-28T19:57:12.191 INFO:tasks.rados.rados.0.smithi059.stdout:3828: read oid 21 snap -1 2022-04-28T19:57:12.191 INFO:tasks.rados.rados.0.smithi059.stdout:3828: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:57:12.191 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 380 2022-04-28T19:57:12.191 INFO:tasks.rados.rados.0.smithi059.stdout:3829: seq_num 1193 ranges {2162688=786432} 2022-04-28T19:57:12.215 INFO:tasks.rados.rados.0.smithi059.stdout:3829: writing smithi05955964-32 from 2162688 to 2949120 tid 1 2022-04-28T19:57:12.216 INFO:tasks.rados.rados.0.smithi059.stdout:3830: snap_remove snap 350 2022-04-28T19:57:12.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:12 smithi059 bash[59745]: cluster 2022-04-28T19:57:11.034973+0000 mgr.y (mgr.24961) 401 : cluster [DBG] pgmap v689: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 836 KiB/s wr, 14 op/s 2022-04-28T19:57:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:12 smithi059 bash[59745]: cluster 2022-04-28T19:57:11.223049+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2022-04-28T19:57:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:12 smithi059 bash[59745]: cluster 2022-04-28T19:57:12.031981+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2022-04-28T19:57:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:12 smithi059 bash[61152]: cluster 2022-04-28T19:57:11.034973+0000 mgr.y (mgr.24961) 401 : cluster [DBG] pgmap v689: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 836 KiB/s wr, 14 op/s 2022-04-28T19:57:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:12 smithi059 bash[61152]: cluster 2022-04-28T19:57:11.223049+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2022-04-28T19:57:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:12 smithi059 bash[61152]: cluster 2022-04-28T19:57:12.031981+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2022-04-28T19:57:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:12 smithi139 bash[43301]: cluster 2022-04-28T19:57:11.034973+0000 mgr.y (mgr.24961) 401 : cluster [DBG] pgmap v689: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 836 KiB/s wr, 14 op/s 2022-04-28T19:57:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:12 smithi139 bash[43301]: cluster 2022-04-28T19:57:11.223049+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2022-04-28T19:57:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:12 smithi139 bash[43301]: cluster 2022-04-28T19:57:12.031981+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2022-04-28T19:57:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:3830: done (2 left) 2022-04-28T19:57:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:3831: read oid 31 snap -1 2022-04-28T19:57:13.047 INFO:tasks.rados.rados.0.smithi059.stdout:3831: expect (ObjNum 1045 snap 324 seq_num 1045) 2022-04-28T19:57:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:3832: read oid 45 snap -1 2022-04-28T19:57:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:3832: expect (ObjNum 1137 snap 354 seq_num 1137) 2022-04-28T19:57:13.048 INFO:tasks.rados.rados.0.smithi059.stdout:3833: snap_remove snap 356 2022-04-28T19:57:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:13 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:13.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:14.048 INFO:tasks.rados.rados.0.smithi059.stdout:3833: done (4 left) 2022-04-28T19:57:14.048 INFO:tasks.rados.rados.0.smithi059.stdout:3834: rollback oid 27 current snap is 380 2022-04-28T19:57:14.048 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 27 to 339 2022-04-28T19:57:14.048 INFO:tasks.rados.rados.0.smithi059.stdout:3828: expect (ObjNum 1093 snap 339 seq_num 1093) 2022-04-28T19:57:14.123 INFO:tasks.rados.rados.0.smithi059.stdout:3829: finishing write tid 1 to smithi05955964-32 2022-04-28T19:57:14.123 INFO:tasks.rados.rados.0.smithi059.stdout:3829: finishing write tid 3 to smithi05955964-32 2022-04-28T19:57:14.123 INFO:tasks.rados.rados.0.smithi059.stdout:3829: finishing write tid 4 to smithi05955964-32 2022-04-28T19:57:14.123 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1608 (ObjNum 1193 snap 380 seq_num 1193) dirty exists 2022-04-28T19:57:14.123 INFO:tasks.rados.rados.0.smithi059.stdout:3829: left oid 32 (ObjNum 1193 snap 380 seq_num 1193) 2022-04-28T19:57:14.124 INFO:tasks.rados.rados.0.smithi059.stdout:3831: expect (ObjNum 1045 snap 324 seq_num 1045) 2022-04-28T19:57:14.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:14 smithi059 bash[59745]: cluster 2022-04-28T19:57:13.035718+0000 mgr.y (mgr.24961) 402 : cluster [DBG] pgmap v692: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 664 KiB/s wr, 13 op/s 2022-04-28T19:57:14.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:14 smithi059 bash[59745]: cluster 2022-04-28T19:57:13.046109+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2022-04-28T19:57:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:14 smithi059 bash[61152]: cluster 2022-04-28T19:57:13.035718+0000 mgr.y (mgr.24961) 402 : cluster [DBG] pgmap v692: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 664 KiB/s wr, 13 op/s 2022-04-28T19:57:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:14 smithi059 bash[61152]: cluster 2022-04-28T19:57:13.046109+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2022-04-28T19:57:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:14 smithi139 bash[43301]: cluster 2022-04-28T19:57:13.035718+0000 mgr.y (mgr.24961) 402 : cluster [DBG] pgmap v692: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 664 KiB/s wr, 13 op/s 2022-04-28T19:57:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:14 smithi139 bash[43301]: cluster 2022-04-28T19:57:13.046109+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2022-04-28T19:57:14.985 INFO:tasks.rados.rados.0.smithi059.stdout:3832: expect (ObjNum 1137 snap 354 seq_num 1137) 2022-04-28T19:57:15.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:15 smithi059 bash[61152]: cluster 2022-04-28T19:57:14.047406+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2022-04-28T19:57:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:15 smithi059 bash[59745]: cluster 2022-04-28T19:57:14.047406+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2022-04-28T19:57:15.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:15 smithi139 bash[43301]: cluster 2022-04-28T19:57:14.047406+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2022-04-28T19:57:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:16 smithi059 bash[61152]: cluster 2022-04-28T19:57:15.036611+0000 mgr.y (mgr.24961) 403 : cluster [DBG] pgmap v695: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 368 KiB/s wr, 10 op/s 2022-04-28T19:57:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:16 smithi059 bash[59745]: cluster 2022-04-28T19:57:15.036611+0000 mgr.y (mgr.24961) 403 : cluster [DBG] pgmap v695: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 368 KiB/s wr, 10 op/s 2022-04-28T19:57:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:16 smithi139 bash[43301]: cluster 2022-04-28T19:57:15.036611+0000 mgr.y (mgr.24961) 403 : cluster [DBG] pgmap v695: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 368 KiB/s wr, 10 op/s 2022-04-28T19:57:17.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:17 smithi059 bash[61152]: audit 2022-04-28T19:57:16.937051+0000 mon.a (mon.0) 1458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:57:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:17 smithi059 bash[61152]: audit 2022-04-28T19:57:16.937748+0000 mon.a (mon.0) 1459 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T19:57:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:17 smithi059 bash[61152]: audit 2022-04-28T19:57:16.938211+0000 mon.a (mon.0) 1460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T19:57:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:17 smithi059 bash[59745]: audit 2022-04-28T19:57:16.937051+0000 mon.a (mon.0) 1458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:57:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:17 smithi059 bash[59745]: audit 2022-04-28T19:57:16.937748+0000 mon.a (mon.0) 1459 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T19:57:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:17 smithi059 bash[59745]: audit 2022-04-28T19:57:16.938211+0000 mon.a (mon.0) 1460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T19:57:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:17 smithi139 bash[43301]: audit 2022-04-28T19:57:16.937051+0000 mon.a (mon.0) 1458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T19:57:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:17 smithi139 bash[43301]: audit 2022-04-28T19:57:16.937748+0000 mon.a (mon.0) 1459 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T19:57:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:17 smithi139 bash[43301]: audit 2022-04-28T19:57:16.938211+0000 mon.a (mon.0) 1460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T19:57:17.481 INFO:tasks.rados.rados.0.smithi059.stdout:3834: finishing rollback tid 1 to smithi05955964-27 2022-04-28T19:57:17.481 INFO:tasks.rados.rados.0.smithi059.stdout:3834: finishing rollback tid 2 to smithi05955964-27 2022-04-28T19:57:17.481 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 27 v 1585 (ObjNum 1078 snap 335 seq_num 1078) dirty exists 2022-04-28T19:57:17.481 INFO:tasks.rados.rados.0.smithi059.stdout:3828: done (4 left) 2022-04-28T19:57:17.482 INFO:tasks.rados.rados.0.smithi059.stdout:3829: done (3 left) 2022-04-28T19:57:17.482 INFO:tasks.rados.rados.0.smithi059.stdout:3831: done (2 left) 2022-04-28T19:57:17.482 INFO:tasks.rados.rados.0.smithi059.stdout:3832: done (1 left) 2022-04-28T19:57:17.482 INFO:tasks.rados.rados.0.smithi059.stdout:3834: done (0 left) 2022-04-28T19:57:17.482 INFO:tasks.rados.rados.0.smithi059.stdout:3835: copy_from oid 1 from oid 28 current snap is 380 2022-04-28T19:57:17.483 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 40 current snap is 380 2022-04-28T19:57:17.483 INFO:tasks.rados.rados.0.smithi059.stdout:3836: seq_num 1194 ranges {4579328=802816} 2022-04-28T19:57:17.521 INFO:tasks.rados.rados.0.smithi059.stdout:3836: writing smithi05955964-40 from 4579328 to 5382144 tid 1 2022-04-28T19:57:17.521 INFO:tasks.rados.rados.0.smithi059.stdout:3837: snap_create 2022-04-28T19:57:17.535 INFO:tasks.rados.rados.0.smithi059.stdout:3836: finishing write tid 1 to smithi05955964-40 2022-04-28T19:57:17.535 INFO:tasks.rados.rados.0.smithi059.stdout:3836: finishing write tid 3 to smithi05955964-40 2022-04-28T19:57:17.546 INFO:tasks.rados.rados.0.smithi059.stdout:3836: finishing write tid 4 to smithi05955964-40 2022-04-28T19:57:17.546 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 40 v 1577 (ObjNum 1194 snap 380 seq_num 1194) dirty exists 2022-04-28T19:57:17.547 INFO:tasks.rados.rados.0.smithi059.stdout:3836: left oid 40 (ObjNum 1194 snap 380 seq_num 1194) 2022-04-28T19:57:17.547 INFO:tasks.rados.rados.0.smithi059.stdout:3835: finishing copy_from to smithi05955964-1 2022-04-28T19:57:17.547 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1458 (ObjNum 1096 snap 339 seq_num 1096) dirty exists 2022-04-28T19:57:17.547 INFO:tasks.rados.rados.0.smithi059.stdout:3835: finishing copy_from racing read to smithi05955964-1 2022-04-28T19:57:18.076 INFO:tasks.rados.rados.0.smithi059.stdout:3835: done (2 left) 2022-04-28T19:57:18.076 INFO:tasks.rados.rados.0.smithi059.stdout:3836: done (1 left) 2022-04-28T19:57:18.077 INFO:tasks.rados.rados.0.smithi059.stdout:3837: done (0 left) 2022-04-28T19:57:18.077 INFO:tasks.rados.rados.0.smithi059.stdout:3838: snap_remove snap 336 2022-04-28T19:57:18.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:18 smithi059 bash[61152]: cluster 2022-04-28T19:57:17.037515+0000 mgr.y (mgr.24961) 404 : cluster [DBG] pgmap v696: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 253 KiB/s wr, 8 op/s 2022-04-28T19:57:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:18 smithi059 bash[61152]: audit 2022-04-28T19:57:17.163470+0000 mon.a (mon.0) 1461 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:57:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:18 smithi059 bash[61152]: audit 2022-04-28T19:57:17.164010+0000 mon.a (mon.0) 1462 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:57:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:18 smithi059 bash[59745]: cluster 2022-04-28T19:57:17.037515+0000 mgr.y (mgr.24961) 404 : cluster [DBG] pgmap v696: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 253 KiB/s wr, 8 op/s 2022-04-28T19:57:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:18 smithi059 bash[59745]: audit 2022-04-28T19:57:17.163470+0000 mon.a (mon.0) 1461 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:57:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:18 smithi059 bash[59745]: audit 2022-04-28T19:57:17.164010+0000 mon.a (mon.0) 1462 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:57:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:18 smithi139 bash[43301]: cluster 2022-04-28T19:57:17.037515+0000 mgr.y (mgr.24961) 404 : cluster [DBG] pgmap v696: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 253 KiB/s wr, 8 op/s 2022-04-28T19:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:18 smithi139 bash[43301]: audit 2022-04-28T19:57:17.163470+0000 mon.a (mon.0) 1461 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:18 smithi139 bash[43301]: audit 2022-04-28T19:57:17.164010+0000 mon.a (mon.0) 1462 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:57:19.077 INFO:tasks.rados.rados.0.smithi059.stdout:3838: done (0 left) 2022-04-28T19:57:19.077 INFO:tasks.rados.rados.0.smithi059.stdout:3839: delete oid 2 current snap is 381 2022-04-28T19:57:19.086 INFO:tasks.rados.rados.0.smithi059.stdout:3839: done (0 left) 2022-04-28T19:57:19.086 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 33 current snap is 381 2022-04-28T19:57:19.087 INFO:tasks.rados.rados.0.smithi059.stdout:3840: seq_num 1196 ranges {12410880=434176,12845056=401408} 2022-04-28T19:57:19.156 INFO:tasks.rados.rados.0.smithi059.stdout:3840: writing smithi05955964-33 from 12410880 to 12845056 tid 1 2022-04-28T19:57:19.158 INFO:tasks.rados.rados.0.smithi059.stdout:3840: writing smithi05955964-33 from 12845056 to 13246464 tid 2 2022-04-28T19:57:19.158 INFO:tasks.rados.rados.0.smithi059.stdout:3841: read oid 39 snap -1 2022-04-28T19:57:19.158 INFO:tasks.rados.rados.0.smithi059.stdout:3841: expect deleted 2022-04-28T19:57:19.159 INFO:tasks.rados.rados.0.smithi059.stdout:3842: read oid 42 snap -1 2022-04-28T19:57:19.159 INFO:tasks.rados.rados.0.smithi059.stdout:3842: expect (ObjNum 1188 snap 377 seq_num 1188) 2022-04-28T19:57:19.159 INFO:tasks.rados.rados.0.smithi059.stdout:3843: copy_from oid 28 from oid 9 current snap is 381 2022-04-28T19:57:19.159 INFO:tasks.rados.rados.0.smithi059.stdout:3844: read oid 6 snap -1 2022-04-28T19:57:19.159 INFO:tasks.rados.rados.0.smithi059.stdout:3844: expect (ObjNum 1158 snap 366 seq_num 1158) 2022-04-28T19:57:19.159 INFO:tasks.rados.rados.0.smithi059.stdout:3845: snap_create 2022-04-28T19:57:19.162 INFO:tasks.rados.rados.0.smithi059.stdout:3844: expect (ObjNum 1158 snap 366 seq_num 1158) 2022-04-28T19:57:19.168 INFO:tasks.rados.rados.0.smithi059.stdout:3842: expect (ObjNum 1188 snap 377 seq_num 1188) 2022-04-28T19:57:19.186 INFO:tasks.rados.rados.0.smithi059.stdout:3840: finishing write tid 1 to smithi05955964-33 2022-04-28T19:57:19.186 INFO:tasks.rados.rados.0.smithi059.stdout:3840: finishing write tid 2 to smithi05955964-33 2022-04-28T19:57:19.186 INFO:tasks.rados.rados.0.smithi059.stdout:3840: finishing write tid 4 to smithi05955964-33 2022-04-28T19:57:19.186 INFO:tasks.rados.rados.0.smithi059.stdout:3840: finishing write tid 5 to smithi05955964-33 2022-04-28T19:57:19.186 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 33 v 1611 (ObjNum 1196 snap 381 seq_num 1196) dirty exists 2022-04-28T19:57:19.186 INFO:tasks.rados.rados.0.smithi059.stdout:3840: left oid 33 (ObjNum 1196 snap 381 seq_num 1196) 2022-04-28T19:57:19.203 INFO:tasks.rados.rados.0.smithi059.stdout:3843: finishing copy_from to smithi05955964-28 2022-04-28T19:57:19.203 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1220 (ObjNum 1000 snap 297 seq_num 1000) dirty exists 2022-04-28T19:57:19.204 INFO:tasks.rados.rados.0.smithi059.stdout:3843: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:57:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:19 smithi059 bash[61152]: cluster 2022-04-28T19:57:18.076532+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2022-04-28T19:57:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:19 smithi059 bash[61152]: cluster 2022-04-28T19:57:19.076940+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2022-04-28T19:57:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:19 smithi059 bash[59745]: cluster 2022-04-28T19:57:18.076532+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2022-04-28T19:57:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:19 smithi059 bash[59745]: cluster 2022-04-28T19:57:19.076940+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2022-04-28T19:57:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:19 smithi139 bash[43301]: cluster 2022-04-28T19:57:18.076532+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2022-04-28T19:57:19.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:19 smithi139 bash[43301]: cluster 2022-04-28T19:57:19.076940+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2022-04-28T19:57:20.097 INFO:tasks.rados.rados.0.smithi059.stdout:3840: done (5 left) 2022-04-28T19:57:20.097 INFO:tasks.rados.rados.0.smithi059.stdout:3841: done (4 left) 2022-04-28T19:57:20.097 INFO:tasks.rados.rados.0.smithi059.stdout:3842: done (3 left) 2022-04-28T19:57:20.097 INFO:tasks.rados.rados.0.smithi059.stdout:3843: done (2 left) 2022-04-28T19:57:20.097 INFO:tasks.rados.rados.0.smithi059.stdout:3844: done (1 left) 2022-04-28T19:57:20.097 INFO:tasks.rados.rados.0.smithi059.stdout:3845: done (0 left) 2022-04-28T19:57:20.098 INFO:tasks.rados.rados.0.smithi059.stdout:3846: setattr oid 23 current snap is 382 2022-04-28T19:57:20.098 INFO:tasks.rados.rados.0.smithi059.stdout:3847: setattr oid 43 current snap is 382 2022-04-28T19:57:20.099 INFO:tasks.rados.rados.0.smithi059.stdout:3848: snap_remove snap 344 2022-04-28T19:57:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:20 smithi139 bash[43301]: cluster 2022-04-28T19:57:19.038402+0000 mgr.y (mgr.24961) 405 : cluster [DBG] pgmap v698: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 245 KiB/s wr, 7 op/s 2022-04-28T19:57:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:20 smithi059 bash[61152]: cluster 2022-04-28T19:57:19.038402+0000 mgr.y (mgr.24961) 405 : cluster [DBG] pgmap v698: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 245 KiB/s wr, 7 op/s 2022-04-28T19:57:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:20 smithi059 bash[59745]: cluster 2022-04-28T19:57:19.038402+0000 mgr.y (mgr.24961) 405 : cluster [DBG] pgmap v698: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 245 KiB/s wr, 7 op/s 2022-04-28T19:57:21.109 INFO:tasks.rados.rados.0.smithi059.stdout:3848: done (2 left) 2022-04-28T19:57:21.110 INFO:tasks.rados.rados.0.smithi059.stdout:3849: setattr oid 2 current snap is 382 2022-04-28T19:57:21.110 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1586 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:57:21.110 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1612 (ObjNum 1183 snap 376 seq_num 1183) dirty exists 2022-04-28T19:57:21.110 INFO:tasks.rados.rados.0.smithi059.stdout:3846: done (2 left) 2022-04-28T19:57:21.110 INFO:tasks.rados.rados.0.smithi059.stdout:3847: done (1 left) 2022-04-28T19:57:21.110 INFO:tasks.rados.rados.0.smithi059.stdout:3850: setattr oid 16 current snap is 382 2022-04-28T19:57:21.114 INFO:tasks.rados.rados.0.smithi059.stdout:3851: read oid 12 snap -1 2022-04-28T19:57:21.115 INFO:tasks.rados.rados.0.smithi059.stdout:3851: expect deleted 2022-04-28T19:57:21.115 INFO:tasks.rados.rados.0.smithi059.stdout:3852: read oid 9 snap -1 2022-04-28T19:57:21.115 INFO:tasks.rados.rados.0.smithi059.stdout:3852: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:21.115 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 382 2022-04-28T19:57:21.115 INFO:tasks.rados.rados.0.smithi059.stdout:3853: seq_num 1197 ranges {5021696=450560,5472256=401408} 2022-04-28T19:57:21.143 INFO:tasks.rados.rados.0.smithi059.stdout:3853: writing smithi05955964-43 from 5021696 to 5472256 tid 1 2022-04-28T19:57:21.266 INFO:tasks.rados.rados.0.smithi059.stdout:3853: writing smithi05955964-43 from 5472256 to 5873664 tid 2 2022-04-28T19:57:21.267 INFO:tasks.rados.rados.0.smithi059.stdout:3854: delete oid 37 current snap is 382 2022-04-28T19:57:21.267 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 2 v 1472 (ObjNum 1953066355 snap 892365160 seq_num 959788345) dirty exists 2022-04-28T19:57:21.267 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1461 (ObjNum 1185 snap 376 seq_num 1185) dirty exists 2022-04-28T19:57:21.267 INFO:tasks.rados.rados.0.smithi059.stdout:3852: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:21 smithi139 bash[43301]: cluster 2022-04-28T19:57:20.095855+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2022-04-28T19:57:21.455 INFO:tasks.rados.rados.0.smithi059.stdout:3853: finishing write tid 1 to smithi05955964-43 2022-04-28T19:57:21.455 INFO:tasks.rados.rados.0.smithi059.stdout:3853: finishing write tid 2 to smithi05955964-43 2022-04-28T19:57:21.455 INFO:tasks.rados.rados.0.smithi059.stdout:3853: finishing write tid 4 to smithi05955964-43 2022-04-28T19:57:21.455 INFO:tasks.rados.rados.0.smithi059.stdout:3853: finishing write tid 5 to smithi05955964-43 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1615 (ObjNum 1197 snap 382 seq_num 1197) dirty exists 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3853: left oid 43 (ObjNum 1197 snap 382 seq_num 1197) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3849: done (5 left) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3850: done (4 left) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3851: done (3 left) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3852: done (2 left) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3853: done (1 left) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3854: done (0 left) 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3855: read oid 15 snap -1 2022-04-28T19:57:21.456 INFO:tasks.rados.rados.0.smithi059.stdout:3855: expect (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:57:21.457 INFO:tasks.rados.rados.0.smithi059.stdout:3856: read oid 43 snap -1 2022-04-28T19:57:21.457 INFO:tasks.rados.rados.0.smithi059.stdout:3856: expect (ObjNum 1197 snap 382 seq_num 1197) 2022-04-28T19:57:21.457 INFO:tasks.rados.rados.0.smithi059.stdout:3857: snap_create 2022-04-28T19:57:21.502 INFO:tasks.rados.rados.0.smithi059.stdout:3856: expect (ObjNum 1197 snap 382 seq_num 1197) 2022-04-28T19:57:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:21 smithi059 bash[61152]: cluster 2022-04-28T19:57:20.095855+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2022-04-28T19:57:21.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:21 smithi059 bash[59745]: cluster 2022-04-28T19:57:20.095855+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2022-04-28T19:57:21.902 INFO:tasks.rados.rados.0.smithi059.stdout:3855: expect (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:57:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:21 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:21.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:22 smithi139 bash[43301]: cluster 2022-04-28T19:57:21.039245+0000 mgr.y (mgr.24961) 406 : cluster [DBG] pgmap v701: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 3 op/s 2022-04-28T19:57:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:22 smithi139 bash[43301]: cluster 2022-04-28T19:57:21.105908+0000 mon.a (mon.0) 1466 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2022-04-28T19:57:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:22 smithi139 bash[43301]: cluster 2022-04-28T19:57:22.036298+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2022-04-28T19:57:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:22 smithi059 bash[61152]: cluster 2022-04-28T19:57:21.039245+0000 mgr.y (mgr.24961) 406 : cluster [DBG] pgmap v701: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 3 op/s 2022-04-28T19:57:22.746 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:22 smithi059 bash[61152]: cluster 2022-04-28T19:57:21.105908+0000 mon.a (mon.0) 1466 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2022-04-28T19:57:22.746 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:22 smithi059 bash[61152]: cluster 2022-04-28T19:57:22.036298+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2022-04-28T19:57:22.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:22 smithi059 bash[59745]: cluster 2022-04-28T19:57:21.039245+0000 mgr.y (mgr.24961) 406 : cluster [DBG] pgmap v701: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.2 MiB/s wr, 3 op/s 2022-04-28T19:57:22.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:22 smithi059 bash[59745]: cluster 2022-04-28T19:57:21.105908+0000 mon.a (mon.0) 1466 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2022-04-28T19:57:22.747 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:22 smithi059 bash[59745]: cluster 2022-04-28T19:57:22.036298+0000 mon.a (mon.0) 1467 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2022-04-28T19:57:23.033 INFO:tasks.rados.rados.0.smithi059.stdout:3855: done (2 left) 2022-04-28T19:57:23.034 INFO:tasks.rados.rados.0.smithi059.stdout:3856: done (1 left) 2022-04-28T19:57:23.034 INFO:tasks.rados.rados.0.smithi059.stdout:3857: done (0 left) 2022-04-28T19:57:23.034 INFO:tasks.rados.rados.0.smithi059.stdout:3858: snap_remove snap 358 2022-04-28T19:57:23.128 INFO:tasks.rados.rados.0.smithi059.stdout:3858: done (0 left) 2022-04-28T19:57:23.129 INFO:tasks.rados.rados.0.smithi059.stdout:3859: snap_remove snap 355 2022-04-28T19:57:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:23 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:23.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:24.140 INFO:tasks.rados.rados.0.smithi059.stdout:3859: done (0 left) 2022-04-28T19:57:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3860: read oid 24 snap 377 2022-04-28T19:57:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3860: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:57:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 20 current snap is 383 2022-04-28T19:57:24.141 INFO:tasks.rados.rados.0.smithi059.stdout:3861: seq_num 1199 ranges {0=483328,483328=401408} 2022-04-28T19:57:24.143 INFO:tasks.rados.rados.0.smithi059.stdout:3861: writing smithi05955964-20 from 0 to 483328 tid 1 2022-04-28T19:57:24.146 INFO:tasks.rados.rados.0.smithi059.stdout:3861: writing smithi05955964-20 from 483328 to 884736 tid 2 2022-04-28T19:57:24.146 INFO:tasks.rados.rados.0.smithi059.stdout:3862: rollback oid 31 current snap is 383 2022-04-28T19:57:24.146 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 31 to 374 2022-04-28T19:57:24.147 INFO:tasks.rados.rados.0.smithi059.stdout:3863: rmattr oid 1 current snap is 383 2022-04-28T19:57:24.147 INFO:tasks.rados.rados.0.smithi059.stdout:3864: read oid 13 snap -1 2022-04-28T19:57:24.147 INFO:tasks.rados.rados.0.smithi059.stdout:3864: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:57:24.148 INFO:tasks.rados.rados.0.smithi059.stdout:3865: read oid 45 snap -1 2022-04-28T19:57:24.148 INFO:tasks.rados.rados.0.smithi059.stdout:3865: expect (ObjNum 1137 snap 354 seq_num 1137) 2022-04-28T19:57:24.148 INFO:tasks.rados.rados.0.smithi059.stdout:3866: read oid 12 snap 338 2022-04-28T19:57:24.149 INFO:tasks.rados.rados.0.smithi059.stdout:3866: expect (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:57:24.162 INFO:tasks.rados.rados.0.smithi059.stdout:3867: setattr oid 6 current snap is 383 2022-04-28T19:57:24.162 INFO:tasks.rados.rados.0.smithi059.stdout:3861: finishing write tid 1 to smithi05955964-20 2022-04-28T19:57:24.162 INFO:tasks.rados.rados.0.smithi059.stdout:3864: expect (ObjNum 1132 snap 352 seq_num 1132) 2022-04-28T19:57:24.219 INFO:tasks.rados.rados.0.smithi059.stdout:3862: finishing rollback tid 0 to smithi05955964-31 2022-04-28T19:57:24.219 INFO:tasks.rados.rados.0.smithi059.stdout:3862: finishing rollback tid 1 to smithi05955964-31 2022-04-28T19:57:24.219 INFO:tasks.rados.rados.0.smithi059.stdout:3862: finishing rollback tid 2 to smithi05955964-31 2022-04-28T19:57:24.219 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 31 v 1410 (ObjNum 1045 snap 324 seq_num 1045) dirty exists 2022-04-28T19:57:24.219 INFO:tasks.rados.rados.0.smithi059.stdout:3860: expect (ObjNum 636 snap 183 seq_num 636) 2022-04-28T19:57:24.477 INFO:tasks.rados.rados.0.smithi059.stdout:3861: finishing write tid 2 to smithi05955964-20 2022-04-28T19:57:24.477 INFO:tasks.rados.rados.0.smithi059.stdout:3861: finishing write tid 4 to smithi05955964-20 2022-04-28T19:57:24.478 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1578 (ObjNum 1096 snap 339 seq_num 1096) dirty exists 2022-04-28T19:57:24.478 INFO:tasks.rados.rados.0.smithi059.stdout:3866: expect (ObjNum 1075 snap 334 seq_num 1075) 2022-04-28T19:57:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:24 smithi059 bash[61152]: cluster 2022-04-28T19:57:23.040022+0000 mgr.y (mgr.24961) 407 : cluster [DBG] pgmap v704: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 4.8 MiB/s wr, 3 op/s 2022-04-28T19:57:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:24 smithi059 bash[61152]: cluster 2022-04-28T19:57:23.125956+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2022-04-28T19:57:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:24 smithi059 bash[59745]: cluster 2022-04-28T19:57:23.040022+0000 mgr.y (mgr.24961) 407 : cluster [DBG] pgmap v704: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 4.8 MiB/s wr, 3 op/s 2022-04-28T19:57:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:24 smithi059 bash[59745]: cluster 2022-04-28T19:57:23.125956+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2022-04-28T19:57:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:24 smithi139 bash[43301]: cluster 2022-04-28T19:57:23.040022+0000 mgr.y (mgr.24961) 407 : cluster [DBG] pgmap v704: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 4.8 MiB/s wr, 3 op/s 2022-04-28T19:57:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:24 smithi139 bash[43301]: cluster 2022-04-28T19:57:23.125956+0000 mon.a (mon.0) 1468 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2022-04-28T19:57:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:25 smithi059 bash[61152]: cluster 2022-04-28T19:57:24.137806+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2022-04-28T19:57:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:25 smithi059 bash[59745]: cluster 2022-04-28T19:57:24.137806+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2022-04-28T19:57:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:25 smithi139 bash[43301]: cluster 2022-04-28T19:57:24.137806+0000 mon.a (mon.0) 1469 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2022-04-28T19:57:25.791 INFO:tasks.rados.rados.0.smithi059.stdout:3865: expect (ObjNum 1137 snap 354 seq_num 1137) 2022-04-28T19:57:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:26 smithi059 bash[61152]: cluster 2022-04-28T19:57:25.040810+0000 mgr.y (mgr.24961) 408 : cluster [DBG] pgmap v707: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 628 KiB/s wr, 30 op/s 2022-04-28T19:57:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:26 smithi059 bash[59745]: cluster 2022-04-28T19:57:25.040810+0000 mgr.y (mgr.24961) 408 : cluster [DBG] pgmap v707: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 628 KiB/s wr, 30 op/s 2022-04-28T19:57:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:26 smithi139 bash[43301]: cluster 2022-04-28T19:57:25.040810+0000 mgr.y (mgr.24961) 408 : cluster [DBG] pgmap v707: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 628 KiB/s wr, 30 op/s 2022-04-28T19:57:28.287 INFO:tasks.rados.rados.0.smithi059.stdout:3861: finishing write tid 5 to smithi05955964-20 2022-04-28T19:57:28.287 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1316 (ObjNum 1199 snap 383 seq_num 1199) dirty exists 2022-04-28T19:57:28.287 INFO:tasks.rados.rados.0.smithi059.stdout:3861: left oid 20 (ObjNum 1199 snap 383 seq_num 1199) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3860: done (7 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3861: done (6 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3862: done (5 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3863: done (4 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3864: done (3 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3865: done (2 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3866: done (1 left) 2022-04-28T19:57:28.288 INFO:tasks.rados.rados.0.smithi059.stdout:3868: delete oid 24 current snap is 383 2022-04-28T19:57:28.289 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1317 (ObjNum 1158 snap 366 seq_num 1158) dirty exists 2022-04-28T19:57:28.290 INFO:tasks.rados.rados.0.smithi059.stdout:3867: done (1 left) 2022-04-28T19:57:28.290 INFO:tasks.rados.rados.0.smithi059.stdout:3868: done (0 left) 2022-04-28T19:57:28.290 INFO:tasks.rados.rados.0.smithi059.stdout:3869: rollback oid 5 current snap is 383 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 327 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:3870: rollback oid 48 current snap is 383 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 337 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:3871: read oid 39 snap -1 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:3871: expect deleted 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:3872: read oid 9 snap -1 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:3872: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:28.291 INFO:tasks.rados.rados.0.smithi059.stdout:3873: delete oid 12 current snap is 383 2022-04-28T19:57:28.296 INFO:tasks.rados.rados.0.smithi059.stdout:3870: finishing rollback tid 0 to smithi05955964-48 2022-04-28T19:57:28.296 INFO:tasks.rados.rados.0.smithi059.stdout:3870: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:57:28.296 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1598 (ObjNum 288 snap 0 seq_num 129) dirty dne 2022-04-28T19:57:28.297 INFO:tasks.rados.rados.0.smithi059.stdout:3870: done (4 left) 2022-04-28T19:57:28.297 INFO:tasks.rados.rados.0.smithi059.stdout:3871: done (3 left) 2022-04-28T19:57:28.297 INFO:tasks.rados.rados.0.smithi059.stdout:3873: done (2 left) 2022-04-28T19:57:28.297 INFO:tasks.rados.rados.0.smithi059.stdout:3874: setattr oid 47 current snap is 383 2022-04-28T19:57:28.298 INFO:tasks.rados.rados.0.smithi059.stdout:3869: finishing rollback tid 0 to smithi05955964-5 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:3869: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1604 (ObjNum 1365732707 snap 1383030113 seq_num 1718049138) dirty dne 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:3869: done (2 left) 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:3875: copy_from oid 11 from oid 32 current snap is 383 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:3876: rollback oid 5 current snap is 383 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 5 to 345 2022-04-28T19:57:28.299 INFO:tasks.rados.rados.0.smithi059.stdout:3877: delete oid 21 current snap is 383 2022-04-28T19:57:28.301 INFO:tasks.rados.rados.0.smithi059.stdout:3876: finishing rollback tid 1 to smithi05955964-5 2022-04-28T19:57:28.301 INFO:tasks.rados.rados.0.smithi059.stdout:3876: finishing rollback tid 2 to smithi05955964-5 2022-04-28T19:57:28.301 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 5 v 1606 (ObjNum 1365732707 snap 1383030113 seq_num 1718049138) dirty exists 2022-04-28T19:57:28.304 INFO:tasks.rados.rados.0.smithi059.stdout:3876: done (4 left) 2022-04-28T19:57:28.304 INFO:tasks.rados.rados.0.smithi059.stdout:3877: done (3 left) 2022-04-28T19:57:28.304 INFO:tasks.rados.rados.0.smithi059.stdout:3878: rollback oid 46 current snap is 383 2022-04-28T19:57:28.304 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 46 to 311 2022-04-28T19:57:28.304 INFO:tasks.rados.rados.0.smithi059.stdout:3879: snap_create 2022-04-28T19:57:28.305 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1587 (ObjNum 1170 snap 370 seq_num 1170) dirty exists 2022-04-28T19:57:28.306 INFO:tasks.rados.rados.0.smithi059.stdout:3878: finishing rollback tid 1 to smithi05955964-46 2022-04-28T19:57:28.307 INFO:tasks.rados.rados.0.smithi059.stdout:3878: finishing rollback tid 2 to smithi05955964-46 2022-04-28T19:57:28.307 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1608 (ObjNum 955 snap 283 seq_num 955) dirty exists 2022-04-28T19:57:28.316 INFO:tasks.rados.rados.0.smithi059.stdout:3875: finishing copy_from to smithi05955964-11 2022-04-28T19:57:28.316 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1608 (ObjNum 1193 snap 380 seq_num 1193) dirty exists 2022-04-28T19:57:28.316 INFO:tasks.rados.rados.0.smithi059.stdout:3875: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:57:28.318 INFO:tasks.rados.rados.0.smithi059.stdout:3872: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:28.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:28 smithi059 bash[61152]: cluster 2022-04-28T19:57:27.037998+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2022-04-28T19:57:28.497 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:28 smithi059 bash[61152]: cluster 2022-04-28T19:57:27.041178+0000 mgr.y (mgr.24961) 409 : cluster [DBG] pgmap v709: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 501 KiB/s wr, 29 op/s 2022-04-28T19:57:28.497 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:28 smithi059 bash[59745]: cluster 2022-04-28T19:57:27.037998+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2022-04-28T19:57:28.498 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:28 smithi059 bash[59745]: cluster 2022-04-28T19:57:27.041178+0000 mgr.y (mgr.24961) 409 : cluster [DBG] pgmap v709: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 501 KiB/s wr, 29 op/s 2022-04-28T19:57:28.499 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:28 smithi139 bash[43301]: cluster 2022-04-28T19:57:27.037998+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2022-04-28T19:57:28.499 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:28 smithi139 bash[43301]: cluster 2022-04-28T19:57:27.041178+0000 mgr.y (mgr.24961) 409 : cluster [DBG] pgmap v709: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 501 KiB/s wr, 29 op/s 2022-04-28T19:57:29.049 INFO:tasks.rados.rados.0.smithi059.stdout:3872: done (4 left) 2022-04-28T19:57:29.049 INFO:tasks.rados.rados.0.smithi059.stdout:3874: done (3 left) 2022-04-28T19:57:29.050 INFO:tasks.rados.rados.0.smithi059.stdout:3875: done (2 left) 2022-04-28T19:57:29.050 INFO:tasks.rados.rados.0.smithi059.stdout:3878: done (1 left) 2022-04-28T19:57:29.050 INFO:tasks.rados.rados.0.smithi059.stdout:3879: done (0 left) 2022-04-28T19:57:29.050 INFO:tasks.rados.rados.0.smithi059.stdout:3880: setattr oid 38 current snap is 384 2022-04-28T19:57:29.051 INFO:tasks.rados.rados.0.smithi059.stdout:3881: read oid 39 snap -1 2022-04-28T19:57:29.051 INFO:tasks.rados.rados.0.smithi059.stdout:3881: expect deleted 2022-04-28T19:57:29.051 INFO:tasks.rados.rados.0.smithi059.stdout:3882: read oid 43 snap 327 2022-04-28T19:57:29.051 INFO:tasks.rados.rados.0.smithi059.stdout:3882: expect deleted 2022-04-28T19:57:29.051 INFO:tasks.rados.rados.0.smithi059.stdout:3883: rollback oid 25 current snap is 384 2022-04-28T19:57:29.051 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 25 to 327 2022-04-28T19:57:29.052 INFO:tasks.rados.rados.0.smithi059.stdout:3884: snap_create 2022-04-28T19:57:29.057 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 38 v 1609 (ObjNum 1182 snap 375 seq_num 1182) dirty exists 2022-04-28T19:57:29.059 INFO:tasks.rados.rados.0.smithi059.stdout:3883: finishing rollback tid 0 to smithi05955964-25 2022-04-28T19:57:29.059 INFO:tasks.rados.rados.0.smithi059.stdout:3883: finishing rollback tid 1 to smithi05955964-25 2022-04-28T19:57:29.060 INFO:tasks.rados.rados.0.smithi059.stdout:3883: finishing rollback tid 2 to smithi05955964-25 2022-04-28T19:57:29.060 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1612 (ObjNum 974 snap 289 seq_num 974) dirty exists 2022-04-28T19:57:30.062 INFO:tasks.rados.rados.0.smithi059.stdout:3880: done (4 left) 2022-04-28T19:57:30.063 INFO:tasks.rados.rados.0.smithi059.stdout:3881: done (3 left) 2022-04-28T19:57:30.063 INFO:tasks.rados.rados.0.smithi059.stdout:3882: done (2 left) 2022-04-28T19:57:30.063 INFO:tasks.rados.rados.0.smithi059.stdout:3883: done (1 left) 2022-04-28T19:57:30.063 INFO:tasks.rados.rados.0.smithi059.stdout:3884: done (0 left) 2022-04-28T19:57:30.064 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 385 2022-04-28T19:57:30.064 INFO:tasks.rados.rados.0.smithi059.stdout:3885: seq_num 1203 ranges {0=557056,557056=401408} 2022-04-28T19:57:30.065 INFO:tasks.rados.rados.0.smithi059.stdout:3885: writing smithi05955964-34 from 0 to 557056 tid 1 2022-04-28T19:57:30.068 INFO:tasks.rados.rados.0.smithi059.stdout:3885: writing smithi05955964-34 from 557056 to 958464 tid 2 2022-04-28T19:57:30.068 INFO:tasks.rados.rados.0.smithi059.stdout:3886: read oid 9 snap 370 2022-04-28T19:57:30.068 INFO:tasks.rados.rados.0.smithi059.stdout:3886: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:30.068 INFO:tasks.rados.rados.0.smithi059.stdout:3887: snap_create 2022-04-28T19:57:30.074 INFO:tasks.rados.rados.0.smithi059.stdout:3885: finishing write tid 1 to smithi05955964-34 2022-04-28T19:57:30.075 INFO:tasks.rados.rados.0.smithi059.stdout:3885: finishing write tid 2 to smithi05955964-34 2022-04-28T19:57:30.075 INFO:tasks.rados.rados.0.smithi059.stdout:3885: finishing write tid 4 to smithi05955964-34 2022-04-28T19:57:30.076 INFO:tasks.rados.rados.0.smithi059.stdout:3885: finishing write tid 5 to smithi05955964-34 2022-04-28T19:57:30.076 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1563 (ObjNum 1203 snap 385 seq_num 1203) dirty exists 2022-04-28T19:57:30.076 INFO:tasks.rados.rados.0.smithi059.stdout:3885: left oid 34 (ObjNum 1203 snap 385 seq_num 1203) 2022-04-28T19:57:30.091 INFO:tasks.rados.rados.0.smithi059.stdout:3886: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:30.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:30 smithi059 bash[59745]: cluster 2022-04-28T19:57:29.041951+0000 mgr.y (mgr.24961) 410 : cluster [DBG] pgmap v710: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 419 KiB/s wr, 26 op/s 2022-04-28T19:57:30.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:30 smithi059 bash[59745]: cluster 2022-04-28T19:57:29.048970+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2022-04-28T19:57:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:30 smithi059 bash[61152]: cluster 2022-04-28T19:57:29.041951+0000 mgr.y (mgr.24961) 410 : cluster [DBG] pgmap v710: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 419 KiB/s wr, 26 op/s 2022-04-28T19:57:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:30 smithi059 bash[61152]: cluster 2022-04-28T19:57:29.048970+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2022-04-28T19:57:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:30 smithi139 bash[43301]: cluster 2022-04-28T19:57:29.041951+0000 mgr.y (mgr.24961) 410 : cluster [DBG] pgmap v710: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 419 KiB/s wr, 26 op/s 2022-04-28T19:57:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:30 smithi139 bash[43301]: cluster 2022-04-28T19:57:29.048970+0000 mon.a (mon.0) 1471 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2022-04-28T19:57:31.068 INFO:tasks.rados.rados.0.smithi059.stdout:3885: done (2 left) 2022-04-28T19:57:31.068 INFO:tasks.rados.rados.0.smithi059.stdout:3886: done (1 left) 2022-04-28T19:57:31.068 INFO:tasks.rados.rados.0.smithi059.stdout:3887: done (0 left) 2022-04-28T19:57:31.068 INFO:tasks.rados.rados.0.smithi059.stdout:3888: rollback oid 48 current snap is 386 2022-04-28T19:57:31.069 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 338 2022-04-28T19:57:31.069 INFO:tasks.rados.rados.0.smithi059.stdout:3889: rollback oid 16 current snap is 386 2022-04-28T19:57:31.069 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 16 to 372 2022-04-28T19:57:31.069 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 41 current snap is 386 2022-04-28T19:57:31.069 INFO:tasks.rados.rados.0.smithi059.stdout:3890: seq_num 1204 ranges {0=573440,573440=401408} 2022-04-28T19:57:31.071 INFO:tasks.rados.rados.0.smithi059.stdout:3890: writing smithi05955964-41 from 0 to 573440 tid 1 2022-04-28T19:57:31.074 INFO:tasks.rados.rados.0.smithi059.stdout:3890: writing smithi05955964-41 from 573440 to 974848 tid 2 2022-04-28T19:57:31.076 INFO:tasks.rados.rados.0.smithi059.stdout:3891: read oid 9 snap -1 2022-04-28T19:57:31.076 INFO:tasks.rados.rados.0.smithi059.stdout:3891: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:31.076 INFO:tasks.rados.rados.0.smithi059.stdout:3892: read oid 2 snap -1 2022-04-28T19:57:31.076 INFO:tasks.rados.rados.0.smithi059.stdout:3892: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:57:31.076 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 21 current snap is 386 2022-04-28T19:57:31.077 INFO:tasks.rados.rados.0.smithi059.stdout:3893: seq_num 1205 ranges {0=589824,589824=401408} 2022-04-28T19:57:31.079 INFO:tasks.rados.rados.0.smithi059.stdout:3893: writing smithi05955964-21 from 0 to 589824 tid 1 2022-04-28T19:57:31.083 INFO:tasks.rados.rados.0.smithi059.stdout:3893: writing smithi05955964-21 from 589824 to 991232 tid 2 2022-04-28T19:57:31.083 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 386 2022-04-28T19:57:31.083 INFO:tasks.rados.rados.0.smithi059.stdout:3894: seq_num 1206 ranges {3784704=606208,4390912=401408} 2022-04-28T19:57:31.107 INFO:tasks.rados.rados.0.smithi059.stdout:3894: writing smithi05955964-25 from 3784704 to 4390912 tid 1 2022-04-28T19:57:31.110 INFO:tasks.rados.rados.0.smithi059.stdout:3894: writing smithi05955964-25 from 4390912 to 4792320 tid 2 2022-04-28T19:57:31.110 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 386 2022-04-28T19:57:31.110 INFO:tasks.rados.rados.0.smithi059.stdout:3895: seq_num 1207 ranges {3883008=622592,4505600=401408} 2022-04-28T19:57:31.136 INFO:tasks.rados.rados.0.smithi059.stdout:3895: writing smithi05955964-49 from 3883008 to 4505600 tid 1 2022-04-28T19:57:31.139 INFO:tasks.rados.rados.0.smithi059.stdout:3895: writing smithi05955964-49 from 4505600 to 4907008 tid 2 2022-04-28T19:57:31.139 INFO:tasks.rados.rados.0.smithi059.stdout:3896: rollback oid 4 current snap is 386 2022-04-28T19:57:31.139 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 4 to 373 2022-04-28T19:57:31.139 INFO:tasks.rados.rados.0.smithi059.stdout:3889: finishing rollback tid 0 to smithi05955964-16 2022-04-28T19:57:31.139 INFO:tasks.rados.rados.0.smithi059.stdout:3892: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:57:31.142 INFO:tasks.rados.rados.0.smithi059.stdout:3892: done (8 left) 2022-04-28T19:57:31.142 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 47 current snap is 386 2022-04-28T19:57:31.142 INFO:tasks.rados.rados.0.smithi059.stdout:3897: seq_num 1208 ranges {2605056=638976,3244032=401408} 2022-04-28T19:57:31.160 INFO:tasks.rados.rados.0.smithi059.stdout:3897: writing smithi05955964-47 from 2605056 to 3244032 tid 1 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3897: writing smithi05955964-47 from 3244032 to 3645440 tid 2 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3898: copy_from oid 22 from oid 36 current snap is 386 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3889: finishing rollback tid 1 to smithi05955964-16 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3899: copy_from oid 13 from oid 20 current snap is 386 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3889: finishing rollback tid 2 to smithi05955964-16 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1464 (ObjNum 1098 snap 339 seq_num 1098) dirty exists 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3888: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1598 (ObjNum -133287760 snap 21893 seq_num 4161356144) dirty dne 2022-04-28T19:57:31.164 INFO:tasks.rados.rados.0.smithi059.stdout:3893: finishing write tid 1 to smithi05955964-21 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3890: finishing write tid 1 to smithi05955964-41 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3890: finishing write tid 2 to smithi05955964-41 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3890: finishing write tid 4 to smithi05955964-41 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3893: finishing write tid 2 to smithi05955964-21 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3893: finishing write tid 4 to smithi05955964-21 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3893: finishing write tid 5 to smithi05955964-21 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1170 (ObjNum 1205 snap 386 seq_num 1205) dirty exists 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3893: left oid 21 (ObjNum 1205 snap 386 seq_num 1205) 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:3890: finishing write tid 5 to smithi05955964-41 2022-04-28T19:57:31.165 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 41 v 1357 (ObjNum 1204 snap 386 seq_num 1204) dirty exists 2022-04-28T19:57:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:3890: left oid 41 (ObjNum 1204 snap 386 seq_num 1204) 2022-04-28T19:57:31.166 INFO:tasks.rados.rados.0.smithi059.stdout:3891: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:31.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:31 smithi059 bash[59745]: cluster 2022-04-28T19:57:30.061781+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2022-04-28T19:57:31.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:31 smithi059 bash[61152]: cluster 2022-04-28T19:57:30.061781+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2022-04-28T19:57:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:31 smithi139 bash[43301]: cluster 2022-04-28T19:57:30.061781+0000 mon.a (mon.0) 1472 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2022-04-28T19:57:31.471 INFO:tasks.rados.rados.0.smithi059.stdout:3894: finishing write tid 1 to smithi05955964-25 2022-04-28T19:57:31.471 INFO:tasks.rados.rados.0.smithi059.stdout:3894: finishing write tid 2 to smithi05955964-25 2022-04-28T19:57:31.471 INFO:tasks.rados.rados.0.smithi059.stdout:3894: finishing write tid 4 to smithi05955964-25 2022-04-28T19:57:31.471 INFO:tasks.rados.rados.0.smithi059.stdout:3894: finishing write tid 5 to smithi05955964-25 2022-04-28T19:57:31.471 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1616 (ObjNum 1206 snap 386 seq_num 1206) dirty exists 2022-04-28T19:57:31.471 INFO:tasks.rados.rados.0.smithi059.stdout:3894: left oid 25 (ObjNum 1206 snap 386 seq_num 1206) 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3896: finishing rollback tid 0 to smithi05955964-4 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3895: finishing write tid 1 to smithi05955964-49 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3896: finishing rollback tid 1 to smithi05955964-4 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3895: finishing write tid 2 to smithi05955964-49 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3896: finishing rollback tid 2 to smithi05955964-4 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1612 (ObjNum 901 snap 268 seq_num 901) dirty exists 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3895: finishing write tid 4 to smithi05955964-49 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3895: finishing write tid 5 to smithi05955964-49 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1581 (ObjNum 1207 snap 386 seq_num 1207) dirty exists 2022-04-28T19:57:31.472 INFO:tasks.rados.rados.0.smithi059.stdout:3895: left oid 49 (ObjNum 1207 snap 386 seq_num 1207) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3897: finishing write tid 1 to smithi05955964-47 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3897: finishing write tid 2 to smithi05955964-47 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3888: done (10 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3889: done (9 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3890: done (8 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3891: done (7 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3893: done (6 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3894: done (5 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3895: done (4 left) 2022-04-28T19:57:31.473 INFO:tasks.rados.rados.0.smithi059.stdout:3896: done (3 left) 2022-04-28T19:57:31.474 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 386 2022-04-28T19:57:31.474 INFO:tasks.rados.rados.0.smithi059.stdout:3900: seq_num 1209 ranges {1605632=655360,2260992=401408} 2022-04-28T19:57:31.493 INFO:tasks.rados.rados.0.smithi059.stdout:3900: writing smithi05955964-46 from 1605632 to 2260992 tid 1 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3900: writing smithi05955964-46 from 2260992 to 2662400 tid 2 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3901: snap_create 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3897: finishing write tid 4 to smithi05955964-47 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3897: finishing write tid 5 to smithi05955964-47 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1590 (ObjNum 1208 snap 386 seq_num 1208) dirty exists 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3897: left oid 47 (ObjNum 1208 snap 386 seq_num 1208) 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3899: finishing copy_from to smithi05955964-13 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 13 v 1316 (ObjNum 1199 snap 383 seq_num 1199) dirty exists 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3899: finishing copy_from racing read to smithi05955964-13 2022-04-28T19:57:31.496 INFO:tasks.rados.rados.0.smithi059.stdout:3898: finishing copy_from to smithi05955964-22 2022-04-28T19:57:31.497 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 22 v 1474 (ObjNum 1138 snap 355 seq_num 1138) dirty exists 2022-04-28T19:57:31.497 INFO:tasks.rados.rados.0.smithi059.stdout:3898: finishing copy_from racing read to smithi05955964-22 2022-04-28T19:57:31.503 INFO:tasks.rados.rados.0.smithi059.stdout:3900: finishing write tid 1 to smithi05955964-46 2022-04-28T19:57:31.503 INFO:tasks.rados.rados.0.smithi059.stdout:3900: finishing write tid 2 to smithi05955964-46 2022-04-28T19:57:31.504 INFO:tasks.rados.rados.0.smithi059.stdout:3900: finishing write tid 4 to smithi05955964-46 2022-04-28T19:57:31.506 INFO:tasks.rados.rados.0.smithi059.stdout:3900: finishing write tid 5 to smithi05955964-46 2022-04-28T19:57:31.506 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1620 (ObjNum 1209 snap 386 seq_num 1209) dirty exists 2022-04-28T19:57:31.506 INFO:tasks.rados.rados.0.smithi059.stdout:3900: left oid 46 (ObjNum 1209 snap 386 seq_num 1209) 2022-04-28T19:57:32.041 INFO:tasks.rados.rados.0.smithi059.stdout:3897: done (4 left) 2022-04-28T19:57:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3898: done (3 left) 2022-04-28T19:57:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3899: done (2 left) 2022-04-28T19:57:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3900: done (1 left) 2022-04-28T19:57:32.065 INFO:tasks.rados.rados.0.smithi059.stdout:3901: done (0 left) 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3902: rollback oid 20 current snap is 387 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 20 to 374 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3903: read oid 37 snap -1 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3903: expect deleted 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3904: read oid 29 snap -1 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3904: expect deleted 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3905: copy_from oid 43 from oid 3 current snap is 387 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 18 current snap is 387 2022-04-28T19:57:32.066 INFO:tasks.rados.rados.0.smithi059.stdout:3906: seq_num 1210 ranges {1605632=671744,2277376=401408} 2022-04-28T19:57:32.068 INFO:tasks.rados.rados.0.smithi059.stdout:3906: writing smithi05955964-18 from 1605632 to 2277376 tid 1 2022-04-28T19:57:32.068 INFO:tasks.rados.rados.0.smithi059.stdout:3906: writing smithi05955964-18 from 2277376 to 2678784 tid 2 2022-04-28T19:57:32.068 INFO:tasks.rados.rados.0.smithi059.stdout:3907: read oid 28 snap -1 2022-04-28T19:57:32.068 INFO:tasks.rados.rados.0.smithi059.stdout:3907: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3908: snap_create 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3902: finishing rollback tid 0 to smithi05955964-20 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3902: finishing rollback tid 1 to smithi05955964-20 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 20 v 1319 (ObjNum 288 snap 0 seq_num 33) dirty dne 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3906: finishing write tid 1 to smithi05955964-18 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3906: finishing write tid 2 to smithi05955964-18 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3906: finishing write tid 4 to smithi05955964-18 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3906: finishing write tid 5 to smithi05955964-18 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 18 v 1566 (ObjNum 1210 snap 387 seq_num 1210) dirty exists 2022-04-28T19:57:32.069 INFO:tasks.rados.rados.0.smithi059.stdout:3906: left oid 18 (ObjNum 1210 snap 387 seq_num 1210) 2022-04-28T19:57:32.070 INFO:tasks.rados.rados.0.smithi059.stdout:3905: finishing copy_from to smithi05955964-43 2022-04-28T19:57:32.070 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1310 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2022-04-28T19:57:32.070 INFO:tasks.rados.rados.0.smithi059.stdout:3905: finishing copy_from racing read to smithi05955964-43 2022-04-28T19:57:32.073 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:31 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:31.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:32.073 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:32 smithi059 bash[59745]: cluster 2022-04-28T19:57:31.042928+0000 mgr.y (mgr.24961) 411 : cluster [DBG] pgmap v713: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 480 KiB/s wr, 14 op/s 2022-04-28T19:57:32.074 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:32 smithi059 bash[59745]: cluster 2022-04-28T19:57:31.067451+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2022-04-28T19:57:32.074 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:32 smithi059 bash[59745]: cluster 2022-04-28T19:57:32.040215+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2022-04-28T19:57:32.095 INFO:tasks.rados.rados.0.smithi059.stdout:3907: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:32.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:32 smithi059 bash[61152]: cluster 2022-04-28T19:57:31.042928+0000 mgr.y (mgr.24961) 411 : cluster [DBG] pgmap v713: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 480 KiB/s wr, 14 op/s 2022-04-28T19:57:32.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:32 smithi059 bash[61152]: cluster 2022-04-28T19:57:31.067451+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2022-04-28T19:57:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:32 smithi059 bash[61152]: cluster 2022-04-28T19:57:32.040215+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2022-04-28T19:57:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:32 smithi139 bash[43301]: cluster 2022-04-28T19:57:31.042928+0000 mgr.y (mgr.24961) 411 : cluster [DBG] pgmap v713: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 480 KiB/s wr, 14 op/s 2022-04-28T19:57:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:32 smithi139 bash[43301]: cluster 2022-04-28T19:57:31.067451+0000 mon.a (mon.0) 1473 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2022-04-28T19:57:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:32 smithi139 bash[43301]: cluster 2022-04-28T19:57:32.040215+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2022-04-28T19:57:33.053 INFO:tasks.rados.rados.0.smithi059.stdout:3902: done (6 left) 2022-04-28T19:57:33.092 INFO:tasks.rados.rados.0.smithi059.stdout:3903: done (5 left) 2022-04-28T19:57:33.092 INFO:tasks.rados.rados.0.smithi059.stdout:3904: done (4 left) 2022-04-28T19:57:33.093 INFO:tasks.rados.rados.0.smithi059.stdout:3905: done (3 left) 2022-04-28T19:57:33.093 INFO:tasks.rados.rados.0.smithi059.stdout:3906: done (2 left) 2022-04-28T19:57:33.093 INFO:tasks.rados.rados.0.smithi059.stdout:3907: done (1 left) 2022-04-28T19:57:33.093 INFO:tasks.rados.rados.0.smithi059.stdout:3908: done (0 left) 2022-04-28T19:57:33.094 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 49 current snap is 388 2022-04-28T19:57:33.094 INFO:tasks.rados.rados.0.smithi059.stdout:3909: seq_num 1211 ranges {4907008=688128,5595136=401408} 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:3909: writing smithi05955964-49 from 4907008 to 5595136 tid 1 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:3909: writing smithi05955964-49 from 5595136 to 5996544 tid 2 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:3910: read oid 20 snap 331 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:3910: expect (ObjNum 989 snap 294 seq_num 989) 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:3911: setattr oid 43 current snap is 388 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 46 current snap is 388 2022-04-28T19:57:33.095 INFO:tasks.rados.rados.0.smithi059.stdout:3912: seq_num 1212 ranges {2662400=704512,3366912=401408} 2022-04-28T19:57:33.110 INFO:tasks.rados.rados.0.smithi059.stdout:3912: writing smithi05955964-46 from 2662400 to 3366912 tid 1 2022-04-28T19:57:33.113 INFO:tasks.rados.rados.0.smithi059.stdout:3912: writing smithi05955964-46 from 3366912 to 3768320 tid 2 2022-04-28T19:57:33.113 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 7 current snap is 388 2022-04-28T19:57:33.113 INFO:tasks.rados.rados.0.smithi059.stdout:3913: seq_num 1213 ranges {630784=720896,1351680=401408} 2022-04-28T19:57:33.121 INFO:tasks.rados.rados.0.smithi059.stdout:3913: writing smithi05955964-7 from 630784 to 1351680 tid 1 2022-04-28T19:57:33.124 INFO:tasks.rados.rados.0.smithi059.stdout:3913: writing smithi05955964-7 from 1351680 to 1753088 tid 2 2022-04-28T19:57:33.124 INFO:tasks.rados.rados.0.smithi059.stdout:3914: read oid 29 snap 382 2022-04-28T19:57:33.124 INFO:tasks.rados.rados.0.smithi059.stdout:3914: expect deleted 2022-04-28T19:57:33.124 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 4 current snap is 388 2022-04-28T19:57:33.124 INFO:tasks.rados.rados.0.smithi059.stdout:3915: seq_num 1214 ranges {7634944=737280,8372224=401408} 2022-04-28T19:57:33.171 INFO:tasks.rados.rados.0.smithi059.stdout:3915: writing smithi05955964-4 from 7634944 to 8372224 tid 1 2022-04-28T19:57:33.174 INFO:tasks.rados.rados.0.smithi059.stdout:3915: writing smithi05955964-4 from 8372224 to 8773632 tid 2 2022-04-28T19:57:33.174 INFO:tasks.rados.rados.0.smithi059.stdout:3916: read oid 13 snap -1 2022-04-28T19:57:33.174 INFO:tasks.rados.rados.0.smithi059.stdout:3916: expect (ObjNum 1199 snap 383 seq_num 1199) 2022-04-28T19:57:33.174 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 9 current snap is 388 2022-04-28T19:57:33.174 INFO:tasks.rados.rados.0.smithi059.stdout:3917: seq_num 1215 ranges {6774784=753664,7528448=401408} 2022-04-28T19:57:33.216 INFO:tasks.rados.rados.0.smithi059.stdout:3917: writing smithi05955964-9 from 6774784 to 7528448 tid 1 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3917: writing smithi05955964-9 from 7528448 to 7929856 tid 2 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3918: read oid 41 snap 338 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3918: expect (ObjNum 1038 snap 321 seq_num 1038) 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3919: rmattr oid 8 current snap is 388 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3919: done (10 left) 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3920: snap_create 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1616 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3909: finishing write tid 1 to smithi05955964-49 2022-04-28T19:57:33.220 INFO:tasks.rados.rados.0.smithi059.stdout:3909: finishing write tid 2 to smithi05955964-49 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3909: finishing write tid 4 to smithi05955964-49 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3909: finishing write tid 5 to smithi05955964-49 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 49 v 1584 (ObjNum 1211 snap 388 seq_num 1211) dirty exists 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3909: left oid 49 (ObjNum 1211 snap 388 seq_num 1211) 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3912: finishing write tid 1 to smithi05955964-46 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3912: finishing write tid 2 to smithi05955964-46 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3912: finishing write tid 4 to smithi05955964-46 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3912: finishing write tid 5 to smithi05955964-46 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 46 v 1624 (ObjNum 1212 snap 388 seq_num 1212) dirty exists 2022-04-28T19:57:33.221 INFO:tasks.rados.rados.0.smithi059.stdout:3912: left oid 46 (ObjNum 1212 snap 388 seq_num 1212) 2022-04-28T19:57:33.222 INFO:tasks.rados.rados.0.smithi059.stdout:3913: finishing write tid 1 to smithi05955964-7 2022-04-28T19:57:33.222 INFO:tasks.rados.rados.0.smithi059.stdout:3913: finishing write tid 2 to smithi05955964-7 2022-04-28T19:57:33.222 INFO:tasks.rados.rados.0.smithi059.stdout:3913: finishing write tid 4 to smithi05955964-7 2022-04-28T19:57:33.222 INFO:tasks.rados.rados.0.smithi059.stdout:3910: expect (ObjNum 989 snap 294 seq_num 989) 2022-04-28T19:57:33.797 INFO:tasks.rados.rados.0.smithi059.stdout:3913: finishing write tid 5 to smithi05955964-7 2022-04-28T19:57:33.797 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 7 v 1413 (ObjNum 1213 snap 388 seq_num 1213) dirty exists 2022-04-28T19:57:33.797 INFO:tasks.rados.rados.0.smithi059.stdout:3913: left oid 7 (ObjNum 1213 snap 388 seq_num 1213) 2022-04-28T19:57:33.797 INFO:tasks.rados.rados.0.smithi059.stdout:3915: finishing write tid 1 to smithi05955964-4 2022-04-28T19:57:33.797 INFO:tasks.rados.rados.0.smithi059.stdout:3915: finishing write tid 2 to smithi05955964-4 2022-04-28T19:57:33.797 INFO:tasks.rados.rados.0.smithi059.stdout:3915: finishing write tid 4 to smithi05955964-4 2022-04-28T19:57:33.798 INFO:tasks.rados.rados.0.smithi059.stdout:3915: finishing write tid 5 to smithi05955964-4 2022-04-28T19:57:33.798 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 4 v 1615 (ObjNum 1214 snap 388 seq_num 1214) dirty exists 2022-04-28T19:57:33.798 INFO:tasks.rados.rados.0.smithi059.stdout:3915: left oid 4 (ObjNum 1214 snap 388 seq_num 1214) 2022-04-28T19:57:33.798 INFO:tasks.rados.rados.0.smithi059.stdout:3916: expect (ObjNum 1199 snap 383 seq_num 1199) 2022-04-28T19:57:33.817 INFO:tasks.rados.rados.0.smithi059.stdout:3917: finishing write tid 1 to smithi05955964-9 2022-04-28T19:57:33.817 INFO:tasks.rados.rados.0.smithi059.stdout:3917: finishing write tid 2 to smithi05955964-9 2022-04-28T19:57:33.817 INFO:tasks.rados.rados.0.smithi059.stdout:3917: finishing write tid 4 to smithi05955964-9 2022-04-28T19:57:33.817 INFO:tasks.rados.rados.0.smithi059.stdout:3918: expect (ObjNum 1038 snap 321 seq_num 1038) 2022-04-28T19:57:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:33 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:33.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:33.876 INFO:tasks.rados.rados.0.smithi059.stdout:3917: finishing write tid 5 to smithi05955964-9 2022-04-28T19:57:33.876 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1223 (ObjNum 1215 snap 388 seq_num 1215) dirty exists 2022-04-28T19:57:33.876 INFO:tasks.rados.rados.0.smithi059.stdout:3917: left oid 9 (ObjNum 1215 snap 388 seq_num 1215) 2022-04-28T19:57:34.085 INFO:tasks.rados.rados.0.smithi059.stdout:3909: done (10 left) 2022-04-28T19:57:34.085 INFO:tasks.rados.rados.0.smithi059.stdout:3910: done (9 left) 2022-04-28T19:57:34.085 INFO:tasks.rados.rados.0.smithi059.stdout:3911: done (8 left) 2022-04-28T19:57:34.085 INFO:tasks.rados.rados.0.smithi059.stdout:3912: done (7 left) 2022-04-28T19:57:34.085 INFO:tasks.rados.rados.0.smithi059.stdout:3913: done (6 left) 2022-04-28T19:57:34.085 INFO:tasks.rados.rados.0.smithi059.stdout:3914: done (5 left) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3915: done (4 left) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3916: done (3 left) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3917: done (2 left) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3918: done (1 left) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3920: done (0 left) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3921: read oid 44 snap -1 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3921: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3922: read oid 18 snap -1 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3922: expect (ObjNum 1210 snap 387 seq_num 1210) 2022-04-28T19:57:34.086 INFO:tasks.rados.rados.0.smithi059.stdout:3923: read oid 33 snap -1 2022-04-28T19:57:34.087 INFO:tasks.rados.rados.0.smithi059.stdout:3923: expect (ObjNum 1196 snap 381 seq_num 1196) 2022-04-28T19:57:34.087 INFO:tasks.rados.rados.0.smithi059.stdout:3924: delete oid 16 current snap is 389 2022-04-28T19:57:34.087 INFO:tasks.rados.rados.0.smithi059.stdout:3921: expect (ObjNum 0 snap 0 seq_num 0) 2022-04-28T19:57:34.092 INFO:tasks.rados.rados.0.smithi059.stdout:3921: done (3 left) 2022-04-28T19:57:34.093 INFO:tasks.rados.rados.0.smithi059.stdout:3924: done (2 left) 2022-04-28T19:57:34.093 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 43 current snap is 389 2022-04-28T19:57:34.093 INFO:tasks.rados.rados.0.smithi059.stdout:3925: seq_num 1217 ranges {2744320=786432,3530752=401408} 2022-04-28T19:57:34.112 INFO:tasks.rados.rados.0.smithi059.stdout:3925: writing smithi05955964-43 from 2744320 to 3530752 tid 1 2022-04-28T19:57:34.115 INFO:tasks.rados.rados.0.smithi059.stdout:3925: writing smithi05955964-43 from 3530752 to 3932160 tid 2 2022-04-28T19:57:34.115 INFO:tasks.rados.rados.0.smithi059.stdout:3926: rollback oid 23 current snap is 389 2022-04-28T19:57:34.115 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 23 to 353 2022-04-28T19:57:34.115 INFO:tasks.rados.rados.0.smithi059.stdout:3922: expect (ObjNum 1210 snap 387 seq_num 1210) 2022-04-28T19:57:34.210 INFO:tasks.rados.rados.0.smithi059.stdout:3926: finishing rollback tid 0 to smithi05955964-23 2022-04-28T19:57:34.313 INFO:tasks.rados.rados.0.smithi059.stdout:3926: finishing rollback tid 1 to smithi05955964-23 2022-04-28T19:57:34.314 INFO:tasks.rados.rados.0.smithi059.stdout:3926: finishing rollback tid 2 to smithi05955964-23 2022-04-28T19:57:34.314 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 23 v 1593 (ObjNum 1049 snap 327 seq_num 1049) dirty exists 2022-04-28T19:57:34.314 INFO:tasks.rados.rados.0.smithi059.stdout:3923: expect (ObjNum 1196 snap 381 seq_num 1196) 2022-04-28T19:57:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:34 smithi059 bash[59745]: cluster 2022-04-28T19:57:33.043680+0000 mgr.y (mgr.24961) 412 : cluster [DBG] pgmap v716: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 720 KiB/s wr, 13 op/s 2022-04-28T19:57:34.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:34 smithi059 bash[59745]: cluster 2022-04-28T19:57:33.052617+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2022-04-28T19:57:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:34 smithi059 bash[61152]: cluster 2022-04-28T19:57:33.043680+0000 mgr.y (mgr.24961) 412 : cluster [DBG] pgmap v716: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 720 KiB/s wr, 13 op/s 2022-04-28T19:57:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:34 smithi059 bash[61152]: cluster 2022-04-28T19:57:33.052617+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2022-04-28T19:57:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:34 smithi139 bash[43301]: cluster 2022-04-28T19:57:33.043680+0000 mgr.y (mgr.24961) 412 : cluster [DBG] pgmap v716: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 720 KiB/s wr, 13 op/s 2022-04-28T19:57:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:34 smithi139 bash[43301]: cluster 2022-04-28T19:57:33.052617+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2022-04-28T19:57:35.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:35 smithi059 bash[59745]: cluster 2022-04-28T19:57:34.083646+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2022-04-28T19:57:35.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:35 smithi059 bash[61152]: cluster 2022-04-28T19:57:34.083646+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2022-04-28T19:57:35.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:35 smithi139 bash[43301]: cluster 2022-04-28T19:57:34.083646+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2022-04-28T19:57:35.563 INFO:tasks.rados.rados.0.smithi059.stdout:3925: finishing write tid 1 to smithi05955964-43 2022-04-28T19:57:35.563 INFO:tasks.rados.rados.0.smithi059.stdout:3925: finishing write tid 2 to smithi05955964-43 2022-04-28T19:57:35.563 INFO:tasks.rados.rados.0.smithi059.stdout:3925: finishing write tid 4 to smithi05955964-43 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3925: finishing write tid 5 to smithi05955964-43 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 43 v 1619 (ObjNum 1217 snap 389 seq_num 1217) dirty exists 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3925: left oid 43 (ObjNum 1217 snap 389 seq_num 1217) 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3922: done (3 left) 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3923: done (2 left) 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3925: done (1 left) 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3926: done (0 left) 2022-04-28T19:57:35.564 INFO:tasks.rados.rados.0.smithi059.stdout:3927: delete oid 31 current snap is 389 2022-04-28T19:57:35.569 INFO:tasks.rados.rados.0.smithi059.stdout:3927: done (0 left) 2022-04-28T19:57:35.569 INFO:tasks.rados.rados.0.smithi059.stdout:3928: read oid 7 snap -1 2022-04-28T19:57:35.569 INFO:tasks.rados.rados.0.smithi059.stdout:3928: expect (ObjNum 1213 snap 388 seq_num 1213) 2022-04-28T19:57:35.570 INFO:tasks.rados.rados.0.smithi059.stdout:3929: copy_from oid 9 from oid 15 current snap is 389 2022-04-28T19:57:35.570 INFO:tasks.rados.rados.0.smithi059.stdout:3930: rmattr oid 46 current snap is 389 2022-04-28T19:57:35.570 INFO:tasks.rados.rados.0.smithi059.stdout:3930: done (2 left) 2022-04-28T19:57:35.570 INFO:tasks.rados.rados.0.smithi059.stdout:3931: rollback oid 25 current snap is 389 2022-04-28T19:57:35.570 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 25 to 376 2022-04-28T19:57:35.570 INFO:tasks.rados.rados.0.smithi059.stdout:3932: snap_remove snap 387 2022-04-28T19:57:36.102 INFO:tasks.rados.rados.0.smithi059.stdout:3932: done (3 left) 2022-04-28T19:57:36.102 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 32 current snap is 389 2022-04-28T19:57:36.102 INFO:tasks.rados.rados.0.smithi059.stdout:3933: seq_num 1219 ranges {2949120=417792,3366912=557056,3923968=245760} 2022-04-28T19:57:36.121 INFO:tasks.rados.rados.0.smithi059.stdout:3933: writing smithi05955964-32 from 2949120 to 3366912 tid 1 2022-04-28T19:57:36.124 INFO:tasks.rados.rados.0.smithi059.stdout:3933: writing smithi05955964-32 from 3366912 to 3923968 tid 2 2022-04-28T19:57:36.126 INFO:tasks.rados.rados.0.smithi059.stdout:3933: writing smithi05955964-32 from 3923968 to 4169728 tid 3 2022-04-28T19:57:36.126 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 34 current snap is 389 2022-04-28T19:57:36.126 INFO:tasks.rados.rados.0.smithi059.stdout:3934: seq_num 1220 ranges {958464=434176,1392640=442368,1835008=360448} 2022-04-28T19:57:36.134 INFO:tasks.rados.rados.0.smithi059.stdout:3934: writing smithi05955964-34 from 958464 to 1392640 tid 1 2022-04-28T19:57:36.136 INFO:tasks.rados.rados.0.smithi059.stdout:3934: writing smithi05955964-34 from 1392640 to 1835008 tid 2 2022-04-28T19:57:36.139 INFO:tasks.rados.rados.0.smithi059.stdout:3934: writing smithi05955964-34 from 1835008 to 2195456 tid 3 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3935: copy_from oid 21 from oid 33 current snap is 389 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3931: finishing rollback tid 0 to smithi05955964-25 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3936: read oid 3 snap -1 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3936: expect (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3937: delete oid 16 current snap is 389 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3931: finishing rollback tid 1 to smithi05955964-25 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3931: finishing rollback tid 2 to smithi05955964-25 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1628 (ObjNum 1159 snap 367 seq_num 1159) dirty exists 2022-04-28T19:57:36.140 INFO:tasks.rados.rados.0.smithi059.stdout:3928: expect (ObjNum 1213 snap 388 seq_num 1213) 2022-04-28T19:57:36.197 INFO:tasks.rados.rados.0.smithi059.stdout:3929: finishing copy_from to smithi05955964-9 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1404 (ObjNum 1107 snap 342 seq_num 1107) dirty exists 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3929: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3933: finishing write tid 1 to smithi05955964-32 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3928: done (7 left) 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3929: done (6 left) 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3931: done (5 left) 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3937: done (4 left) 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3938: read oid 9 snap -1 2022-04-28T19:57:36.198 INFO:tasks.rados.rados.0.smithi059.stdout:3938: expect (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3939: snap_create 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3933: finishing write tid 2 to smithi05955964-32 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3933: finishing write tid 3 to smithi05955964-32 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3933: finishing write tid 5 to smithi05955964-32 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3933: finishing write tid 6 to smithi05955964-32 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1623 (ObjNum 1219 snap 389 seq_num 1219) dirty exists 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3933: left oid 32 (ObjNum 1219 snap 389 seq_num 1219) 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3934: finishing write tid 1 to smithi05955964-34 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3934: finishing write tid 2 to smithi05955964-34 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3934: finishing write tid 3 to smithi05955964-34 2022-04-28T19:57:36.199 INFO:tasks.rados.rados.0.smithi059.stdout:3934: finishing write tid 5 to smithi05955964-34 2022-04-28T19:57:36.200 INFO:tasks.rados.rados.0.smithi059.stdout:3934: finishing write tid 6 to smithi05955964-34 2022-04-28T19:57:36.200 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 34 v 1570 (ObjNum 1220 snap 389 seq_num 1220) dirty exists 2022-04-28T19:57:36.200 INFO:tasks.rados.rados.0.smithi059.stdout:3934: left oid 34 (ObjNum 1220 snap 389 seq_num 1220) 2022-04-28T19:57:36.200 INFO:tasks.rados.rados.0.smithi059.stdout:3936: expect (ObjNum 1015 snap 314 seq_num 1015) 2022-04-28T19:57:36.335 INFO:tasks.rados.rados.0.smithi059.stdout:3935: finishing copy_from to smithi05955964-21 2022-04-28T19:57:36.335 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1611 (ObjNum 1196 snap 381 seq_num 1196) dirty exists 2022-04-28T19:57:36.335 INFO:tasks.rados.rados.0.smithi059.stdout:3935: finishing copy_from racing read to smithi05955964-21 2022-04-28T19:57:36.335 INFO:tasks.rados.rados.0.smithi059.stdout:3938: expect (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:57:36.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:36 smithi059 bash[61152]: cluster 2022-04-28T19:57:35.044511+0000 mgr.y (mgr.24961) 413 : cluster [DBG] pgmap v719: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.6 MiB/s wr, 25 op/s 2022-04-28T19:57:36.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:36 smithi059 bash[59745]: cluster 2022-04-28T19:57:35.044511+0000 mgr.y (mgr.24961) 413 : cluster [DBG] pgmap v719: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.6 MiB/s wr, 25 op/s 2022-04-28T19:57:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:36 smithi139 bash[43301]: cluster 2022-04-28T19:57:35.044511+0000 mgr.y (mgr.24961) 413 : cluster [DBG] pgmap v719: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 3.6 MiB/s wr, 25 op/s 2022-04-28T19:57:37.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:37 smithi139 bash[43301]: cluster 2022-04-28T19:57:36.102034+0000 mon.a (mon.0) 1477 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2022-04-28T19:57:37.467 INFO:tasks.rados.rados.0.smithi059.stdout:3933: done (5 left) 2022-04-28T19:57:37.467 INFO:tasks.rados.rados.0.smithi059.stdout:3934: done (4 left) 2022-04-28T19:57:37.468 INFO:tasks.rados.rados.0.smithi059.stdout:3935: done (3 left) 2022-04-28T19:57:37.468 INFO:tasks.rados.rados.0.smithi059.stdout:3936: done (2 left) 2022-04-28T19:57:37.468 INFO:tasks.rados.rados.0.smithi059.stdout:3938: done (1 left) 2022-04-28T19:57:37.468 INFO:tasks.rados.rados.0.smithi059.stdout:3939: done (0 left) 2022-04-28T19:57:37.468 INFO:tasks.rados.rados.0.smithi059.stdout:3940: copy_from oid 34 from oid 31 current snap is 390 2022-04-28T19:57:37.468 INFO:tasks.rados.rados.0.smithi059.stdout:3941: setattr oid 24 current snap is 390 2022-04-28T19:57:37.469 INFO:tasks.rados.rados.0.smithi059.stdout:3940: finishing copy_from to smithi05955964-34 2022-04-28T19:57:37.469 INFO:tasks.rados.rados.0.smithi059.stdout:3940: got expected ENOENT (src dne) 2022-04-28T19:57:37.469 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 25 current snap is 390 2022-04-28T19:57:37.469 INFO:tasks.rados.rados.0.smithi059.stdout:3942: seq_num 1222 ranges {212992=475136,688128=606208,1294336=196608} 2022-04-28T19:57:37.474 INFO:tasks.rados.rados.0.smithi059.stdout:3942: writing smithi05955964-25 from 212992 to 688128 tid 1 2022-04-28T19:57:37.478 INFO:tasks.rados.rados.0.smithi059.stdout:3942: writing smithi05955964-25 from 688128 to 1294336 tid 2 2022-04-28T19:57:37.479 INFO:tasks.rados.rados.0.smithi059.stdout:3942: writing smithi05955964-25 from 1294336 to 1490944 tid 3 2022-04-28T19:57:37.480 INFO:tasks.rados.rados.0.smithi059.stdout:3943: rmattr oid 47 current snap is 390 2022-04-28T19:57:37.480 INFO:tasks.rados.rados.0.smithi059.stdout:3940: finishing copy_from racing read to smithi05955964-34 2022-04-28T19:57:37.480 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1616 (ObjNum -156550248 snap 21893 seq_num 3) dirty exists 2022-04-28T19:57:37.484 INFO:tasks.rados.rados.0.smithi059.stdout:3940: done (3 left) 2022-04-28T19:57:37.484 INFO:tasks.rados.rados.0.smithi059.stdout:3941: done (2 left) 2022-04-28T19:57:37.484 INFO:tasks.rados.rados.0.smithi059.stdout:3944: copy_from oid 7 from oid 39 current snap is 390 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3945: delete oid 1 current snap is 390 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3944: finishing copy_from to smithi05955964-7 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3944: got expected ENOENT (src dne) 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3944: finishing copy_from racing read to smithi05955964-7 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3944: done (3 left) 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3945: done (2 left) 2022-04-28T19:57:37.485 INFO:tasks.rados.rados.0.smithi059.stdout:3946: snap_create 2022-04-28T19:57:37.486 INFO:tasks.rados.rados.0.smithi059.stdout:3942: finishing write tid 1 to smithi05955964-25 2022-04-28T19:57:37.486 INFO:tasks.rados.rados.0.smithi059.stdout:3942: finishing write tid 2 to smithi05955964-25 2022-04-28T19:57:37.486 INFO:tasks.rados.rados.0.smithi059.stdout:3942: finishing write tid 3 to smithi05955964-25 2022-04-28T19:57:37.486 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1594 (ObjNum 1208 snap 386 seq_num 1208) dirty exists 2022-04-28T19:57:37.486 INFO:tasks.rados.rados.0.smithi059.stdout:3942: finishing write tid 5 to smithi05955964-25 2022-04-28T19:57:37.487 INFO:tasks.rados.rados.0.smithi059.stdout:3942: finishing write tid 6 to smithi05955964-25 2022-04-28T19:57:37.487 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 25 v 1633 (ObjNum 1222 snap 390 seq_num 1222) dirty exists 2022-04-28T19:57:37.487 INFO:tasks.rados.rados.0.smithi059.stdout:3942: left oid 25 (ObjNum 1222 snap 390 seq_num 1222) 2022-04-28T19:57:37.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:37 smithi059 bash[59745]: cluster 2022-04-28T19:57:36.102034+0000 mon.a (mon.0) 1477 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2022-04-28T19:57:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:37 smithi059 bash[61152]: cluster 2022-04-28T19:57:36.102034+0000 mon.a (mon.0) 1477 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2022-04-28T19:57:38.124 INFO:tasks.rados.rados.0.smithi059.stdout:3942: done (2 left) 2022-04-28T19:57:38.125 INFO:tasks.rados.rados.0.smithi059.stdout:3943: done (1 left) 2022-04-28T19:57:38.125 INFO:tasks.rados.rados.0.smithi059.stdout:3946: done (0 left) 2022-04-28T19:57:38.125 INFO:tasks.rados.rados.0.smithi059.stdout:3947: snap_remove snap 383 2022-04-28T19:57:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:38 smithi139 bash[43301]: cluster 2022-04-28T19:57:37.044933+0000 mgr.y (mgr.24961) 414 : cluster [DBG] pgmap v721: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2022-04-28T19:57:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:38 smithi139 bash[43301]: cluster 2022-04-28T19:57:37.109929+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2022-04-28T19:57:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:38 smithi059 bash[61152]: cluster 2022-04-28T19:57:37.044933+0000 mgr.y (mgr.24961) 414 : cluster [DBG] pgmap v721: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2022-04-28T19:57:38.647 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:38 smithi059 bash[61152]: cluster 2022-04-28T19:57:37.109929+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2022-04-28T19:57:38.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:38 smithi059 bash[59745]: cluster 2022-04-28T19:57:37.044933+0000 mgr.y (mgr.24961) 414 : cluster [DBG] pgmap v721: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2022-04-28T19:57:38.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:38 smithi059 bash[59745]: cluster 2022-04-28T19:57:37.109929+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2022-04-28T19:57:39.133 INFO:tasks.rados.rados.0.smithi059.stdout:3947: done (0 left) 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3948: read oid 19 snap -1 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3948: expect (ObjNum 1164 snap 368 seq_num 1164) 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3949: read oid 5 snap 362 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3949: expect (ObjNum -131958064 snap 21893 seq_num 4163301248) 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3950: copy_from oid 28 from oid 10 current snap is 391 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3951: read oid 26 snap 381 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3951: expect (ObjNum 1070 snap 334 seq_num 1070) 2022-04-28T19:57:39.134 INFO:tasks.rados.rados.0.smithi059.stdout:3952: read oid 9 snap -1 2022-04-28T19:57:39.135 INFO:tasks.rados.rados.0.smithi059.stdout:3952: expect (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:57:39.135 INFO:tasks.rados.rados.0.smithi059.stdout:3953: snap_create 2022-04-28T19:57:39.140 INFO:tasks.rados.rados.0.smithi059.stdout:3949: expect (ObjNum -131958064 snap 21893 seq_num 4163301248) 2022-04-28T19:57:39.146 INFO:tasks.rados.rados.0.smithi059.stdout:3948: expect (ObjNum 1164 snap 368 seq_num 1164) 2022-04-28T19:57:39.158 INFO:tasks.rados.rados.0.smithi059.stdout:3950: finishing copy_from to smithi05955964-28 2022-04-28T19:57:39.158 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1583 (ObjNum 1192 snap 378 seq_num 1192) dirty exists 2022-04-28T19:57:39.158 INFO:tasks.rados.rados.0.smithi059.stdout:3950: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:57:39.164 INFO:tasks.rados.rados.0.smithi059.stdout:3951: expect (ObjNum 1070 snap 334 seq_num 1070) 2022-04-28T19:57:39.385 INFO:tasks.rados.rados.0.smithi059.stdout:3952: expect (ObjNum 1107 snap 342 seq_num 1107) 2022-04-28T19:57:39.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:39 smithi059 bash[59745]: cluster 2022-04-28T19:57:38.120987+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2022-04-28T19:57:39.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:39 smithi059 bash[61152]: cluster 2022-04-28T19:57:38.120987+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2022-04-28T19:57:39.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:39 smithi139 bash[43301]: cluster 2022-04-28T19:57:38.120987+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2022-04-28T19:57:40.525 INFO:tasks.rados.rados.0.smithi059.stdout:3948: done (5 left) 2022-04-28T19:57:40.525 INFO:tasks.rados.rados.0.smithi059.stdout:3949: done (4 left) 2022-04-28T19:57:40.525 INFO:tasks.rados.rados.0.smithi059.stdout:3950: done (3 left) 2022-04-28T19:57:40.525 INFO:tasks.rados.rados.0.smithi059.stdout:3951: done (2 left) 2022-04-28T19:57:40.526 INFO:tasks.rados.rados.0.smithi059.stdout:3952: done (1 left) 2022-04-28T19:57:40.526 INFO:tasks.rados.rados.0.smithi059.stdout:3953: done (0 left) 2022-04-28T19:57:40.526 INFO:tasks.rados.rados.0.smithi059.stdout:3954: snap_remove snap 341 2022-04-28T19:57:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:40 smithi059 bash[61152]: cluster 2022-04-28T19:57:39.045841+0000 mgr.y (mgr.24961) 415 : cluster [DBG] pgmap v724: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2022-04-28T19:57:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:40 smithi059 bash[61152]: cluster 2022-04-28T19:57:39.131579+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2022-04-28T19:57:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:40 smithi059 bash[59745]: cluster 2022-04-28T19:57:39.045841+0000 mgr.y (mgr.24961) 415 : cluster [DBG] pgmap v724: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2022-04-28T19:57:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:40 smithi059 bash[59745]: cluster 2022-04-28T19:57:39.131579+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2022-04-28T19:57:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:40 smithi139 bash[43301]: cluster 2022-04-28T19:57:39.045841+0000 mgr.y (mgr.24961) 415 : cluster [DBG] pgmap v724: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2022-04-28T19:57:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:40 smithi139 bash[43301]: cluster 2022-04-28T19:57:39.131579+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2022-04-28T19:57:41.152 INFO:tasks.rados.rados.0.smithi059.stdout:3954: done (0 left) 2022-04-28T19:57:41.153 INFO:tasks.rados.rados.0.smithi059.stdout:3955: copy_from oid 10 from oid 17 current snap is 392 2022-04-28T19:57:41.153 INFO:tasks.rados.rados.0.smithi059.stdout:3956: read oid 42 snap -1 2022-04-28T19:57:41.153 INFO:tasks.rados.rados.0.smithi059.stdout:3956: expect (ObjNum 1188 snap 377 seq_num 1188) 2022-04-28T19:57:41.153 INFO:tasks.rados.rados.0.smithi059.stdout:3957: delete oid 31 current snap is 392 2022-04-28T19:57:41.158 INFO:tasks.rados.rados.0.smithi059.stdout:3957: done (2 left) 2022-04-28T19:57:41.159 INFO:tasks.rados.rados.0.smithi059.stdout:3958: read oid 11 snap -1 2022-04-28T19:57:41.159 INFO:tasks.rados.rados.0.smithi059.stdout:3958: expect (ObjNum 1193 snap 380 seq_num 1193) 2022-04-28T19:57:41.159 INFO:tasks.rados.rados.0.smithi059.stdout:3959: rmattr oid 8 current snap is 392 2022-04-28T19:57:41.159 INFO:tasks.rados.rados.0.smithi059.stdout:3959: done (3 left) 2022-04-28T19:57:41.159 INFO:tasks.rados.rados.0.smithi059.stdout:3960: snap_remove snap 346 2022-04-28T19:57:41.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:41 smithi059 bash[61152]: cluster 2022-04-28T19:57:40.134243+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2022-04-28T19:57:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:41 smithi059 bash[59745]: cluster 2022-04-28T19:57:40.134243+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2022-04-28T19:57:41.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:41 smithi139 bash[43301]: cluster 2022-04-28T19:57:40.134243+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2022-04-28T19:57:42.048 INFO:tasks.rados.rados.0.smithi059.stdout:3960: done (3 left) 2022-04-28T19:57:42.048 INFO:tasks.rados.rados.0.smithi059.stdout:3961: snap_remove snap 388 2022-04-28T19:57:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:41 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:41.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:42 smithi059 bash[61152]: cluster 2022-04-28T19:57:41.046764+0000 mgr.y (mgr.24961) 416 : cluster [DBG] pgmap v727: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 7.5 MiB/s wr, 23 op/s 2022-04-28T19:57:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:42 smithi059 bash[61152]: cluster 2022-04-28T19:57:41.149361+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2022-04-28T19:57:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:42 smithi059 bash[61152]: cluster 2022-04-28T19:57:42.045256+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2022-04-28T19:57:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:42 smithi059 bash[59745]: cluster 2022-04-28T19:57:41.046764+0000 mgr.y (mgr.24961) 416 : cluster [DBG] pgmap v727: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 7.5 MiB/s wr, 23 op/s 2022-04-28T19:57:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:42 smithi059 bash[59745]: cluster 2022-04-28T19:57:41.149361+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2022-04-28T19:57:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:42 smithi059 bash[59745]: cluster 2022-04-28T19:57:42.045256+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2022-04-28T19:57:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:42 smithi139 bash[43301]: cluster 2022-04-28T19:57:41.046764+0000 mgr.y (mgr.24961) 416 : cluster [DBG] pgmap v727: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 MiB/s rd, 7.5 MiB/s wr, 23 op/s 2022-04-28T19:57:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:42 smithi139 bash[43301]: cluster 2022-04-28T19:57:41.149361+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2022-04-28T19:57:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:42 smithi139 bash[43301]: cluster 2022-04-28T19:57:42.045256+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2022-04-28T19:57:43.059 INFO:tasks.rados.rados.0.smithi059.stdout:3961: done (3 left) 2022-04-28T19:57:43.059 INFO:tasks.rados.rados.0.smithi059.stdout:3962: delete oid 16 current snap is 392 2022-04-28T19:57:43.060 INFO:tasks.rados.rados.0.smithi059.stdout:3956: expect (ObjNum 1188 snap 377 seq_num 1188) 2022-04-28T19:57:43.077 INFO:tasks.rados.rados.0.smithi059.stdout:3955: finishing copy_from to smithi05955964-10 2022-04-28T19:57:43.077 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 10 v 1087 (ObjNum 1143 snap 357 seq_num 1143) dirty exists 2022-04-28T19:57:43.077 INFO:tasks.rados.rados.0.smithi059.stdout:3955: finishing copy_from racing read to smithi05955964-10 2022-04-28T19:57:43.077 INFO:tasks.rados.rados.0.smithi059.stdout:3958: expect (ObjNum 1193 snap 380 seq_num 1193) 2022-04-28T19:57:43.238 INFO:tasks.rados.rados.0.smithi059.stdout:3955: done (3 left) 2022-04-28T19:57:43.238 INFO:tasks.rados.rados.0.smithi059.stdout:3956: done (2 left) 2022-04-28T19:57:43.238 INFO:tasks.rados.rados.0.smithi059.stdout:3958: done (1 left) 2022-04-28T19:57:43.238 INFO:tasks.rados.rados.0.smithi059.stdout:3962: done (0 left) 2022-04-28T19:57:43.238 INFO:tasks.rados.rados.0.smithi059.stdout:3963: snap_create 2022-04-28T19:57:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:43 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:43.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:44.159 INFO:tasks.rados.rados.0.smithi059.stdout:3963: done (0 left) 2022-04-28T19:57:44.159 INFO:tasks.rados.rados.0.smithi059.stdout:3964: read oid 34 snap -1 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3964: expect (ObjNum 1220 snap 389 seq_num 1220) 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3965: read oid 4 snap 375 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3965: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3966: rollback oid 45 current snap is 393 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 45 to 371 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3967: read oid 9 snap 378 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3967: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3968: rollback oid 48 current snap is 393 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 48 to 381 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3969: read oid 42 snap 368 2022-04-28T19:57:44.160 INFO:tasks.rados.rados.0.smithi059.stdout:3969: expect deleted 2022-04-28T19:57:44.161 INFO:tasks.rados.rados.0.smithi059.stdout:3970: snap_remove snap 295 2022-04-28T19:57:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:44 smithi059 bash[61152]: cluster 2022-04-28T19:57:43.047412+0000 mgr.y (mgr.24961) 417 : cluster [DBG] pgmap v730: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 7.2 MiB/s wr, 19 op/s 2022-04-28T19:57:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:44 smithi059 bash[61152]: cluster 2022-04-28T19:57:43.059007+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2022-04-28T19:57:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:44 smithi059 bash[59745]: cluster 2022-04-28T19:57:43.047412+0000 mgr.y (mgr.24961) 417 : cluster [DBG] pgmap v730: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 7.2 MiB/s wr, 19 op/s 2022-04-28T19:57:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:44 smithi059 bash[59745]: cluster 2022-04-28T19:57:43.059007+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2022-04-28T19:57:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:44 smithi139 bash[43301]: cluster 2022-04-28T19:57:43.047412+0000 mgr.y (mgr.24961) 417 : cluster [DBG] pgmap v730: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 MiB/s rd, 7.2 MiB/s wr, 19 op/s 2022-04-28T19:57:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:44 smithi139 bash[43301]: cluster 2022-04-28T19:57:43.059007+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2022-04-28T19:57:45.169 INFO:tasks.rados.rados.0.smithi059.stdout:3970: done (6 left) 2022-04-28T19:57:45.169 INFO:tasks.rados.rados.0.smithi059.stdout:3971: snap_remove snap 310 2022-04-28T19:57:45.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:45 smithi059 bash[59745]: cluster 2022-04-28T19:57:44.157002+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2022-04-28T19:57:45.625 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:45 smithi059 bash[61152]: cluster 2022-04-28T19:57:44.157002+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2022-04-28T19:57:45.626 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:45 smithi139 bash[43301]: cluster 2022-04-28T19:57:44.157002+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2022-04-28T19:57:46.178 INFO:tasks.rados.rados.0.smithi059.stdout:3971: done (6 left) 2022-04-28T19:57:46.178 INFO:tasks.rados.rados.0.smithi059.stdout:3972: rollback oid 3 current snap is 393 2022-04-28T19:57:46.179 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 3 to 331 2022-04-28T19:57:46.179 INFO:tasks.rados.rados.0.smithi059.stdout:3968: finishing rollback tid 1 to smithi05955964-48 2022-04-28T19:57:46.179 INFO:tasks.rados.rados.0.smithi059.stdout:3968: finishing rollback tid 2 to smithi05955964-48 2022-04-28T19:57:46.179 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 48 v 1600 (ObjNum 1151 snap 360 seq_num 1151) dirty exists 2022-04-28T19:57:46.179 INFO:tasks.rados.rados.0.smithi059.stdout:3966: finishing rollback tid 0 to smithi05955964-45 2022-04-28T19:57:46.179 INFO:tasks.rados.rados.0.smithi059.stdout:3966: finishing rollback tid 1 to smithi05955964-45 2022-04-28T19:57:46.180 INFO:tasks.rados.rados.0.smithi059.stdout:3966: finishing rollback tid 2 to smithi05955964-45 2022-04-28T19:57:46.180 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 45 v 1322 (ObjNum 1137 snap 354 seq_num 1137) dirty exists 2022-04-28T19:57:46.180 INFO:tasks.rados.rados.0.smithi059.stdout:3964: expect (ObjNum 1220 snap 389 seq_num 1220) 2022-04-28T19:57:46.249 INFO:tasks.rados.rados.0.smithi059.stdout:3964: done (6 left) 2022-04-28T19:57:46.249 INFO:tasks.rados.rados.0.smithi059.stdout:3966: done (5 left) 2022-04-28T19:57:46.249 INFO:tasks.rados.rados.0.smithi059.stdout:3968: done (4 left) 2022-04-28T19:57:46.250 INFO:tasks.rados.rados.0.smithi059.stdout:3969: done (3 left) 2022-04-28T19:57:46.250 INFO:tasks.rados.rados.0.smithi059.stdout:3973: snap_remove snap 362 2022-04-28T19:57:46.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:46 smithi059 bash[59745]: cluster 2022-04-28T19:57:45.048251+0000 mgr.y (mgr.24961) 418 : cluster [DBG] pgmap v733: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 576 KiB/s wr, 3 op/s 2022-04-28T19:57:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:46 smithi059 bash[59745]: cluster 2022-04-28T19:57:45.168073+0000 mon.a (mon.0) 1486 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2022-04-28T19:57:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:46 smithi059 bash[61152]: cluster 2022-04-28T19:57:45.048251+0000 mgr.y (mgr.24961) 418 : cluster [DBG] pgmap v733: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 576 KiB/s wr, 3 op/s 2022-04-28T19:57:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:46 smithi059 bash[61152]: cluster 2022-04-28T19:57:45.168073+0000 mon.a (mon.0) 1486 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2022-04-28T19:57:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:46 smithi139 bash[43301]: cluster 2022-04-28T19:57:45.048251+0000 mgr.y (mgr.24961) 418 : cluster [DBG] pgmap v733: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 576 KiB/s wr, 3 op/s 2022-04-28T19:57:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:46 smithi139 bash[43301]: cluster 2022-04-28T19:57:45.168073+0000 mon.a (mon.0) 1486 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2022-04-28T19:57:47.050 INFO:tasks.rados.rados.0.smithi059.stdout:3973: done (3 left) 2022-04-28T19:57:47.199 INFO:tasks.rados.rados.0.smithi059.stdout:3974: read oid 14 snap -1 2022-04-28T19:57:47.199 INFO:tasks.rados.rados.0.smithi059.stdout:3974: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:57:47.199 INFO:tasks.rados.rados.0.smithi059.stdout:3975: delete oid 50 current snap is 393 2022-04-28T19:57:47.199 INFO:tasks.rados.rados.0.smithi059.stdout:3967: expect (ObjNum 1000 snap 297 seq_num 1000) 2022-04-28T19:57:47.352 INFO:tasks.rados.rados.0.smithi059.stdout:3965: expect (ObjNum 901 snap 268 seq_num 901) 2022-04-28T19:57:47.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:47 smithi059 bash[59745]: cluster 2022-04-28T19:57:46.177418+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2022-04-28T19:57:47.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:47 smithi059 bash[59745]: cluster 2022-04-28T19:57:47.048084+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2022-04-28T19:57:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:47 smithi059 bash[61152]: cluster 2022-04-28T19:57:46.177418+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2022-04-28T19:57:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:47 smithi059 bash[61152]: cluster 2022-04-28T19:57:47.048084+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2022-04-28T19:57:47.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:47 smithi139 bash[43301]: cluster 2022-04-28T19:57:46.177418+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2022-04-28T19:57:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:47 smithi139 bash[43301]: cluster 2022-04-28T19:57:47.048084+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2022-04-28T19:57:47.954 INFO:tasks.rados.rados.0.smithi059.stdout:3972: finishing rollback tid 0 to smithi05955964-3 2022-04-28T19:57:47.955 INFO:tasks.rados.rados.0.smithi059.stdout:3972: finishing rollback tid 1 to smithi05955964-3 2022-04-28T19:57:47.955 INFO:tasks.rados.rados.0.smithi059.stdout:3972: finishing rollback tid 2 to smithi05955964-3 2022-04-28T19:57:47.955 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 3 v 1325 (ObjNum 1015 snap 314 seq_num 1015) dirty exists 2022-04-28T19:57:47.955 INFO:tasks.rados.rados.0.smithi059.stdout:3974: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:57:48.232 INFO:tasks.rados.rados.0.smithi059.stdout:3965: done (4 left) 2022-04-28T19:57:48.232 INFO:tasks.rados.rados.0.smithi059.stdout:3967: done (3 left) 2022-04-28T19:57:48.232 INFO:tasks.rados.rados.0.smithi059.stdout:3972: done (2 left) 2022-04-28T19:57:48.232 INFO:tasks.rados.rados.0.smithi059.stdout:3974: done (1 left) 2022-04-28T19:57:48.232 INFO:tasks.rados.rados.0.smithi059.stdout:3975: done (0 left) 2022-04-28T19:57:48.233 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 12 current snap is 393 2022-04-28T19:57:48.233 INFO:tasks.rados.rados.0.smithi059.stdout:3976: seq_num 1227 ranges {0=557056,557056=483328,1040384=319488} 2022-04-28T19:57:48.240 INFO:tasks.rados.rados.0.smithi059.stdout:3976: writing smithi05955964-12 from 0 to 557056 tid 1 2022-04-28T19:57:48.247 INFO:tasks.rados.rados.0.smithi059.stdout:3976: writing smithi05955964-12 from 557056 to 1040384 tid 2 2022-04-28T19:57:48.250 INFO:tasks.rados.rados.0.smithi059.stdout:3976: writing smithi05955964-12 from 1040384 to 1359872 tid 3 2022-04-28T19:57:48.250 INFO:tasks.rados.rados.0.smithi059.stdout:3977: copy_from oid 9 from oid 13 current snap is 393 2022-04-28T19:57:48.250 INFO:tasks.rados.rados.0.smithi059.stdout:3978: read oid 27 snap -1 2022-04-28T19:57:48.250 INFO:tasks.rados.rados.0.smithi059.stdout:3978: expect (ObjNum 1078 snap 335 seq_num 1078) 2022-04-28T19:57:48.250 INFO:tasks.rados.rados.0.smithi059.stdout:append oid 24 current snap is 393 2022-04-28T19:57:48.250 INFO:tasks.rados.rados.0.smithi059.stdout:3979: seq_num 1228 ranges {0=573440,573440=761856,1335296=40960} 2022-04-28T19:57:48.252 INFO:tasks.rados.rados.0.smithi059.stdout:3979: writing smithi05955964-24 from 0 to 573440 tid 1 2022-04-28T19:57:48.257 INFO:tasks.rados.rados.0.smithi059.stdout:3979: writing smithi05955964-24 from 573440 to 1335296 tid 2 2022-04-28T19:57:48.259 INFO:tasks.rados.rados.0.smithi059.stdout:3979: writing smithi05955964-24 from 1335296 to 1376256 tid 3 2022-04-28T19:57:48.259 INFO:tasks.rados.rados.0.smithi059.stdout:3980: read oid 16 snap -1 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3980: expect deleted 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3981: copy_from oid 37 from oid 45 current snap is 393 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3976: finishing write tid 1 to smithi05955964-12 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3982: copy_from oid 11 from oid 8 current snap is 393 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3976: finishing write tid 2 to smithi05955964-12 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3983: copy_from oid 6 from oid 44 current snap is 393 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3976: finishing write tid 3 to smithi05955964-12 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3976: finishing write tid 5 to smithi05955964-12 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3976: finishing write tid 6 to smithi05955964-12 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 12 v 1418 (ObjNum 1227 snap 393 seq_num 1227) dirty exists 2022-04-28T19:57:48.260 INFO:tasks.rados.rados.0.smithi059.stdout:3976: left oid 12 (ObjNum 1227 snap 393 seq_num 1227) 2022-04-28T19:57:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:3976: done (7 left) 2022-04-28T19:57:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:3984: copy_from oid 32 from oid 36 current snap is 393 2022-04-28T19:57:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:3985: snap_create 2022-04-28T19:57:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:3979: finishing write tid 1 to smithi05955964-24 2022-04-28T19:57:48.261 INFO:tasks.rados.rados.0.smithi059.stdout:3978: expect (ObjNum 1078 snap 335 seq_num 1078) 2022-04-28T19:57:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:3979: finishing write tid 2 to smithi05955964-24 2022-04-28T19:57:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:3979: finishing write tid 3 to smithi05955964-24 2022-04-28T19:57:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:3977: finishing copy_from to smithi05955964-9 2022-04-28T19:57:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 9 v 1316 (ObjNum 1199 snap 383 seq_num 1199) dirty exists 2022-04-28T19:57:48.296 INFO:tasks.rados.rados.0.smithi059.stdout:3977: finishing copy_from racing read to smithi05955964-9 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3979: finishing write tid 5 to smithi05955964-24 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3979: finishing write tid 6 to smithi05955964-24 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 24 v 1620 (ObjNum 1228 snap 393 seq_num 1228) dirty exists 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3979: left oid 24 (ObjNum 1228 snap 393 seq_num 1228) 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3983: finishing copy_from to smithi05955964-6 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 6 v 1075 (ObjNum 288 snap 0 seq_num 33) dirty exists 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3983: finishing copy_from racing read to smithi05955964-6 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3984: finishing copy_from to smithi05955964-32 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 32 v 1474 (ObjNum 1138 snap 355 seq_num 1138) dirty exists 2022-04-28T19:57:48.297 INFO:tasks.rados.rados.0.smithi059.stdout:3984: finishing copy_from racing read to smithi05955964-32 2022-04-28T19:57:48.298 INFO:tasks.rados.rados.0.smithi059.stdout:3982: finishing copy_from to smithi05955964-11 2022-04-28T19:57:48.298 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 11 v 1596 (ObjNum 1166 snap 369 seq_num 1166) dirty exists 2022-04-28T19:57:48.298 INFO:tasks.rados.rados.0.smithi059.stdout:3982: finishing copy_from racing read to smithi05955964-11 2022-04-28T19:57:48.354 INFO:tasks.rados.rados.0.smithi059.stdout:3981: finishing copy_from to smithi05955964-37 2022-04-28T19:57:48.354 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1322 (ObjNum 1137 snap 354 seq_num 1137) dirty exists 2022-04-28T19:57:48.355 INFO:tasks.rados.rados.0.smithi059.stdout:3981: finishing copy_from racing read to smithi05955964-37 2022-04-28T19:57:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:48 smithi059 bash[61152]: cluster 2022-04-28T19:57:47.048969+0000 mgr.y (mgr.24961) 419 : cluster [DBG] pgmap v737: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 577 KiB/s wr, 6 op/s 2022-04-28T19:57:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:48 smithi059 bash[59745]: cluster 2022-04-28T19:57:47.048969+0000 mgr.y (mgr.24961) 419 : cluster [DBG] pgmap v737: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 577 KiB/s wr, 6 op/s 2022-04-28T19:57:48.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:48 smithi139 bash[43301]: cluster 2022-04-28T19:57:47.048969+0000 mgr.y (mgr.24961) 419 : cluster [DBG] pgmap v737: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 577 KiB/s wr, 6 op/s 2022-04-28T19:57:49.190 INFO:tasks.rados.rados.0.smithi059.stdout:3977: done (8 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3978: done (7 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3979: done (6 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3980: done (5 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3981: done (4 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3982: done (3 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3983: done (2 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3984: done (1 left) 2022-04-28T19:57:49.191 INFO:tasks.rados.rados.0.smithi059.stdout:3985: done (0 left) 2022-04-28T19:57:49.192 INFO:tasks.rados.rados.0.smithi059.stdout:3986: rollback oid 1 current snap is 394 2022-04-28T19:57:49.192 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 1 to 370 2022-04-28T19:57:49.192 INFO:tasks.rados.rados.0.smithi059.stdout:3987: delete oid 16 current snap is 394 2022-04-28T19:57:49.193 INFO:tasks.rados.rados.0.smithi059.stdout:3986: finishing rollback tid 1 to smithi05955964-1 2022-04-28T19:57:49.194 INFO:tasks.rados.rados.0.smithi059.stdout:3986: finishing rollback tid 2 to smithi05955964-1 2022-04-28T19:57:49.194 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 1 v 1587 (ObjNum 933 snap 276 seq_num 933) dirty exists 2022-04-28T19:57:49.195 INFO:tasks.rados.rados.0.smithi059.stdout:3986: done (1 left) 2022-04-28T19:57:49.195 INFO:tasks.rados.rados.0.smithi059.stdout:3987: done (0 left) 2022-04-28T19:57:49.195 INFO:tasks.rados.rados.0.smithi059.stdout:3988: setattr oid 37 current snap is 394 2022-04-28T19:57:49.196 INFO:tasks.rados.rados.0.smithi059.stdout:3989: snap_create 2022-04-28T19:57:49.201 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 37 v 1571 (ObjNum 1137 snap 354 seq_num 1137) dirty exists 2022-04-28T19:57:50.200 INFO:tasks.rados.rados.0.smithi059.stdout:3988: done (1 left) 2022-04-28T19:57:50.200 INFO:tasks.rados.rados.0.smithi059.stdout:3989: done (0 left) 2022-04-28T19:57:50.200 INFO:tasks.rados.rados.0.smithi059.stdout:3990: read oid 14 snap -1 2022-04-28T19:57:50.200 INFO:tasks.rados.rados.0.smithi059.stdout:3990: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:57:50.200 INFO:tasks.rados.rados.0.smithi059.stdout:3991: snap_create 2022-04-28T19:57:50.219 INFO:tasks.rados.rados.0.smithi059.stdout:3990: expect (ObjNum 898 snap 268 seq_num 898) 2022-04-28T19:57:50.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:50 smithi059 bash[61152]: cluster 2022-04-28T19:57:49.049825+0000 mgr.y (mgr.24961) 420 : cluster [DBG] pgmap v738: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.6 MiB/s wr, 5 op/s 2022-04-28T19:57:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:50 smithi059 bash[61152]: cluster 2022-04-28T19:57:49.190349+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2022-04-28T19:57:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:50 smithi059 bash[59745]: cluster 2022-04-28T19:57:49.049825+0000 mgr.y (mgr.24961) 420 : cluster [DBG] pgmap v738: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.6 MiB/s wr, 5 op/s 2022-04-28T19:57:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:50 smithi059 bash[59745]: cluster 2022-04-28T19:57:49.190349+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2022-04-28T19:57:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:50 smithi139 bash[43301]: cluster 2022-04-28T19:57:49.049825+0000 mgr.y (mgr.24961) 420 : cluster [DBG] pgmap v738: 89 pgs: 89 active+clean; 1.3 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.6 MiB/s wr, 5 op/s 2022-04-28T19:57:50.775 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:50 smithi139 bash[43301]: cluster 2022-04-28T19:57:49.190349+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2022-04-28T19:57:51.214 INFO:tasks.rados.rados.0.smithi059.stdout:3990: done (1 left) 2022-04-28T19:57:51.225 INFO:tasks.rados.rados.0.smithi059.stdout:3991: done (0 left) 2022-04-28T19:57:51.225 INFO:tasks.rados.rados.0.smithi059.stdout:3992: snap_remove snap 327 2022-04-28T19:57:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[61152]: cluster 2022-04-28T19:57:50.198559+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2022-04-28T19:57:51.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[61152]: audit 2022-04-28T19:57:50.946257+0000 mon.a (mon.0) 1491 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:57:51.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[61152]: audit 2022-04-28T19:57:50.949249+0000 mon.a (mon.0) 1492 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:57:51.616 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[61152]: audit 2022-04-28T19:57:50.951401+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:57:51.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[59745]: cluster 2022-04-28T19:57:50.198559+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2022-04-28T19:57:51.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[59745]: audit 2022-04-28T19:57:50.946257+0000 mon.a (mon.0) 1491 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:57:51.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[59745]: audit 2022-04-28T19:57:50.949249+0000 mon.a (mon.0) 1492 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:57:51.617 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[59745]: audit 2022-04-28T19:57:50.951401+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:57:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:51 smithi139 bash[43301]: cluster 2022-04-28T19:57:50.198559+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2022-04-28T19:57:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:51 smithi139 bash[43301]: audit 2022-04-28T19:57:50.946257+0000 mon.a (mon.0) 1491 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:57:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:51 smithi139 bash[43301]: audit 2022-04-28T19:57:50.949249+0000 mon.a (mon.0) 1492 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:57:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:51 smithi139 bash[43301]: audit 2022-04-28T19:57:50.951401+0000 mon.a (mon.0) 1493 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:57:52.054 INFO:tasks.rados.rados.0.smithi059.stdout:3992: done (0 left) 2022-04-28T19:57:52.054 INFO:tasks.rados.rados.0.smithi059.stdout:3993: read oid 18 snap -1 2022-04-28T19:57:52.054 INFO:tasks.rados.rados.0.smithi059.stdout:3993: expect (ObjNum 1210 snap 387 seq_num 1210) 2022-04-28T19:57:52.054 INFO:tasks.rados.rados.0.smithi059.stdout:3994: copy_from oid 16 from oid 2 current snap is 396 2022-04-28T19:57:52.054 INFO:tasks.rados.rados.0.smithi059.stdout:3995: copy_from oid 47 from oid 5 current snap is 396 2022-04-28T19:57:52.054 INFO:tasks.rados.rados.0.smithi059.stdout:3996: copy_from oid 28 from oid 49 current snap is 396 2022-04-28T19:57:52.055 INFO:tasks.rados.rados.0.smithi059.stdout:3997: snap_remove snap 370 2022-04-28T19:57:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:51 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:51.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: cephadm 2022-04-28T19:57:50.953204+0000 mgr.y (mgr.24961) 421 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:57:52.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: cluster 2022-04-28T19:57:51.050712+0000 mgr.y (mgr.24961) 422 : cluster [DBG] pgmap v741: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.7 MiB/s wr, 7 op/s 2022-04-28T19:57:52.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: cluster 2022-04-28T19:57:51.212028+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2022-04-28T19:57:52.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: audit 2022-04-28T19:57:51.259658+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: audit 2022-04-28T19:57:51.266194+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: audit 2022-04-28T19:57:51.570018+0000 mon.a (mon.0) 1497 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.650 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[61152]: cluster 2022-04-28T19:57:52.052795+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: cephadm 2022-04-28T19:57:50.953204+0000 mgr.y (mgr.24961) 421 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: cluster 2022-04-28T19:57:51.050712+0000 mgr.y (mgr.24961) 422 : cluster [DBG] pgmap v741: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.7 MiB/s wr, 7 op/s 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: cluster 2022-04-28T19:57:51.212028+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: audit 2022-04-28T19:57:51.259658+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: audit 2022-04-28T19:57:51.266194+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: audit 2022-04-28T19:57:51.570018+0000 mon.a (mon.0) 1497 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.651 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:52 smithi059 bash[59745]: cluster 2022-04-28T19:57:52.052795+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2022-04-28T19:57:52.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: cephadm 2022-04-28T19:57:50.953204+0000 mgr.y (mgr.24961) 421 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:57:52.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: cluster 2022-04-28T19:57:51.050712+0000 mgr.y (mgr.24961) 422 : cluster [DBG] pgmap v741: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.7 MiB/s wr, 7 op/s 2022-04-28T19:57:52.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: cluster 2022-04-28T19:57:51.212028+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2022-04-28T19:57:52.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: audit 2022-04-28T19:57:51.259658+0000 mon.a (mon.0) 1495 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: audit 2022-04-28T19:57:51.266194+0000 mon.a (mon.0) 1496 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.652 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: audit 2022-04-28T19:57:51.570018+0000 mon.a (mon.0) 1497 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:57:52.653 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:52 smithi139 bash[43301]: cluster 2022-04-28T19:57:52.052795+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2022-04-28T19:57:53.068 INFO:tasks.rados.rados.0.smithi059.stdout:3997: done (4 left) 2022-04-28T19:57:53.068 INFO:tasks.rados.rados.0.smithi059.stdout:3998: snap_remove snap 365 2022-04-28T19:57:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:57:53 smithi059 bash[72035]: level=error ts=2022-04-28T19:57:53.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:57:54.067 INFO:tasks.rados.rados.0.smithi059.stdout:3998: done (4 left) 2022-04-28T19:57:54.067 INFO:tasks.rados.rados.0.smithi059.stdout:3999: rollback oid 21 current snap is 396 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:rollback oid 21 to 311 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 5 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:3995: finishing copy_from to smithi05955964-47 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 47 v 1606 (ObjNum 1 snap 21893 seq_num 4161576128) dirty exists 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:3995: finishing copy_from racing read to smithi05955964-47 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:3994: finishing copy_from to smithi05955964-16 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 16 v 1472 (ObjNum 0 snap 0 seq_num 0) dirty exists 2022-04-28T19:57:54.068 INFO:tasks.rados.rados.0.smithi059.stdout:3994: finishing copy_from racing read to smithi05955964-16 2022-04-28T19:57:54.069 INFO:tasks.rados.rados.0.smithi059.stdout:3993: expect (ObjNum 1210 snap 387 seq_num 1210) 2022-04-28T19:57:54.160 INFO:tasks.rados.rados.0.smithi059.stdout:3996: finishing copy_from to smithi05955964-28 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 28 v 1584 (ObjNum 1211 snap 388 seq_num 1211) dirty exists 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3996: finishing copy_from racing read to smithi05955964-28 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3993: done (4 left) 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3994: done (3 left) 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3995: done (2 left) 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3996: done (1 left) 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout: waiting on 1 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3999: finishing rollback tid 0 to smithi05955964-21 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3999: finishing rollback tid 1 to smithi05955964-21 2022-04-28T19:57:54.161 INFO:tasks.rados.rados.0.smithi059.stdout:3999: finishing rollback tid 2 to smithi05955964-21 2022-04-28T19:57:54.162 INFO:tasks.rados.rados.0.smithi059.stdout:update_object_version oid 21 v 1614 (ObjNum 916 snap 273 seq_num 916) dirty exists 2022-04-28T19:57:54.162 INFO:tasks.rados.rados.0.smithi059.stdout:3999: done (0 left) 2022-04-28T19:57:54.170 INFO:tasks.rados.rados.0.smithi059.stderr:0 errors. 2022-04-28T19:57:54.170 INFO:tasks.rados.rados.0.smithi059.stderr: 2022-04-28T19:57:54.184 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd dump --format=json 2022-04-28T19:57:54.326 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:57:54.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:54 smithi059 bash[59745]: cluster 2022-04-28T19:57:53.051425+0000 mgr.y (mgr.24961) 423 : cluster [DBG] pgmap v744: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2022-04-28T19:57:54.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:54 smithi059 bash[59745]: cluster 2022-04-28T19:57:53.067734+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2022-04-28T19:57:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:54 smithi059 bash[61152]: cluster 2022-04-28T19:57:53.051425+0000 mgr.y (mgr.24961) 423 : cluster [DBG] pgmap v744: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2022-04-28T19:57:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:54 smithi059 bash[61152]: cluster 2022-04-28T19:57:53.067734+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2022-04-28T19:57:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:54 smithi139 bash[43301]: cluster 2022-04-28T19:57:53.051425+0000 mgr.y (mgr.24961) 423 : cluster [DBG] pgmap v744: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.8 MiB/s wr, 6 op/s 2022-04-28T19:57:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:54 smithi139 bash[43301]: cluster 2022-04-28T19:57:53.067734+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2022-04-28T19:57:55.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:55 smithi059 bash[61152]: cluster 2022-04-28T19:57:54.067240+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2022-04-28T19:57:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:55 smithi059 bash[59745]: cluster 2022-04-28T19:57:54.067240+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2022-04-28T19:57:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:55 smithi139 bash[43301]: cluster 2022-04-28T19:57:54.067240+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2022-04-28T19:57:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:56 smithi059 bash[61152]: cluster 2022-04-28T19:57:55.052375+0000 mgr.y (mgr.24961) 424 : cluster [DBG] pgmap v747: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2022-04-28T19:57:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:56 smithi059 bash[59745]: cluster 2022-04-28T19:57:55.052375+0000 mgr.y (mgr.24961) 424 : cluster [DBG] pgmap v747: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2022-04-28T19:57:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:56 smithi139 bash[43301]: cluster 2022-04-28T19:57:55.052375+0000 mgr.y (mgr.24961) 424 : cluster [DBG] pgmap v747: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.4 MiB/s wr, 7 op/s 2022-04-28T19:57:57.623 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:57:57.624 INFO:teuthology.orchestra.run.smithi059.stdout:{"epoch":834,"fsid":"20a1061c-c72a-11ec-8c39-001a4aab830c","created":"2022-04-28T19:34:30.427578+0000","modified":"2022-04-28T19:57:54.056763+0000","last_up_change":"2022-04-28T19:54:10.043467+0000","last_in_change":"2022-04-28T19:40:01.137034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"jewel","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2022-04-28T19:36:09.679269+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"352","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2022-04-28T19:40:38.422278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"51","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2022-04-28T19:42:26.047647+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2022-04-28T19:42:26.668967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"56","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2022-04-28T19:44:21.750840+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"834","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":759,"snap_epoch":834,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"923b6dc7-5112-4753-8e1f-7f260c42bcb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":10,"last_clean_end":411,"up_from":423,"up_thru":627,"down_at":413,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6802","nonce":787267435},{"type":"v1","addr":"172.21.15.59:6803","nonce":787267435}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6804","nonce":787267435},{"type":"v1","addr":"172.21.15.59:6805","nonce":787267435}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6808","nonce":787267435},{"type":"v1","addr":"172.21.15.59:6809","nonce":787267435}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6806","nonce":787267435},{"type":"v1","addr":"172.21.15.59:6807","nonce":787267435}]},"public_addr":"172.21.15.59:6803/787267435","cluster_addr":"172.21.15.59:6805/787267435","heartbeat_back_addr":"172.21.15.59:6809/787267435","heartbeat_front_addr":"172.21.15.59:6807/787267435","state":["exists","up"]},{"osd":1,"uuid":"5173783f-e72e-45f9-943b-bfa78f399cf4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":16,"last_clean_end":440,"up_from":453,"up_thru":627,"down_at":441,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6810","nonce":518678580},{"type":"v1","addr":"172.21.15.59:6811","nonce":518678580}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6812","nonce":518678580},{"type":"v1","addr":"172.21.15.59:6813","nonce":518678580}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6816","nonce":518678580},{"type":"v1","addr":"172.21.15.59:6817","nonce":518678580}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6814","nonce":518678580},{"type":"v1","addr":"172.21.15.59:6815","nonce":518678580}]},"public_addr":"172.21.15.59:6811/518678580","cluster_addr":"172.21.15.59:6813/518678580","heartbeat_back_addr":"172.21.15.59:6817/518678580","heartbeat_front_addr":"172.21.15.59:6815/518678580","state":["exists","up"]},{"osd":2,"uuid":"a3832226-ed91-463c-917e-209d876b2656","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":21,"last_clean_end":470,"up_from":482,"up_thru":627,"down_at":471,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6818","nonce":3675900974},{"type":"v1","addr":"172.21.15.59:6819","nonce":3675900974}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6820","nonce":3675900974},{"type":"v1","addr":"172.21.15.59:6821","nonce":3675900974}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6824","nonce":3675900974},{"type":"v1","addr":"172.21.15.59:6825","nonce":3675900974}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6822","nonce":3675900974},{"type":"v1","addr":"172.21.15.59:6823","nonce":3675900974}]},"public_addr":"172.21.15.59:6819/3675900974","cluster_addr":"172.21.15.59:6821/3675900974","heartbeat_back_addr":"172.21.15.59:6825/3675900974","heartbeat_front_addr":"172.21.15.59:6823/3675900974","state":["exists","up"]},{"osd":3,"uuid":"2e34b16e-61a7-4684-a305-7d3c3a64f393","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":499,"up_from":512,"up_thru":627,"down_at":501,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6826","nonce":3357327104},{"type":"v1","addr":"172.21.15.59:6827","nonce":3357327104}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6828","nonce":3357327104},{"type":"v1","addr":"172.21.15.59:6829","nonce":3357327104}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6832","nonce":3357327104},{"type":"v1","addr":"172.21.15.59:6833","nonce":3357327104}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.59:6830","nonce":3357327104},{"type":"v1","addr":"172.21.15.59:6831","nonce":3357327104}]},"public_addr":"172.21.15.59:6827/3357327104","cluster_addr":"172.21.15.59:6829/3357327104","heartbeat_back_addr":"172.21.15.59:6833/3357327104","heartbeat_front_addr":"172.21.15.59:6831/3357327104","state":["exists","up"]},{"osd":4,"uuid":"fb8003d3-2e2b-49dc-9c09-7951da79c977","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":529,"up_from":541,"up_thru":627,"down_at":530,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6800","nonce":3795640394},{"type":"v1","addr":"172.21.15.139:6801","nonce":3795640394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6802","nonce":3795640394},{"type":"v1","addr":"172.21.15.139:6803","nonce":3795640394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6806","nonce":3795640394},{"type":"v1","addr":"172.21.15.139:6807","nonce":3795640394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6804","nonce":3795640394},{"type":"v1","addr":"172.21.15.139:6805","nonce":3795640394}]},"public_addr":"172.21.15.139:6801/3795640394","cluster_addr":"172.21.15.139:6803/3795640394","heartbeat_back_addr":"172.21.15.139:6807/3795640394","heartbeat_front_addr":"172.21.15.139:6805/3795640394","state":["exists","up"]},{"osd":5,"uuid":"c6583c6c-c41c-45b4-8035-9c5158a4fc9d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":559,"up_from":571,"up_thru":627,"down_at":560,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6808","nonce":2702638567},{"type":"v1","addr":"172.21.15.139:6809","nonce":2702638567}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6810","nonce":2702638567},{"type":"v1","addr":"172.21.15.139:6811","nonce":2702638567}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6814","nonce":2702638567},{"type":"v1","addr":"172.21.15.139:6815","nonce":2702638567}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6812","nonce":2702638567},{"type":"v1","addr":"172.21.15.139:6813","nonce":2702638567}]},"public_addr":"172.21.15.139:6809/2702638567","cluster_addr":"172.21.15.139:6811/2702638567","heartbeat_back_addr":"172.21.15.139:6815/2702638567","heartbeat_front_addr":"172.21.15.139:6813/2702638567","state":["exists","up"]},{"osd":6,"uuid":"ab0065ec-bbb4-403d-97b7-d4756f91e824","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":41,"last_clean_end":589,"up_from":602,"up_thru":627,"down_at":590,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6816","nonce":3450296303},{"type":"v1","addr":"172.21.15.139:6817","nonce":3450296303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6818","nonce":3450296303},{"type":"v1","addr":"172.21.15.139:6819","nonce":3450296303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6822","nonce":3450296303},{"type":"v1","addr":"172.21.15.139:6823","nonce":3450296303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6820","nonce":3450296303},{"type":"v1","addr":"172.21.15.139:6821","nonce":3450296303}]},"public_addr":"172.21.15.139:6817/3450296303","cluster_addr":"172.21.15.139:6819/3450296303","heartbeat_back_addr":"172.21.15.139:6823/3450296303","heartbeat_front_addr":"172.21.15.139:6821/3450296303","state":["exists","up"]},{"osd":7,"uuid":"04593407-3ff3-42d0-bc54-fe0e3e96c835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":46,"last_clean_end":615,"up_from":627,"up_thru":627,"down_at":617,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6824","nonce":1889887116},{"type":"v1","addr":"172.21.15.139:6825","nonce":1889887116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6826","nonce":1889887116},{"type":"v1","addr":"172.21.15.139:6827","nonce":1889887116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6830","nonce":1889887116},{"type":"v1","addr":"172.21.15.139:6831","nonce":1889887116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.139:6828","nonce":1889887116},{"type":"v1","addr":"172.21.15.139:6829","nonce":1889887116}]},"public_addr":"172.21.15.139:6825/1889887116","cluster_addr":"172.21.15.139:6827/1889887116","heartbeat_back_addr":"172.21.15.139:6831/1889887116","heartbeat_front_addr":"172.21.15.139:6829/1889887116","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:15.357824+0000","dead_epoch":413},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:37:40.048601+0000","dead_epoch":441},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:02.747600+0000","dead_epoch":471},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:27.304844+0000","dead_epoch":501},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:38:49.708141+0000","dead_epoch":530},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:12.852749+0000","dead_epoch":560},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:39:37.626704+0000","dead_epoch":590},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2022-04-28T19:40:00.076999+0000","dead_epoch":617}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.139:6833/1765843380":"2022-04-29T19:54:26.109459+0000","172.21.15.139:6832/1765843380":"2022-04-29T19:54:26.109459+0000","172.21.15.59:0/2473162909":"2022-04-29T19:49:09.961834+0000","172.21.15.59:0/154241392":"2022-04-29T19:49:09.961834+0000","172.21.15.59:6801/3431119152":"2022-04-29T19:49:09.961834+0000","172.21.15.59:0/3662111215":"2022-04-29T19:34:54.197243+0000","172.21.15.139:6835/182856780":"2022-04-29T19:48:17.574794+0000","172.21.15.59:0/1269652554":"2022-04-29T19:34:54.197243+0000","172.21.15.59:0/3545965256":"2022-04-29T19:49:09.961834+0000","172.21.15.59:6800/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/245946828":"2022-04-29T19:49:09.961834+0000","172.21.15.59:6801/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.139:0/836182014":"2022-04-29T19:48:17.574794+0000","172.21.15.139:0/650349500":"2022-04-29T19:48:17.574794+0000","172.21.15.139:6834/182856780":"2022-04-29T19:48:17.574794+0000","172.21.15.59:0/3381593068":"2022-04-29T19:47:26.061303+0000","172.21.15.59:0/1235412511":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/2735512980":"2022-04-29T19:35:15.837313+0000","172.21.15.59:0/397086884":"2022-04-29T19:47:26.061303+0000","172.21.15.59:6800/1632668962":"2022-04-29T19:34:54.197243+0000","172.21.15.59:6800/103934933":"2022-04-29T19:47:26.061303+0000","172.21.15.59:0/2161102763":"2022-04-29T19:35:15.837313+0000","172.21.15.59:6801/103934933":"2022-04-29T19:47:26.061303+0000","172.21.15.139:0/3147865049":"2022-04-29T19:48:17.574794+0000","172.21.15.59:6800/3431119152":"2022-04-29T19:49:09.961834+0000","172.21.15.139:0/3603577324":"2022-04-29T19:48:17.574794+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","ruleset-failure-domain":"osd","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":628,"length":1},{"begin":701,"length":1},{"begin":713,"length":1}]}],"new_removed_snaps":[{"pool":5,"snaps":[{"begin":701,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2022-04-28T19:57:58.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:58 smithi059 bash[61152]: cluster 2022-04-28T19:57:57.053415+0000 mgr.y (mgr.24961) 425 : cluster [DBG] pgmap v748: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1002 KiB/s wr, 6 op/s 2022-04-28T19:57:58.337 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:57:58 smithi059 bash[61152]: audit 2022-04-28T19:57:57.621892+0000 mon.a (mon.0) 1501 : audit [DBG] from='client.? 172.21.15.59:0/3260595616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:57:58.337 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:58 smithi059 bash[59745]: cluster 2022-04-28T19:57:57.053415+0000 mgr.y (mgr.24961) 425 : cluster [DBG] pgmap v748: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1002 KiB/s wr, 6 op/s 2022-04-28T19:57:58.338 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:57:58 smithi059 bash[59745]: audit 2022-04-28T19:57:57.621892+0000 mon.a (mon.0) 1501 : audit [DBG] from='client.? 172.21.15.59:0/3260595616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:57:58.339 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph pg dump --format=json 2022-04-28T19:57:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:58 smithi139 bash[43301]: cluster 2022-04-28T19:57:57.053415+0000 mgr.y (mgr.24961) 425 : cluster [DBG] pgmap v748: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1002 KiB/s wr, 6 op/s 2022-04-28T19:57:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:57:58 smithi139 bash[43301]: audit 2022-04-28T19:57:57.621892+0000 mon.a (mon.0) 1501 : audit [DBG] from='client.? 172.21.15.59:0/3260595616' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2022-04-28T19:57:58.480 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:58:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:00 smithi139 bash[43301]: cluster 2022-04-28T19:57:59.054320+0000 mgr.y (mgr.24961) 426 : cluster [DBG] pgmap v749: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 835 KiB/s wr, 5 op/s 2022-04-28T19:58:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:00 smithi059 bash[61152]: cluster 2022-04-28T19:57:59.054320+0000 mgr.y (mgr.24961) 426 : cluster [DBG] pgmap v749: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 835 KiB/s wr, 5 op/s 2022-04-28T19:58:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:00 smithi059 bash[59745]: cluster 2022-04-28T19:57:59.054320+0000 mgr.y (mgr.24961) 426 : cluster [DBG] pgmap v749: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 835 KiB/s wr, 5 op/s 2022-04-28T19:58:01.771 INFO:teuthology.orchestra.run.smithi059.stdout: 2022-04-28T19:58:01.873 INFO:teuthology.orchestra.run.smithi059.stderr:dumped all 2022-04-28T19:58:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:01 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:01.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:02 smithi139 bash[43301]: cluster 2022-04-28T19:58:01.055442+0000 mgr.y (mgr.24961) 427 : cluster [DBG] pgmap v750: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 732 KiB/s wr, 4 op/s 2022-04-28T19:58:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:02 smithi139 bash[43301]: cluster 2022-04-28T19:58:02.054200+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2022-04-28T19:58:02.400 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:02 smithi059 bash[61152]: cluster 2022-04-28T19:58:01.055442+0000 mgr.y (mgr.24961) 427 : cluster [DBG] pgmap v750: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 732 KiB/s wr, 4 op/s 2022-04-28T19:58:02.400 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:02 smithi059 bash[61152]: cluster 2022-04-28T19:58:02.054200+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2022-04-28T19:58:02.400 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:02 smithi059 bash[59745]: cluster 2022-04-28T19:58:01.055442+0000 mgr.y (mgr.24961) 427 : cluster [DBG] pgmap v750: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 732 KiB/s wr, 4 op/s 2022-04-28T19:58:02.400 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:02 smithi059 bash[59745]: cluster 2022-04-28T19:58:02.054200+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2022-04-28T19:58:02.401 INFO:teuthology.orchestra.run.smithi059.stdout:{"pg_ready":true,"pg_map":{"version":750,"stamp":"2022-04-28T19:58:01.054635+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1389347220,"num_objects":780,"num_object_clones":708,"num_object_copies":2340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":780,"num_whiteouts":5,"num_read":3932,"num_read_kb":6624074,"num_write":9090,"num_write_kb":2082306,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":160,"num_bytes_recovered":538370048,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15000,"ondisk_log_size":15000,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2587420,"kb_used_data":1927132,"kb_used_omap":1695,"kb_used_meta":658528,"kb_avail":747373796,"statfs":{"total":767960285184,"available":765310767104,"internally_reserved":0,"allocated":1973383168,"data_stored":5000288396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1735708,"internal_metadata":674333668},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":6946816,"num_objects":4,"num_object_clones":4,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":-1,"num_read":11,"num_read_kb":23547,"num_write":25,"num_write_kb":5856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003597"},"pg_stats":[{"pgid":"3.1a","version":"643'2","reported_seq":811,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059954+0000","last_change":"2022-04-28T19:52:40.937469+0000","last_active":"2022-04-28T19:57:53.059954+0000","last_peered":"2022-04-28T19:57:53.059954+0000","last_clean":"2022-04-28T19:57:53.059954+0000","last_became_active":"2022-04-28T19:52:40.936965+0000","last_became_peered":"2022-04-28T19:52:40.936965+0000","last_unstale":"2022-04-28T19:57:53.059954+0000","last_undegraded":"2022-04-28T19:57:53.059954+0000","last_fullsized":"2022-04-28T19:57:53.059954+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.067293+0000","last_change":"2022-04-28T19:52:40.941143+0000","last_active":"2022-04-28T19:57:53.067293+0000","last_peered":"2022-04-28T19:57:53.067293+0000","last_clean":"2022-04-28T19:57:53.067293+0000","last_became_active":"2022-04-28T19:52:40.940705+0000","last_became_peered":"2022-04-28T19:52:40.940705+0000","last_unstale":"2022-04-28T19:57:53.067293+0000","last_undegraded":"2022-04-28T19:57:53.067293+0000","last_fullsized":"2022-04-28T19:57:53.067293+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.076617+0000","last_change":"2022-04-28T19:52:40.940241+0000","last_active":"2022-04-28T19:57:53.076617+0000","last_peered":"2022-04-28T19:57:53.076617+0000","last_clean":"2022-04-28T19:57:53.076617+0000","last_became_active":"2022-04-28T19:52:40.939750+0000","last_became_peered":"2022-04-28T19:52:40.939750+0000","last_unstale":"2022-04-28T19:57:53.076617+0000","last_undegraded":"2022-04-28T19:57:53.076617+0000","last_fullsized":"2022-04-28T19:57:53.076617+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557447+0000","last_change":"2022-04-28T19:54:11.071443+0000","last_active":"2022-04-28T19:57:53.557447+0000","last_peered":"2022-04-28T19:57:53.557447+0000","last_clean":"2022-04-28T19:57:53.557447+0000","last_became_active":"2022-04-28T19:54:11.070890+0000","last_became_peered":"2022-04-28T19:54:11.070890+0000","last_unstale":"2022-04-28T19:57:53.557447+0000","last_undegraded":"2022-04-28T19:57:53.557447+0000","last_fullsized":"2022-04-28T19:57:53.557447+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":800,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.065350+0000","last_change":"2022-04-28T19:52:12.052926+0000","last_active":"2022-04-28T19:57:53.065350+0000","last_peered":"2022-04-28T19:57:53.065350+0000","last_clean":"2022-04-28T19:57:53.065350+0000","last_became_active":"2022-04-28T19:52:12.052446+0000","last_became_peered":"2022-04-28T19:52:12.052446+0000","last_unstale":"2022-04-28T19:57:53.065350+0000","last_undegraded":"2022-04-28T19:57:53.065350+0000","last_fullsized":"2022-04-28T19:57:53.065350+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":786,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.757771+0000","last_change":"2022-04-28T19:53:41.973937+0000","last_active":"2022-04-28T19:57:53.757771+0000","last_peered":"2022-04-28T19:57:53.757771+0000","last_clean":"2022-04-28T19:57:53.757771+0000","last_became_active":"2022-04-28T19:53:41.973609+0000","last_became_peered":"2022-04-28T19:53:41.973609+0000","last_unstale":"2022-04-28T19:57:53.757771+0000","last_undegraded":"2022-04-28T19:57:53.757771+0000","last_fullsized":"2022-04-28T19:57:53.757771+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557555+0000","last_change":"2022-04-28T19:53:10.974221+0000","last_active":"2022-04-28T19:57:53.557555+0000","last_peered":"2022-04-28T19:57:53.557555+0000","last_clean":"2022-04-28T19:57:53.557555+0000","last_became_active":"2022-04-28T19:53:10.973750+0000","last_became_peered":"2022-04-28T19:53:10.973750+0000","last_unstale":"2022-04-28T19:57:53.557555+0000","last_undegraded":"2022-04-28T19:57:53.557555+0000","last_fullsized":"2022-04-28T19:57:53.557555+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.065117+0000","last_change":"2022-04-28T19:53:41.976817+0000","last_active":"2022-04-28T19:57:53.065117+0000","last_peered":"2022-04-28T19:57:53.065117+0000","last_clean":"2022-04-28T19:57:53.065117+0000","last_became_active":"2022-04-28T19:53:41.976485+0000","last_became_peered":"2022-04-28T19:53:41.976485+0000","last_unstale":"2022-04-28T19:57:53.065117+0000","last_undegraded":"2022-04-28T19:57:53.065117+0000","last_fullsized":"2022-04-28T19:57:53.065117+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":781,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.900895+0000","last_change":"2022-04-28T19:54:11.066241+0000","last_active":"2022-04-28T19:57:53.900895+0000","last_peered":"2022-04-28T19:57:53.900895+0000","last_clean":"2022-04-28T19:57:53.900895+0000","last_became_active":"2022-04-28T19:54:11.065870+0000","last_became_peered":"2022-04-28T19:54:11.065870+0000","last_unstale":"2022-04-28T19:57:53.900895+0000","last_undegraded":"2022-04-28T19:57:53.900895+0000","last_fullsized":"2022-04-28T19:57:53.900895+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":769,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.065730+0000","last_change":"2022-04-28T19:54:11.066456+0000","last_active":"2022-04-28T19:57:53.065730+0000","last_peered":"2022-04-28T19:57:53.065730+0000","last_clean":"2022-04-28T19:57:53.065730+0000","last_became_active":"2022-04-28T19:54:11.066085+0000","last_became_peered":"2022-04-28T19:54:11.066085+0000","last_unstale":"2022-04-28T19:57:53.065730+0000","last_undegraded":"2022-04-28T19:57:53.065730+0000","last_fullsized":"2022-04-28T19:57:53.065730+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"56'3","reported_seq":772,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.997789+0000","last_change":"2022-04-28T19:54:11.074169+0000","last_active":"2022-04-28T19:57:53.997789+0000","last_peered":"2022-04-28T19:57:53.997789+0000","last_clean":"2022-04-28T19:57:53.997789+0000","last_became_active":"2022-04-28T19:54:11.073666+0000","last_became_peered":"2022-04-28T19:54:11.073666+0000","last_unstale":"2022-04-28T19:57:53.997789+0000","last_undegraded":"2022-04-28T19:57:53.997789+0000","last_fullsized":"2022-04-28T19:57:53.997789+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.076519+0000","last_change":"2022-04-28T19:53:41.984984+0000","last_active":"2022-04-28T19:57:53.076519+0000","last_peered":"2022-04-28T19:57:53.076519+0000","last_clean":"2022-04-28T19:57:53.076519+0000","last_became_active":"2022-04-28T19:53:41.984645+0000","last_became_peered":"2022-04-28T19:53:41.984645+0000","last_unstale":"2022-04-28T19:57:53.076519+0000","last_undegraded":"2022-04-28T19:57:53.076519+0000","last_fullsized":"2022-04-28T19:57:53.076519+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"645'34","reported_seq":833,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059109+0000","last_change":"2022-04-28T19:54:11.064961+0000","last_active":"2022-04-28T19:57:53.059109+0000","last_peered":"2022-04-28T19:57:53.059109+0000","last_clean":"2022-04-28T19:57:53.059109+0000","last_became_active":"2022-04-28T19:54:11.064574+0000","last_became_peered":"2022-04-28T19:54:11.064574+0000","last_unstale":"2022-04-28T19:57:53.059109+0000","last_undegraded":"2022-04-28T19:57:53.059109+0000","last_fullsized":"2022-04-28T19:57:53.059109+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":34,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":24211,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":33,"num_write_kb":40,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":779,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.058675+0000","last_change":"2022-04-28T19:53:41.975729+0000","last_active":"2022-04-28T19:57:53.058675+0000","last_peered":"2022-04-28T19:57:53.058675+0000","last_clean":"2022-04-28T19:57:53.058675+0000","last_became_active":"2022-04-28T19:53:41.975387+0000","last_became_peered":"2022-04-28T19:53:41.975387+0000","last_unstale":"2022-04-28T19:57:53.058675+0000","last_undegraded":"2022-04-28T19:57:53.058675+0000","last_fullsized":"2022-04-28T19:57:53.058675+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"56'2","reported_seq":772,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.997291+0000","last_change":"2022-04-28T19:54:11.070700+0000","last_active":"2022-04-28T19:57:53.997291+0000","last_peered":"2022-04-28T19:57:53.997291+0000","last_clean":"2022-04-28T19:57:53.997291+0000","last_became_active":"2022-04-28T19:54:11.070241+0000","last_became_peered":"2022-04-28T19:54:11.070241+0000","last_unstale":"2022-04-28T19:57:53.997291+0000","last_undegraded":"2022-04-28T19:57:53.997291+0000","last_fullsized":"2022-04-28T19:57:53.997291+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":3,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.066449+0000","last_change":"2022-04-28T19:54:11.066800+0000","last_active":"2022-04-28T19:57:53.066449+0000","last_peered":"2022-04-28T19:57:53.066449+0000","last_clean":"2022-04-28T19:57:53.066449+0000","last_became_active":"2022-04-28T19:54:11.066250+0000","last_became_peered":"2022-04-28T19:54:11.066250+0000","last_unstale":"2022-04-28T19:57:53.066449+0000","last_undegraded":"2022-04-28T19:57:53.066449+0000","last_fullsized":"2022-04-28T19:57:53.066449+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.556925+0000","last_change":"2022-04-28T19:54:11.067245+0000","last_active":"2022-04-28T19:57:53.556925+0000","last_peered":"2022-04-28T19:57:53.556925+0000","last_clean":"2022-04-28T19:57:53.556925+0000","last_became_active":"2022-04-28T19:54:11.066853+0000","last_became_peered":"2022-04-28T19:54:11.066853+0000","last_unstale":"2022-04-28T19:57:53.556925+0000","last_undegraded":"2022-04-28T19:57:53.556925+0000","last_fullsized":"2022-04-28T19:57:53.556925+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":798,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.900886+0000","last_change":"2022-04-28T19:54:11.067054+0000","last_active":"2022-04-28T19:57:53.900886+0000","last_peered":"2022-04-28T19:57:53.900886+0000","last_clean":"2022-04-28T19:57:53.900886+0000","last_became_active":"2022-04-28T19:54:11.066742+0000","last_became_peered":"2022-04-28T19:54:11.066742+0000","last_unstale":"2022-04-28T19:57:53.900886+0000","last_undegraded":"2022-04-28T19:57:53.900886+0000","last_fullsized":"2022-04-28T19:57:53.900886+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":786,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.996818+0000","last_change":"2022-04-28T19:54:11.072784+0000","last_active":"2022-04-28T19:57:53.996818+0000","last_peered":"2022-04-28T19:57:53.996818+0000","last_clean":"2022-04-28T19:57:53.996818+0000","last_became_active":"2022-04-28T19:54:11.070834+0000","last_became_peered":"2022-04-28T19:54:11.070834+0000","last_unstale":"2022-04-28T19:57:53.996818+0000","last_undegraded":"2022-04-28T19:57:53.996818+0000","last_fullsized":"2022-04-28T19:57:53.996818+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557137+0000","last_change":"2022-04-28T19:54:11.073390+0000","last_active":"2022-04-28T19:57:53.557137+0000","last_peered":"2022-04-28T19:57:53.557137+0000","last_clean":"2022-04-28T19:57:53.557137+0000","last_became_active":"2022-04-28T19:54:11.073035+0000","last_became_peered":"2022-04-28T19:54:11.073035+0000","last_unstale":"2022-04-28T19:57:53.557137+0000","last_undegraded":"2022-04-28T19:57:53.557137+0000","last_fullsized":"2022-04-28T19:57:53.557137+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"56'8","reported_seq":795,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.756888+0000","last_change":"2022-04-28T19:53:41.975792+0000","last_active":"2022-04-28T19:57:53.756888+0000","last_peered":"2022-04-28T19:57:53.756888+0000","last_clean":"2022-04-28T19:57:53.756888+0000","last_became_active":"2022-04-28T19:53:41.973690+0000","last_became_peered":"2022-04-28T19:53:41.973690+0000","last_unstale":"2022-04-28T19:57:53.756888+0000","last_undegraded":"2022-04-28T19:57:53.756888+0000","last_fullsized":"2022-04-28T19:57:53.756888+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.067964+0000","last_change":"2022-04-28T19:54:11.066714+0000","last_active":"2022-04-28T19:57:53.067964+0000","last_peered":"2022-04-28T19:57:53.067964+0000","last_clean":"2022-04-28T19:57:53.067964+0000","last_became_active":"2022-04-28T19:54:11.066336+0000","last_became_peered":"2022-04-28T19:54:11.066336+0000","last_unstale":"2022-04-28T19:57:53.067964+0000","last_undegraded":"2022-04-28T19:57:53.067964+0000","last_fullsized":"2022-04-28T19:57:53.067964+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"818'1633","reported_seq":3951,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.131034+0000","last_change":"2022-04-28T19:57:54.131573+0000","last_active":"2022-04-28T19:57:54.131034+0000","last_peered":"2022-04-28T19:57:54.131034+0000","last_clean":"2022-04-28T19:57:54.131034+0000","last_became_active":"2022-04-28T19:54:11.071626+0000","last_became_peered":"2022-04-28T19:54:11.071626+0000","last_unstale":"2022-04-28T19:57:54.131034+0000","last_undegraded":"2022-04-28T19:57:54.131034+0000","last_fullsized":"2022-04-28T19:57:54.131034+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1633,"ondisk_log_size":1633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":109117440,"num_objects":74,"num_object_clones":69,"num_object_copies":222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":74,"num_whiteouts":1,"num_read":290,"num_read_kb":348436,"num_write":864,"num_write_kb":192952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":51191808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.f","version":"56'1","reported_seq":770,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.998714+0000","last_change":"2022-04-28T19:54:11.069889+0000","last_active":"2022-04-28T19:57:53.998714+0000","last_peered":"2022-04-28T19:57:53.998714+0000","last_clean":"2022-04-28T19:57:53.998714+0000","last_became_active":"2022-04-28T19:54:11.069457+0000","last_became_peered":"2022-04-28T19:54:11.069457+0000","last_unstale":"2022-04-28T19:57:53.998714+0000","last_undegraded":"2022-04-28T19:57:53.998714+0000","last_fullsized":"2022-04-28T19:57:53.998714+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":790,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.900256+0000","last_change":"2022-04-28T19:54:11.066247+0000","last_active":"2022-04-28T19:57:53.900256+0000","last_peered":"2022-04-28T19:57:53.900256+0000","last_clean":"2022-04-28T19:57:53.900256+0000","last_became_active":"2022-04-28T19:54:11.065897+0000","last_became_peered":"2022-04-28T19:54:11.065897+0000","last_unstale":"2022-04-28T19:57:53.900256+0000","last_undegraded":"2022-04-28T19:57:53.900256+0000","last_fullsized":"2022-04-28T19:57:53.900256+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"828'1142","reported_seq":3584,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.062767+0000","last_change":"2022-04-28T19:57:54.063116+0000","last_active":"2022-04-28T19:57:54.062767+0000","last_peered":"2022-04-28T19:57:54.062767+0000","last_clean":"2022-04-28T19:57:54.062767+0000","last_became_active":"2022-04-28T19:51:41.935942+0000","last_became_peered":"2022-04-28T19:51:41.935942+0000","last_unstale":"2022-04-28T19:57:54.062767+0000","last_undegraded":"2022-04-28T19:57:54.062767+0000","last_fullsized":"2022-04-28T19:57:54.062767+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1142,"ondisk_log_size":1142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":144023552,"num_objects":71,"num_object_clones":67,"num_object_copies":213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71,"num_whiteouts":1,"num_read":323,"num_read_kb":953870,"num_write":674,"num_write_kb":127800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":79093760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.e","version":"56'2","reported_seq":770,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.998537+0000","last_change":"2022-04-28T19:54:11.070435+0000","last_active":"2022-04-28T19:57:53.998537+0000","last_peered":"2022-04-28T19:57:53.998537+0000","last_clean":"2022-04-28T19:57:53.998537+0000","last_became_active":"2022-04-28T19:54:11.069526+0000","last_became_peered":"2022-04-28T19:54:11.069526+0000","last_unstale":"2022-04-28T19:57:53.998537+0000","last_undegraded":"2022-04-28T19:57:53.998537+0000","last_fullsized":"2022-04-28T19:57:53.998537+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":796,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.058646+0000","last_change":"2022-04-28T19:52:40.940635+0000","last_active":"2022-04-28T19:57:53.058646+0000","last_peered":"2022-04-28T19:57:53.058646+0000","last_clean":"2022-04-28T19:57:53.058646+0000","last_became_active":"2022-04-28T19:52:40.940316+0000","last_became_peered":"2022-04-28T19:52:40.940316+0000","last_unstale":"2022-04-28T19:57:53.058646+0000","last_undegraded":"2022-04-28T19:57:53.058646+0000","last_fullsized":"2022-04-28T19:57:53.058646+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"829'1174","reported_seq":3592,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.064723+0000","last_change":"2022-04-28T19:57:54.065033+0000","last_active":"2022-04-28T19:57:54.064723+0000","last_peered":"2022-04-28T19:57:54.064723+0000","last_clean":"2022-04-28T19:57:54.064723+0000","last_became_active":"2022-04-28T19:53:10.979637+0000","last_became_peered":"2022-04-28T19:53:10.979637+0000","last_unstale":"2022-04-28T19:57:54.064723+0000","last_undegraded":"2022-04-28T19:57:54.064723+0000","last_fullsized":"2022-04-28T19:57:54.064723+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1174,"ondisk_log_size":1174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":123625472,"num_objects":66,"num_object_clones":62,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":329,"num_read_kb":527884,"num_write":930,"num_write_kb":171288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":31588352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.d","version":"56'1","reported_seq":788,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.997643+0000","last_change":"2022-04-28T19:54:11.072852+0000","last_active":"2022-04-28T19:57:53.997643+0000","last_peered":"2022-04-28T19:57:53.997643+0000","last_clean":"2022-04-28T19:57:53.997643+0000","last_became_active":"2022-04-28T19:54:11.070546+0000","last_became_peered":"2022-04-28T19:54:11.070546+0000","last_unstale":"2022-04-28T19:57:53.997643+0000","last_undegraded":"2022-04-28T19:57:53.997643+0000","last_fullsized":"2022-04-28T19:57:53.997643+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.757416+0000","last_change":"2022-04-28T19:54:11.070926+0000","last_active":"2022-04-28T19:57:53.757416+0000","last_peered":"2022-04-28T19:57:53.757416+0000","last_clean":"2022-04-28T19:57:53.757416+0000","last_became_active":"2022-04-28T19:54:11.070543+0000","last_became_peered":"2022-04-28T19:54:11.070543+0000","last_unstale":"2022-04-28T19:57:53.757416+0000","last_undegraded":"2022-04-28T19:57:53.757416+0000","last_fullsized":"2022-04-28T19:57:53.757416+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"832'1285","reported_seq":3690,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.065490+0000","last_change":"2022-04-28T19:57:54.066382+0000","last_active":"2022-04-28T19:57:54.065490+0000","last_peered":"2022-04-28T19:57:54.065490+0000","last_clean":"2022-04-28T19:57:54.065490+0000","last_became_active":"2022-04-28T19:52:40.943432+0000","last_became_peered":"2022-04-28T19:52:40.943432+0000","last_unstale":"2022-04-28T19:57:54.065490+0000","last_undegraded":"2022-04-28T19:57:54.065490+0000","last_fullsized":"2022-04-28T19:57:54.065490+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1285,"ondisk_log_size":1285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":75145216,"num_objects":63,"num_object_clones":59,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":343,"num_read_kb":389713,"num_write":851,"num_write_kb":145824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":12845056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.c","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.901397+0000","last_change":"2022-04-28T19:53:41.975595+0000","last_active":"2022-04-28T19:57:53.901397+0000","last_peered":"2022-04-28T19:57:53.901397+0000","last_clean":"2022-04-28T19:57:53.901397+0000","last_became_active":"2022-04-28T19:53:41.975215+0000","last_became_peered":"2022-04-28T19:53:41.975215+0000","last_unstale":"2022-04-28T19:57:53.901397+0000","last_undegraded":"2022-04-28T19:57:53.901397+0000","last_fullsized":"2022-04-28T19:57:53.901397+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.556916+0000","last_change":"2022-04-28T19:52:40.946085+0000","last_active":"2022-04-28T19:57:53.556916+0000","last_peered":"2022-04-28T19:57:53.556916+0000","last_clean":"2022-04-28T19:57:53.556916+0000","last_became_active":"2022-04-28T19:52:40.945748+0000","last_became_peered":"2022-04-28T19:52:40.945748+0000","last_unstale":"2022-04-28T19:57:53.556916+0000","last_undegraded":"2022-04-28T19:57:53.556916+0000","last_fullsized":"2022-04-28T19:57:53.556916+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"813'290","reported_seq":2708,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.064948+0000","last_change":"2022-04-28T19:57:54.065319+0000","last_active":"2022-04-28T19:57:54.064948+0000","last_peered":"2022-04-28T19:57:54.064948+0000","last_clean":"2022-04-28T19:57:54.064948+0000","last_became_active":"2022-04-28T19:54:11.072518+0000","last_became_peered":"2022-04-28T19:54:11.072518+0000","last_unstale":"2022-04-28T19:57:54.064948+0000","last_undegraded":"2022-04-28T19:57:54.064948+0000","last_fullsized":"2022-04-28T19:57:54.064948+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":290,"ondisk_log_size":290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":33390592,"num_objects":13,"num_object_clones":12,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":79,"num_read_kb":96690,"num_write":196,"num_write_kb":44160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":10526720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.b","version":"643'3","reported_seq":775,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.064397+0000","last_change":"2022-04-28T19:52:40.943026+0000","last_active":"2022-04-28T19:57:53.064397+0000","last_peered":"2022-04-28T19:57:53.064397+0000","last_clean":"2022-04-28T19:57:53.064397+0000","last_became_active":"2022-04-28T19:52:40.941837+0000","last_became_peered":"2022-04-28T19:52:40.941837+0000","last_unstale":"2022-04-28T19:57:53.064397+0000","last_undegraded":"2022-04-28T19:57:53.064397+0000","last_fullsized":"2022-04-28T19:57:53.064397+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":796,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059486+0000","last_change":"2022-04-28T19:53:41.980840+0000","last_active":"2022-04-28T19:57:53.059486+0000","last_peered":"2022-04-28T19:57:53.059486+0000","last_clean":"2022-04-28T19:57:53.059486+0000","last_became_active":"2022-04-28T19:53:41.980340+0000","last_became_peered":"2022-04-28T19:53:41.980340+0000","last_unstale":"2022-04-28T19:57:53.059486+0000","last_undegraded":"2022-04-28T19:57:53.059486+0000","last_fullsized":"2022-04-28T19:57:53.059486+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"834'307","reported_seq":2736,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.078802+0000","last_change":"2022-04-28T19:57:54.075081+0000","last_active":"2022-04-28T19:57:54.078802+0000","last_peered":"2022-04-28T19:57:54.078802+0000","last_clean":"2022-04-28T19:57:54.078802+0000","last_became_active":"2022-04-28T19:52:40.936586+0000","last_became_peered":"2022-04-28T19:52:40.936586+0000","last_unstale":"2022-04-28T19:57:54.078802+0000","last_undegraded":"2022-04-28T19:57:54.078802+0000","last_fullsized":"2022-04-28T19:57:54.078802+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":307,"ondisk_log_size":307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":39149568,"num_objects":23,"num_object_clones":22,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":111,"num_read_kb":114985,"num_write":204,"num_write_kb":51144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":9158656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.a","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.756708+0000","last_change":"2022-04-28T19:53:41.981238+0000","last_active":"2022-04-28T19:57:53.756708+0000","last_peered":"2022-04-28T19:57:53.756708+0000","last_clean":"2022-04-28T19:57:53.756708+0000","last_became_active":"2022-04-28T19:53:41.978469+0000","last_became_peered":"2022-04-28T19:53:41.978469+0000","last_unstale":"2022-04-28T19:57:53.756708+0000","last_undegraded":"2022-04-28T19:57:53.756708+0000","last_fullsized":"2022-04-28T19:57:53.756708+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":808,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059316+0000","last_change":"2022-04-28T19:52:40.937782+0000","last_active":"2022-04-28T19:57:53.059316+0000","last_peered":"2022-04-28T19:57:53.059316+0000","last_clean":"2022-04-28T19:57:53.059316+0000","last_became_active":"2022-04-28T19:52:40.936522+0000","last_became_peered":"2022-04-28T19:52:40.936522+0000","last_unstale":"2022-04-28T19:57:53.059316+0000","last_undegraded":"2022-04-28T19:57:53.059316+0000","last_fullsized":"2022-04-28T19:57:53.059316+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"828'1250","reported_seq":3653,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.131092+0000","last_change":"2022-04-28T19:57:54.131348+0000","last_active":"2022-04-28T19:57:54.131092+0000","last_peered":"2022-04-28T19:57:54.131092+0000","last_clean":"2022-04-28T19:57:54.131092+0000","last_became_active":"2022-04-28T19:53:41.975658+0000","last_became_peered":"2022-04-28T19:53:41.975658+0000","last_unstale":"2022-04-28T19:57:54.131092+0000","last_undegraded":"2022-04-28T19:57:54.131092+0000","last_fullsized":"2022-04-28T19:57:54.131092+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1250,"ondisk_log_size":1250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":121782272,"num_objects":53,"num_object_clones":49,"num_object_copies":159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53,"num_whiteouts":1,"num_read":324,"num_read_kb":690748,"num_write":590,"num_write_kb":177664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":39927808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.9","version":"0'0","reported_seq":809,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.058828+0000","last_change":"2022-04-28T19:54:11.064794+0000","last_active":"2022-04-28T19:57:53.058828+0000","last_peered":"2022-04-28T19:57:53.058828+0000","last_clean":"2022-04-28T19:57:53.058828+0000","last_became_active":"2022-04-28T19:54:11.064446+0000","last_became_peered":"2022-04-28T19:54:11.064446+0000","last_unstale":"2022-04-28T19:57:53.058828+0000","last_undegraded":"2022-04-28T19:57:53.058828+0000","last_fullsized":"2022-04-28T19:57:53.058828+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":779,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059324+0000","last_change":"2022-04-28T19:53:41.975905+0000","last_active":"2022-04-28T19:57:53.059324+0000","last_peered":"2022-04-28T19:57:53.059324+0000","last_clean":"2022-04-28T19:57:53.059324+0000","last_became_active":"2022-04-28T19:53:41.975575+0000","last_became_peered":"2022-04-28T19:53:41.975575+0000","last_unstale":"2022-04-28T19:57:53.059324+0000","last_undegraded":"2022-04-28T19:57:53.059324+0000","last_fullsized":"2022-04-28T19:57:53.059324+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"813'657","reported_seq":3061,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.072932+0000","last_change":"2022-04-28T19:57:54.073273+0000","last_active":"2022-04-28T19:57:54.072932+0000","last_peered":"2022-04-28T19:57:54.072932+0000","last_clean":"2022-04-28T19:57:54.072932+0000","last_became_active":"2022-04-28T19:53:10.977040+0000","last_became_peered":"2022-04-28T19:53:10.977040+0000","last_unstale":"2022-04-28T19:57:54.072932+0000","last_undegraded":"2022-04-28T19:57:54.072932+0000","last_fullsized":"2022-04-28T19:57:54.072932+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":657,"ondisk_log_size":657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":63152128,"num_objects":23,"num_object_clones":21,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":151,"num_read_kb":241287,"num_write":329,"num_write_kb":79136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":60350464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.8","version":"0'0","reported_seq":800,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.065863+0000","last_change":"2022-04-28T19:54:11.077276+0000","last_active":"2022-04-28T19:57:53.065863+0000","last_peered":"2022-04-28T19:57:53.065863+0000","last_clean":"2022-04-28T19:57:53.065863+0000","last_became_active":"2022-04-28T19:54:11.074043+0000","last_became_peered":"2022-04-28T19:54:11.074043+0000","last_unstale":"2022-04-28T19:57:53.065863+0000","last_undegraded":"2022-04-28T19:57:53.065863+0000","last_fullsized":"2022-04-28T19:57:53.065863+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.077198+0000","last_change":"2022-04-28T19:52:40.938159+0000","last_active":"2022-04-28T19:57:53.077198+0000","last_peered":"2022-04-28T19:57:53.077198+0000","last_clean":"2022-04-28T19:57:53.077198+0000","last_became_active":"2022-04-28T19:52:40.937793+0000","last_became_peered":"2022-04-28T19:52:40.937793+0000","last_unstale":"2022-04-28T19:57:53.077198+0000","last_undegraded":"2022-04-28T19:57:53.077198+0000","last_fullsized":"2022-04-28T19:57:53.077198+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":797,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.077627+0000","last_change":"2022-04-28T19:54:11.073004+0000","last_active":"2022-04-28T19:57:53.077627+0000","last_peered":"2022-04-28T19:57:53.077627+0000","last_clean":"2022-04-28T19:57:53.077627+0000","last_became_active":"2022-04-28T19:54:11.072646+0000","last_became_peered":"2022-04-28T19:54:11.072646+0000","last_unstale":"2022-04-28T19:57:53.077627+0000","last_undegraded":"2022-04-28T19:57:53.077627+0000","last_fullsized":"2022-04-28T19:57:53.077627+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":799,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.901174+0000","last_change":"2022-04-28T19:53:10.978612+0000","last_active":"2022-04-28T19:57:53.901174+0000","last_peered":"2022-04-28T19:57:53.901174+0000","last_clean":"2022-04-28T19:57:53.901174+0000","last_became_active":"2022-04-28T19:53:10.977982+0000","last_became_peered":"2022-04-28T19:53:10.977982+0000","last_unstale":"2022-04-28T19:57:53.901174+0000","last_undegraded":"2022-04-28T19:57:53.901174+0000","last_fullsized":"2022-04-28T19:57:53.901174+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.076452+0000","last_change":"2022-04-28T19:53:10.984750+0000","last_active":"2022-04-28T19:57:53.076452+0000","last_peered":"2022-04-28T19:57:53.076452+0000","last_clean":"2022-04-28T19:57:53.076452+0000","last_became_active":"2022-04-28T19:53:10.984235+0000","last_became_peered":"2022-04-28T19:53:10.984235+0000","last_unstale":"2022-04-28T19:57:53.076452+0000","last_undegraded":"2022-04-28T19:57:53.076452+0000","last_fullsized":"2022-04-28T19:57:53.076452+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"828'1065","reported_seq":3478,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.075344+0000","last_change":"2022-04-28T19:57:54.075746+0000","last_active":"2022-04-28T19:57:54.075344+0000","last_peered":"2022-04-28T19:57:54.075344+0000","last_clean":"2022-04-28T19:57:54.075344+0000","last_became_active":"2022-04-28T19:53:41.976266+0000","last_became_peered":"2022-04-28T19:53:41.976266+0000","last_unstale":"2022-04-28T19:57:54.075344+0000","last_undegraded":"2022-04-28T19:57:54.075344+0000","last_fullsized":"2022-04-28T19:57:54.075344+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1065,"ondisk_log_size":1065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":115253248,"num_objects":80,"num_object_clones":76,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":271,"num_read_kb":423638,"num_write":799,"num_write_kb":162896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":54894592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"5.5","version":"806'214","reported_seq":2623,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.075630+0000","last_change":"2022-04-28T19:57:54.075874+0000","last_active":"2022-04-28T19:57:54.075630+0000","last_peered":"2022-04-28T19:57:54.075630+0000","last_clean":"2022-04-28T19:57:54.075630+0000","last_became_active":"2022-04-28T19:52:40.942467+0000","last_became_peered":"2022-04-28T19:52:40.942467+0000","last_unstale":"2022-04-28T19:57:54.075630+0000","last_undegraded":"2022-04-28T19:57:54.075630+0000","last_fullsized":"2022-04-28T19:57:54.075630+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":214,"ondisk_log_size":214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":24240128,"num_objects":12,"num_object_clones":11,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":75,"num_read_kb":101094,"num_write":133,"num_write_kb":37184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"2.2","version":"0'0","reported_seq":785,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.900631+0000","last_change":"2022-04-28T19:53:41.979038+0000","last_active":"2022-04-28T19:57:53.900631+0000","last_peered":"2022-04-28T19:57:53.900631+0000","last_clean":"2022-04-28T19:57:53.900631+0000","last_became_active":"2022-04-28T19:53:41.978702+0000","last_became_peered":"2022-04-28T19:53:41.978702+0000","last_unstale":"2022-04-28T19:57:53.900631+0000","last_undegraded":"2022-04-28T19:57:53.900631+0000","last_fullsized":"2022-04-28T19:57:53.900631+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":791,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059335+0000","last_change":"2022-04-28T19:53:41.981234+0000","last_active":"2022-04-28T19:57:53.059335+0000","last_peered":"2022-04-28T19:57:53.059335+0000","last_clean":"2022-04-28T19:57:53.059335+0000","last_became_active":"2022-04-28T19:53:41.979792+0000","last_became_peered":"2022-04-28T19:53:41.979792+0000","last_unstale":"2022-04-28T19:57:53.059335+0000","last_undegraded":"2022-04-28T19:57:53.059335+0000","last_fullsized":"2022-04-28T19:57:53.059335+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.076925+0000","last_change":"2022-04-28T19:52:12.050710+0000","last_active":"2022-04-28T19:57:53.076925+0000","last_peered":"2022-04-28T19:57:53.076925+0000","last_clean":"2022-04-28T19:57:53.076925+0000","last_became_active":"2022-04-28T19:52:12.050045+0000","last_became_peered":"2022-04-28T19:52:12.050045+0000","last_unstale":"2022-04-28T19:57:53.076925+0000","last_undegraded":"2022-04-28T19:57:53.076925+0000","last_fullsized":"2022-04-28T19:57:53.076925+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"828'582","reported_seq":3042,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.065318+0000","last_change":"2022-04-28T19:57:54.065649+0000","last_active":"2022-04-28T19:57:54.065318+0000","last_peered":"2022-04-28T19:57:54.065318+0000","last_clean":"2022-04-28T19:57:54.065318+0000","last_became_active":"2022-04-28T19:53:10.977049+0000","last_became_peered":"2022-04-28T19:53:10.977049+0000","last_unstale":"2022-04-28T19:57:54.065318+0000","last_undegraded":"2022-04-28T19:57:54.065318+0000","last_fullsized":"2022-04-28T19:57:54.065318+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":582,"ondisk_log_size":582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":39534592,"num_objects":35,"num_object_clones":33,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":216,"num_read_kb":455261,"num_write":313,"num_write_kb":74744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":29179904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":811,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.063952+0000","last_change":"2022-04-28T19:52:12.044457+0000","last_active":"2022-04-28T19:57:53.063952+0000","last_peered":"2022-04-28T19:57:53.063952+0000","last_clean":"2022-04-28T19:57:53.063952+0000","last_became_active":"2022-04-28T19:52:12.043854+0000","last_became_peered":"2022-04-28T19:52:12.043854+0000","last_unstale":"2022-04-28T19:57:53.063952+0000","last_undegraded":"2022-04-28T19:57:53.063952+0000","last_fullsized":"2022-04-28T19:57:53.063952+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.076991+0000","last_change":"2022-04-28T19:53:41.978188+0000","last_active":"2022-04-28T19:57:53.076991+0000","last_peered":"2022-04-28T19:57:53.076991+0000","last_clean":"2022-04-28T19:57:53.076991+0000","last_became_active":"2022-04-28T19:53:41.977530+0000","last_became_peered":"2022-04-28T19:53:41.977530+0000","last_unstale":"2022-04-28T19:57:53.076991+0000","last_undegraded":"2022-04-28T19:57:53.076991+0000","last_fullsized":"2022-04-28T19:57:53.076991+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.077493+0000","last_change":"2022-04-28T19:53:10.985009+0000","last_active":"2022-04-28T19:57:53.077493+0000","last_peered":"2022-04-28T19:57:53.077493+0000","last_clean":"2022-04-28T19:57:53.077493+0000","last_became_active":"2022-04-28T19:53:10.984688+0000","last_became_peered":"2022-04-28T19:53:10.984688+0000","last_unstale":"2022-04-28T19:57:53.077493+0000","last_undegraded":"2022-04-28T19:57:53.077493+0000","last_fullsized":"2022-04-28T19:57:53.077493+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":773,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.900639+0000","last_change":"2022-04-28T19:54:11.066102+0000","last_active":"2022-04-28T19:57:53.900639+0000","last_peered":"2022-04-28T19:57:53.900639+0000","last_clean":"2022-04-28T19:57:53.900639+0000","last_became_active":"2022-04-28T19:54:11.065783+0000","last_became_peered":"2022-04-28T19:54:11.065783+0000","last_unstale":"2022-04-28T19:57:53.900639+0000","last_undegraded":"2022-04-28T19:57:53.900639+0000","last_fullsized":"2022-04-28T19:57:53.900639+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":793,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.996594+0000","last_change":"2022-04-28T19:54:11.071033+0000","last_active":"2022-04-28T19:57:53.996594+0000","last_peered":"2022-04-28T19:57:53.996594+0000","last_clean":"2022-04-28T19:57:53.996594+0000","last_became_active":"2022-04-28T19:54:11.070192+0000","last_became_peered":"2022-04-28T19:54:11.070192+0000","last_unstale":"2022-04-28T19:57:53.996594+0000","last_undegraded":"2022-04-28T19:57:53.996594+0000","last_fullsized":"2022-04-28T19:57:53.996594+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":5,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:36:10.237993+0000","last_clean_scrub_stamp":"2022-04-28T19:36:10.237993+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":769,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.064161+0000","last_change":"2022-04-28T19:53:41.980799+0000","last_active":"2022-04-28T19:57:53.064161+0000","last_peered":"2022-04-28T19:57:53.064161+0000","last_clean":"2022-04-28T19:57:53.064161+0000","last_became_active":"2022-04-28T19:53:41.980461+0000","last_became_peered":"2022-04-28T19:53:41.980461+0000","last_unstale":"2022-04-28T19:57:53.064161+0000","last_undegraded":"2022-04-28T19:57:53.064161+0000","last_fullsized":"2022-04-28T19:57:53.064161+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":798,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.757299+0000","last_change":"2022-04-28T19:54:11.076097+0000","last_active":"2022-04-28T19:57:53.757299+0000","last_peered":"2022-04-28T19:57:53.757299+0000","last_clean":"2022-04-28T19:57:53.757299+0000","last_became_active":"2022-04-28T19:54:11.075823+0000","last_became_peered":"2022-04-28T19:54:11.075823+0000","last_unstale":"2022-04-28T19:57:53.757299+0000","last_undegraded":"2022-04-28T19:57:53.757299+0000","last_fullsized":"2022-04-28T19:57:53.757299+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"828'1601","reported_seq":3981,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.130439+0000","last_change":"2022-04-28T19:57:54.130852+0000","last_active":"2022-04-28T19:57:54.130439+0000","last_peered":"2022-04-28T19:57:54.130439+0000","last_clean":"2022-04-28T19:57:54.130439+0000","last_became_active":"2022-04-28T19:54:11.071686+0000","last_became_peered":"2022-04-28T19:54:11.071686+0000","last_unstale":"2022-04-28T19:57:54.130439+0000","last_undegraded":"2022-04-28T19:57:54.130439+0000","last_fullsized":"2022-04-28T19:57:54.130439+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1601,"ondisk_log_size":1601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":104660992,"num_objects":69,"num_object_clones":64,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":2,"num_read":367,"num_read_kb":517296,"num_write":922,"num_write_kb":184432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":52314112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"5.7","version":"829'1333","reported_seq":3810,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.130931+0000","last_change":"2022-04-28T19:57:54.131348+0000","last_active":"2022-04-28T19:57:54.130931+0000","last_peered":"2022-04-28T19:57:54.130931+0000","last_clean":"2022-04-28T19:57:54.130931+0000","last_became_active":"2022-04-28T19:53:10.986003+0000","last_became_peered":"2022-04-28T19:53:10.986003+0000","last_unstale":"2022-04-28T19:57:54.130931+0000","last_undegraded":"2022-04-28T19:57:54.130931+0000","last_fullsized":"2022-04-28T19:57:54.130931+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1333,"ondisk_log_size":1333,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":122077184,"num_objects":52,"num_object_clones":47,"num_object_copies":156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52,"num_whiteouts":0,"num_read":440,"num_read_kb":928531,"num_write":703,"num_write_kb":218552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":26738688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":766,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.996593+0000","last_change":"2022-04-28T19:54:11.064329+0000","last_active":"2022-04-28T19:57:53.996593+0000","last_peered":"2022-04-28T19:57:53.996593+0000","last_clean":"2022-04-28T19:57:53.996593+0000","last_became_active":"2022-04-28T19:54:11.063970+0000","last_became_peered":"2022-04-28T19:54:11.063970+0000","last_unstale":"2022-04-28T19:57:53.996593+0000","last_undegraded":"2022-04-28T19:57:53.996593+0000","last_fullsized":"2022-04-28T19:57:53.996593+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.556542+0000","last_change":"2022-04-28T19:52:40.945779+0000","last_active":"2022-04-28T19:57:53.556542+0000","last_peered":"2022-04-28T19:57:53.556542+0000","last_clean":"2022-04-28T19:57:53.556542+0000","last_became_active":"2022-04-28T19:52:40.945429+0000","last_became_peered":"2022-04-28T19:52:40.945429+0000","last_unstale":"2022-04-28T19:57:53.556542+0000","last_undegraded":"2022-04-28T19:57:53.556542+0000","last_fullsized":"2022-04-28T19:57:53.556542+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557166+0000","last_change":"2022-04-28T19:51:41.932291+0000","last_active":"2022-04-28T19:57:53.557166+0000","last_peered":"2022-04-28T19:57:53.557166+0000","last_clean":"2022-04-28T19:57:53.557166+0000","last_became_active":"2022-04-28T19:51:41.931953+0000","last_became_peered":"2022-04-28T19:51:41.931953+0000","last_unstale":"2022-04-28T19:57:53.557166+0000","last_undegraded":"2022-04-28T19:57:53.557166+0000","last_fullsized":"2022-04-28T19:57:53.557166+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":772,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.997770+0000","last_change":"2022-04-28T19:54:11.073643+0000","last_active":"2022-04-28T19:57:53.997770+0000","last_peered":"2022-04-28T19:57:53.997770+0000","last_clean":"2022-04-28T19:57:53.997770+0000","last_became_active":"2022-04-28T19:54:11.072207+0000","last_became_peered":"2022-04-28T19:54:11.072207+0000","last_unstale":"2022-04-28T19:57:53.997770+0000","last_undegraded":"2022-04-28T19:57:53.997770+0000","last_fullsized":"2022-04-28T19:57:53.997770+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.077211+0000","last_change":"2022-04-28T19:53:10.984431+0000","last_active":"2022-04-28T19:57:53.077211+0000","last_peered":"2022-04-28T19:57:53.077211+0000","last_clean":"2022-04-28T19:57:53.077211+0000","last_became_active":"2022-04-28T19:53:10.983992+0000","last_became_peered":"2022-04-28T19:53:10.983992+0000","last_unstale":"2022-04-28T19:57:53.077211+0000","last_undegraded":"2022-04-28T19:57:53.077211+0000","last_fullsized":"2022-04-28T19:57:53.077211+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":780,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557230+0000","last_change":"2022-04-28T19:54:11.071168+0000","last_active":"2022-04-28T19:57:53.557230+0000","last_peered":"2022-04-28T19:57:53.557230+0000","last_clean":"2022-04-28T19:57:53.557230+0000","last_became_active":"2022-04-28T19:54:11.070546+0000","last_became_peered":"2022-04-28T19:54:11.070546+0000","last_unstale":"2022-04-28T19:57:53.557230+0000","last_undegraded":"2022-04-28T19:57:53.557230+0000","last_fullsized":"2022-04-28T19:57:53.557230+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"832'715","reported_seq":3094,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.063414+0000","last_change":"2022-04-28T19:57:54.063656+0000","last_active":"2022-04-28T19:57:54.063414+0000","last_peered":"2022-04-28T19:57:54.063414+0000","last_clean":"2022-04-28T19:57:54.063414+0000","last_became_active":"2022-04-28T19:53:41.979001+0000","last_became_peered":"2022-04-28T19:53:41.979001+0000","last_unstale":"2022-04-28T19:57:54.063414+0000","last_undegraded":"2022-04-28T19:57:54.063414+0000","last_fullsized":"2022-04-28T19:57:54.063414+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":715,"ondisk_log_size":715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":71155712,"num_objects":28,"num_object_clones":26,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":167,"num_read_kb":258750,"num_write":421,"num_write_kb":137352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":20307968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.7","version":"56'2","reported_seq":773,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.067969+0000","last_change":"2022-04-28T19:54:11.066667+0000","last_active":"2022-04-28T19:57:53.067969+0000","last_peered":"2022-04-28T19:57:53.067969+0000","last_clean":"2022-04-28T19:57:53.067969+0000","last_became_active":"2022-04-28T19:54:11.066236+0000","last_became_peered":"2022-04-28T19:54:11.066236+0000","last_unstale":"2022-04-28T19:57:53.067969+0000","last_undegraded":"2022-04-28T19:57:53.067969+0000","last_fullsized":"2022-04-28T19:57:53.067969+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":792,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.077247+0000","last_change":"2022-04-28T19:53:41.973530+0000","last_active":"2022-04-28T19:57:53.077247+0000","last_peered":"2022-04-28T19:57:53.077247+0000","last_clean":"2022-04-28T19:57:53.077247+0000","last_became_active":"2022-04-28T19:53:41.973059+0000","last_became_peered":"2022-04-28T19:53:41.973059+0000","last_unstale":"2022-04-28T19:57:53.077247+0000","last_undegraded":"2022-04-28T19:57:53.077247+0000","last_fullsized":"2022-04-28T19:57:53.077247+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":770,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.065545+0000","last_change":"2022-04-28T19:54:11.072966+0000","last_active":"2022-04-28T19:57:53.065545+0000","last_peered":"2022-04-28T19:57:53.065545+0000","last_clean":"2022-04-28T19:57:53.065545+0000","last_became_active":"2022-04-28T19:54:11.072604+0000","last_became_peered":"2022-04-28T19:54:11.072604+0000","last_unstale":"2022-04-28T19:57:53.065545+0000","last_undegraded":"2022-04-28T19:57:53.065545+0000","last_fullsized":"2022-04-28T19:57:53.065545+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"798'349","reported_seq":2753,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.071963+0000","last_change":"2022-04-28T19:57:54.072418+0000","last_active":"2022-04-28T19:57:54.071963+0000","last_peered":"2022-04-28T19:57:54.071963+0000","last_clean":"2022-04-28T19:57:54.071963+0000","last_became_active":"2022-04-28T19:54:11.065562+0000","last_became_peered":"2022-04-28T19:54:11.065562+0000","last_unstale":"2022-04-28T19:57:54.071963+0000","last_undegraded":"2022-04-28T19:57:54.071963+0000","last_fullsized":"2022-04-28T19:57:54.071963+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":349,"ondisk_log_size":349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":56934400,"num_objects":20,"num_object_clones":19,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":82,"num_read_kb":183032,"num_write":247,"num_write_kb":85792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":9551872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"2.7","version":"0'0","reported_seq":785,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.757677+0000","last_change":"2022-04-28T19:54:11.075480+0000","last_active":"2022-04-28T19:57:53.757677+0000","last_peered":"2022-04-28T19:57:53.757677+0000","last_clean":"2022-04-28T19:57:53.757677+0000","last_became_active":"2022-04-28T19:54:11.075133+0000","last_became_peered":"2022-04-28T19:54:11.075133+0000","last_unstale":"2022-04-28T19:57:53.757677+0000","last_undegraded":"2022-04-28T19:57:53.757677+0000","last_fullsized":"2022-04-28T19:57:53.757677+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":48,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:40:38.817723+0000","last_clean_scrub_stamp":"2022-04-28T19:40:38.817723+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"648'2","reported_seq":786,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557587+0000","last_change":"2022-04-28T19:52:40.944842+0000","last_active":"2022-04-28T19:57:53.557587+0000","last_peered":"2022-04-28T19:57:53.557587+0000","last_clean":"2022-04-28T19:57:53.557587+0000","last_became_active":"2022-04-28T19:52:40.944382+0000","last_became_peered":"2022-04-28T19:52:40.944382+0000","last_unstale":"2022-04-28T19:57:53.557587+0000","last_undegraded":"2022-04-28T19:57:53.557587+0000","last_fullsized":"2022-04-28T19:57:53.557587+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":791,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.058399+0000","last_change":"2022-04-28T19:53:41.979858+0000","last_active":"2022-04-28T19:57:53.058399+0000","last_peered":"2022-04-28T19:57:53.058399+0000","last_clean":"2022-04-28T19:57:53.058399+0000","last_became_active":"2022-04-28T19:53:41.979426+0000","last_became_peered":"2022-04-28T19:53:41.979426+0000","last_unstale":"2022-04-28T19:57:53.058399+0000","last_undegraded":"2022-04-28T19:57:53.058399+0000","last_fullsized":"2022-04-28T19:57:53.058399+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"825'1330","reported_seq":3753,"reported_epoch":834,"state":"active+clean","last_fresh":"2022-04-28T19:57:54.064357+0000","last_change":"2022-04-28T19:57:54.064646+0000","last_active":"2022-04-28T19:57:54.064357+0000","last_peered":"2022-04-28T19:57:54.064357+0000","last_clean":"2022-04-28T19:57:54.064357+0000","last_became_active":"2022-04-28T19:52:40.937133+0000","last_became_peered":"2022-04-28T19:52:40.937133+0000","last_unstale":"2022-04-28T19:57:54.064357+0000","last_undegraded":"2022-04-28T19:57:54.064357+0000","last_fullsized":"2022-04-28T19:57:54.064357+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:44:22.305655+0000","last_clean_scrub_stamp":"2022-04-28T19:44:22.305655+0000","log_size":1330,"ondisk_log_size":1330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":146079744,"num_objects":76,"num_object_clones":71,"num_object_copies":228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76,"num_whiteouts":0,"num_read":345,"num_read_kb":392838,"num_write":841,"num_write_kb":191336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":50700288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"274","length":"1"},{"start":"2bd","length":"1"},{"start":"2c9","length":"1"}]},{"pgid":"3.1b","version":"643'2","reported_seq":785,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557115+0000","last_change":"2022-04-28T19:54:11.070097+0000","last_active":"2022-04-28T19:57:53.557115+0000","last_peered":"2022-04-28T19:57:53.557115+0000","last_clean":"2022-04-28T19:57:53.557115+0000","last_became_active":"2022-04-28T19:54:11.069684+0000","last_became_peered":"2022-04-28T19:54:11.069684+0000","last_unstale":"2022-04-28T19:57:53.557115+0000","last_undegraded":"2022-04-28T19:57:53.557115+0000","last_fullsized":"2022-04-28T19:57:53.557115+0000","mapping_epoch":627,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":628,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":794,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.063950+0000","last_change":"2022-04-28T19:52:12.042586+0000","last_active":"2022-04-28T19:57:53.063950+0000","last_peered":"2022-04-28T19:57:53.063950+0000","last_clean":"2022-04-28T19:57:53.063950+0000","last_became_active":"2022-04-28T19:52:12.042219+0000","last_became_peered":"2022-04-28T19:52:12.042219+0000","last_unstale":"2022-04-28T19:57:53.063950+0000","last_undegraded":"2022-04-28T19:57:53.063950+0000","last_fullsized":"2022-04-28T19:57:53.063950+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"56'2","reported_seq":788,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.900671+0000","last_change":"2022-04-28T19:53:10.979336+0000","last_active":"2022-04-28T19:57:53.900671+0000","last_peered":"2022-04-28T19:57:53.900671+0000","last_clean":"2022-04-28T19:57:53.900671+0000","last_became_active":"2022-04-28T19:53:10.979039+0000","last_became_peered":"2022-04-28T19:53:10.979039+0000","last_unstale":"2022-04-28T19:57:53.900671+0000","last_undegraded":"2022-04-28T19:57:53.900671+0000","last_fullsized":"2022-04-28T19:57:53.900671+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":796,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.059133+0000","last_change":"2022-04-28T19:52:40.937409+0000","last_active":"2022-04-28T19:57:53.059133+0000","last_peered":"2022-04-28T19:57:53.059133+0000","last_clean":"2022-04-28T19:57:53.059133+0000","last_became_active":"2022-04-28T19:52:40.936369+0000","last_became_peered":"2022-04-28T19:52:40.936369+0000","last_unstale":"2022-04-28T19:57:53.059133+0000","last_undegraded":"2022-04-28T19:57:53.059133+0000","last_fullsized":"2022-04-28T19:57:53.059133+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"56'2","reported_seq":788,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.901070+0000","last_change":"2022-04-28T19:53:41.973566+0000","last_active":"2022-04-28T19:57:53.901070+0000","last_peered":"2022-04-28T19:57:53.901070+0000","last_clean":"2022-04-28T19:57:53.901070+0000","last_became_active":"2022-04-28T19:53:41.973242+0000","last_became_peered":"2022-04-28T19:53:41.973242+0000","last_unstale":"2022-04-28T19:57:53.901070+0000","last_undegraded":"2022-04-28T19:57:53.901070+0000","last_fullsized":"2022-04-28T19:57:53.901070+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":796,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.058441+0000","last_change":"2022-04-28T19:52:40.936354+0000","last_active":"2022-04-28T19:57:53.058441+0000","last_peered":"2022-04-28T19:57:53.058441+0000","last_clean":"2022-04-28T19:57:53.058441+0000","last_became_active":"2022-04-28T19:52:40.935957+0000","last_became_peered":"2022-04-28T19:52:40.935957+0000","last_unstale":"2022-04-28T19:57:53.058441+0000","last_undegraded":"2022-04-28T19:57:53.058441+0000","last_fullsized":"2022-04-28T19:57:53.058441+0000","mapping_epoch":541,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":542,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"643'2","reported_seq":790,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.067932+0000","last_change":"2022-04-28T19:53:41.975625+0000","last_active":"2022-04-28T19:57:53.067932+0000","last_peered":"2022-04-28T19:57:53.067932+0000","last_clean":"2022-04-28T19:57:53.067932+0000","last_became_active":"2022-04-28T19:53:41.975148+0000","last_became_peered":"2022-04-28T19:53:41.975148+0000","last_unstale":"2022-04-28T19:57:53.067932+0000","last_undegraded":"2022-04-28T19:57:53.067932+0000","last_fullsized":"2022-04-28T19:57:53.067932+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":787,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.065949+0000","last_change":"2022-04-28T19:52:12.052418+0000","last_active":"2022-04-28T19:57:53.065949+0000","last_peered":"2022-04-28T19:57:53.065949+0000","last_clean":"2022-04-28T19:57:53.065949+0000","last_became_active":"2022-04-28T19:52:12.051604+0000","last_became_peered":"2022-04-28T19:52:12.051604+0000","last_unstale":"2022-04-28T19:57:53.065949+0000","last_undegraded":"2022-04-28T19:57:53.065949+0000","last_fullsized":"2022-04-28T19:57:53.065949+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"643'5","reported_seq":790,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.557446+0000","last_change":"2022-04-28T19:53:10.976027+0000","last_active":"2022-04-28T19:57:53.557446+0000","last_peered":"2022-04-28T19:57:53.557446+0000","last_clean":"2022-04-28T19:57:53.557446+0000","last_became_active":"2022-04-28T19:53:10.975248+0000","last_became_peered":"2022-04-28T19:53:10.975248+0000","last_unstale":"2022-04-28T19:57:53.557446+0000","last_undegraded":"2022-04-28T19:57:53.557446+0000","last_fullsized":"2022-04-28T19:57:53.557446+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:26.207836+0000","last_clean_scrub_stamp":"2022-04-28T19:42:26.207836+0000","log_size":5,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":553,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":779,"reported_epoch":833,"state":"active+clean","last_fresh":"2022-04-28T19:57:53.058316+0000","last_change":"2022-04-28T19:53:41.976384+0000","last_active":"2022-04-28T19:57:53.058316+0000","last_peered":"2022-04-28T19:57:53.058316+0000","last_clean":"2022-04-28T19:57:53.058316+0000","last_became_active":"2022-04-28T19:53:41.976035+0000","last_became_peered":"2022-04-28T19:53:41.976035+0000","last_unstale":"2022-04-28T19:57:53.058316+0000","last_undegraded":"2022-04-28T19:57:53.058316+0000","last_fullsized":"2022-04-28T19:57:53.058316+0000","mapping_epoch":602,"log_start":"0'0","ondisk_log_start":"0'0","created":53,"last_epoch_clean":603,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2022-04-28T19:42:27.217071+0000","last_clean_scrub_stamp":"2022-04-28T19:42:27.217071+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1389322240,"num_objects":758,"num_object_clones":708,"num_object_copies":2274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":758,"num_whiteouts":5,"num_read":3913,"num_read_kb":6624053,"num_write":9017,"num_write_kb":2082256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":160,"num_bytes_recovered":538370048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1916239872,"data_stored":4960751616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":14927,"ondisk_log_size":14927,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":24980,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":19,"num_read_kb":21,"num_write":73,"num_write_kb":50,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":159744,"data_stored":74940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":767535,"internal_metadata":0},"log_size":73,"ondisk_log_size":73,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3799,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":627,"seq":2692944494639,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":183488,"kb_used_data":143864,"kb_used_omap":188,"kb_used_meta":39427,"kb_avail":93561664,"statfs":{"total":95995035648,"available":95807143936,"internally_reserved":0,"allocated":147316736,"data_stored":340878309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":192583,"internal_metadata":40374201},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.94499999999999995,"15min":0.95399999999999996},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.341,"5min":1.341,"15min":1.341},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.96099999999999997,"15min":0.96699999999999997},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.925,"5min":1.925,"15min":1.925},"last":0.85399999999999998}]},{"osd":1,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.95599999999999996,"15min":0.97399999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.2929999999999999,"5min":1.375,"15min":1.375},"last":0.94499999999999995},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.98899999999999999,"15min":0.996},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":2.198,"5min":2.198,"15min":2.198},"last":1.119}]},{"osd":2,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":1.0069999999999999,"15min":1.03},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0229999999999999,"15min":1.0389999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":2.0190000000000001,"5min":2.0190000000000001,"15min":2.0190000000000001},"last":1.258}]},{"osd":3,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.038,"15min":1.054},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":2.1509999999999998,"5min":2.1509999999999998,"15min":2.1509999999999998},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.004,"15min":1.02},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.282,"5min":1.349,"15min":1.349},"last":0.80900000000000005}]},{"osd":4,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.95599999999999996,"15min":0.98799999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.2210000000000001,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.202},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.97099999999999997,"15min":0.98699999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.373,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":1.2230000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.99399999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.333,"5min":1.345,"15min":1.345},"last":0.995},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.98299999999999998,"15min":0.98899999999999999},"min":{"1min":0.51200000000000001,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.569,"5min":1.639,"15min":1.639},"last":1.175}]},{"osd":6,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.038,"15min":1.05},"min":{"1min":0.55000000000000004,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.393,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":1.1479999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.069,"15min":1.071},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.5,"5min":1.52,"15min":1.52},"last":1.278}]}]},{"osd":6,"up_from":602,"seq":2585570312245,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":259984,"kb_used_data":206472,"kb_used_omap":222,"kb_used_meta":53281,"kb_avail":93485168,"statfs":{"total":95995035648,"available":95728812032,"internally_reserved":0,"allocated":211427328,"data_stored":499159891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":228122,"internal_metadata":54559974},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:48 2022","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.89600000000000002,"15min":0.89000000000000001},"min":{"1min":0.53300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":1.1819999999999999,"5min":1.855,"15min":1.855},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.96199999999999997,"15min":0.91000000000000003},"min":{"1min":0.63800000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.345,"5min":3.4180000000000001,"15min":3.4180000000000001},"last":1.2589999999999999}]},{"osd":1,"last update":"Thu Apr 28 19:57:48 2022","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.89600000000000002,"15min":0.84199999999999997},"min":{"1min":0.56499999999999995,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3169999999999999,"5min":3.3300000000000001,"15min":3.3300000000000001},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.85999999999999999,"15min":0.78600000000000003},"min":{"1min":0.58499999999999996,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.256,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.80600000000000005}]},{"osd":2,"last update":"Thu Apr 28 19:57:48 2022","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":0.96499999999999997,"15min":0.90300000000000002},"min":{"1min":0.63300000000000001,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.9730000000000001,"5min":3.4550000000000001,"15min":3.4550000000000001},"last":0.998},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0449999999999999,"15min":1.115},"min":{"1min":0.58599999999999997,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.4399999999999999,"5min":6.5880000000000001,"15min":6.5880000000000001},"last":0.68200000000000005}]},{"osd":3,"last update":"Thu Apr 28 19:57:48 2022","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":0.94299999999999995,"15min":0.877},"min":{"1min":0.61499999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":2.4100000000000001,"5min":2.4100000000000001,"15min":2.4100000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0549999999999999,"15min":1.0049999999999999},"min":{"1min":0.69799999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":2.0459999999999998,"5min":3.25,"15min":3.25},"last":1.1000000000000001}]},{"osd":4,"last update":"Thu Apr 28 19:57:48 2022","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.94499999999999995,"15min":0.94799999999999995},"min":{"1min":0.46200000000000002,"5min":0.37,"15min":0.37},"max":{"1min":2.46,"5min":2.46,"15min":2.46},"last":0.97199999999999998},{"interface":"front","average":{"1min":1.083,"5min":1.0089999999999999,"15min":0.93700000000000006},"min":{"1min":0.53200000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.085,"5min":3.375,"15min":3.375},"last":1.2030000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:57:48 2022","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.94399999999999995,"15min":0.92300000000000004},"min":{"1min":0.64700000000000002,"5min":0.44,"15min":0.44},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.079,"15min":1.0640000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.4950000000000001,"5min":2.4950000000000001,"15min":2.4950000000000001},"last":1.157}]},{"osd":7,"last update":"Thu Apr 28 19:57:16 2022","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.0589999999999999,"15min":1.054},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.706,"5min":1.706,"15min":1.706},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.141,"15min":1.153},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.7949999999999999,"5min":2.29,"15min":2.29},"last":1.2410000000000001}]}]},{"osd":1,"up_from":453,"seq":1945620185171,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":398692,"kb_used_data":265052,"kb_used_omap":186,"kb_used_meta":133445,"kb_avail":93346460,"statfs":{"total":95995035648,"available":95586775040,"internally_reserved":0,"allocated":271413248,"data_stored":783549562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":191292,"internal_metadata":136647876},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:21 2022","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.69499999999999995,"15min":0.65800000000000003},"min":{"1min":0.30499999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.496,"5min":1.496,"15min":1.496},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.80700000000000005,"15min":0.77200000000000002},"min":{"1min":0.42399999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.304,"5min":3.778,"15min":3.778},"last":0.65100000000000002}]},{"osd":2,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.83699999999999997,"15min":0.80700000000000005},"min":{"1min":0.45900000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.081,"5min":4.0069999999999997,"15min":4.0069999999999997},"last":0.871},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.81000000000000005,"15min":0.81999999999999995},"min":{"1min":0.497,"5min":0.374,"15min":0.374},"max":{"1min":1.127,"5min":1.78,"15min":1.78},"last":0.83799999999999997}]},{"osd":3,"last update":"Thu Apr 28 19:57:21 2022","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.81399999999999995,"15min":0.82199999999999995},"min":{"1min":0.51400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.3169999999999999,"5min":1.869,"15min":1.869},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.84799999999999998,"15min":0.81399999999999995},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.6160000000000001,"5min":2.0790000000000002,"15min":2.0790000000000002},"last":0.97899999999999998}]},{"osd":4,"last update":"Thu Apr 28 19:57:46 2022","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":1.0589999999999999,"15min":1.069},"min":{"1min":0.48599999999999999,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.5740000000000001,"5min":4.0739999999999998,"15min":4.0739999999999998},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1040000000000001,"15min":1.0860000000000001},"min":{"1min":0.69199999999999995,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.4339999999999999,"5min":3.0259999999999998,"15min":3.0259999999999998},"last":1.107}]},{"osd":5,"last update":"Thu Apr 28 19:57:15 2022","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.1539999999999999,"15min":1.143},"min":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.873,"5min":4.8109999999999999,"15min":4.8109999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.2609999999999999,"5min":1.181,"15min":1.1719999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.73,"5min":1.774,"15min":1.774},"last":1.2270000000000001}]},{"osd":6,"last update":"Thu Apr 28 19:57:46 2022","interfaces":[{"interface":"back","average":{"1min":1.181,"5min":1.2230000000000001,"15min":1.276},"min":{"1min":0.59199999999999997,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.7350000000000001,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.2509999999999999,"15min":1.2729999999999999},"min":{"1min":0.86899999999999999,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":2.1749999999999998,"5min":2.1749999999999998,"15min":2.1749999999999998},"last":1.149}]},{"osd":7,"last update":"Thu Apr 28 19:57:15 2022","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.27,"15min":1.274},"min":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"max":{"1min":2.0579999999999998,"5min":2.0579999999999998,"15min":2.0579999999999998},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.4039999999999999,"5min":1.2689999999999999,"15min":1.2509999999999999},"min":{"1min":0.60299999999999998,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":1.3}]}]},{"osd":0,"up_from":423,"seq":1816771166297,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":435724,"kb_used_data":303940,"kb_used_omap":192,"kb_used_meta":131583,"kb_avail":93309428,"statfs":{"total":95995035648,"available":95548854272,"internally_reserved":0,"allocated":311234560,"data_stored":878675641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":196661,"internal_metadata":134741963},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Apr 28 19:57:19 2022","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.72199999999999998,"15min":0.754},"min":{"1min":0.54900000000000004,"5min":0.316,"15min":0.316},"max":{"1min":0.99199999999999999,"5min":1.1040000000000001,"15min":2.2690000000000001},"last":0.753},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.91200000000000003,"15min":0.93000000000000005},"min":{"1min":0.60099999999999998,"5min":0.55100000000000005,"15min":0.50900000000000001},"max":{"1min":1.296,"5min":1.5449999999999999,"15min":2.343},"last":1.0069999999999999}]},{"osd":2,"last update":"Thu Apr 28 19:57:00 2022","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.73599999999999999,"15min":0.76400000000000001},"min":{"1min":0.495,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.1419999999999999,"5min":2.0569999999999999,"15min":2.0569999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.88500000000000001,"15min":0.871},"min":{"1min":0.51500000000000001,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.3400000000000001,"5min":1.972,"15min":1.972},"last":0.93899999999999995}]},{"osd":3,"last update":"Thu Apr 28 19:57:19 2022","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.90700000000000003,"15min":0.91100000000000003},"min":{"1min":0.66600000000000004,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.83599999999999997,"15min":0.80400000000000005},"min":{"1min":0.629,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.96199999999999997}]},{"osd":4,"last update":"Thu Apr 28 19:57:49 2022","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.008,"15min":0.98599999999999999},"min":{"1min":0.78500000000000003,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.355,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.109},{"interface":"front","average":{"1min":1.196,"5min":1.081,"15min":1.0600000000000001},"min":{"1min":0.92100000000000004,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":1.052}]},{"osd":5,"last update":"Thu Apr 28 19:57:19 2022","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.1100000000000001,"15min":1.157},"min":{"1min":0.86699999999999999,"5min":0.623,"15min":0.623},"max":{"1min":1.804,"5min":1.804,"15min":1.804},"last":1.196},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1699999999999999,"15min":1.173},"min":{"1min":1.0529999999999999,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.526,"5min":1.526,"15min":1.526},"last":1.175}]},{"osd":6,"last update":"Thu Apr 28 19:57:49 2022","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.1599999999999999,"15min":1.1679999999999999},"min":{"1min":0.88700000000000001,"5min":0.751,"15min":0.751},"max":{"1min":1.4219999999999999,"5min":1.724,"15min":1.724},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.258,"15min":1.274},"min":{"1min":0.88600000000000001,"5min":0.85899999999999999,"15min":0.85899999999999999},"max":{"1min":1.5609999999999999,"5min":2.0390000000000001,"15min":2.0390000000000001},"last":1.294}]},{"osd":7,"last update":"Thu Apr 28 19:57:19 2022","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":1.254,"15min":1.226},"min":{"1min":1.2230000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"max":{"1min":1.5880000000000001,"5min":1.6990000000000001,"15min":1.6990000000000001},"last":1.2569999999999999},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.353,"15min":1.325},"min":{"1min":1.272,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.734,"5min":1.78,"15min":1.78},"last":1.3340000000000001}]}]},{"osd":2,"up_from":482,"seq":2070174236749,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":327368,"kb_used_data":236416,"kb_used_omap":203,"kb_used_meta":90740,"kb_avail":93417784,"statfs":{"total":95995035648,"available":95659810816,"internally_reserved":0,"allocated":242089984,"data_stored":704607941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":208104,"internal_metadata":92918552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.71799999999999997,"15min":0.753},"min":{"1min":0.47199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.96699999999999997,"5min":2.0539999999999998,"15min":2.0539999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.75900000000000001,"15min":0.73499999999999999},"min":{"1min":0.57399999999999995,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.073,"5min":1.7989999999999999,"15min":1.7989999999999999},"last":0.65700000000000003}]},{"osd":1,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.73799999999999999,"15min":0.73599999999999999},"min":{"1min":0.496,"5min":0.33400000000000002,"15min":0.27100000000000002},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.83299999999999996,"15min":0.84299999999999997},"min":{"1min":0.55900000000000005,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.206,"5min":1.351,"15min":1.3560000000000001},"last":1.0129999999999999}]},{"osd":3,"last update":"Thu Apr 28 19:57:22 2022","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.86399999999999999,"15min":0.82999999999999996},"min":{"1min":0.55200000000000005,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.6910000000000001,"5min":1.6910000000000001,"15min":1.6910000000000001},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.91600000000000004,"15min":0.91200000000000003},"min":{"1min":0.63400000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.296,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.89800000000000002}]},{"osd":4,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.96399999999999997,"15min":1.0329999999999999},"min":{"1min":0.71099999999999997,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.4890000000000001,"5min":1.946,"15min":1.946},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.21,"5min":1.125,"15min":1.1040000000000001},"min":{"1min":0.73499999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.724,"5min":1.8160000000000001,"15min":1.8160000000000001},"last":1.1850000000000001}]},{"osd":5,"last update":"Thu Apr 28 19:57:22 2022","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.139,"15min":1.151},"min":{"1min":0.71399999999999997,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.5720000000000001,"5min":1.722,"15min":1.722},"last":1.216},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.1830000000000001,"15min":1.165},"min":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":1.0640000000000001}]},{"osd":6,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.222,"15min":1.1799999999999999},"min":{"1min":0.89000000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.617,"5min":1.762,"15min":1.762},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.218,"15min":1.2050000000000001},"min":{"1min":0.77500000000000002,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.6180000000000001,"5min":1.8120000000000001,"15min":1.8120000000000001},"last":1.105}]},{"osd":7,"last update":"Thu Apr 28 19:57:15 2022","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.3380000000000001,"15min":1.3140000000000001},"min":{"1min":0.89000000000000001,"5min":0.81299999999999994,"15min":0.81299999999999994},"max":{"1min":1.833,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":1.139},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.423,"15min":1.3959999999999999},"min":{"1min":1.004,"5min":0.84699999999999998,"15min":0.84699999999999998},"max":{"1min":1.9259999999999999,"5min":1.9370000000000001,"15min":1.9370000000000001},"last":1.2689999999999999}]}]},{"osd":3,"up_from":512,"seq":2199023255623,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":246948,"kb_used_data":185564,"kb_used_omap":155,"kb_used_meta":61220,"kb_avail":93498204,"statfs":{"total":95995035648,"available":95742160896,"internally_reserved":0,"allocated":190017536,"data_stored":474334420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":159647,"internal_metadata":62689377},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:26 2022","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.70799999999999996,"15min":0.65900000000000003},"min":{"1min":0.52100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":1.4850000000000001,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.81299999999999994,"15min":0.80100000000000005},"min":{"1min":0.39700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":2.4380000000000002,"5min":2.4380000000000002,"15min":2.4380000000000002},"last":0.88800000000000001}]},{"osd":1,"last update":"Thu Apr 28 19:57:26 2022","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.78600000000000003,"15min":0.79200000000000004},"min":{"1min":0.42599999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.609,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.90900000000000003,"15min":0.95199999999999996},"min":{"1min":0.44900000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.3819999999999999,"5min":3.0779999999999998,"15min":3.0779999999999998},"last":1.0740000000000001}]},{"osd":2,"last update":"Thu Apr 28 19:57:26 2022","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.91600000000000004,"15min":0.90500000000000003},"min":{"1min":0.51700000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.569,"5min":1.569,"15min":1.569},"last":1.139},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.81100000000000005,"15min":0.76800000000000002},"min":{"1min":0.40699999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.5820000000000001,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.1559999999999999}]},{"osd":4,"last update":"Thu Apr 28 19:57:45 2022","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.0940000000000001,"15min":1.081},"min":{"1min":0.55900000000000005,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.5269999999999999,"5min":1.661,"15min":1.661},"last":1.228},{"interface":"front","average":{"1min":1.048,"5min":0.96299999999999997,"15min":0.93500000000000005},"min":{"1min":0.77600000000000002,"5min":0.37,"15min":0.37},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.94499999999999995}]},{"osd":5,"last update":"Thu Apr 28 19:57:19 2022","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.1539999999999999,"15min":1.1799999999999999},"min":{"1min":0.81100000000000005,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.7909999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.55},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.111,"15min":1.1180000000000001},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":1.472}]},{"osd":6,"last update":"Thu Apr 28 19:57:45 2022","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.3,"15min":1.2809999999999999},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":1.7609999999999999,"5min":6.8479999999999999,"15min":6.8479999999999999},"last":1.3720000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.1599999999999999,"15min":1.196},"min":{"1min":0.63600000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.526,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.5049999999999999}]},{"osd":7,"last update":"Thu Apr 28 19:57:19 2022","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.1859999999999999,"15min":1.1970000000000001},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.9430000000000001,"5min":1.9430000000000001,"15min":1.9430000000000001},"last":1.343},{"interface":"front","average":{"1min":1.2809999999999999,"5min":1.2070000000000001,"15min":1.224},"min":{"1min":0.88500000000000001,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.595}]}]},{"osd":4,"up_from":541,"seq":2323577307201,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307928,"kb_used_data":246224,"kb_used_omap":233,"kb_used_meta":61462,"kb_avail":93437224,"statfs":{"total":95995035648,"available":95679717376,"internally_reserved":0,"allocated":252133376,"data_stored":565334943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":238790,"internal_metadata":62937914},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.875,"15min":0.88300000000000001},"min":{"1min":0.66500000000000004,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.3140000000000001,"5min":2.2210000000000001,"15min":2.2210000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.95399999999999996,"15min":1},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.179,"5min":3.0209999999999999,"15min":3.0209999999999999},"last":0.88}]},{"osd":1,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.96399999999999997,"15min":0.98999999999999999},"min":{"1min":0.60399999999999998,"5min":0.378,"15min":0.378},"max":{"1min":1.5369999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.878,"5min":0.93200000000000005,"15min":0.95299999999999996},"min":{"1min":0.45500000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.242,"5min":2.444,"15min":2.444},"last":0.96199999999999997}]},{"osd":2,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.92300000000000004,"15min":0.995},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.2210000000000001,"5min":2.2530000000000001,"15min":2.2530000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.044,"5min":0.996,"15min":1.03},"min":{"1min":0.58499999999999996,"5min":0.443,"15min":0.443},"max":{"1min":1.373,"5min":2.6869999999999998,"15min":2.6869999999999998},"last":1.1100000000000001}]},{"osd":3,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.98299999999999998,"15min":1.044},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.4830000000000001,"5min":2.5920000000000001,"15min":2.5920000000000001},"last":1.294},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.004,"15min":1.0569999999999999},"min":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.3100000000000001,"5min":2.7370000000000001,"15min":2.7370000000000001},"last":0.79900000000000004}]},{"osd":5,"last update":"Thu Apr 28 19:57:21 2022","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.876,"15min":0.85799999999999998},"min":{"1min":0.46000000000000002,"5min":0.376,"15min":0.376},"max":{"1min":1.2430000000000001,"5min":1.9670000000000001,"15min":1.9670000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.0349999999999999,"15min":1.0189999999999999},"min":{"1min":0.56999999999999995,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.385,"5min":1.52,"15min":1.52},"last":1.2010000000000001}]},{"osd":6,"last update":"Thu Apr 28 19:57:50 2022","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.94899999999999995,"15min":0.91500000000000004},"min":{"1min":0.56699999999999995,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.1850000000000001,"5min":1.946,"15min":1.946},"last":1.141},{"interface":"front","average":{"1min":1.1990000000000001,"5min":1.0880000000000001,"15min":1.0489999999999999},"min":{"1min":0.67800000000000005,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.4990000000000001,"5min":1.752,"15min":1.752},"last":1.018}]},{"osd":7,"last update":"Thu Apr 28 19:57:12 2022","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.091,"15min":1.131},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.5049999999999999,"5min":3.0049999999999999,"15min":3.0049999999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.115,"5min":1.206,"15min":1.2410000000000001},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.403,"5min":3.6160000000000001,"15min":3.6160000000000001},"last":1.0509999999999999}]}]},{"osd":5,"up_from":571,"seq":2452426326075,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":427288,"kb_used_data":339600,"kb_used_omap":312,"kb_used_meta":87367,"kb_avail":93317864,"statfs":{"total":95995035648,"available":95557492736,"internally_reserved":0,"allocated":347750400,"data_stored":753747689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":320509,"internal_metadata":89463811},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Apr 28 19:57:18 2022","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.90500000000000003,"15min":0.99199999999999999},"min":{"1min":0.56999999999999995,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.45,"5min":4.9359999999999999,"15min":4.9359999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":0.90900000000000003,"15min":0.85699999999999998},"min":{"1min":0.626,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.762,"5min":1.835,"15min":1.835},"last":0.66500000000000004}]},{"osd":1,"last update":"Thu Apr 28 19:57:18 2022","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":0.94999999999999996,"15min":0.91600000000000004},"min":{"1min":0.67600000000000005,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.881,"5min":1.911,"15min":1.911},"last":0.496},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.89600000000000002,"15min":0.84399999999999997},"min":{"1min":0.71199999999999997,"5min":0.504,"15min":0.504},"max":{"1min":1.7889999999999999,"5min":2.0609999999999999,"15min":2.0609999999999999},"last":0.38}]},{"osd":2,"last update":"Thu Apr 28 19:57:18 2022","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.97699999999999998,"15min":1.0389999999999999},"min":{"1min":0.68300000000000005,"5min":0.432,"15min":0.432},"max":{"1min":1.23,"5min":4.984,"15min":4.984},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.075,"5min":0.98999999999999999,"15min":0.97599999999999998},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.579,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.61599999999999999}]},{"osd":3,"last update":"Thu Apr 28 19:57:18 2022","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":0.99199999999999999,"15min":0.99399999999999999},"min":{"1min":0.51900000000000002,"5min":0.439,"15min":0.439},"max":{"1min":1.6220000000000001,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.022,"5min":0.97699999999999998,"15min":0.95499999999999996},"min":{"1min":0.63100000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.4379999999999999,"5min":2.3079999999999998,"15min":2.3079999999999998},"last":0.63200000000000001}]},{"osd":4,"last update":"Thu Apr 28 19:57:18 2022","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.92000000000000004,"15min":0.90600000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.5349999999999999,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0069999999999999,"15min":1.0720000000000001},"min":{"1min":0.42299999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.3859999999999999,"5min":4.8109999999999999,"15min":4.8109999999999999},"last":0.55100000000000005}]},{"osd":6,"last update":"Thu Apr 28 19:57:45 2022","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.028,"15min":1.077},"min":{"1min":0.64600000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.353,"5min":4.9039999999999999,"15min":4.9039999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.0509999999999999,"15min":1.0409999999999999},"min":{"1min":0.71599999999999997,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.897,"5min":1.984,"15min":1.984},"last":0.56699999999999995}]},{"osd":7,"last update":"Thu Apr 28 19:57:18 2022","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.0820000000000001,"15min":1.1080000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.3879999999999999,"5min":2.3780000000000001,"15min":2.3780000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.1160000000000001,"15min":1.123},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.54,"5min":1.599,"15min":1.599},"last":0.60299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3799,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":92267,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":86761,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":99169,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71432,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":24357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":102263,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":139376,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":24281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":91844,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":24427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":84423,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":304103424,"data_stored":873742336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":264290304,"data_stored":778616832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":234958848,"data_stored":699674624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":182890496,"data_stored":469401600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":244965376,"data_stored":560377856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":340619264,"data_stored":748814336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":204267520,"data_stored":494202880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":140144640,"data_stored":335921152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2022-04-28T19:58:02.440 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2022-04-28T19:58:02.441 DEBUG:teuthology.orchestra.run.smithi059:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:octopus shell --fsid 20a1061c-c72a-11ec-8c39-001a4aab830c -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2022-04-28T19:58:02.590 INFO:teuthology.orchestra.run.smithi059.stderr:Inferring config /var/lib/ceph/20a1061c-c72a-11ec-8c39-001a4aab830c/mon.a/config 2022-04-28T19:58:03.535 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:03 smithi059 bash[59745]: audit 2022-04-28T19:58:01.749422+0000 mgr.y (mgr.24961) 428 : audit [DBG] from='client.54403 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:58:03.535 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:03 smithi059 bash[61152]: audit 2022-04-28T19:58:01.749422+0000 mgr.y (mgr.24961) 428 : audit [DBG] from='client.54403 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:58:03.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:03 smithi139 bash[43301]: audit 2022-04-28T19:58:01.749422+0000 mgr.y (mgr.24961) 428 : audit [DBG] from='client.54403 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2022-04-28T19:58:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:03 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:03.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:04.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:04 smithi059 bash[59745]: cluster 2022-04-28T19:58:03.056193+0000 mgr.y (mgr.24961) 429 : cluster [DBG] pgmap v752: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 651 KiB/s wr, 1 op/s 2022-04-28T19:58:04.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:04 smithi059 bash[61152]: cluster 2022-04-28T19:58:03.056193+0000 mgr.y (mgr.24961) 429 : cluster [DBG] pgmap v752: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 651 KiB/s wr, 1 op/s 2022-04-28T19:58:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:04 smithi139 bash[43301]: cluster 2022-04-28T19:58:03.056193+0000 mgr.y (mgr.24961) 429 : cluster [DBG] pgmap v752: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 651 KiB/s wr, 1 op/s 2022-04-28T19:58:06.159 INFO:teuthology.orchestra.run.smithi059.stderr:pool 'unique_pool_0' removed 2022-04-28T19:58:06.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:06 smithi059 bash[59745]: cluster 2022-04-28T19:58:05.057227+0000 mgr.y (mgr.24961) 430 : cluster [DBG] pgmap v753: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 0 op/s 2022-04-28T19:58:06.665 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:06 smithi059 bash[59745]: audit 2022-04-28T19:58:05.791569+0000 mon.a (mon.0) 1503 : audit [INF] from='client.? 172.21.15.59:0/3869704152' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2022-04-28T19:58:06.666 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:06 smithi059 bash[61152]: cluster 2022-04-28T19:58:05.057227+0000 mgr.y (mgr.24961) 430 : cluster [DBG] pgmap v753: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 0 op/s 2022-04-28T19:58:06.666 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:06 smithi059 bash[61152]: audit 2022-04-28T19:58:05.791569+0000 mon.a (mon.0) 1503 : audit [INF] from='client.? 172.21.15.59:0/3869704152' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2022-04-28T19:58:06.666 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:06 smithi139 bash[43301]: cluster 2022-04-28T19:58:05.057227+0000 mgr.y (mgr.24961) 430 : cluster [DBG] pgmap v753: 89 pgs: 89 active+clean; 1.3 GiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 0 B/s wr, 0 op/s 2022-04-28T19:58:06.667 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:06 smithi139 bash[43301]: audit 2022-04-28T19:58:05.791569+0000 mon.a (mon.0) 1503 : audit [INF] from='client.? 172.21.15.59:0/3869704152' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2022-04-28T19:58:06.842 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2022-04-28T19:58:06.842 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2022-04-28T19:58:06.842 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2022-04-28T19:58:06.842 INFO:teuthology.task.print:**** done start rados_api.yaml 2022-04-28T19:58:06.843 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2022-04-28T19:58:06.848 INFO:tasks.workunit:Pulling workunits from ref octopus 2022-04-28T19:58:06.849 INFO:tasks.workunit:Making a separate scratch dir for every client... 2022-04-28T19:58:06.850 DEBUG:teuthology.orchestra.run.smithi059:> stat -- /home/ubuntu/cephtest/mnt.0 2022-04-28T19:58:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-04-28T19:58:06.857 INFO:teuthology.orchestra.run.smithi059.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2022-04-28T19:58:06.857 DEBUG:teuthology.orchestra.run.smithi059:> mkdir -- /home/ubuntu/cephtest/mnt.0 2022-04-28T19:58:06.904 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2022-04-28T19:58:06.905 DEBUG:teuthology.orchestra.run.smithi059:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2022-04-28T19:58:06.953 INFO:tasks.workunit:timeout=3h 2022-04-28T19:58:06.953 INFO:tasks.workunit:cleanup=True 2022-04-28T19:58:06.954 DEBUG:teuthology.orchestra.run.smithi059:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch octopus https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2022-04-28T19:58:07.002 INFO:tasks.workunit.client.0.smithi059.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2022-04-28T19:58:07.144 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:07 smithi059 bash[59745]: audit 2022-04-28T19:58:06.145015+0000 mon.a (mon.0) 1504 : audit [INF] from='client.? 172.21.15.59:0/3869704152' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2022-04-28T19:58:07.145 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:07 smithi059 bash[59745]: cluster 2022-04-28T19:58:06.145250+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2022-04-28T19:58:07.375 INFO:tasks.workunit.client.0.smithi059.stderr:warning: Could not find remote branch octopus to clone. 2022-04-28T19:58:07.375 INFO:tasks.workunit.client.0.smithi059.stderr:fatal: Remote branch octopus not found in upstream origin 2022-04-28T19:58:07.377 DEBUG:teuthology.orchestra.run:got remote process result: 128 2022-04-28T19:58:07.377 INFO:tasks.workunit:failed to check out 'octopus' from https://github.com/ceph/ceph-ci.git; will also try in https://github.com/ceph/ceph.git 2022-04-28T19:58:07.378 DEBUG:teuthology.orchestra.run.smithi059:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch octopus https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2022-04-28T19:58:07.427 INFO:tasks.workunit.client.0.smithi059.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2022-04-28T19:58:07.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:07 smithi059 bash[61152]: audit 2022-04-28T19:58:06.145015+0000 mon.a (mon.0) 1504 : audit [INF] from='client.? 172.21.15.59:0/3869704152' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2022-04-28T19:58:07.735 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:07 smithi059 bash[61152]: cluster 2022-04-28T19:58:06.145250+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2022-04-28T19:58:07.735 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:07 smithi139 bash[43301]: audit 2022-04-28T19:58:06.145015+0000 mon.a (mon.0) 1504 : audit [INF] from='client.? 172.21.15.59:0/3869704152' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2022-04-28T19:58:07.735 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:07 smithi139 bash[43301]: cluster 2022-04-28T19:58:06.145250+0000 mon.a (mon.0) 1505 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2022-04-28T19:58:08.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:08 smithi059 bash[59745]: cluster 2022-04-28T19:58:07.057868+0000 mgr.y (mgr.24961) 431 : cluster [DBG] pgmap v755: 73 pgs: 73 active+clean; 24 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2022-04-28T19:58:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:08 smithi059 bash[61152]: cluster 2022-04-28T19:58:07.057868+0000 mgr.y (mgr.24961) 431 : cluster [DBG] pgmap v755: 73 pgs: 73 active+clean; 24 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2022-04-28T19:58:08.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:08 smithi139 bash[43301]: cluster 2022-04-28T19:58:07.057868+0000 mgr.y (mgr.24961) 431 : cluster [DBG] pgmap v755: 73 pgs: 73 active+clean; 24 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2022-04-28T19:58:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:10 smithi059 bash[61152]: cluster 2022-04-28T19:58:09.058693+0000 mgr.y (mgr.24961) 432 : cluster [DBG] pgmap v756: 73 pgs: 73 active+clean; 24 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2022-04-28T19:58:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:10 smithi059 bash[59745]: cluster 2022-04-28T19:58:09.058693+0000 mgr.y (mgr.24961) 432 : cluster [DBG] pgmap v756: 73 pgs: 73 active+clean; 24 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2022-04-28T19:58:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:10 smithi139 bash[43301]: cluster 2022-04-28T19:58:09.058693+0000 mgr.y (mgr.24961) 432 : cluster [DBG] pgmap v756: 73 pgs: 73 active+clean; 24 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2022-04-28T19:58:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:11 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:11.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:12 smithi059 bash[61152]: cluster 2022-04-28T19:58:11.059714+0000 mgr.y (mgr.24961) 433 : cluster [DBG] pgmap v757: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:12 smithi059 bash[59745]: cluster 2022-04-28T19:58:11.059714+0000 mgr.y (mgr.24961) 433 : cluster [DBG] pgmap v757: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:12 smithi139 bash[43301]: cluster 2022-04-28T19:58:11.059714+0000 mgr.y (mgr.24961) 433 : cluster [DBG] pgmap v757: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:13 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:13.539Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:14.396 DEBUG:teuthology.orchestra.run.smithi059:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2022-04-28T19:58:14.445 INFO:tasks.workunit.client.0.smithi059.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2022-04-28T19:58:14.565 INFO:tasks.workunit.client.0.smithi059.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2022-04-28T19:58:14.566 INFO:tasks.workunit.client.0.smithi059.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2022-04-28T19:58:14.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:14 smithi059 bash[59745]: cluster 2022-04-28T19:58:13.060307+0000 mgr.y (mgr.24961) 434 : cluster [DBG] pgmap v758: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:14 smithi059 bash[61152]: cluster 2022-04-28T19:58:13.060307+0000 mgr.y (mgr.24961) 434 : cluster [DBG] pgmap v758: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:14.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:14 smithi139 bash[43301]: cluster 2022-04-28T19:58:13.060307+0000 mgr.y (mgr.24961) 434 : cluster [DBG] pgmap v758: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:15.083 INFO:tasks.workunit.client.0.smithi059.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2022-04-28T19:58:15.214 INFO:tasks.workunit.client.0.smithi059.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2022-04-28T19:58:15.269 INFO:tasks.workunit.client.0.smithi059.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2022-04-28T19:58:15.271 INFO:tasks.workunit.client.0.smithi059.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2022-04-28T19:58:15.271 INFO:tasks.workunit.client.0.smithi059.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2022-04-28T19:58:15.324 INFO:tasks.workunit.client.0.smithi059.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2022-04-28T19:58:15.329 DEBUG:teuthology.orchestra.run.smithi059:> set -ex 2022-04-28T19:58:15.329 DEBUG:teuthology.orchestra.run.smithi059:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2022-04-28T19:58:15.377 INFO:tasks.workunit:Running workunits matching cls on client.0... 2022-04-28T19:58:15.493 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2022-04-28T19:58:15.493 DEBUG:teuthology.orchestra.run.smithi059:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=octopus TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2022-04-28T19:58:15.523 INFO:tasks.workunit.client.0.smithi059.stdout:Running main() from gmock_main.cc 2022-04-28T19:58:15.524 INFO:tasks.workunit.client.0.smithi059.stdout:[==========] Running 6 tests from 1 test suite. 2022-04-28T19:58:15.524 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] Global test environment set-up. 2022-04-28T19:58:15.524 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] 6 tests from ClsHello 2022-04-28T19:58:15.524 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] ClsHello.SayHello 2022-04-28T19:58:16.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:16 smithi059 bash[59745]: cluster 2022-04-28T19:58:15.061241+0000 mgr.y (mgr.24961) 435 : cluster [DBG] pgmap v759: 73 pgs: 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:16 smithi059 bash[61152]: cluster 2022-04-28T19:58:15.061241+0000 mgr.y (mgr.24961) 435 : cluster [DBG] pgmap v759: 73 pgs: 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:16 smithi139 bash[43301]: cluster 2022-04-28T19:58:15.061241+0000 mgr.y (mgr.24961) 435 : cluster [DBG] pgmap v759: 73 pgs: 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[61152]: cluster 2022-04-28T19:58:16.181505+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2022-04-28T19:58:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[61152]: audit 2022-04-28T19:58:16.183163+0000 mon.c (mon.1) 16 : audit [INF] from='client.? 172.21.15.59:0/2147412904' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[61152]: audit 2022-04-28T19:58:16.184703+0000 mon.a (mon.0) 1507 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[61152]: audit 2022-04-28T19:58:17.181046+0000 mon.a (mon.0) 1508 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[61152]: cluster 2022-04-28T19:58:17.181503+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2022-04-28T19:58:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[59745]: cluster 2022-04-28T19:58:16.181505+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2022-04-28T19:58:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[59745]: audit 2022-04-28T19:58:16.183163+0000 mon.c (mon.1) 16 : audit [INF] from='client.? 172.21.15.59:0/2147412904' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[59745]: audit 2022-04-28T19:58:16.184703+0000 mon.a (mon.0) 1507 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[59745]: audit 2022-04-28T19:58:17.181046+0000 mon.a (mon.0) 1508 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:17 smithi059 bash[59745]: cluster 2022-04-28T19:58:17.181503+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2022-04-28T19:58:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:17 smithi139 bash[43301]: cluster 2022-04-28T19:58:16.181505+0000 mon.a (mon.0) 1506 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2022-04-28T19:58:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:17 smithi139 bash[43301]: audit 2022-04-28T19:58:16.183163+0000 mon.c (mon.1) 16 : audit [INF] from='client.? 172.21.15.59:0/2147412904' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:17 smithi139 bash[43301]: audit 2022-04-28T19:58:16.184703+0000 mon.a (mon.0) 1507 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:17 smithi139 bash[43301]: audit 2022-04-28T19:58:17.181046+0000 mon.a (mon.0) 1508 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:17 smithi139 bash[43301]: cluster 2022-04-28T19:58:17.181503+0000 mon.a (mon.0) 1509 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2022-04-28T19:58:18.190 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] ClsHello.SayHello (2667 ms) 2022-04-28T19:58:18.191 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] ClsHello.RecordHello 2022-04-28T19:58:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[61152]: cluster 2022-04-28T19:58:17.061955+0000 mgr.y (mgr.24961) 436 : cluster [DBG] pgmap v761: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[61152]: audit 2022-04-28T19:58:17.183064+0000 mon.a (mon.0) 1510 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:58:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[61152]: audit 2022-04-28T19:58:17.183364+0000 mon.a (mon.0) 1511 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:58:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[61152]: cluster 2022-04-28T19:58:18.185024+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2022-04-28T19:58:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[59745]: cluster 2022-04-28T19:58:17.061955+0000 mgr.y (mgr.24961) 436 : cluster [DBG] pgmap v761: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[59745]: audit 2022-04-28T19:58:17.183064+0000 mon.a (mon.0) 1510 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:58:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[59745]: audit 2022-04-28T19:58:17.183364+0000 mon.a (mon.0) 1511 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:58:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:18 smithi059 bash[59745]: cluster 2022-04-28T19:58:18.185024+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2022-04-28T19:58:18.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:18 smithi139 bash[43301]: cluster 2022-04-28T19:58:17.061955+0000 mgr.y (mgr.24961) 436 : cluster [DBG] pgmap v761: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:18.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:18 smithi139 bash[43301]: audit 2022-04-28T19:58:17.183064+0000 mon.a (mon.0) 1510 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:58:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:18 smithi139 bash[43301]: audit 2022-04-28T19:58:17.183364+0000 mon.a (mon.0) 1511 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:58:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:18 smithi139 bash[43301]: cluster 2022-04-28T19:58:18.185024+0000 mon.a (mon.0) 1512 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2022-04-28T19:58:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:20 smithi059 bash[61152]: cluster 2022-04-28T19:58:19.062799+0000 mgr.y (mgr.24961) 437 : cluster [DBG] pgmap v764: 73 pgs: 73 active+clean; 24 KiB data, 705 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:20.715 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:20 smithi059 bash[61152]: cluster 2022-04-28T19:58:19.220703+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2022-04-28T19:58:20.715 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:20 smithi059 bash[61152]: audit 2022-04-28T19:58:19.222105+0000 mon.a (mon.0) 1514 : audit [INF] from='client.? 172.21.15.59:0/3516648978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:20.715 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:20 smithi059 bash[59745]: cluster 2022-04-28T19:58:19.062799+0000 mgr.y (mgr.24961) 437 : cluster [DBG] pgmap v764: 73 pgs: 73 active+clean; 24 KiB data, 705 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:20.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:20 smithi059 bash[59745]: cluster 2022-04-28T19:58:19.220703+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2022-04-28T19:58:20.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:20 smithi059 bash[59745]: audit 2022-04-28T19:58:19.222105+0000 mon.a (mon.0) 1514 : audit [INF] from='client.? 172.21.15.59:0/3516648978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:20.716 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:20 smithi139 bash[43301]: cluster 2022-04-28T19:58:19.062799+0000 mgr.y (mgr.24961) 437 : cluster [DBG] pgmap v764: 73 pgs: 73 active+clean; 24 KiB data, 705 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:20.717 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:20 smithi139 bash[43301]: cluster 2022-04-28T19:58:19.220703+0000 mon.a (mon.0) 1513 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2022-04-28T19:58:20.717 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:20 smithi139 bash[43301]: audit 2022-04-28T19:58:19.222105+0000 mon.a (mon.0) 1514 : audit [INF] from='client.? 172.21.15.59:0/3516648978' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:21.242 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] ClsHello.RecordHello (3052 ms) 2022-04-28T19:58:21.243 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] ClsHello.WriteReturnData 2022-04-28T19:58:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:21 smithi059 bash[61152]: audit 2022-04-28T19:58:20.231387+0000 mon.a (mon.0) 1515 : audit [INF] from='client.? 172.21.15.59:0/3516648978' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:21 smithi059 bash[61152]: cluster 2022-04-28T19:58:20.231733+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2022-04-28T19:58:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:21 smithi059 bash[59745]: audit 2022-04-28T19:58:20.231387+0000 mon.a (mon.0) 1515 : audit [INF] from='client.? 172.21.15.59:0/3516648978' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:21 smithi059 bash[59745]: cluster 2022-04-28T19:58:20.231733+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2022-04-28T19:58:21.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:21 smithi139 bash[43301]: audit 2022-04-28T19:58:20.231387+0000 mon.a (mon.0) 1515 : audit [INF] from='client.? 172.21.15.59:0/3516648978' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:21.763 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:21 smithi139 bash[43301]: cluster 2022-04-28T19:58:20.231733+0000 mon.a (mon.0) 1516 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2022-04-28T19:58:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:21 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:21.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:22.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:22 smithi059 bash[59745]: cluster 2022-04-28T19:58:21.063767+0000 mgr.y (mgr.24961) 438 : cluster [DBG] pgmap v767: 105 pgs: 13 creating+peering, 9 unknown, 83 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:22.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:22 smithi059 bash[59745]: cluster 2022-04-28T19:58:21.240543+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2022-04-28T19:58:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:22 smithi059 bash[61152]: cluster 2022-04-28T19:58:21.063767+0000 mgr.y (mgr.24961) 438 : cluster [DBG] pgmap v767: 105 pgs: 13 creating+peering, 9 unknown, 83 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:22 smithi059 bash[61152]: cluster 2022-04-28T19:58:21.240543+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2022-04-28T19:58:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:22 smithi139 bash[43301]: cluster 2022-04-28T19:58:21.063767+0000 mgr.y (mgr.24961) 438 : cluster [DBG] pgmap v767: 105 pgs: 13 creating+peering, 9 unknown, 83 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:22 smithi139 bash[43301]: cluster 2022-04-28T19:58:21.240543+0000 mon.a (mon.0) 1517 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2022-04-28T19:58:23.269 INFO:tasks.workunit.client.0.smithi059.stdout:require_osd_release = pacific 2022-04-28T19:58:23.284 INFO:tasks.workunit.client.0.smithi059.stdout:0 2022-04-28T19:58:23.286 INFO:tasks.workunit.client.0.smithi059.stdout:1 2022-04-28T19:58:23.287 INFO:tasks.workunit.client.0.smithi059.stdout:2 2022-04-28T19:58:23.288 INFO:tasks.workunit.client.0.smithi059.stdout:3 2022-04-28T19:58:23.289 INFO:tasks.workunit.client.0.smithi059.stdout:4 2022-04-28T19:58:23.290 INFO:tasks.workunit.client.0.smithi059.stdout:5 2022-04-28T19:58:23.291 INFO:tasks.workunit.client.0.smithi059.stdout:6 2022-04-28T19:58:23.292 INFO:tasks.workunit.client.0.smithi059.stdout:7 2022-04-28T19:58:23.293 INFO:tasks.workunit.client.0.smithi059.stdout:8 2022-04-28T19:58:23.294 INFO:tasks.workunit.client.0.smithi059.stdout:9 2022-04-28T19:58:23.295 INFO:tasks.workunit.client.0.smithi059.stdout:10 2022-04-28T19:58:23.296 INFO:tasks.workunit.client.0.smithi059.stdout:11 2022-04-28T19:58:23.297 INFO:tasks.workunit.client.0.smithi059.stdout:12 2022-04-28T19:58:23.298 INFO:tasks.workunit.client.0.smithi059.stdout:13 2022-04-28T19:58:23.299 INFO:tasks.workunit.client.0.smithi059.stdout:14 2022-04-28T19:58:23.300 INFO:tasks.workunit.client.0.smithi059.stdout:15 2022-04-28T19:58:23.301 INFO:tasks.workunit.client.0.smithi059.stdout:16 2022-04-28T19:58:23.302 INFO:tasks.workunit.client.0.smithi059.stdout:17 2022-04-28T19:58:23.303 INFO:tasks.workunit.client.0.smithi059.stdout:18 2022-04-28T19:58:23.304 INFO:tasks.workunit.client.0.smithi059.stdout:19 2022-04-28T19:58:23.305 INFO:tasks.workunit.client.0.smithi059.stdout:20 2022-04-28T19:58:23.306 INFO:tasks.workunit.client.0.smithi059.stdout:21 2022-04-28T19:58:23.307 INFO:tasks.workunit.client.0.smithi059.stdout:22 2022-04-28T19:58:23.308 INFO:tasks.workunit.client.0.smithi059.stdout:23 2022-04-28T19:58:23.309 INFO:tasks.workunit.client.0.smithi059.stdout:24 2022-04-28T19:58:23.310 INFO:tasks.workunit.client.0.smithi059.stdout:25 2022-04-28T19:58:23.311 INFO:tasks.workunit.client.0.smithi059.stdout:26 2022-04-28T19:58:23.312 INFO:tasks.workunit.client.0.smithi059.stdout:27 2022-04-28T19:58:23.313 INFO:tasks.workunit.client.0.smithi059.stdout:28 2022-04-28T19:58:23.314 INFO:tasks.workunit.client.0.smithi059.stdout:29 2022-04-28T19:58:23.315 INFO:tasks.workunit.client.0.smithi059.stdout:30 2022-04-28T19:58:23.316 INFO:tasks.workunit.client.0.smithi059.stdout:31 2022-04-28T19:58:23.317 INFO:tasks.workunit.client.0.smithi059.stdout:32 2022-04-28T19:58:23.318 INFO:tasks.workunit.client.0.smithi059.stdout:33 2022-04-28T19:58:23.320 INFO:tasks.workunit.client.0.smithi059.stdout:34 2022-04-28T19:58:23.320 INFO:tasks.workunit.client.0.smithi059.stdout:35 2022-04-28T19:58:23.322 INFO:tasks.workunit.client.0.smithi059.stdout:36 2022-04-28T19:58:23.323 INFO:tasks.workunit.client.0.smithi059.stdout:37 2022-04-28T19:58:23.324 INFO:tasks.workunit.client.0.smithi059.stdout:38 2022-04-28T19:58:23.325 INFO:tasks.workunit.client.0.smithi059.stdout:39 2022-04-28T19:58:23.326 INFO:tasks.workunit.client.0.smithi059.stdout:40 2022-04-28T19:58:23.327 INFO:tasks.workunit.client.0.smithi059.stdout:41 2022-04-28T19:58:23.328 INFO:tasks.workunit.client.0.smithi059.stdout:42 2022-04-28T19:58:23.329 INFO:tasks.workunit.client.0.smithi059.stdout:43 2022-04-28T19:58:23.330 INFO:tasks.workunit.client.0.smithi059.stdout:44 2022-04-28T19:58:23.332 INFO:tasks.workunit.client.0.smithi059.stdout:45 2022-04-28T19:58:23.333 INFO:tasks.workunit.client.0.smithi059.stdout:46 2022-04-28T19:58:23.334 INFO:tasks.workunit.client.0.smithi059.stdout:47 2022-04-28T19:58:23.335 INFO:tasks.workunit.client.0.smithi059.stdout:48 2022-04-28T19:58:23.336 INFO:tasks.workunit.client.0.smithi059.stdout:49 2022-04-28T19:58:23.337 INFO:tasks.workunit.client.0.smithi059.stdout:50 2022-04-28T19:58:23.338 INFO:tasks.workunit.client.0.smithi059.stdout:51 2022-04-28T19:58:23.339 INFO:tasks.workunit.client.0.smithi059.stdout:52 2022-04-28T19:58:23.340 INFO:tasks.workunit.client.0.smithi059.stdout:53 2022-04-28T19:58:23.341 INFO:tasks.workunit.client.0.smithi059.stdout:54 2022-04-28T19:58:23.342 INFO:tasks.workunit.client.0.smithi059.stdout:55 2022-04-28T19:58:23.343 INFO:tasks.workunit.client.0.smithi059.stdout:56 2022-04-28T19:58:23.344 INFO:tasks.workunit.client.0.smithi059.stdout:57 2022-04-28T19:58:23.345 INFO:tasks.workunit.client.0.smithi059.stdout:58 2022-04-28T19:58:23.347 INFO:tasks.workunit.client.0.smithi059.stdout:59 2022-04-28T19:58:23.347 INFO:tasks.workunit.client.0.smithi059.stdout:60 2022-04-28T19:58:23.348 INFO:tasks.workunit.client.0.smithi059.stdout:61 2022-04-28T19:58:23.349 INFO:tasks.workunit.client.0.smithi059.stdout:62 2022-04-28T19:58:23.350 INFO:tasks.workunit.client.0.smithi059.stdout:63 2022-04-28T19:58:23.351 INFO:tasks.workunit.client.0.smithi059.stdout:64 2022-04-28T19:58:23.353 INFO:tasks.workunit.client.0.smithi059.stdout:65 2022-04-28T19:58:23.354 INFO:tasks.workunit.client.0.smithi059.stdout:66 2022-04-28T19:58:23.355 INFO:tasks.workunit.client.0.smithi059.stdout:67 2022-04-28T19:58:23.356 INFO:tasks.workunit.client.0.smithi059.stdout:68 2022-04-28T19:58:23.357 INFO:tasks.workunit.client.0.smithi059.stdout:69 2022-04-28T19:58:23.358 INFO:tasks.workunit.client.0.smithi059.stdout:70 2022-04-28T19:58:23.359 INFO:tasks.workunit.client.0.smithi059.stdout:71 2022-04-28T19:58:23.360 INFO:tasks.workunit.client.0.smithi059.stdout:72 2022-04-28T19:58:23.361 INFO:tasks.workunit.client.0.smithi059.stdout:73 2022-04-28T19:58:23.362 INFO:tasks.workunit.client.0.smithi059.stdout:74 2022-04-28T19:58:23.363 INFO:tasks.workunit.client.0.smithi059.stdout:75 2022-04-28T19:58:23.364 INFO:tasks.workunit.client.0.smithi059.stdout:76 2022-04-28T19:58:23.365 INFO:tasks.workunit.client.0.smithi059.stdout:77 2022-04-28T19:58:23.366 INFO:tasks.workunit.client.0.smithi059.stdout:78 2022-04-28T19:58:23.367 INFO:tasks.workunit.client.0.smithi059.stdout:79 2022-04-28T19:58:23.368 INFO:tasks.workunit.client.0.smithi059.stdout:80 2022-04-28T19:58:23.369 INFO:tasks.workunit.client.0.smithi059.stdout:81 2022-04-28T19:58:23.370 INFO:tasks.workunit.client.0.smithi059.stdout:82 2022-04-28T19:58:23.371 INFO:tasks.workunit.client.0.smithi059.stdout:83 2022-04-28T19:58:23.372 INFO:tasks.workunit.client.0.smithi059.stdout:84 2022-04-28T19:58:23.373 INFO:tasks.workunit.client.0.smithi059.stdout:85 2022-04-28T19:58:23.374 INFO:tasks.workunit.client.0.smithi059.stdout:86 2022-04-28T19:58:23.375 INFO:tasks.workunit.client.0.smithi059.stdout:87 2022-04-28T19:58:23.376 INFO:tasks.workunit.client.0.smithi059.stdout:88 2022-04-28T19:58:23.377 INFO:tasks.workunit.client.0.smithi059.stdout:89 2022-04-28T19:58:23.378 INFO:tasks.workunit.client.0.smithi059.stdout:90 2022-04-28T19:58:23.379 INFO:tasks.workunit.client.0.smithi059.stdout:91 2022-04-28T19:58:23.380 INFO:tasks.workunit.client.0.smithi059.stdout:92 2022-04-28T19:58:23.381 INFO:tasks.workunit.client.0.smithi059.stdout:93 2022-04-28T19:58:23.382 INFO:tasks.workunit.client.0.smithi059.stdout:94 2022-04-28T19:58:23.383 INFO:tasks.workunit.client.0.smithi059.stdout:95 2022-04-28T19:58:23.384 INFO:tasks.workunit.client.0.smithi059.stdout:96 2022-04-28T19:58:23.385 INFO:tasks.workunit.client.0.smithi059.stdout:97 2022-04-28T19:58:23.386 INFO:tasks.workunit.client.0.smithi059.stdout:98 2022-04-28T19:58:23.387 INFO:tasks.workunit.client.0.smithi059.stdout:99 2022-04-28T19:58:23.389 INFO:tasks.workunit.client.0.smithi059.stdout:100 2022-04-28T19:58:23.390 INFO:tasks.workunit.client.0.smithi059.stdout:101 2022-04-28T19:58:23.391 INFO:tasks.workunit.client.0.smithi059.stdout:102 2022-04-28T19:58:23.392 INFO:tasks.workunit.client.0.smithi059.stdout:103 2022-04-28T19:58:23.393 INFO:tasks.workunit.client.0.smithi059.stdout:104 2022-04-28T19:58:23.394 INFO:tasks.workunit.client.0.smithi059.stdout:105 2022-04-28T19:58:23.395 INFO:tasks.workunit.client.0.smithi059.stdout:106 2022-04-28T19:58:23.396 INFO:tasks.workunit.client.0.smithi059.stdout:107 2022-04-28T19:58:23.397 INFO:tasks.workunit.client.0.smithi059.stdout:108 2022-04-28T19:58:23.398 INFO:tasks.workunit.client.0.smithi059.stdout:109 2022-04-28T19:58:23.399 INFO:tasks.workunit.client.0.smithi059.stdout:110 2022-04-28T19:58:23.400 INFO:tasks.workunit.client.0.smithi059.stdout:111 2022-04-28T19:58:23.401 INFO:tasks.workunit.client.0.smithi059.stdout:112 2022-04-28T19:58:23.402 INFO:tasks.workunit.client.0.smithi059.stdout:113 2022-04-28T19:58:23.403 INFO:tasks.workunit.client.0.smithi059.stdout:114 2022-04-28T19:58:23.404 INFO:tasks.workunit.client.0.smithi059.stdout:115 2022-04-28T19:58:23.405 INFO:tasks.workunit.client.0.smithi059.stdout:116 2022-04-28T19:58:23.406 INFO:tasks.workunit.client.0.smithi059.stdout:117 2022-04-28T19:58:23.407 INFO:tasks.workunit.client.0.smithi059.stdout:118 2022-04-28T19:58:23.408 INFO:tasks.workunit.client.0.smithi059.stdout:119 2022-04-28T19:58:23.409 INFO:tasks.workunit.client.0.smithi059.stdout:120 2022-04-28T19:58:23.410 INFO:tasks.workunit.client.0.smithi059.stdout:121 2022-04-28T19:58:23.411 INFO:tasks.workunit.client.0.smithi059.stdout:122 2022-04-28T19:58:23.412 INFO:tasks.workunit.client.0.smithi059.stdout:123 2022-04-28T19:58:23.413 INFO:tasks.workunit.client.0.smithi059.stdout:124 2022-04-28T19:58:23.414 INFO:tasks.workunit.client.0.smithi059.stdout:125 2022-04-28T19:58:23.415 INFO:tasks.workunit.client.0.smithi059.stdout:126 2022-04-28T19:58:23.416 INFO:tasks.workunit.client.0.smithi059.stdout:127 2022-04-28T19:58:23.417 INFO:tasks.workunit.client.0.smithi059.stdout:128 2022-04-28T19:58:23.418 INFO:tasks.workunit.client.0.smithi059.stdout:129 2022-04-28T19:58:23.419 INFO:tasks.workunit.client.0.smithi059.stdout:130 2022-04-28T19:58:23.420 INFO:tasks.workunit.client.0.smithi059.stdout:131 2022-04-28T19:58:23.421 INFO:tasks.workunit.client.0.smithi059.stdout:132 2022-04-28T19:58:23.422 INFO:tasks.workunit.client.0.smithi059.stdout:133 2022-04-28T19:58:23.424 INFO:tasks.workunit.client.0.smithi059.stdout:134 2022-04-28T19:58:23.425 INFO:tasks.workunit.client.0.smithi059.stdout:135 2022-04-28T19:58:23.425 INFO:tasks.workunit.client.0.smithi059.stdout:136 2022-04-28T19:58:23.427 INFO:tasks.workunit.client.0.smithi059.stdout:137 2022-04-28T19:58:23.428 INFO:tasks.workunit.client.0.smithi059.stdout:138 2022-04-28T19:58:23.429 INFO:tasks.workunit.client.0.smithi059.stdout:139 2022-04-28T19:58:23.430 INFO:tasks.workunit.client.0.smithi059.stdout:140 2022-04-28T19:58:23.431 INFO:tasks.workunit.client.0.smithi059.stdout:141 2022-04-28T19:58:23.433 INFO:tasks.workunit.client.0.smithi059.stdout:142 2022-04-28T19:58:23.433 INFO:tasks.workunit.client.0.smithi059.stdout:143 2022-04-28T19:58:23.434 INFO:tasks.workunit.client.0.smithi059.stdout:144 2022-04-28T19:58:23.435 INFO:tasks.workunit.client.0.smithi059.stdout:145 2022-04-28T19:58:23.436 INFO:tasks.workunit.client.0.smithi059.stdout:146 2022-04-28T19:58:23.437 INFO:tasks.workunit.client.0.smithi059.stdout:147 2022-04-28T19:58:23.438 INFO:tasks.workunit.client.0.smithi059.stdout:148 2022-04-28T19:58:23.439 INFO:tasks.workunit.client.0.smithi059.stdout:149 2022-04-28T19:58:23.440 INFO:tasks.workunit.client.0.smithi059.stdout:150 2022-04-28T19:58:23.441 INFO:tasks.workunit.client.0.smithi059.stdout:151 2022-04-28T19:58:23.442 INFO:tasks.workunit.client.0.smithi059.stdout:152 2022-04-28T19:58:23.443 INFO:tasks.workunit.client.0.smithi059.stdout:153 2022-04-28T19:58:23.446 INFO:tasks.workunit.client.0.smithi059.stdout:154 2022-04-28T19:58:23.446 INFO:tasks.workunit.client.0.smithi059.stdout:155 2022-04-28T19:58:23.446 INFO:tasks.workunit.client.0.smithi059.stdout:156 2022-04-28T19:58:23.447 INFO:tasks.workunit.client.0.smithi059.stdout:157 2022-04-28T19:58:23.447 INFO:tasks.workunit.client.0.smithi059.stdout:158 2022-04-28T19:58:23.449 INFO:tasks.workunit.client.0.smithi059.stdout:159 2022-04-28T19:58:23.450 INFO:tasks.workunit.client.0.smithi059.stdout:160 2022-04-28T19:58:23.451 INFO:tasks.workunit.client.0.smithi059.stdout:161 2022-04-28T19:58:23.452 INFO:tasks.workunit.client.0.smithi059.stdout:162 2022-04-28T19:58:23.453 INFO:tasks.workunit.client.0.smithi059.stdout:163 2022-04-28T19:58:23.454 INFO:tasks.workunit.client.0.smithi059.stdout:164 2022-04-28T19:58:23.455 INFO:tasks.workunit.client.0.smithi059.stdout:165 2022-04-28T19:58:23.456 INFO:tasks.workunit.client.0.smithi059.stdout:166 2022-04-28T19:58:23.457 INFO:tasks.workunit.client.0.smithi059.stdout:167 2022-04-28T19:58:23.458 INFO:tasks.workunit.client.0.smithi059.stdout:168 2022-04-28T19:58:23.459 INFO:tasks.workunit.client.0.smithi059.stdout:169 2022-04-28T19:58:23.460 INFO:tasks.workunit.client.0.smithi059.stdout:170 2022-04-28T19:58:23.461 INFO:tasks.workunit.client.0.smithi059.stdout:171 2022-04-28T19:58:23.462 INFO:tasks.workunit.client.0.smithi059.stdout:172 2022-04-28T19:58:23.463 INFO:tasks.workunit.client.0.smithi059.stdout:173 2022-04-28T19:58:23.465 INFO:tasks.workunit.client.0.smithi059.stdout:174 2022-04-28T19:58:23.466 INFO:tasks.workunit.client.0.smithi059.stdout:175 2022-04-28T19:58:23.467 INFO:tasks.workunit.client.0.smithi059.stdout:176 2022-04-28T19:58:23.468 INFO:tasks.workunit.client.0.smithi059.stdout:177 2022-04-28T19:58:23.469 INFO:tasks.workunit.client.0.smithi059.stdout:178 2022-04-28T19:58:23.470 INFO:tasks.workunit.client.0.smithi059.stdout:179 2022-04-28T19:58:23.471 INFO:tasks.workunit.client.0.smithi059.stdout:180 2022-04-28T19:58:23.472 INFO:tasks.workunit.client.0.smithi059.stdout:181 2022-04-28T19:58:23.473 INFO:tasks.workunit.client.0.smithi059.stdout:182 2022-04-28T19:58:23.474 INFO:tasks.workunit.client.0.smithi059.stdout:183 2022-04-28T19:58:23.475 INFO:tasks.workunit.client.0.smithi059.stdout:184 2022-04-28T19:58:23.476 INFO:tasks.workunit.client.0.smithi059.stdout:185 2022-04-28T19:58:23.477 INFO:tasks.workunit.client.0.smithi059.stdout:186 2022-04-28T19:58:23.478 INFO:tasks.workunit.client.0.smithi059.stdout:187 2022-04-28T19:58:23.479 INFO:tasks.workunit.client.0.smithi059.stdout:188 2022-04-28T19:58:23.480 INFO:tasks.workunit.client.0.smithi059.stdout:189 2022-04-28T19:58:23.481 INFO:tasks.workunit.client.0.smithi059.stdout:190 2022-04-28T19:58:23.482 INFO:tasks.workunit.client.0.smithi059.stdout:191 2022-04-28T19:58:23.483 INFO:tasks.workunit.client.0.smithi059.stdout:192 2022-04-28T19:58:23.484 INFO:tasks.workunit.client.0.smithi059.stdout:193 2022-04-28T19:58:23.484 INFO:tasks.workunit.client.0.smithi059.stdout:194 2022-04-28T19:58:23.485 INFO:tasks.workunit.client.0.smithi059.stdout:195 2022-04-28T19:58:23.486 INFO:tasks.workunit.client.0.smithi059.stdout:196 2022-04-28T19:58:23.487 INFO:tasks.workunit.client.0.smithi059.stdout:197 2022-04-28T19:58:23.488 INFO:tasks.workunit.client.0.smithi059.stdout:198 2022-04-28T19:58:23.492 INFO:tasks.workunit.client.0.smithi059.stdout:199 2022-04-28T19:58:23.492 INFO:tasks.workunit.client.0.smithi059.stdout:200 2022-04-28T19:58:23.493 INFO:tasks.workunit.client.0.smithi059.stdout:201 2022-04-28T19:58:23.493 INFO:tasks.workunit.client.0.smithi059.stdout:202 2022-04-28T19:58:23.494 INFO:tasks.workunit.client.0.smithi059.stdout:203 2022-04-28T19:58:23.494 INFO:tasks.workunit.client.0.smithi059.stdout:204 2022-04-28T19:58:23.495 INFO:tasks.workunit.client.0.smithi059.stdout:205 2022-04-28T19:58:23.496 INFO:tasks.workunit.client.0.smithi059.stdout:206 2022-04-28T19:58:23.497 INFO:tasks.workunit.client.0.smithi059.stdout:207 2022-04-28T19:58:23.498 INFO:tasks.workunit.client.0.smithi059.stdout:208 2022-04-28T19:58:23.500 INFO:tasks.workunit.client.0.smithi059.stdout:209 2022-04-28T19:58:23.501 INFO:tasks.workunit.client.0.smithi059.stdout:210 2022-04-28T19:58:23.502 INFO:tasks.workunit.client.0.smithi059.stdout:211 2022-04-28T19:58:23.503 INFO:tasks.workunit.client.0.smithi059.stdout:212 2022-04-28T19:58:23.504 INFO:tasks.workunit.client.0.smithi059.stdout:213 2022-04-28T19:58:23.637 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:23 smithi139 bash[43301]: cluster 2022-04-28T19:58:22.247652+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2022-04-28T19:58:23.701 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:23 smithi139 bash[43301]: audit 2022-04-28T19:58:22.252945+0000 mon.b (mon.2) 11 : audit [INF] from='client.? 172.21.15.59:0/2773853503' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:23.701 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:23 smithi139 bash[43301]: audit 2022-04-28T19:58:22.258185+0000 mon.a (mon.0) 1519 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:23.704 INFO:tasks.workunit.client.0.smithi059.stdout:214 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:215 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:216 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:217 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:218 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:219 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:220 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:221 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:222 2022-04-28T19:58:23.705 INFO:tasks.workunit.client.0.smithi059.stdout:223 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:224 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:225 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:226 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:227 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:228 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:229 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:230 2022-04-28T19:58:23.706 INFO:tasks.workunit.client.0.smithi059.stdout:231 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:232 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:233 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:234 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:235 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:236 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:237 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:238 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:239 2022-04-28T19:58:23.707 INFO:tasks.workunit.client.0.smithi059.stdout:240 2022-04-28T19:58:23.708 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[61152]: cluster 2022-04-28T19:58:22.247652+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2022-04-28T19:58:23.708 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[61152]: audit 2022-04-28T19:58:22.252945+0000 mon.b (mon.2) 11 : audit [INF] from='client.? 172.21.15.59:0/2773853503' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:23.708 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[61152]: audit 2022-04-28T19:58:22.258185+0000 mon.a (mon.0) 1519 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:23.708 INFO:tasks.workunit.client.0.smithi059.stdout:241 2022-04-28T19:58:23.708 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[59745]: cluster 2022-04-28T19:58:22.247652+0000 mon.a (mon.0) 1518 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2022-04-28T19:58:23.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[59745]: audit 2022-04-28T19:58:22.252945+0000 mon.b (mon.2) 11 : audit [INF] from='client.? 172.21.15.59:0/2773853503' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:23.709 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[59745]: audit 2022-04-28T19:58:22.258185+0000 mon.a (mon.0) 1519 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:23.709 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:23 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:23.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:242 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:243 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:244 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:245 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:246 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:247 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:248 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:249 2022-04-28T19:58:23.726 INFO:tasks.workunit.client.0.smithi059.stdout:250 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:251 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:252 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:253 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:254 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:255 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:256 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:257 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:258 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:259 2022-04-28T19:58:23.727 INFO:tasks.workunit.client.0.smithi059.stdout:260 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:261 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:262 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:263 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:264 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:265 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:266 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:267 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:268 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:269 2022-04-28T19:58:23.728 INFO:tasks.workunit.client.0.smithi059.stdout:270 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:271 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:272 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:273 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:274 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:275 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:276 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:277 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:278 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:279 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:280 2022-04-28T19:58:23.729 INFO:tasks.workunit.client.0.smithi059.stdout:281 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:282 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:283 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:284 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:285 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:286 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:287 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:288 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:289 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:290 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:291 2022-04-28T19:58:23.730 INFO:tasks.workunit.client.0.smithi059.stdout:292 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:293 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:294 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:295 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:296 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:297 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:298 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:299 2022-04-28T19:58:23.731 INFO:tasks.workunit.client.0.smithi059.stdout:300 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:301 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:302 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:303 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:304 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:305 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:306 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:307 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:308 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:309 2022-04-28T19:58:23.732 INFO:tasks.workunit.client.0.smithi059.stdout:310 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:311 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:312 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:313 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:314 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:315 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:316 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:317 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:318 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:319 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:320 2022-04-28T19:58:23.733 INFO:tasks.workunit.client.0.smithi059.stdout:321 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:322 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:323 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:324 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:325 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:326 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:327 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:328 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:329 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:330 2022-04-28T19:58:23.734 INFO:tasks.workunit.client.0.smithi059.stdout:331 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:332 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:333 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:334 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:335 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:336 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:337 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:338 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:339 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:340 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:341 2022-04-28T19:58:23.735 INFO:tasks.workunit.client.0.smithi059.stdout:342 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:343 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:344 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:345 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:346 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:347 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:348 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:349 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:350 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:351 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:352 2022-04-28T19:58:23.736 INFO:tasks.workunit.client.0.smithi059.stdout:353 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:354 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:355 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:356 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:357 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:358 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:359 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:360 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:361 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:362 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:363 2022-04-28T19:58:23.737 INFO:tasks.workunit.client.0.smithi059.stdout:364 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:365 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:366 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:367 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:368 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:369 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:370 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:371 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:372 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:373 2022-04-28T19:58:23.738 INFO:tasks.workunit.client.0.smithi059.stdout:374 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:375 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:376 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:377 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:378 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:379 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:380 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:381 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:382 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:383 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:384 2022-04-28T19:58:23.739 INFO:tasks.workunit.client.0.smithi059.stdout:385 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:386 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:387 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:388 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:389 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:390 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:391 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:392 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:393 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:394 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:395 2022-04-28T19:58:23.740 INFO:tasks.workunit.client.0.smithi059.stdout:396 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:397 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:398 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:399 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:400 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:401 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:402 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:403 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:404 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:405 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:406 2022-04-28T19:58:23.741 INFO:tasks.workunit.client.0.smithi059.stdout:407 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:408 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:409 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:410 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:411 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:412 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:413 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:414 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:415 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:416 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:417 2022-04-28T19:58:23.742 INFO:tasks.workunit.client.0.smithi059.stdout:418 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:419 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:420 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:421 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:422 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:423 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:424 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:425 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:426 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:427 2022-04-28T19:58:23.745 INFO:tasks.workunit.client.0.smithi059.stdout:428 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:429 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:430 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:431 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:432 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:433 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:434 2022-04-28T19:58:23.746 INFO:tasks.workunit.client.0.smithi059.stdout:435 2022-04-28T19:58:23.747 INFO:tasks.workunit.client.0.smithi059.stdout:436 2022-04-28T19:58:23.747 INFO:tasks.workunit.client.0.smithi059.stdout:437 2022-04-28T19:58:23.747 INFO:tasks.workunit.client.0.smithi059.stdout:438 2022-04-28T19:58:23.748 INFO:tasks.workunit.client.0.smithi059.stdout:439 2022-04-28T19:58:23.749 INFO:tasks.workunit.client.0.smithi059.stdout:440 2022-04-28T19:58:23.750 INFO:tasks.workunit.client.0.smithi059.stdout:441 2022-04-28T19:58:23.751 INFO:tasks.workunit.client.0.smithi059.stdout:442 2022-04-28T19:58:23.752 INFO:tasks.workunit.client.0.smithi059.stdout:443 2022-04-28T19:58:23.753 INFO:tasks.workunit.client.0.smithi059.stdout:444 2022-04-28T19:58:23.754 INFO:tasks.workunit.client.0.smithi059.stdout:445 2022-04-28T19:58:23.755 INFO:tasks.workunit.client.0.smithi059.stdout:446 2022-04-28T19:58:23.756 INFO:tasks.workunit.client.0.smithi059.stdout:447 2022-04-28T19:58:23.757 INFO:tasks.workunit.client.0.smithi059.stdout:448 2022-04-28T19:58:23.758 INFO:tasks.workunit.client.0.smithi059.stdout:449 2022-04-28T19:58:23.759 INFO:tasks.workunit.client.0.smithi059.stdout:450 2022-04-28T19:58:23.760 INFO:tasks.workunit.client.0.smithi059.stdout:451 2022-04-28T19:58:23.761 INFO:tasks.workunit.client.0.smithi059.stdout:452 2022-04-28T19:58:23.762 INFO:tasks.workunit.client.0.smithi059.stdout:453 2022-04-28T19:58:23.763 INFO:tasks.workunit.client.0.smithi059.stdout:454 2022-04-28T19:58:23.764 INFO:tasks.workunit.client.0.smithi059.stdout:455 2022-04-28T19:58:23.765 INFO:tasks.workunit.client.0.smithi059.stdout:456 2022-04-28T19:58:23.766 INFO:tasks.workunit.client.0.smithi059.stdout:457 2022-04-28T19:58:23.767 INFO:tasks.workunit.client.0.smithi059.stdout:458 2022-04-28T19:58:23.768 INFO:tasks.workunit.client.0.smithi059.stdout:459 2022-04-28T19:58:23.769 INFO:tasks.workunit.client.0.smithi059.stdout:460 2022-04-28T19:58:23.770 INFO:tasks.workunit.client.0.smithi059.stdout:461 2022-04-28T19:58:23.771 INFO:tasks.workunit.client.0.smithi059.stdout:462 2022-04-28T19:58:23.772 INFO:tasks.workunit.client.0.smithi059.stdout:463 2022-04-28T19:58:23.773 INFO:tasks.workunit.client.0.smithi059.stdout:464 2022-04-28T19:58:23.775 INFO:tasks.workunit.client.0.smithi059.stdout:465 2022-04-28T19:58:23.776 INFO:tasks.workunit.client.0.smithi059.stdout:466 2022-04-28T19:58:23.777 INFO:tasks.workunit.client.0.smithi059.stdout:467 2022-04-28T19:58:23.778 INFO:tasks.workunit.client.0.smithi059.stdout:468 2022-04-28T19:58:23.779 INFO:tasks.workunit.client.0.smithi059.stdout:469 2022-04-28T19:58:23.779 INFO:tasks.workunit.client.0.smithi059.stdout:470 2022-04-28T19:58:23.781 INFO:tasks.workunit.client.0.smithi059.stdout:471 2022-04-28T19:58:23.782 INFO:tasks.workunit.client.0.smithi059.stdout:472 2022-04-28T19:58:23.783 INFO:tasks.workunit.client.0.smithi059.stdout:473 2022-04-28T19:58:23.784 INFO:tasks.workunit.client.0.smithi059.stdout:474 2022-04-28T19:58:23.785 INFO:tasks.workunit.client.0.smithi059.stdout:475 2022-04-28T19:58:23.786 INFO:tasks.workunit.client.0.smithi059.stdout:476 2022-04-28T19:58:23.787 INFO:tasks.workunit.client.0.smithi059.stdout:477 2022-04-28T19:58:23.787 INFO:tasks.workunit.client.0.smithi059.stdout:478 2022-04-28T19:58:23.789 INFO:tasks.workunit.client.0.smithi059.stdout:479 2022-04-28T19:58:23.790 INFO:tasks.workunit.client.0.smithi059.stdout:480 2022-04-28T19:58:23.791 INFO:tasks.workunit.client.0.smithi059.stdout:481 2022-04-28T19:58:23.792 INFO:tasks.workunit.client.0.smithi059.stdout:482 2022-04-28T19:58:23.793 INFO:tasks.workunit.client.0.smithi059.stdout:483 2022-04-28T19:58:23.794 INFO:tasks.workunit.client.0.smithi059.stdout:484 2022-04-28T19:58:23.795 INFO:tasks.workunit.client.0.smithi059.stdout:485 2022-04-28T19:58:23.796 INFO:tasks.workunit.client.0.smithi059.stdout:486 2022-04-28T19:58:23.797 INFO:tasks.workunit.client.0.smithi059.stdout:487 2022-04-28T19:58:23.798 INFO:tasks.workunit.client.0.smithi059.stdout:488 2022-04-28T19:58:23.799 INFO:tasks.workunit.client.0.smithi059.stdout:489 2022-04-28T19:58:23.800 INFO:tasks.workunit.client.0.smithi059.stdout:490 2022-04-28T19:58:23.801 INFO:tasks.workunit.client.0.smithi059.stdout:491 2022-04-28T19:58:23.802 INFO:tasks.workunit.client.0.smithi059.stdout:492 2022-04-28T19:58:23.803 INFO:tasks.workunit.client.0.smithi059.stdout:493 2022-04-28T19:58:23.804 INFO:tasks.workunit.client.0.smithi059.stdout:494 2022-04-28T19:58:23.805 INFO:tasks.workunit.client.0.smithi059.stdout:495 2022-04-28T19:58:23.806 INFO:tasks.workunit.client.0.smithi059.stdout:496 2022-04-28T19:58:23.807 INFO:tasks.workunit.client.0.smithi059.stdout:497 2022-04-28T19:58:23.808 INFO:tasks.workunit.client.0.smithi059.stdout:498 2022-04-28T19:58:23.809 INFO:tasks.workunit.client.0.smithi059.stdout:499 2022-04-28T19:58:23.810 INFO:tasks.workunit.client.0.smithi059.stdout:500 2022-04-28T19:58:23.811 INFO:tasks.workunit.client.0.smithi059.stdout:501 2022-04-28T19:58:23.813 INFO:tasks.workunit.client.0.smithi059.stdout:502 2022-04-28T19:58:23.814 INFO:tasks.workunit.client.0.smithi059.stdout:503 2022-04-28T19:58:23.815 INFO:tasks.workunit.client.0.smithi059.stdout:504 2022-04-28T19:58:23.816 INFO:tasks.workunit.client.0.smithi059.stdout:505 2022-04-28T19:58:23.817 INFO:tasks.workunit.client.0.smithi059.stdout:506 2022-04-28T19:58:23.818 INFO:tasks.workunit.client.0.smithi059.stdout:507 2022-04-28T19:58:23.819 INFO:tasks.workunit.client.0.smithi059.stdout:508 2022-04-28T19:58:23.820 INFO:tasks.workunit.client.0.smithi059.stdout:509 2022-04-28T19:58:23.821 INFO:tasks.workunit.client.0.smithi059.stdout:510 2022-04-28T19:58:23.822 INFO:tasks.workunit.client.0.smithi059.stdout:511 2022-04-28T19:58:23.823 INFO:tasks.workunit.client.0.smithi059.stdout:512 2022-04-28T19:58:23.824 INFO:tasks.workunit.client.0.smithi059.stdout:513 2022-04-28T19:58:23.826 INFO:tasks.workunit.client.0.smithi059.stdout:514 2022-04-28T19:58:23.826 INFO:tasks.workunit.client.0.smithi059.stdout:515 2022-04-28T19:58:23.827 INFO:tasks.workunit.client.0.smithi059.stdout:516 2022-04-28T19:58:23.828 INFO:tasks.workunit.client.0.smithi059.stdout:517 2022-04-28T19:58:23.829 INFO:tasks.workunit.client.0.smithi059.stdout:518 2022-04-28T19:58:23.830 INFO:tasks.workunit.client.0.smithi059.stdout:519 2022-04-28T19:58:23.832 INFO:tasks.workunit.client.0.smithi059.stdout:520 2022-04-28T19:58:23.832 INFO:tasks.workunit.client.0.smithi059.stdout:521 2022-04-28T19:58:23.833 INFO:tasks.workunit.client.0.smithi059.stdout:522 2022-04-28T19:58:23.835 INFO:tasks.workunit.client.0.smithi059.stdout:523 2022-04-28T19:58:23.835 INFO:tasks.workunit.client.0.smithi059.stdout:524 2022-04-28T19:58:23.837 INFO:tasks.workunit.client.0.smithi059.stdout:525 2022-04-28T19:58:23.838 INFO:tasks.workunit.client.0.smithi059.stdout:526 2022-04-28T19:58:23.839 INFO:tasks.workunit.client.0.smithi059.stdout:527 2022-04-28T19:58:23.840 INFO:tasks.workunit.client.0.smithi059.stdout:528 2022-04-28T19:58:23.841 INFO:tasks.workunit.client.0.smithi059.stdout:529 2022-04-28T19:58:23.842 INFO:tasks.workunit.client.0.smithi059.stdout:530 2022-04-28T19:58:23.843 INFO:tasks.workunit.client.0.smithi059.stdout:531 2022-04-28T19:58:23.844 INFO:tasks.workunit.client.0.smithi059.stdout:532 2022-04-28T19:58:23.845 INFO:tasks.workunit.client.0.smithi059.stdout:533 2022-04-28T19:58:23.846 INFO:tasks.workunit.client.0.smithi059.stdout:534 2022-04-28T19:58:23.847 INFO:tasks.workunit.client.0.smithi059.stdout:535 2022-04-28T19:58:23.848 INFO:tasks.workunit.client.0.smithi059.stdout:536 2022-04-28T19:58:23.849 INFO:tasks.workunit.client.0.smithi059.stdout:537 2022-04-28T19:58:23.850 INFO:tasks.workunit.client.0.smithi059.stdout:538 2022-04-28T19:58:23.851 INFO:tasks.workunit.client.0.smithi059.stdout:539 2022-04-28T19:58:23.852 INFO:tasks.workunit.client.0.smithi059.stdout:540 2022-04-28T19:58:23.853 INFO:tasks.workunit.client.0.smithi059.stdout:541 2022-04-28T19:58:23.855 INFO:tasks.workunit.client.0.smithi059.stdout:542 2022-04-28T19:58:23.856 INFO:tasks.workunit.client.0.smithi059.stdout:543 2022-04-28T19:58:23.857 INFO:tasks.workunit.client.0.smithi059.stdout:544 2022-04-28T19:58:23.858 INFO:tasks.workunit.client.0.smithi059.stdout:545 2022-04-28T19:58:23.859 INFO:tasks.workunit.client.0.smithi059.stdout:546 2022-04-28T19:58:23.860 INFO:tasks.workunit.client.0.smithi059.stdout:547 2022-04-28T19:58:23.861 INFO:tasks.workunit.client.0.smithi059.stdout:548 2022-04-28T19:58:23.862 INFO:tasks.workunit.client.0.smithi059.stdout:549 2022-04-28T19:58:23.863 INFO:tasks.workunit.client.0.smithi059.stdout:550 2022-04-28T19:58:23.864 INFO:tasks.workunit.client.0.smithi059.stdout:551 2022-04-28T19:58:23.865 INFO:tasks.workunit.client.0.smithi059.stdout:552 2022-04-28T19:58:23.866 INFO:tasks.workunit.client.0.smithi059.stdout:553 2022-04-28T19:58:23.867 INFO:tasks.workunit.client.0.smithi059.stdout:554 2022-04-28T19:58:23.868 INFO:tasks.workunit.client.0.smithi059.stdout:555 2022-04-28T19:58:23.869 INFO:tasks.workunit.client.0.smithi059.stdout:556 2022-04-28T19:58:23.870 INFO:tasks.workunit.client.0.smithi059.stdout:557 2022-04-28T19:58:23.871 INFO:tasks.workunit.client.0.smithi059.stdout:558 2022-04-28T19:58:23.872 INFO:tasks.workunit.client.0.smithi059.stdout:559 2022-04-28T19:58:23.873 INFO:tasks.workunit.client.0.smithi059.stdout:560 2022-04-28T19:58:23.874 INFO:tasks.workunit.client.0.smithi059.stdout:561 2022-04-28T19:58:23.875 INFO:tasks.workunit.client.0.smithi059.stdout:562 2022-04-28T19:58:23.876 INFO:tasks.workunit.client.0.smithi059.stdout:563 2022-04-28T19:58:23.877 INFO:tasks.workunit.client.0.smithi059.stdout:564 2022-04-28T19:58:23.878 INFO:tasks.workunit.client.0.smithi059.stdout:565 2022-04-28T19:58:23.879 INFO:tasks.workunit.client.0.smithi059.stdout:566 2022-04-28T19:58:23.881 INFO:tasks.workunit.client.0.smithi059.stdout:567 2022-04-28T19:58:23.882 INFO:tasks.workunit.client.0.smithi059.stdout:568 2022-04-28T19:58:23.883 INFO:tasks.workunit.client.0.smithi059.stdout:569 2022-04-28T19:58:23.885 INFO:tasks.workunit.client.0.smithi059.stdout:570 2022-04-28T19:58:23.886 INFO:tasks.workunit.client.0.smithi059.stdout:571 2022-04-28T19:58:23.888 INFO:tasks.workunit.client.0.smithi059.stdout:572 2022-04-28T19:58:23.889 INFO:tasks.workunit.client.0.smithi059.stdout:573 2022-04-28T19:58:23.891 INFO:tasks.workunit.client.0.smithi059.stdout:574 2022-04-28T19:58:23.892 INFO:tasks.workunit.client.0.smithi059.stdout:575 2022-04-28T19:58:23.894 INFO:tasks.workunit.client.0.smithi059.stdout:576 2022-04-28T19:58:23.895 INFO:tasks.workunit.client.0.smithi059.stdout:577 2022-04-28T19:58:23.896 INFO:tasks.workunit.client.0.smithi059.stdout:578 2022-04-28T19:58:23.898 INFO:tasks.workunit.client.0.smithi059.stdout:579 2022-04-28T19:58:23.899 INFO:tasks.workunit.client.0.smithi059.stdout:580 2022-04-28T19:58:23.901 INFO:tasks.workunit.client.0.smithi059.stdout:581 2022-04-28T19:58:23.902 INFO:tasks.workunit.client.0.smithi059.stdout:582 2022-04-28T19:58:23.905 INFO:tasks.workunit.client.0.smithi059.stdout:583 2022-04-28T19:58:23.905 INFO:tasks.workunit.client.0.smithi059.stdout:584 2022-04-28T19:58:23.907 INFO:tasks.workunit.client.0.smithi059.stdout:585 2022-04-28T19:58:23.908 INFO:tasks.workunit.client.0.smithi059.stdout:586 2022-04-28T19:58:23.910 INFO:tasks.workunit.client.0.smithi059.stdout:587 2022-04-28T19:58:23.912 INFO:tasks.workunit.client.0.smithi059.stdout:588 2022-04-28T19:58:23.913 INFO:tasks.workunit.client.0.smithi059.stdout:589 2022-04-28T19:58:23.915 INFO:tasks.workunit.client.0.smithi059.stdout:590 2022-04-28T19:58:23.917 INFO:tasks.workunit.client.0.smithi059.stdout:591 2022-04-28T19:58:23.918 INFO:tasks.workunit.client.0.smithi059.stdout:592 2022-04-28T19:58:23.919 INFO:tasks.workunit.client.0.smithi059.stdout:593 2022-04-28T19:58:23.920 INFO:tasks.workunit.client.0.smithi059.stdout:594 2022-04-28T19:58:23.921 INFO:tasks.workunit.client.0.smithi059.stdout:595 2022-04-28T19:58:23.923 INFO:tasks.workunit.client.0.smithi059.stdout:596 2022-04-28T19:58:23.924 INFO:tasks.workunit.client.0.smithi059.stdout:597 2022-04-28T19:58:23.925 INFO:tasks.workunit.client.0.smithi059.stdout:598 2022-04-28T19:58:23.927 INFO:tasks.workunit.client.0.smithi059.stdout:599 2022-04-28T19:58:23.928 INFO:tasks.workunit.client.0.smithi059.stdout:600 2022-04-28T19:58:23.929 INFO:tasks.workunit.client.0.smithi059.stdout:601 2022-04-28T19:58:23.931 INFO:tasks.workunit.client.0.smithi059.stdout:602 2022-04-28T19:58:23.932 INFO:tasks.workunit.client.0.smithi059.stdout:603 2022-04-28T19:58:23.933 INFO:tasks.workunit.client.0.smithi059.stdout:604 2022-04-28T19:58:23.934 INFO:tasks.workunit.client.0.smithi059.stdout:605 2022-04-28T19:58:23.935 INFO:tasks.workunit.client.0.smithi059.stdout:606 2022-04-28T19:58:23.936 INFO:tasks.workunit.client.0.smithi059.stdout:607 2022-04-28T19:58:23.938 INFO:tasks.workunit.client.0.smithi059.stdout:608 2022-04-28T19:58:23.939 INFO:tasks.workunit.client.0.smithi059.stdout:609 2022-04-28T19:58:23.940 INFO:tasks.workunit.client.0.smithi059.stdout:610 2022-04-28T19:58:23.942 INFO:tasks.workunit.client.0.smithi059.stdout:611 2022-04-28T19:58:23.943 INFO:tasks.workunit.client.0.smithi059.stdout:612 2022-04-28T19:58:23.945 INFO:tasks.workunit.client.0.smithi059.stdout:613 2022-04-28T19:58:23.946 INFO:tasks.workunit.client.0.smithi059.stdout:614 2022-04-28T19:58:23.947 INFO:tasks.workunit.client.0.smithi059.stdout:615 2022-04-28T19:58:23.949 INFO:tasks.workunit.client.0.smithi059.stdout:616 2022-04-28T19:58:23.951 INFO:tasks.workunit.client.0.smithi059.stdout:617 2022-04-28T19:58:23.952 INFO:tasks.workunit.client.0.smithi059.stdout:618 2022-04-28T19:58:23.954 INFO:tasks.workunit.client.0.smithi059.stdout:619 2022-04-28T19:58:23.955 INFO:tasks.workunit.client.0.smithi059.stdout:620 2022-04-28T19:58:23.957 INFO:tasks.workunit.client.0.smithi059.stdout:621 2022-04-28T19:58:23.958 INFO:tasks.workunit.client.0.smithi059.stdout:622 2022-04-28T19:58:23.960 INFO:tasks.workunit.client.0.smithi059.stdout:623 2022-04-28T19:58:23.962 INFO:tasks.workunit.client.0.smithi059.stdout:624 2022-04-28T19:58:23.963 INFO:tasks.workunit.client.0.smithi059.stdout:625 2022-04-28T19:58:23.965 INFO:tasks.workunit.client.0.smithi059.stdout:626 2022-04-28T19:58:23.967 INFO:tasks.workunit.client.0.smithi059.stdout:627 2022-04-28T19:58:23.968 INFO:tasks.workunit.client.0.smithi059.stdout:628 2022-04-28T19:58:23.970 INFO:tasks.workunit.client.0.smithi059.stdout:629 2022-04-28T19:58:23.971 INFO:tasks.workunit.client.0.smithi059.stdout:630 2022-04-28T19:58:23.973 INFO:tasks.workunit.client.0.smithi059.stdout:631 2022-04-28T19:58:23.974 INFO:tasks.workunit.client.0.smithi059.stdout:632 2022-04-28T19:58:23.975 INFO:tasks.workunit.client.0.smithi059.stdout:633 2022-04-28T19:58:23.976 INFO:tasks.workunit.client.0.smithi059.stdout:634 2022-04-28T19:58:23.978 INFO:tasks.workunit.client.0.smithi059.stdout:635 2022-04-28T19:58:23.979 INFO:tasks.workunit.client.0.smithi059.stdout:636 2022-04-28T19:58:23.980 INFO:tasks.workunit.client.0.smithi059.stdout:637 2022-04-28T19:58:23.982 INFO:tasks.workunit.client.0.smithi059.stdout:638 2022-04-28T19:58:23.983 INFO:tasks.workunit.client.0.smithi059.stdout:639 2022-04-28T19:58:23.984 INFO:tasks.workunit.client.0.smithi059.stdout:640 2022-04-28T19:58:23.985 INFO:tasks.workunit.client.0.smithi059.stdout:641 2022-04-28T19:58:23.986 INFO:tasks.workunit.client.0.smithi059.stdout:642 2022-04-28T19:58:23.987 INFO:tasks.workunit.client.0.smithi059.stdout:643 2022-04-28T19:58:23.989 INFO:tasks.workunit.client.0.smithi059.stdout:644 2022-04-28T19:58:23.990 INFO:tasks.workunit.client.0.smithi059.stdout:645 2022-04-28T19:58:23.991 INFO:tasks.workunit.client.0.smithi059.stdout:646 2022-04-28T19:58:23.992 INFO:tasks.workunit.client.0.smithi059.stdout:647 2022-04-28T19:58:23.994 INFO:tasks.workunit.client.0.smithi059.stdout:648 2022-04-28T19:58:23.995 INFO:tasks.workunit.client.0.smithi059.stdout:649 2022-04-28T19:58:23.996 INFO:tasks.workunit.client.0.smithi059.stdout:650 2022-04-28T19:58:23.997 INFO:tasks.workunit.client.0.smithi059.stdout:651 2022-04-28T19:58:23.998 INFO:tasks.workunit.client.0.smithi059.stdout:652 2022-04-28T19:58:24.000 INFO:tasks.workunit.client.0.smithi059.stdout:653 2022-04-28T19:58:24.001 INFO:tasks.workunit.client.0.smithi059.stdout:654 2022-04-28T19:58:24.002 INFO:tasks.workunit.client.0.smithi059.stdout:655 2022-04-28T19:58:24.003 INFO:tasks.workunit.client.0.smithi059.stdout:656 2022-04-28T19:58:24.004 INFO:tasks.workunit.client.0.smithi059.stdout:657 2022-04-28T19:58:24.006 INFO:tasks.workunit.client.0.smithi059.stdout:658 2022-04-28T19:58:24.007 INFO:tasks.workunit.client.0.smithi059.stdout:659 2022-04-28T19:58:24.008 INFO:tasks.workunit.client.0.smithi059.stdout:660 2022-04-28T19:58:24.009 INFO:tasks.workunit.client.0.smithi059.stdout:661 2022-04-28T19:58:24.010 INFO:tasks.workunit.client.0.smithi059.stdout:662 2022-04-28T19:58:24.012 INFO:tasks.workunit.client.0.smithi059.stdout:663 2022-04-28T19:58:24.013 INFO:tasks.workunit.client.0.smithi059.stdout:664 2022-04-28T19:58:24.014 INFO:tasks.workunit.client.0.smithi059.stdout:665 2022-04-28T19:58:24.015 INFO:tasks.workunit.client.0.smithi059.stdout:666 2022-04-28T19:58:24.016 INFO:tasks.workunit.client.0.smithi059.stdout:667 2022-04-28T19:58:24.018 INFO:tasks.workunit.client.0.smithi059.stdout:668 2022-04-28T19:58:24.019 INFO:tasks.workunit.client.0.smithi059.stdout:669 2022-04-28T19:58:24.021 INFO:tasks.workunit.client.0.smithi059.stdout:670 2022-04-28T19:58:24.022 INFO:tasks.workunit.client.0.smithi059.stdout:671 2022-04-28T19:58:24.024 INFO:tasks.workunit.client.0.smithi059.stdout:672 2022-04-28T19:58:24.025 INFO:tasks.workunit.client.0.smithi059.stdout:673 2022-04-28T19:58:24.027 INFO:tasks.workunit.client.0.smithi059.stdout:674 2022-04-28T19:58:24.029 INFO:tasks.workunit.client.0.smithi059.stdout:675 2022-04-28T19:58:24.030 INFO:tasks.workunit.client.0.smithi059.stdout:676 2022-04-28T19:58:24.031 INFO:tasks.workunit.client.0.smithi059.stdout:677 2022-04-28T19:58:24.032 INFO:tasks.workunit.client.0.smithi059.stdout:678 2022-04-28T19:58:24.033 INFO:tasks.workunit.client.0.smithi059.stdout:679 2022-04-28T19:58:24.034 INFO:tasks.workunit.client.0.smithi059.stdout:680 2022-04-28T19:58:24.036 INFO:tasks.workunit.client.0.smithi059.stdout:681 2022-04-28T19:58:24.037 INFO:tasks.workunit.client.0.smithi059.stdout:682 2022-04-28T19:58:24.038 INFO:tasks.workunit.client.0.smithi059.stdout:683 2022-04-28T19:58:24.040 INFO:tasks.workunit.client.0.smithi059.stdout:684 2022-04-28T19:58:24.041 INFO:tasks.workunit.client.0.smithi059.stdout:685 2022-04-28T19:58:24.043 INFO:tasks.workunit.client.0.smithi059.stdout:686 2022-04-28T19:58:24.044 INFO:tasks.workunit.client.0.smithi059.stdout:687 2022-04-28T19:58:24.046 INFO:tasks.workunit.client.0.smithi059.stdout:688 2022-04-28T19:58:24.047 INFO:tasks.workunit.client.0.smithi059.stdout:689 2022-04-28T19:58:24.049 INFO:tasks.workunit.client.0.smithi059.stdout:690 2022-04-28T19:58:24.050 INFO:tasks.workunit.client.0.smithi059.stdout:691 2022-04-28T19:58:24.052 INFO:tasks.workunit.client.0.smithi059.stdout:692 2022-04-28T19:58:24.053 INFO:tasks.workunit.client.0.smithi059.stdout:693 2022-04-28T19:58:24.055 INFO:tasks.workunit.client.0.smithi059.stdout:694 2022-04-28T19:58:24.056 INFO:tasks.workunit.client.0.smithi059.stdout:695 2022-04-28T19:58:24.058 INFO:tasks.workunit.client.0.smithi059.stdout:696 2022-04-28T19:58:24.060 INFO:tasks.workunit.client.0.smithi059.stdout:697 2022-04-28T19:58:24.061 INFO:tasks.workunit.client.0.smithi059.stdout:698 2022-04-28T19:58:24.063 INFO:tasks.workunit.client.0.smithi059.stdout:699 2022-04-28T19:58:24.065 INFO:tasks.workunit.client.0.smithi059.stdout:700 2022-04-28T19:58:24.066 INFO:tasks.workunit.client.0.smithi059.stdout:701 2022-04-28T19:58:24.068 INFO:tasks.workunit.client.0.smithi059.stdout:702 2022-04-28T19:58:24.070 INFO:tasks.workunit.client.0.smithi059.stdout:703 2022-04-28T19:58:24.071 INFO:tasks.workunit.client.0.smithi059.stdout:704 2022-04-28T19:58:24.073 INFO:tasks.workunit.client.0.smithi059.stdout:705 2022-04-28T19:58:24.075 INFO:tasks.workunit.client.0.smithi059.stdout:706 2022-04-28T19:58:24.076 INFO:tasks.workunit.client.0.smithi059.stdout:707 2022-04-28T19:58:24.078 INFO:tasks.workunit.client.0.smithi059.stdout:708 2022-04-28T19:58:24.079 INFO:tasks.workunit.client.0.smithi059.stdout:709 2022-04-28T19:58:24.081 INFO:tasks.workunit.client.0.smithi059.stdout:710 2022-04-28T19:58:24.083 INFO:tasks.workunit.client.0.smithi059.stdout:711 2022-04-28T19:58:24.084 INFO:tasks.workunit.client.0.smithi059.stdout:712 2022-04-28T19:58:24.086 INFO:tasks.workunit.client.0.smithi059.stdout:713 2022-04-28T19:58:24.088 INFO:tasks.workunit.client.0.smithi059.stdout:714 2022-04-28T19:58:24.090 INFO:tasks.workunit.client.0.smithi059.stdout:715 2022-04-28T19:58:24.091 INFO:tasks.workunit.client.0.smithi059.stdout:716 2022-04-28T19:58:24.093 INFO:tasks.workunit.client.0.smithi059.stdout:717 2022-04-28T19:58:24.095 INFO:tasks.workunit.client.0.smithi059.stdout:718 2022-04-28T19:58:24.097 INFO:tasks.workunit.client.0.smithi059.stdout:719 2022-04-28T19:58:24.099 INFO:tasks.workunit.client.0.smithi059.stdout:720 2022-04-28T19:58:24.100 INFO:tasks.workunit.client.0.smithi059.stdout:721 2022-04-28T19:58:24.102 INFO:tasks.workunit.client.0.smithi059.stdout:722 2022-04-28T19:58:24.104 INFO:tasks.workunit.client.0.smithi059.stdout:723 2022-04-28T19:58:24.106 INFO:tasks.workunit.client.0.smithi059.stdout:724 2022-04-28T19:58:24.108 INFO:tasks.workunit.client.0.smithi059.stdout:725 2022-04-28T19:58:24.110 INFO:tasks.workunit.client.0.smithi059.stdout:726 2022-04-28T19:58:24.111 INFO:tasks.workunit.client.0.smithi059.stdout:727 2022-04-28T19:58:24.113 INFO:tasks.workunit.client.0.smithi059.stdout:728 2022-04-28T19:58:24.115 INFO:tasks.workunit.client.0.smithi059.stdout:729 2022-04-28T19:58:24.116 INFO:tasks.workunit.client.0.smithi059.stdout:730 2022-04-28T19:58:24.118 INFO:tasks.workunit.client.0.smithi059.stdout:731 2022-04-28T19:58:24.119 INFO:tasks.workunit.client.0.smithi059.stdout:732 2022-04-28T19:58:24.121 INFO:tasks.workunit.client.0.smithi059.stdout:733 2022-04-28T19:58:24.123 INFO:tasks.workunit.client.0.smithi059.stdout:734 2022-04-28T19:58:24.124 INFO:tasks.workunit.client.0.smithi059.stdout:735 2022-04-28T19:58:24.126 INFO:tasks.workunit.client.0.smithi059.stdout:736 2022-04-28T19:58:24.128 INFO:tasks.workunit.client.0.smithi059.stdout:737 2022-04-28T19:58:24.129 INFO:tasks.workunit.client.0.smithi059.stdout:738 2022-04-28T19:58:24.131 INFO:tasks.workunit.client.0.smithi059.stdout:739 2022-04-28T19:58:24.133 INFO:tasks.workunit.client.0.smithi059.stdout:740 2022-04-28T19:58:24.135 INFO:tasks.workunit.client.0.smithi059.stdout:741 2022-04-28T19:58:24.136 INFO:tasks.workunit.client.0.smithi059.stdout:742 2022-04-28T19:58:24.138 INFO:tasks.workunit.client.0.smithi059.stdout:743 2022-04-28T19:58:24.140 INFO:tasks.workunit.client.0.smithi059.stdout:744 2022-04-28T19:58:24.142 INFO:tasks.workunit.client.0.smithi059.stdout:745 2022-04-28T19:58:24.143 INFO:tasks.workunit.client.0.smithi059.stdout:746 2022-04-28T19:58:24.145 INFO:tasks.workunit.client.0.smithi059.stdout:747 2022-04-28T19:58:24.147 INFO:tasks.workunit.client.0.smithi059.stdout:748 2022-04-28T19:58:24.149 INFO:tasks.workunit.client.0.smithi059.stdout:749 2022-04-28T19:58:24.151 INFO:tasks.workunit.client.0.smithi059.stdout:750 2022-04-28T19:58:24.153 INFO:tasks.workunit.client.0.smithi059.stdout:751 2022-04-28T19:58:24.154 INFO:tasks.workunit.client.0.smithi059.stdout:752 2022-04-28T19:58:24.156 INFO:tasks.workunit.client.0.smithi059.stdout:753 2022-04-28T19:58:24.157 INFO:tasks.workunit.client.0.smithi059.stdout:754 2022-04-28T19:58:24.159 INFO:tasks.workunit.client.0.smithi059.stdout:755 2022-04-28T19:58:24.160 INFO:tasks.workunit.client.0.smithi059.stdout:756 2022-04-28T19:58:24.162 INFO:tasks.workunit.client.0.smithi059.stdout:757 2022-04-28T19:58:24.163 INFO:tasks.workunit.client.0.smithi059.stdout:758 2022-04-28T19:58:24.164 INFO:tasks.workunit.client.0.smithi059.stdout:759 2022-04-28T19:58:24.166 INFO:tasks.workunit.client.0.smithi059.stdout:760 2022-04-28T19:58:24.167 INFO:tasks.workunit.client.0.smithi059.stdout:761 2022-04-28T19:58:24.168 INFO:tasks.workunit.client.0.smithi059.stdout:762 2022-04-28T19:58:24.169 INFO:tasks.workunit.client.0.smithi059.stdout:763 2022-04-28T19:58:24.170 INFO:tasks.workunit.client.0.smithi059.stdout:764 2022-04-28T19:58:24.172 INFO:tasks.workunit.client.0.smithi059.stdout:765 2022-04-28T19:58:24.173 INFO:tasks.workunit.client.0.smithi059.stdout:766 2022-04-28T19:58:24.174 INFO:tasks.workunit.client.0.smithi059.stdout:767 2022-04-28T19:58:24.176 INFO:tasks.workunit.client.0.smithi059.stdout:768 2022-04-28T19:58:24.177 INFO:tasks.workunit.client.0.smithi059.stdout:769 2022-04-28T19:58:24.178 INFO:tasks.workunit.client.0.smithi059.stdout:770 2022-04-28T19:58:24.180 INFO:tasks.workunit.client.0.smithi059.stdout:771 2022-04-28T19:58:24.182 INFO:tasks.workunit.client.0.smithi059.stdout:772 2022-04-28T19:58:24.183 INFO:tasks.workunit.client.0.smithi059.stdout:773 2022-04-28T19:58:24.184 INFO:tasks.workunit.client.0.smithi059.stdout:774 2022-04-28T19:58:24.186 INFO:tasks.workunit.client.0.smithi059.stdout:775 2022-04-28T19:58:24.187 INFO:tasks.workunit.client.0.smithi059.stdout:776 2022-04-28T19:58:24.189 INFO:tasks.workunit.client.0.smithi059.stdout:777 2022-04-28T19:58:24.190 INFO:tasks.workunit.client.0.smithi059.stdout:778 2022-04-28T19:58:24.191 INFO:tasks.workunit.client.0.smithi059.stdout:779 2022-04-28T19:58:24.192 INFO:tasks.workunit.client.0.smithi059.stdout:780 2022-04-28T19:58:24.194 INFO:tasks.workunit.client.0.smithi059.stdout:781 2022-04-28T19:58:24.195 INFO:tasks.workunit.client.0.smithi059.stdout:782 2022-04-28T19:58:24.196 INFO:tasks.workunit.client.0.smithi059.stdout:783 2022-04-28T19:58:24.198 INFO:tasks.workunit.client.0.smithi059.stdout:784 2022-04-28T19:58:24.199 INFO:tasks.workunit.client.0.smithi059.stdout:785 2022-04-28T19:58:24.201 INFO:tasks.workunit.client.0.smithi059.stdout:786 2022-04-28T19:58:24.202 INFO:tasks.workunit.client.0.smithi059.stdout:787 2022-04-28T19:58:24.204 INFO:tasks.workunit.client.0.smithi059.stdout:788 2022-04-28T19:58:24.205 INFO:tasks.workunit.client.0.smithi059.stdout:789 2022-04-28T19:58:24.207 INFO:tasks.workunit.client.0.smithi059.stdout:790 2022-04-28T19:58:24.208 INFO:tasks.workunit.client.0.smithi059.stdout:791 2022-04-28T19:58:24.210 INFO:tasks.workunit.client.0.smithi059.stdout:792 2022-04-28T19:58:24.212 INFO:tasks.workunit.client.0.smithi059.stdout:793 2022-04-28T19:58:24.213 INFO:tasks.workunit.client.0.smithi059.stdout:794 2022-04-28T19:58:24.215 INFO:tasks.workunit.client.0.smithi059.stdout:795 2022-04-28T19:58:24.216 INFO:tasks.workunit.client.0.smithi059.stdout:796 2022-04-28T19:58:24.218 INFO:tasks.workunit.client.0.smithi059.stdout:797 2022-04-28T19:58:24.220 INFO:tasks.workunit.client.0.smithi059.stdout:798 2022-04-28T19:58:24.221 INFO:tasks.workunit.client.0.smithi059.stdout:799 2022-04-28T19:58:24.223 INFO:tasks.workunit.client.0.smithi059.stdout:800 2022-04-28T19:58:24.225 INFO:tasks.workunit.client.0.smithi059.stdout:801 2022-04-28T19:58:24.226 INFO:tasks.workunit.client.0.smithi059.stdout:802 2022-04-28T19:58:24.228 INFO:tasks.workunit.client.0.smithi059.stdout:803 2022-04-28T19:58:24.229 INFO:tasks.workunit.client.0.smithi059.stdout:804 2022-04-28T19:58:24.231 INFO:tasks.workunit.client.0.smithi059.stdout:805 2022-04-28T19:58:24.232 INFO:tasks.workunit.client.0.smithi059.stdout:806 2022-04-28T19:58:24.233 INFO:tasks.workunit.client.0.smithi059.stdout:807 2022-04-28T19:58:24.235 INFO:tasks.workunit.client.0.smithi059.stdout:808 2022-04-28T19:58:24.236 INFO:tasks.workunit.client.0.smithi059.stdout:809 2022-04-28T19:58:24.238 INFO:tasks.workunit.client.0.smithi059.stdout:810 2022-04-28T19:58:24.239 INFO:tasks.workunit.client.0.smithi059.stdout:811 2022-04-28T19:58:24.240 INFO:tasks.workunit.client.0.smithi059.stdout:812 2022-04-28T19:58:24.242 INFO:tasks.workunit.client.0.smithi059.stdout:813 2022-04-28T19:58:24.243 INFO:tasks.workunit.client.0.smithi059.stdout:814 2022-04-28T19:58:24.245 INFO:tasks.workunit.client.0.smithi059.stdout:815 2022-04-28T19:58:24.246 INFO:tasks.workunit.client.0.smithi059.stdout:816 2022-04-28T19:58:24.248 INFO:tasks.workunit.client.0.smithi059.stdout:817 2022-04-28T19:58:24.249 INFO:tasks.workunit.client.0.smithi059.stdout:818 2022-04-28T19:58:24.251 INFO:tasks.workunit.client.0.smithi059.stdout:819 2022-04-28T19:58:24.253 INFO:tasks.workunit.client.0.smithi059.stdout:820 2022-04-28T19:58:24.254 INFO:tasks.workunit.client.0.smithi059.stdout:821 2022-04-28T19:58:24.256 INFO:tasks.workunit.client.0.smithi059.stdout:822 2022-04-28T19:58:24.258 INFO:tasks.workunit.client.0.smithi059.stdout:823 2022-04-28T19:58:24.261 INFO:tasks.workunit.client.0.smithi059.stdout:824 2022-04-28T19:58:24.267 INFO:tasks.workunit.client.0.smithi059.stdout:825 2022-04-28T19:58:24.270 INFO:tasks.workunit.client.0.smithi059.stdout:826 2022-04-28T19:58:24.272 INFO:tasks.workunit.client.0.smithi059.stdout:827 2022-04-28T19:58:24.274 INFO:tasks.workunit.client.0.smithi059.stdout:828 2022-04-28T19:58:24.275 INFO:tasks.workunit.client.0.smithi059.stdout:829 2022-04-28T19:58:24.276 INFO:tasks.workunit.client.0.smithi059.stdout:830 2022-04-28T19:58:24.278 INFO:tasks.workunit.client.0.smithi059.stdout:831 2022-04-28T19:58:24.279 INFO:tasks.workunit.client.0.smithi059.stdout:832 2022-04-28T19:58:24.281 INFO:tasks.workunit.client.0.smithi059.stdout:833 2022-04-28T19:58:24.282 INFO:tasks.workunit.client.0.smithi059.stdout:834 2022-04-28T19:58:24.283 INFO:tasks.workunit.client.0.smithi059.stdout:835 2022-04-28T19:58:24.284 INFO:tasks.workunit.client.0.smithi059.stdout:836 2022-04-28T19:58:24.286 INFO:tasks.workunit.client.0.smithi059.stdout:837 2022-04-28T19:58:24.287 INFO:tasks.workunit.client.0.smithi059.stdout:838 2022-04-28T19:58:24.289 INFO:tasks.workunit.client.0.smithi059.stdout:839 2022-04-28T19:58:24.290 INFO:tasks.workunit.client.0.smithi059.stdout:840 2022-04-28T19:58:24.291 INFO:tasks.workunit.client.0.smithi059.stdout:841 2022-04-28T19:58:24.292 INFO:tasks.workunit.client.0.smithi059.stdout:842 2022-04-28T19:58:24.294 INFO:tasks.workunit.client.0.smithi059.stdout:843 2022-04-28T19:58:24.295 INFO:tasks.workunit.client.0.smithi059.stdout:844 2022-04-28T19:58:24.297 INFO:tasks.workunit.client.0.smithi059.stdout:845 2022-04-28T19:58:24.298 INFO:tasks.workunit.client.0.smithi059.stdout:846 2022-04-28T19:58:24.300 INFO:tasks.workunit.client.0.smithi059.stdout:847 2022-04-28T19:58:24.301 INFO:tasks.workunit.client.0.smithi059.stdout:848 2022-04-28T19:58:24.302 INFO:tasks.workunit.client.0.smithi059.stdout:849 2022-04-28T19:58:24.304 INFO:tasks.workunit.client.0.smithi059.stdout:850 2022-04-28T19:58:24.305 INFO:tasks.workunit.client.0.smithi059.stdout:851 2022-04-28T19:58:24.307 INFO:tasks.workunit.client.0.smithi059.stdout:852 2022-04-28T19:58:24.308 INFO:tasks.workunit.client.0.smithi059.stdout:853 2022-04-28T19:58:24.310 INFO:tasks.workunit.client.0.smithi059.stdout:854 2022-04-28T19:58:24.311 INFO:tasks.workunit.client.0.smithi059.stdout:855 2022-04-28T19:58:24.312 INFO:tasks.workunit.client.0.smithi059.stdout:856 2022-04-28T19:58:24.314 INFO:tasks.workunit.client.0.smithi059.stdout:857 2022-04-28T19:58:24.315 INFO:tasks.workunit.client.0.smithi059.stdout:858 2022-04-28T19:58:24.316 INFO:tasks.workunit.client.0.smithi059.stdout:859 2022-04-28T19:58:24.318 INFO:tasks.workunit.client.0.smithi059.stdout:860 2022-04-28T19:58:24.319 INFO:tasks.workunit.client.0.smithi059.stdout:861 2022-04-28T19:58:24.321 INFO:tasks.workunit.client.0.smithi059.stdout:862 2022-04-28T19:58:24.322 INFO:tasks.workunit.client.0.smithi059.stdout:863 2022-04-28T19:58:24.323 INFO:tasks.workunit.client.0.smithi059.stdout:864 2022-04-28T19:58:24.324 INFO:tasks.workunit.client.0.smithi059.stdout:865 2022-04-28T19:58:24.326 INFO:tasks.workunit.client.0.smithi059.stdout:866 2022-04-28T19:58:24.327 INFO:tasks.workunit.client.0.smithi059.stdout:867 2022-04-28T19:58:24.328 INFO:tasks.workunit.client.0.smithi059.stdout:868 2022-04-28T19:58:24.329 INFO:tasks.workunit.client.0.smithi059.stdout:869 2022-04-28T19:58:24.331 INFO:tasks.workunit.client.0.smithi059.stdout:870 2022-04-28T19:58:24.332 INFO:tasks.workunit.client.0.smithi059.stdout:871 2022-04-28T19:58:24.333 INFO:tasks.workunit.client.0.smithi059.stdout:872 2022-04-28T19:58:24.335 INFO:tasks.workunit.client.0.smithi059.stdout:873 2022-04-28T19:58:24.336 INFO:tasks.workunit.client.0.smithi059.stdout:874 2022-04-28T19:58:24.337 INFO:tasks.workunit.client.0.smithi059.stdout:875 2022-04-28T19:58:24.339 INFO:tasks.workunit.client.0.smithi059.stdout:876 2022-04-28T19:58:24.340 INFO:tasks.workunit.client.0.smithi059.stdout:877 2022-04-28T19:58:24.342 INFO:tasks.workunit.client.0.smithi059.stdout:878 2022-04-28T19:58:24.343 INFO:tasks.workunit.client.0.smithi059.stdout:879 2022-04-28T19:58:24.344 INFO:tasks.workunit.client.0.smithi059.stdout:880 2022-04-28T19:58:24.346 INFO:tasks.workunit.client.0.smithi059.stdout:881 2022-04-28T19:58:24.347 INFO:tasks.workunit.client.0.smithi059.stdout:882 2022-04-28T19:58:24.349 INFO:tasks.workunit.client.0.smithi059.stdout:883 2022-04-28T19:58:24.350 INFO:tasks.workunit.client.0.smithi059.stdout:884 2022-04-28T19:58:24.351 INFO:tasks.workunit.client.0.smithi059.stdout:885 2022-04-28T19:58:24.353 INFO:tasks.workunit.client.0.smithi059.stdout:886 2022-04-28T19:58:24.354 INFO:tasks.workunit.client.0.smithi059.stdout:887 2022-04-28T19:58:24.355 INFO:tasks.workunit.client.0.smithi059.stdout:888 2022-04-28T19:58:24.357 INFO:tasks.workunit.client.0.smithi059.stdout:889 2022-04-28T19:58:24.358 INFO:tasks.workunit.client.0.smithi059.stdout:890 2022-04-28T19:58:24.359 INFO:tasks.workunit.client.0.smithi059.stdout:891 2022-04-28T19:58:24.361 INFO:tasks.workunit.client.0.smithi059.stdout:892 2022-04-28T19:58:24.362 INFO:tasks.workunit.client.0.smithi059.stdout:893 2022-04-28T19:58:24.363 INFO:tasks.workunit.client.0.smithi059.stdout:894 2022-04-28T19:58:24.365 INFO:tasks.workunit.client.0.smithi059.stdout:895 2022-04-28T19:58:24.366 INFO:tasks.workunit.client.0.smithi059.stdout:896 2022-04-28T19:58:24.373 INFO:tasks.workunit.client.0.smithi059.stdout:897 2022-04-28T19:58:24.373 INFO:tasks.workunit.client.0.smithi059.stdout:898 2022-04-28T19:58:24.373 INFO:tasks.workunit.client.0.smithi059.stdout:899 2022-04-28T19:58:24.373 INFO:tasks.workunit.client.0.smithi059.stdout:900 2022-04-28T19:58:24.373 INFO:tasks.workunit.client.0.smithi059.stdout:901 2022-04-28T19:58:24.374 INFO:tasks.workunit.client.0.smithi059.stdout:902 2022-04-28T19:58:24.376 INFO:tasks.workunit.client.0.smithi059.stdout:903 2022-04-28T19:58:24.377 INFO:tasks.workunit.client.0.smithi059.stdout:904 2022-04-28T19:58:24.378 INFO:tasks.workunit.client.0.smithi059.stdout:905 2022-04-28T19:58:24.379 INFO:tasks.workunit.client.0.smithi059.stdout:906 2022-04-28T19:58:24.381 INFO:tasks.workunit.client.0.smithi059.stdout:907 2022-04-28T19:58:24.382 INFO:tasks.workunit.client.0.smithi059.stdout:908 2022-04-28T19:58:24.383 INFO:tasks.workunit.client.0.smithi059.stdout:909 2022-04-28T19:58:24.385 INFO:tasks.workunit.client.0.smithi059.stdout:910 2022-04-28T19:58:24.386 INFO:tasks.workunit.client.0.smithi059.stdout:911 2022-04-28T19:58:24.387 INFO:tasks.workunit.client.0.smithi059.stdout:912 2022-04-28T19:58:24.388 INFO:tasks.workunit.client.0.smithi059.stdout:913 2022-04-28T19:58:24.390 INFO:tasks.workunit.client.0.smithi059.stdout:914 2022-04-28T19:58:24.391 INFO:tasks.workunit.client.0.smithi059.stdout:915 2022-04-28T19:58:24.392 INFO:tasks.workunit.client.0.smithi059.stdout:916 2022-04-28T19:58:24.393 INFO:tasks.workunit.client.0.smithi059.stdout:917 2022-04-28T19:58:24.395 INFO:tasks.workunit.client.0.smithi059.stdout:918 2022-04-28T19:58:24.396 INFO:tasks.workunit.client.0.smithi059.stdout:919 2022-04-28T19:58:24.398 INFO:tasks.workunit.client.0.smithi059.stdout:920 2022-04-28T19:58:24.399 INFO:tasks.workunit.client.0.smithi059.stdout:921 2022-04-28T19:58:24.401 INFO:tasks.workunit.client.0.smithi059.stdout:922 2022-04-28T19:58:24.402 INFO:tasks.workunit.client.0.smithi059.stdout:923 2022-04-28T19:58:24.403 INFO:tasks.workunit.client.0.smithi059.stdout:924 2022-04-28T19:58:24.405 INFO:tasks.workunit.client.0.smithi059.stdout:925 2022-04-28T19:58:24.406 INFO:tasks.workunit.client.0.smithi059.stdout:926 2022-04-28T19:58:24.407 INFO:tasks.workunit.client.0.smithi059.stdout:927 2022-04-28T19:58:24.409 INFO:tasks.workunit.client.0.smithi059.stdout:928 2022-04-28T19:58:24.410 INFO:tasks.workunit.client.0.smithi059.stdout:929 2022-04-28T19:58:24.412 INFO:tasks.workunit.client.0.smithi059.stdout:930 2022-04-28T19:58:24.413 INFO:tasks.workunit.client.0.smithi059.stdout:931 2022-04-28T19:58:24.414 INFO:tasks.workunit.client.0.smithi059.stdout:932 2022-04-28T19:58:24.416 INFO:tasks.workunit.client.0.smithi059.stdout:933 2022-04-28T19:58:24.417 INFO:tasks.workunit.client.0.smithi059.stdout:934 2022-04-28T19:58:24.418 INFO:tasks.workunit.client.0.smithi059.stdout:935 2022-04-28T19:58:24.420 INFO:tasks.workunit.client.0.smithi059.stdout:936 2022-04-28T19:58:24.421 INFO:tasks.workunit.client.0.smithi059.stdout:937 2022-04-28T19:58:24.422 INFO:tasks.workunit.client.0.smithi059.stdout:938 2022-04-28T19:58:24.424 INFO:tasks.workunit.client.0.smithi059.stdout:939 2022-04-28T19:58:24.425 INFO:tasks.workunit.client.0.smithi059.stdout:940 2022-04-28T19:58:24.426 INFO:tasks.workunit.client.0.smithi059.stdout:941 2022-04-28T19:58:24.428 INFO:tasks.workunit.client.0.smithi059.stdout:942 2022-04-28T19:58:24.429 INFO:tasks.workunit.client.0.smithi059.stdout:943 2022-04-28T19:58:24.431 INFO:tasks.workunit.client.0.smithi059.stdout:944 2022-04-28T19:58:24.432 INFO:tasks.workunit.client.0.smithi059.stdout:945 2022-04-28T19:58:24.433 INFO:tasks.workunit.client.0.smithi059.stdout:946 2022-04-28T19:58:24.435 INFO:tasks.workunit.client.0.smithi059.stdout:947 2022-04-28T19:58:24.436 INFO:tasks.workunit.client.0.smithi059.stdout:948 2022-04-28T19:58:24.438 INFO:tasks.workunit.client.0.smithi059.stdout:949 2022-04-28T19:58:24.439 INFO:tasks.workunit.client.0.smithi059.stdout:950 2022-04-28T19:58:24.440 INFO:tasks.workunit.client.0.smithi059.stdout:951 2022-04-28T19:58:24.442 INFO:tasks.workunit.client.0.smithi059.stdout:952 2022-04-28T19:58:24.443 INFO:tasks.workunit.client.0.smithi059.stdout:953 2022-04-28T19:58:24.444 INFO:tasks.workunit.client.0.smithi059.stdout:954 2022-04-28T19:58:24.446 INFO:tasks.workunit.client.0.smithi059.stdout:955 2022-04-28T19:58:24.447 INFO:tasks.workunit.client.0.smithi059.stdout:956 2022-04-28T19:58:24.449 INFO:tasks.workunit.client.0.smithi059.stdout:957 2022-04-28T19:58:24.450 INFO:tasks.workunit.client.0.smithi059.stdout:958 2022-04-28T19:58:24.451 INFO:tasks.workunit.client.0.smithi059.stdout:959 2022-04-28T19:58:24.453 INFO:tasks.workunit.client.0.smithi059.stdout:960 2022-04-28T19:58:24.454 INFO:tasks.workunit.client.0.smithi059.stdout:961 2022-04-28T19:58:24.456 INFO:tasks.workunit.client.0.smithi059.stdout:962 2022-04-28T19:58:24.457 INFO:tasks.workunit.client.0.smithi059.stdout:963 2022-04-28T19:58:24.458 INFO:tasks.workunit.client.0.smithi059.stdout:964 2022-04-28T19:58:24.460 INFO:tasks.workunit.client.0.smithi059.stdout:965 2022-04-28T19:58:24.461 INFO:tasks.workunit.client.0.smithi059.stdout:966 2022-04-28T19:58:24.462 INFO:tasks.workunit.client.0.smithi059.stdout:967 2022-04-28T19:58:24.464 INFO:tasks.workunit.client.0.smithi059.stdout:968 2022-04-28T19:58:24.465 INFO:tasks.workunit.client.0.smithi059.stdout:969 2022-04-28T19:58:24.466 INFO:tasks.workunit.client.0.smithi059.stdout:970 2022-04-28T19:58:24.468 INFO:tasks.workunit.client.0.smithi059.stdout:971 2022-04-28T19:58:24.469 INFO:tasks.workunit.client.0.smithi059.stdout:972 2022-04-28T19:58:24.470 INFO:tasks.workunit.client.0.smithi059.stdout:973 2022-04-28T19:58:24.472 INFO:tasks.workunit.client.0.smithi059.stdout:974 2022-04-28T19:58:24.473 INFO:tasks.workunit.client.0.smithi059.stdout:975 2022-04-28T19:58:24.474 INFO:tasks.workunit.client.0.smithi059.stdout:976 2022-04-28T19:58:24.476 INFO:tasks.workunit.client.0.smithi059.stdout:977 2022-04-28T19:58:24.477 INFO:tasks.workunit.client.0.smithi059.stdout:978 2022-04-28T19:58:24.479 INFO:tasks.workunit.client.0.smithi059.stdout:979 2022-04-28T19:58:24.480 INFO:tasks.workunit.client.0.smithi059.stdout:980 2022-04-28T19:58:24.481 INFO:tasks.workunit.client.0.smithi059.stdout:981 2022-04-28T19:58:24.483 INFO:tasks.workunit.client.0.smithi059.stdout:982 2022-04-28T19:58:24.484 INFO:tasks.workunit.client.0.smithi059.stdout:983 2022-04-28T19:58:24.485 INFO:tasks.workunit.client.0.smithi059.stdout:984 2022-04-28T19:58:24.487 INFO:tasks.workunit.client.0.smithi059.stdout:985 2022-04-28T19:58:24.488 INFO:tasks.workunit.client.0.smithi059.stdout:986 2022-04-28T19:58:24.489 INFO:tasks.workunit.client.0.smithi059.stdout:987 2022-04-28T19:58:24.490 INFO:tasks.workunit.client.0.smithi059.stdout:988 2022-04-28T19:58:24.492 INFO:tasks.workunit.client.0.smithi059.stdout:989 2022-04-28T19:58:24.493 INFO:tasks.workunit.client.0.smithi059.stdout:990 2022-04-28T19:58:24.494 INFO:tasks.workunit.client.0.smithi059.stdout:991 2022-04-28T19:58:24.496 INFO:tasks.workunit.client.0.smithi059.stdout:992 2022-04-28T19:58:24.497 INFO:tasks.workunit.client.0.smithi059.stdout:993 2022-04-28T19:58:24.499 INFO:tasks.workunit.client.0.smithi059.stdout:994 2022-04-28T19:58:24.500 INFO:tasks.workunit.client.0.smithi059.stdout:995 2022-04-28T19:58:24.501 INFO:tasks.workunit.client.0.smithi059.stdout:996 2022-04-28T19:58:24.503 INFO:tasks.workunit.client.0.smithi059.stdout:997 2022-04-28T19:58:24.505 INFO:tasks.workunit.client.0.smithi059.stdout:998 2022-04-28T19:58:24.506 INFO:tasks.workunit.client.0.smithi059.stdout:999 2022-04-28T19:58:24.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[59745]: cluster 2022-04-28T19:58:23.064459+0000 mgr.y (mgr.24961) 439 : cluster [DBG] pgmap v770: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:24.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[59745]: audit 2022-04-28T19:58:23.262659+0000 mon.a (mon.0) 1520 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:24.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[59745]: cluster 2022-04-28T19:58:23.263116+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2022-04-28T19:58:24.682 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[59745]: audit 2022-04-28T19:58:23.265338+0000 mon.b (mon.2) 12 : audit [DBG] from='client.? 172.21.15.59:0/2773853503' entity='client.admin' cmd=[{"prefix": "osd dump","format":"json"}]: dispatch 2022-04-28T19:58:24.682 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[61152]: cluster 2022-04-28T19:58:23.064459+0000 mgr.y (mgr.24961) 439 : cluster [DBG] pgmap v770: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:24.682 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[61152]: audit 2022-04-28T19:58:23.262659+0000 mon.a (mon.0) 1520 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:24.683 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[61152]: cluster 2022-04-28T19:58:23.263116+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2022-04-28T19:58:24.683 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:24 smithi059 bash[61152]: audit 2022-04-28T19:58:23.265338+0000 mon.b (mon.2) 12 : audit [DBG] from='client.? 172.21.15.59:0/2773853503' entity='client.admin' cmd=[{"prefix": "osd dump","format":"json"}]: dispatch 2022-04-28T19:58:24.683 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:24 smithi139 bash[43301]: cluster 2022-04-28T19:58:23.064459+0000 mgr.y (mgr.24961) 439 : cluster [DBG] pgmap v770: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:24.683 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:24 smithi139 bash[43301]: audit 2022-04-28T19:58:23.262659+0000 mon.a (mon.0) 1520 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:24.683 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:24 smithi139 bash[43301]: cluster 2022-04-28T19:58:23.263116+0000 mon.a (mon.0) 1521 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2022-04-28T19:58:24.684 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:24 smithi139 bash[43301]: audit 2022-04-28T19:58:23.265338+0000 mon.b (mon.2) 12 : audit [DBG] from='client.? 172.21.15.59:0/2773853503' entity='client.admin' cmd=[{"prefix": "osd dump","format":"json"}]: dispatch 2022-04-28T19:58:25.286 INFO:tasks.workunit.client.0.smithi059.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:00000010 69 73 |is| 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:00000012 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:00000010 69 73 |is| 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:00000012 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] ClsHello.WriteReturnData (4043 ms) 2022-04-28T19:58:25.449 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] ClsHello.Loud 2022-04-28T19:58:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:25 smithi059 bash[61152]: cluster 2022-04-28T19:58:24.264713+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2022-04-28T19:58:25.618 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:25 smithi059 bash[59745]: cluster 2022-04-28T19:58:24.264713+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2022-04-28T19:58:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:25 smithi139 bash[43301]: cluster 2022-04-28T19:58:24.264713+0000 mon.a (mon.0) 1522 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2022-04-28T19:58:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:26 smithi059 bash[61152]: cluster 2022-04-28T19:58:25.065366+0000 mgr.y (mgr.24961) 440 : cluster [DBG] pgmap v773: 105 pgs: 16 unknown, 89 active+clean; 24 KiB data, 700 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:58:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:26 smithi059 bash[61152]: cluster 2022-04-28T19:58:25.283257+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2022-04-28T19:58:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:26 smithi059 bash[59745]: cluster 2022-04-28T19:58:25.065366+0000 mgr.y (mgr.24961) 440 : cluster [DBG] pgmap v773: 105 pgs: 16 unknown, 89 active+clean; 24 KiB data, 700 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:58:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:26 smithi059 bash[59745]: cluster 2022-04-28T19:58:25.283257+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2022-04-28T19:58:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:26 smithi139 bash[43301]: cluster 2022-04-28T19:58:25.065366+0000 mgr.y (mgr.24961) 440 : cluster [DBG] pgmap v773: 105 pgs: 16 unknown, 89 active+clean; 24 KiB data, 700 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:58:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:26 smithi139 bash[43301]: cluster 2022-04-28T19:58:25.283257+0000 mon.a (mon.0) 1523 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2022-04-28T19:58:27.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:27 smithi059 bash[61152]: cluster 2022-04-28T19:58:26.292032+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2022-04-28T19:58:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:27 smithi059 bash[61152]: audit 2022-04-28T19:58:26.297324+0000 mon.c (mon.1) 17 : audit [INF] from='client.? 172.21.15.59:0/2906244520' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:27 smithi059 bash[61152]: audit 2022-04-28T19:58:26.302179+0000 mon.a (mon.0) 1525 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:27 smithi059 bash[59745]: cluster 2022-04-28T19:58:26.292032+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2022-04-28T19:58:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:27 smithi059 bash[59745]: audit 2022-04-28T19:58:26.297324+0000 mon.c (mon.1) 17 : audit [INF] from='client.? 172.21.15.59:0/2906244520' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:27 smithi059 bash[59745]: audit 2022-04-28T19:58:26.302179+0000 mon.a (mon.0) 1525 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:27.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:27 smithi139 bash[43301]: cluster 2022-04-28T19:58:26.292032+0000 mon.a (mon.0) 1524 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2022-04-28T19:58:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:27 smithi139 bash[43301]: audit 2022-04-28T19:58:26.297324+0000 mon.c (mon.1) 17 : audit [INF] from='client.? 172.21.15.59:0/2906244520' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:27 smithi139 bash[43301]: audit 2022-04-28T19:58:26.302179+0000 mon.a (mon.0) 1525 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:28.332 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] ClsHello.Loud (3047 ms) 2022-04-28T19:58:28.333 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] ClsHello.BadMethods 2022-04-28T19:58:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:28 smithi059 bash[59745]: cluster 2022-04-28T19:58:27.066068+0000 mgr.y (mgr.24961) 441 : cluster [DBG] pgmap v776: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:28 smithi059 bash[59745]: audit 2022-04-28T19:58:27.299624+0000 mon.a (mon.0) 1526 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:28 smithi059 bash[59745]: cluster 2022-04-28T19:58:27.299877+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2022-04-28T19:58:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:28 smithi059 bash[61152]: cluster 2022-04-28T19:58:27.066068+0000 mgr.y (mgr.24961) 441 : cluster [DBG] pgmap v776: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:28 smithi059 bash[61152]: audit 2022-04-28T19:58:27.299624+0000 mon.a (mon.0) 1526 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:28 smithi059 bash[61152]: cluster 2022-04-28T19:58:27.299877+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2022-04-28T19:58:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:28 smithi139 bash[43301]: cluster 2022-04-28T19:58:27.066068+0000 mgr.y (mgr.24961) 441 : cluster [DBG] pgmap v776: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:28 smithi139 bash[43301]: audit 2022-04-28T19:58:27.299624+0000 mon.a (mon.0) 1526 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:28 smithi139 bash[43301]: cluster 2022-04-28T19:58:27.299877+0000 mon.a (mon.0) 1527 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2022-04-28T19:58:29.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:29 smithi059 bash[59745]: cluster 2022-04-28T19:58:28.330695+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2022-04-28T19:58:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:29 smithi059 bash[61152]: cluster 2022-04-28T19:58:28.330695+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2022-04-28T19:58:29.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:29 smithi139 bash[43301]: cluster 2022-04-28T19:58:28.330695+0000 mon.a (mon.0) 1528 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2022-04-28T19:58:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:30 smithi139 bash[43301]: cluster 2022-04-28T19:58:29.066872+0000 mgr.y (mgr.24961) 442 : cluster [DBG] pgmap v779: 73 pgs: 73 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:30.677 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:30 smithi139 bash[43301]: cluster 2022-04-28T19:58:29.339422+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2022-04-28T19:58:30.677 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:30 smithi139 bash[43301]: audit 2022-04-28T19:58:29.340985+0000 mon.a (mon.0) 1530 : audit [INF] from='client.? 172.21.15.59:0/1694334060' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:30.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[61152]: cluster 2022-04-28T19:58:29.066872+0000 mgr.y (mgr.24961) 442 : cluster [DBG] pgmap v779: 73 pgs: 73 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:30.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[61152]: cluster 2022-04-28T19:58:29.339422+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2022-04-28T19:58:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[61152]: audit 2022-04-28T19:58:29.340985+0000 mon.a (mon.0) 1530 : audit [INF] from='client.? 172.21.15.59:0/1694334060' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[59745]: cluster 2022-04-28T19:58:29.066872+0000 mgr.y (mgr.24961) 442 : cluster [DBG] pgmap v779: 73 pgs: 73 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[59745]: cluster 2022-04-28T19:58:29.339422+0000 mon.a (mon.0) 1529 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2022-04-28T19:58:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[59745]: audit 2022-04-28T19:58:29.340985+0000 mon.a (mon.0) 1530 : audit [INF] from='client.? 172.21.15.59:0/1694334060' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:30.841 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[65202]: debug 2022-04-28T19:58:30.363+0000 7f6e9f435700 -1 osd.1 pg_epoch: 851 pg[10.2( v 851'1 (0'0,851'1] local-lis/les=850/851 n=1 ec=850/850 lis/c=850/850 les/c/f=851/851/0 sis=850) [1,4,0] r=0 lpr=850 crt=851'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2022-04-28T19:58:30.842 INFO:journalctl@ceph.osd.1.smithi059.stdout:Apr 28 19:58:30 smithi059 bash[65202]: debug 2022-04-28T19:58:30.363+0000 7f6e9f435700 -1 osd.1 pg_epoch: 851 pg[10.2( v 851'2 (0'0,851'2] local-lis/les=850/851 n=1 ec=850/850 lis/c=850/850 les/c/f=851/851/0 sis=850) [1,4,0] r=0 lpr=850 crt=851'1 lcod 851'1 mlcod 851'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2022-04-28T19:58:31.374 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] ClsHello.BadMethods (3036 ms) 2022-04-28T19:58:31.375 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] ClsHello.Filter 2022-04-28T19:58:31.646 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:31 smithi059 bash[61152]: audit 2022-04-28T19:58:30.349672+0000 mon.a (mon.0) 1531 : audit [INF] from='client.? 172.21.15.59:0/1694334060' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:31.646 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:31 smithi059 bash[61152]: cluster 2022-04-28T19:58:30.350393+0000 mon.a (mon.0) 1532 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2022-04-28T19:58:31.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:31 smithi059 bash[59745]: audit 2022-04-28T19:58:30.349672+0000 mon.a (mon.0) 1531 : audit [INF] from='client.? 172.21.15.59:0/1694334060' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:31.647 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:31 smithi059 bash[59745]: cluster 2022-04-28T19:58:30.350393+0000 mon.a (mon.0) 1532 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2022-04-28T19:58:31.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:31 smithi139 bash[43301]: audit 2022-04-28T19:58:30.349672+0000 mon.a (mon.0) 1531 : audit [INF] from='client.? 172.21.15.59:0/1694334060' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:31 smithi139 bash[43301]: cluster 2022-04-28T19:58:30.350393+0000 mon.a (mon.0) 1532 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2022-04-28T19:58:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:31 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:31.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:32.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:32 smithi059 bash[61152]: cluster 2022-04-28T19:58:31.067784+0000 mgr.y (mgr.24961) 443 : cluster [DBG] pgmap v782: 105 pgs: 15 creating+peering, 10 unknown, 80 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:58:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:32 smithi059 bash[61152]: cluster 2022-04-28T19:58:31.362546+0000 mon.a (mon.0) 1533 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2022-04-28T19:58:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:32 smithi059 bash[59745]: cluster 2022-04-28T19:58:31.067784+0000 mgr.y (mgr.24961) 443 : cluster [DBG] pgmap v782: 105 pgs: 15 creating+peering, 10 unknown, 80 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:58:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:32 smithi059 bash[59745]: cluster 2022-04-28T19:58:31.362546+0000 mon.a (mon.0) 1533 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2022-04-28T19:58:32.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:32 smithi139 bash[43301]: cluster 2022-04-28T19:58:31.067784+0000 mgr.y (mgr.24961) 443 : cluster [DBG] pgmap v782: 105 pgs: 15 creating+peering, 10 unknown, 80 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2022-04-28T19:58:32.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:32 smithi139 bash[43301]: cluster 2022-04-28T19:58:31.362546+0000 mon.a (mon.0) 1533 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2022-04-28T19:58:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:33.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[61152]: cluster 2022-04-28T19:58:32.383141+0000 mon.a (mon.0) 1534 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2022-04-28T19:58:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[61152]: audit 2022-04-28T19:58:32.387993+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.59:0/920199537' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[61152]: audit 2022-04-28T19:58:32.389608+0000 mon.a (mon.0) 1535 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[59745]: cluster 2022-04-28T19:58:32.383141+0000 mon.a (mon.0) 1534 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2022-04-28T19:58:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[59745]: audit 2022-04-28T19:58:32.387993+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.59:0/920199537' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:33 smithi059 bash[59745]: audit 2022-04-28T19:58:32.389608+0000 mon.a (mon.0) 1535 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:33.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:33 smithi139 bash[43301]: cluster 2022-04-28T19:58:32.383141+0000 mon.a (mon.0) 1534 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2022-04-28T19:58:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:33 smithi139 bash[43301]: audit 2022-04-28T19:58:32.387993+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.59:0/920199537' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:33 smithi139 bash[43301]: audit 2022-04-28T19:58:32.389608+0000 mon.a (mon.0) 1535 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:34.403 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] ClsHello.Filter (3034 ms) 2022-04-28T19:58:34.403 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] 6 tests from ClsHello (18879 ms total) 2022-04-28T19:58:34.403 INFO:tasks.workunit.client.0.smithi059.stdout: 2022-04-28T19:58:34.403 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] Global test environment tear-down 2022-04-28T19:58:34.403 INFO:tasks.workunit.client.0.smithi059.stdout:[==========] 6 tests from 1 test suite ran. (18879 ms total) 2022-04-28T19:58:34.403 INFO:tasks.workunit.client.0.smithi059.stdout:[ PASSED ] 6 tests. 2022-04-28T19:58:34.405 INFO:teuthology.orchestra.run:Running command with timeout 3600 2022-04-28T19:58:34.405 DEBUG:teuthology.orchestra.run.smithi059:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2022-04-28T19:58:34.460 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2022-04-28T19:58:34.461 DEBUG:teuthology.orchestra.run.smithi059:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=octopus TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2022-04-28T19:58:34.526 INFO:tasks.workunit.client.0.smithi059.stdout:Running main() from gmock_main.cc 2022-04-28T19:58:34.526 INFO:tasks.workunit.client.0.smithi059.stdout:[==========] Running 27 tests from 1 test suite. 2022-04-28T19:58:34.526 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] Global test environment set-up. 2022-04-28T19:58:34.527 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] 27 tests from TestClsJournal 2022-04-28T19:58:34.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:34 smithi059 bash[59745]: cluster 2022-04-28T19:58:33.068444+0000 mgr.y (mgr.24961) 444 : cluster [DBG] pgmap v785: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:34.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:34 smithi059 bash[59745]: audit 2022-04-28T19:58:33.392501+0000 mon.a (mon.0) 1536 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:34.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:34 smithi059 bash[59745]: cluster 2022-04-28T19:58:33.392649+0000 mon.a (mon.0) 1537 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2022-04-28T19:58:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:34 smithi059 bash[61152]: cluster 2022-04-28T19:58:33.068444+0000 mgr.y (mgr.24961) 444 : cluster [DBG] pgmap v785: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:34 smithi059 bash[61152]: audit 2022-04-28T19:58:33.392501+0000 mon.a (mon.0) 1536 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:34 smithi059 bash[61152]: cluster 2022-04-28T19:58:33.392649+0000 mon.a (mon.0) 1537 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2022-04-28T19:58:34.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:34 smithi139 bash[43301]: cluster 2022-04-28T19:58:33.068444+0000 mgr.y (mgr.24961) 444 : cluster [DBG] pgmap v785: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:34.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:34 smithi139 bash[43301]: audit 2022-04-28T19:58:33.392501+0000 mon.a (mon.0) 1536 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74416-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:34.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:34 smithi139 bash[43301]: cluster 2022-04-28T19:58:33.392649+0000 mon.a (mon.0) 1537 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2022-04-28T19:58:35.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:35 smithi059 bash[61152]: cluster 2022-04-28T19:58:34.396237+0000 mon.a (mon.0) 1538 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2022-04-28T19:58:35.956 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:35 smithi059 bash[61152]: cluster 2022-04-28T19:58:35.069162+0000 mgr.y (mgr.24961) 445 : cluster [DBG] pgmap v788: 73 pgs: 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:35.957 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:35 smithi059 bash[59745]: cluster 2022-04-28T19:58:34.396237+0000 mon.a (mon.0) 1538 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2022-04-28T19:58:35.957 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:35 smithi059 bash[59745]: cluster 2022-04-28T19:58:35.069162+0000 mgr.y (mgr.24961) 445 : cluster [DBG] pgmap v788: 73 pgs: 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:35.959 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:35 smithi139 bash[43301]: cluster 2022-04-28T19:58:34.396237+0000 mon.a (mon.0) 1538 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2022-04-28T19:58:35.959 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:35 smithi139 bash[43301]: cluster 2022-04-28T19:58:35.069162+0000 mgr.y (mgr.24961) 445 : cluster [DBG] pgmap v788: 73 pgs: 73 active+clean; 24 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:36.425 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.Create 2022-04-28T19:58:36.570 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.Create (23 ms) 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.MinimumSet 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.MinimumSet (5 ms) 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.MinimumSetStale (5 ms) 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (4 ms) 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ActiveSet 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ActiveSet (4 ms) 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2022-04-28T19:58:36.571 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ActiveSetStale (4 ms) 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.CreateDuplicate (2 ms) 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.GetClient 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.GetClient (4 ms) 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientRegister 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:249: Failure 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout: expected_clients 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [id=id1, commit_position=[positions=[]], state=connected] } 2022-04-28T19:58:36.572 INFO:tasks.workunit.client.0.smithi059.stdout: clients 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientRegister (3 ms) 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (3 ms) 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientUpdateData (4 ms) 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientUpdateState (6 ms) 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientUnregister 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientUnregister (3 ms) 2022-04-28T19:58:36.573 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2022-04-28T19:58:36.574 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (4 ms) 2022-04-28T19:58:36.574 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2022-04-28T19:58:36.607 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:365: Failure 2022-04-28T19:58:36.607 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.608 INFO:tasks.workunit.client.0.smithi059.stdout: expected_tags 2022-04-28T19:58:36.608 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [tid=0, tag_class=0, data=], [tid=32, tag_class=1, data=], [tid=33, tag_class=1, data=], [tid=34, tag_class=1, data=], [tid=35, tag_class=1, data=], [tid=36, tag_class=1, data=], [tid=37, tag_class=1, data=], [tid=38, tag_class=1, data=], [tid=39, tag_class=1, data=], [tid=40, tag_class=1, data=], [tid=41, tag_class=1, data=], [tid=42, tag_class=1, data=], [tid=43, tag_class=1, data=], [tid=44, tag_class=1, data=], [tid=45, tag_class=1, data=], [tid=46, tag_class=1, data=], [tid=47, tag_class=1, data=], [tid=48, tag_class=1, data=], [tid=49, tag_class=1, data=], [tid=50, tag_class=1, data=], [tid=51, tag_class=1, data=], [tid=52, tag_class=1, data=], [tid=53, tag_class=1, data=], [tid=54, tag_class=1, data=], [tid=55, tag_class=1, data=], [tid=56, tag_class=1, data=], [tid=57, tag_class=1, data=], [tid=58, tag_class=1, data=], [tid=59, tag_class=1, data=], [tid=60, tag_class=1, data=], [tid=61, tag_class=1, data=], [tid=62, tag_class=1, data=], ... } 2022-04-28T19:58:36.608 INFO:tasks.workunit.client.0.smithi059.stdout: tags 2022-04-28T19:58:36.608 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.608 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientUnregisterPruneTags (107 ms) 2022-04-28T19:58:36.608 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientCommit 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:393: Failure 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout: expected_clients 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [id=id1, commit_position=[positions=[[object_number=0, tag_tid=234, entry_tid=120], [object_number=3, tag_tid=235, entry_tid=121]]], state=connected] } 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout: clients 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientCommit (8 ms) 2022-04-28T19:58:36.616 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2022-04-28T19:58:36.620 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (5 ms) 2022-04-28T19:58:36.620 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2022-04-28T19:58:36.621 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.ClientCommitDNE (1 ms) 2022-04-28T19:58:36.621 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.ClientList 2022-04-28T19:58:36.659 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:450: Failure 2022-04-28T19:58:36.659 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.659 INFO:tasks.workunit.client.0.smithi059.stdout: expected_clients 2022-04-28T19:58:36.659 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [id=id1, commit_position=[positions=[]], state=connected], [id=id10, commit_position=[positions=[]], state=connected], [id=id100, commit_position=[positions=[]], state=connected], [id=id101, commit_position=[positions=[]], state=connected], [id=id102, commit_position=[positions=[]], state=connected], [id=id103, commit_position=[positions=[]], state=connected], [id=id104, commit_position=[positions=[]], state=connected], [id=id105, commit_position=[positions=[]], state=connected], [id=id106, commit_position=[positions=[]], state=connected], [id=id107, commit_position=[positions=[]], state=connected], [id=id108, commit_position=[positions=[]], state=connected], [id=id109, commit_position=[positions=[]], state=connected], [id=id11, commit_position=[positions=[]], state=connected], [id=id110, commit_position=[positions=[]], state=connected], [id=id111, commit_position=[positions=[]], state=connected], [id=id112, commit_position=[positions=[]], state=connected], [id=id113, commit_position=[positions=[]], state=connected], [id=id114, commit_position=[positions=[]], state=connected], [id=id115, commit_position=[positions=[]], state=connected], [id=id116, commit_position=[positions=[]], state=connected], [id=id117, commit_position=[positions=[]], state=connected], [id=id118, commit_position=[positions=[]], state=connected], [id=id119, commit_position=[positions=[]], state=connected], [id=id12, commit_position=[positions=[]], state=connected], [id=id120, commit_position=[positions=[]], state=connected], [id=id121, commit_position=[positions=[]], state=connected], [id=id122, commit_position=[positions=[]], state=connected], [id=id123, commit_position=[positions=[]], state=connected], [id=id124, commit_position=[positions=[]], state=connected], [id=id125, commit_position=[positions=[]], state=connected], [id=id126, commit_position=[positions=[]], state=connected], [id=id127, commit_position=[positions=[]], state=connected], ... } 2022-04-28T19:58:36.659 INFO:tasks.workunit.client.0.smithi059.stdout: clients 2022-04-28T19:58:36.660 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.660 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientList (37 ms) 2022-04-28T19:58:36.660 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2022-04-28T19:58:36.665 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.GetNextTagTid (7 ms) 2022-04-28T19:58:36.665 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.TagCreate 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:512: Failure 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout: expected_tags 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [tid=0, tag_class=0, data=], [tid=1, tag_class=1, data=], [tid=2, tag_class=1, data=] } 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout: tags 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.TagCreate (11 ms) 2022-04-28T19:58:36.677 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2022-04-28T19:58:36.684 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:541: Failure 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout: expected_tags 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [tid=0, tag_class=0, data=], [tid=2, tag_class=1, data=], [tid=3, tag_class=0, data=] } 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout: tags 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.TagCreatePrunesTags (8 ms) 2022-04-28T19:58:36.685 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.TagList 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout:/build/ceph-15.2.16/src/test/cls_journal/test_cls_journal.cc:573: Failure 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout:Expected equality of these values: 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout: expected_all_tags 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: { [tid=0, tag_class=0, data=], [tid=1, tag_class=1, data=], [tid=2, tag_class=0, data=], [tid=3, tag_class=1, data=], [tid=4, tag_class=0, data=], [tid=5, tag_class=1, data=], [tid=6, tag_class=0, data=], [tid=7, tag_class=1, data=], [tid=8, tag_class=0, data=], [tid=9, tag_class=1, data=], [tid=10, tag_class=0, data=], [tid=11, tag_class=1, data=], [tid=12, tag_class=0, data=], [tid=13, tag_class=1, data=], [tid=14, tag_class=0, data=], [tid=15, tag_class=1, data=], [tid=16, tag_class=0, data=], [tid=17, tag_class=1, data=], [tid=18, tag_class=0, data=], [tid=19, tag_class=1, data=], [tid=20, tag_class=0, data=], [tid=21, tag_class=1, data=], [tid=22, tag_class=0, data=], [tid=23, tag_class=1, data=], [tid=24, tag_class=0, data=], [tid=25, tag_class=1, data=], [tid=26, tag_class=0, data=], [tid=27, tag_class=1, data=], [tid=28, tag_class=0, data=], [tid=29, tag_class=1, data=], [tid=30, tag_class=0, data=], [tid=31, tag_class=1, data=], ... } 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout: tags 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout: Which is: {} 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.TagList (103 ms) 2022-04-28T19:58:36.788 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.GuardAppend 2022-04-28T19:58:36.789 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.GuardAppend (2 ms) 2022-04-28T19:58:36.790 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2022-04-28T19:58:36.791 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.GuardAppendDNE (1 ms) 2022-04-28T19:58:36.791 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2022-04-28T19:58:36.793 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (3 ms) 2022-04-28T19:58:36.793 INFO:tasks.workunit.client.0.smithi059.stdout:[ RUN ] TestClsJournal.Append 2022-04-28T19:58:36.806 INFO:tasks.workunit.client.0.smithi059.stdout:padding is not enabled 2022-04-28T19:58:36.806 INFO:tasks.workunit.client.0.smithi059.stdout:[ OK ] TestClsJournal.Append (13 ms) 2022-04-28T19:58:36.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:36 smithi059 bash[61152]: cluster 2022-04-28T19:58:35.420305+0000 mon.a (mon.0) 1539 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2022-04-28T19:58:36.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:36 smithi059 bash[61152]: audit 2022-04-28T19:58:35.421611+0000 mon.c (mon.1) 19 : audit [INF] from='client.? 172.21.15.59:0/1160522368' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:36.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:36 smithi059 bash[61152]: audit 2022-04-28T19:58:35.424855+0000 mon.a (mon.0) 1540 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:36 smithi059 bash[59745]: cluster 2022-04-28T19:58:35.420305+0000 mon.a (mon.0) 1539 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2022-04-28T19:58:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:36 smithi059 bash[59745]: audit 2022-04-28T19:58:35.421611+0000 mon.c (mon.1) 19 : audit [INF] from='client.? 172.21.15.59:0/1160522368' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:36 smithi059 bash[59745]: audit 2022-04-28T19:58:35.424855+0000 mon.a (mon.0) 1540 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:36 smithi139 bash[43301]: cluster 2022-04-28T19:58:35.420305+0000 mon.a (mon.0) 1539 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2022-04-28T19:58:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:36 smithi139 bash[43301]: audit 2022-04-28T19:58:35.421611+0000 mon.c (mon.1) 19 : audit [INF] from='client.? 172.21.15.59:0/1160522368' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:36 smithi139 bash[43301]: audit 2022-04-28T19:58:35.424855+0000 mon.a (mon.0) 1540 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T19:58:37.451 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] 27 tests from TestClsJournal (381 ms total) 2022-04-28T19:58:37.566 INFO:tasks.workunit.client.0.smithi059.stdout: 2022-04-28T19:58:37.566 INFO:tasks.workunit.client.0.smithi059.stdout:[----------] Global test environment tear-down 2022-04-28T19:58:37.566 INFO:tasks.workunit.client.0.smithi059.stdout:[==========] 27 tests from 1 test suite ran. (2924 ms total) 2022-04-28T19:58:37.566 INFO:tasks.workunit.client.0.smithi059.stdout:[ PASSED ] 20 tests. 2022-04-28T19:58:37.566 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] 7 tests, listed below: 2022-04-28T19:58:37.566 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientRegister 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientUnregisterPruneTags 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientCommit 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.ClientList 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.TagCreate 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.TagCreatePrunesTags 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout:[ FAILED ] TestClsJournal.TagList 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout: 2022-04-28T19:58:37.567 INFO:tasks.workunit.client.0.smithi059.stdout: 7 FAILED TESTS 2022-04-28T19:58:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2022-04-28T19:58:37.569 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2022-04-28T19:58:37.569 DEBUG:teuthology.orchestra.run.smithi059:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2022-04-28T19:58:37.768 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:37 smithi059 bash[61152]: audit 2022-04-28T19:58:36.424213+0000 mon.a (mon.0) 1541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:37.769 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:37 smithi059 bash[61152]: cluster 2022-04-28T19:58:36.424840+0000 mon.a (mon.0) 1542 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2022-04-28T19:58:37.769 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:37 smithi059 bash[61152]: cluster 2022-04-28T19:58:37.069727+0000 mgr.y (mgr.24961) 446 : cluster [DBG] pgmap v791: 105 pgs: 11 creating+peering, 21 unknown, 73 active+clean; 24 KiB data, 689 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:37.769 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:37 smithi059 bash[59745]: audit 2022-04-28T19:58:36.424213+0000 mon.a (mon.0) 1541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:37.769 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:37 smithi059 bash[59745]: cluster 2022-04-28T19:58:36.424840+0000 mon.a (mon.0) 1542 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2022-04-28T19:58:37.769 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:37 smithi059 bash[59745]: cluster 2022-04-28T19:58:37.069727+0000 mgr.y (mgr.24961) 446 : cluster [DBG] pgmap v791: 105 pgs: 11 creating+peering, 21 unknown, 73 active+clean; 24 KiB data, 689 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:37.770 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 103, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 82, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 82, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 82, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa/tasks/workunit.py", line 134, in task coverage_and_limits=not config.get('no_coverage_and_limits', None)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa/tasks/workunit.py", line 426, in _run_tests label="workunit test {workunit}".format(workunit=workunit) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/remote.py", line 510, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/run.py", line 183, in _raise_for_status node=self.hostname, label=self.label teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_journal.sh) on smithi059 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=octopus TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh' 2022-04-28T19:58:37.876 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=dc10aacaaee046d8aa787534546d12df Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 103, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 82, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 82, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/run_tasks.py", line 82, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa/tasks/workunit.py", line 134, in task coverage_and_limits=not config.get('no_coverage_and_limits', None)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph-c_4fa079ba14503defa8dc257d7c2d506ebefcfe6d/qa/tasks/workunit.py", line 426, in _run_tests label="workunit test {workunit}".format(workunit=workunit) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/remote.py", line 510, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_git_teuthology_004701def07b67dd914d7b1775457fec6788a495/teuthology/orchestra/run.py", line 183, in _raise_for_status node=self.hostname, label=self.label teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_journal.sh) on smithi059 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=octopus TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh' 2022-04-28T19:58:37.879 INFO:teuthology.run_tasks:Sleeping for 100000 seconds before unwinding because --sleep-before-teardown was given... 2022-04-28T19:58:37.891 INFO:teuthology.run_tasks:Sending no to idryomov@gmail.com: Teuthology job dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495 has fallen asleep at 2022-04-28 19:58:37 for 1 day, 3 hours and 46 minutes Owner: scheduled_dis@teuthology Suite Name: upgrade:octopus-x Sleep Date: 2022-04-28 19:58:37 Sleep Time: 100000 seconds (1 day, 3 hours and 46 minutes) Job Info: http://pulpito.ceph.com/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/ Job Logs: http://qa-proxy.ceph.com/teuthology/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495/ Task Stack: console_log/internal.base/internal.archive_upload/internal.archive/internal.coredump/internal.sudo/internal.syslog/internal.timer/pcp/selinux/ansible.cephlab/clock/install/cephadm Current Status: fail 2022-04-28T19:58:37.892 DEBUG:teuthology.run_tasks:sending email Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: teuthology job dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495 has fallen asleep at 2022-04-28 19:58:37 From: ceph-qa@ceph.com To: idryomov@gmail.com Teuthology job dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495 has fallen asleep at 2022-04-28 19:58:37 for 1 day, 3 hours and 46 minutes Owner: scheduled_dis@teuthology Suite Name: upgrade:octopus-x Sleep Date: 2022-04-28 19:58:37 Sleep Time: 100000 seconds (1 day, 3 hours and 46 minutes) Job Info: http://pulpito.ceph.com/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/ Job Logs: http://qa-proxy.ceph.com/teuthology/dis-2022-04-28_19:20:58-upgrade:octopus-x-pacific-distro-default-smithi/6811495/ Task Stack: console_log/internal.base/internal.archive_upload/internal.archive/internal.coredump/internal.sudo/internal.syslog/internal.timer/pcp/selinux/ansible.cephlab/clock/install/cephadm Current Status: fail 2022-04-28T19:58:37.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:37 smithi139 bash[43301]: audit 2022-04-28T19:58:36.424213+0000 mon.a (mon.0) 1541 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74538-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T19:58:37.919 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:37 smithi139 bash[43301]: cluster 2022-04-28T19:58:36.424840+0000 mon.a (mon.0) 1542 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2022-04-28T19:58:37.920 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:37 smithi139 bash[43301]: cluster 2022-04-28T19:58:37.069727+0000 mgr.y (mgr.24961) 446 : cluster [DBG] pgmap v791: 105 pgs: 11 creating+peering, 21 unknown, 73 active+clean; 24 KiB data, 689 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:38.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:38 smithi059 bash[61152]: cluster 2022-04-28T19:58:37.448536+0000 mon.a (mon.0) 1543 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2022-04-28T19:58:38.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:38 smithi059 bash[59745]: cluster 2022-04-28T19:58:37.448536+0000 mon.a (mon.0) 1543 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2022-04-28T19:58:38.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:38 smithi139 bash[43301]: cluster 2022-04-28T19:58:37.448536+0000 mon.a (mon.0) 1543 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2022-04-28T19:58:39.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:39 smithi059 bash[61152]: cluster 2022-04-28T19:58:39.070505+0000 mgr.y (mgr.24961) 447 : cluster [DBG] pgmap v793: 73 pgs: 73 active+clean; 24 KiB data, 689 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:39 smithi059 bash[59745]: cluster 2022-04-28T19:58:39.070505+0000 mgr.y (mgr.24961) 447 : cluster [DBG] pgmap v793: 73 pgs: 73 active+clean; 24 KiB data, 689 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:39.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:39 smithi139 bash[43301]: cluster 2022-04-28T19:58:39.070505+0000 mgr.y (mgr.24961) 447 : cluster [DBG] pgmap v793: 73 pgs: 73 active+clean; 24 KiB data, 689 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:41 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:41.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:42 smithi059 bash[61152]: cluster 2022-04-28T19:58:41.071578+0000 mgr.y (mgr.24961) 448 : cluster [DBG] pgmap v794: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:42 smithi059 bash[59745]: cluster 2022-04-28T19:58:41.071578+0000 mgr.y (mgr.24961) 448 : cluster [DBG] pgmap v794: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:42 smithi139 bash[43301]: cluster 2022-04-28T19:58:41.071578+0000 mgr.y (mgr.24961) 448 : cluster [DBG] pgmap v794: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:43 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:43.541Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:44.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:44 smithi059 bash[59745]: cluster 2022-04-28T19:58:43.072308+0000 mgr.y (mgr.24961) 449 : cluster [DBG] pgmap v795: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:44 smithi059 bash[61152]: cluster 2022-04-28T19:58:43.072308+0000 mgr.y (mgr.24961) 449 : cluster [DBG] pgmap v795: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:44.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:44 smithi139 bash[43301]: cluster 2022-04-28T19:58:43.072308+0000 mgr.y (mgr.24961) 449 : cluster [DBG] pgmap v795: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:46.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:46 smithi059 bash[61152]: cluster 2022-04-28T19:58:45.073252+0000 mgr.y (mgr.24961) 450 : cluster [DBG] pgmap v796: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:46 smithi059 bash[59745]: cluster 2022-04-28T19:58:45.073252+0000 mgr.y (mgr.24961) 450 : cluster [DBG] pgmap v796: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:46 smithi139 bash[43301]: cluster 2022-04-28T19:58:45.073252+0000 mgr.y (mgr.24961) 450 : cluster [DBG] pgmap v796: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:48 smithi059 bash[61152]: cluster 2022-04-28T19:58:47.074163+0000 mgr.y (mgr.24961) 451 : cluster [DBG] pgmap v797: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:48.455 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:48 smithi059 bash[59745]: cluster 2022-04-28T19:58:47.074163+0000 mgr.y (mgr.24961) 451 : cluster [DBG] pgmap v797: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:48.455 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:48 smithi139 bash[43301]: cluster 2022-04-28T19:58:47.074163+0000 mgr.y (mgr.24961) 451 : cluster [DBG] pgmap v797: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:50.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:50 smithi059 bash[61152]: cluster 2022-04-28T19:58:49.074983+0000 mgr.y (mgr.24961) 452 : cluster [DBG] pgmap v798: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:50 smithi059 bash[59745]: cluster 2022-04-28T19:58:49.074983+0000 mgr.y (mgr.24961) 452 : cluster [DBG] pgmap v798: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:50 smithi139 bash[43301]: cluster 2022-04-28T19:58:49.074983+0000 mgr.y (mgr.24961) 452 : cluster [DBG] pgmap v798: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:51 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:51.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[59745]: cluster 2022-04-28T19:58:51.076005+0000 mgr.y (mgr.24961) 453 : cluster [DBG] pgmap v799: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[59745]: audit 2022-04-28T19:58:51.576639+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:58:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[59745]: audit 2022-04-28T19:58:51.579436+0000 mon.a (mon.0) 1545 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:58:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[59745]: audit 2022-04-28T19:58:51.581203+0000 mon.a (mon.0) 1546 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:58:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[59745]: audit 2022-04-28T19:58:51.908540+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:58:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[61152]: cluster 2022-04-28T19:58:51.076005+0000 mgr.y (mgr.24961) 453 : cluster [DBG] pgmap v799: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[61152]: audit 2022-04-28T19:58:51.576639+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:58:52.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[61152]: audit 2022-04-28T19:58:51.579436+0000 mon.a (mon.0) 1545 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:58:52.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[61152]: audit 2022-04-28T19:58:51.581203+0000 mon.a (mon.0) 1546 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:58:52.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:52 smithi059 bash[61152]: audit 2022-04-28T19:58:51.908540+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:58:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:52 smithi139 bash[43301]: cluster 2022-04-28T19:58:51.076005+0000 mgr.y (mgr.24961) 453 : cluster [DBG] pgmap v799: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:52 smithi139 bash[43301]: audit 2022-04-28T19:58:51.576639+0000 mon.a (mon.0) 1544 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:58:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:52 smithi139 bash[43301]: audit 2022-04-28T19:58:51.579436+0000 mon.a (mon.0) 1545 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:58:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:52 smithi139 bash[43301]: audit 2022-04-28T19:58:51.581203+0000 mon.a (mon.0) 1546 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:58:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:52 smithi139 bash[43301]: audit 2022-04-28T19:58:51.908540+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:58:53.543 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:53 smithi059 bash[61152]: cephadm 2022-04-28T19:58:51.582757+0000 mgr.y (mgr.24961) 454 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:58:53.543 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:53 smithi059 bash[59745]: cephadm 2022-04-28T19:58:51.582757+0000 mgr.y (mgr.24961) 454 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:58:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:53 smithi139 bash[43301]: cephadm 2022-04-28T19:58:51.582757+0000 mgr.y (mgr.24961) 454 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:58:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:58:53 smithi059 bash[72035]: level=error ts=2022-04-28T19:58:53.542Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:58:54.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:54 smithi059 bash[61152]: cluster 2022-04-28T19:58:53.076647+0000 mgr.y (mgr.24961) 455 : cluster [DBG] pgmap v800: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:54 smithi059 bash[59745]: cluster 2022-04-28T19:58:53.076647+0000 mgr.y (mgr.24961) 455 : cluster [DBG] pgmap v800: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:54.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:54 smithi139 bash[43301]: cluster 2022-04-28T19:58:53.076647+0000 mgr.y (mgr.24961) 455 : cluster [DBG] pgmap v800: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:56 smithi059 bash[61152]: cluster 2022-04-28T19:58:55.077579+0000 mgr.y (mgr.24961) 456 : cluster [DBG] pgmap v801: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:56.716 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:56 smithi059 bash[59745]: cluster 2022-04-28T19:58:55.077579+0000 mgr.y (mgr.24961) 456 : cluster [DBG] pgmap v801: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:56.716 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:56 smithi139 bash[43301]: cluster 2022-04-28T19:58:55.077579+0000 mgr.y (mgr.24961) 456 : cluster [DBG] pgmap v801: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:58:58 smithi059 bash[61152]: cluster 2022-04-28T19:58:57.078484+0000 mgr.y (mgr.24961) 457 : cluster [DBG] pgmap v802: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:58:58 smithi059 bash[59745]: cluster 2022-04-28T19:58:57.078484+0000 mgr.y (mgr.24961) 457 : cluster [DBG] pgmap v802: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:58:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:58:58 smithi139 bash[43301]: cluster 2022-04-28T19:58:57.078484+0000 mgr.y (mgr.24961) 457 : cluster [DBG] pgmap v802: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:00 smithi059 bash[61152]: cluster 2022-04-28T19:58:59.079850+0000 mgr.y (mgr.24961) 458 : cluster [DBG] pgmap v803: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:00 smithi059 bash[59745]: cluster 2022-04-28T19:58:59.079850+0000 mgr.y (mgr.24961) 458 : cluster [DBG] pgmap v803: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:00 smithi139 bash[43301]: cluster 2022-04-28T19:58:59.079850+0000 mgr.y (mgr.24961) 458 : cluster [DBG] pgmap v803: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:01 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:01.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:02 smithi059 bash[61152]: cluster 2022-04-28T19:59:01.080910+0000 mgr.y (mgr.24961) 459 : cluster [DBG] pgmap v804: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:02 smithi059 bash[59745]: cluster 2022-04-28T19:59:01.080910+0000 mgr.y (mgr.24961) 459 : cluster [DBG] pgmap v804: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:02 smithi139 bash[43301]: cluster 2022-04-28T19:59:01.080910+0000 mgr.y (mgr.24961) 459 : cluster [DBG] pgmap v804: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:03 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:03.543Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:04.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:04 smithi059 bash[61152]: cluster 2022-04-28T19:59:03.081524+0000 mgr.y (mgr.24961) 460 : cluster [DBG] pgmap v805: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:04 smithi059 bash[59745]: cluster 2022-04-28T19:59:03.081524+0000 mgr.y (mgr.24961) 460 : cluster [DBG] pgmap v805: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:04 smithi139 bash[43301]: cluster 2022-04-28T19:59:03.081524+0000 mgr.y (mgr.24961) 460 : cluster [DBG] pgmap v805: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:06.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:06 smithi059 bash[61152]: cluster 2022-04-28T19:59:05.082424+0000 mgr.y (mgr.24961) 461 : cluster [DBG] pgmap v806: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:06 smithi059 bash[59745]: cluster 2022-04-28T19:59:05.082424+0000 mgr.y (mgr.24961) 461 : cluster [DBG] pgmap v806: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:06 smithi139 bash[43301]: cluster 2022-04-28T19:59:05.082424+0000 mgr.y (mgr.24961) 461 : cluster [DBG] pgmap v806: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:08.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:08 smithi059 bash[61152]: cluster 2022-04-28T19:59:07.083399+0000 mgr.y (mgr.24961) 462 : cluster [DBG] pgmap v807: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:08 smithi059 bash[59745]: cluster 2022-04-28T19:59:07.083399+0000 mgr.y (mgr.24961) 462 : cluster [DBG] pgmap v807: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:08 smithi139 bash[43301]: cluster 2022-04-28T19:59:07.083399+0000 mgr.y (mgr.24961) 462 : cluster [DBG] pgmap v807: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:10 smithi059 bash[61152]: cluster 2022-04-28T19:59:09.084261+0000 mgr.y (mgr.24961) 463 : cluster [DBG] pgmap v808: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:10 smithi059 bash[59745]: cluster 2022-04-28T19:59:09.084261+0000 mgr.y (mgr.24961) 463 : cluster [DBG] pgmap v808: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:10 smithi139 bash[43301]: cluster 2022-04-28T19:59:09.084261+0000 mgr.y (mgr.24961) 463 : cluster [DBG] pgmap v808: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:11 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:11.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:12 smithi059 bash[61152]: cluster 2022-04-28T19:59:11.085292+0000 mgr.y (mgr.24961) 464 : cluster [DBG] pgmap v809: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:12 smithi059 bash[59745]: cluster 2022-04-28T19:59:11.085292+0000 mgr.y (mgr.24961) 464 : cluster [DBG] pgmap v809: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:12 smithi139 bash[43301]: cluster 2022-04-28T19:59:11.085292+0000 mgr.y (mgr.24961) 464 : cluster [DBG] pgmap v809: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:13 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:13.544Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:14 smithi059 bash[61152]: cluster 2022-04-28T19:59:13.085939+0000 mgr.y (mgr.24961) 465 : cluster [DBG] pgmap v810: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:14 smithi059 bash[59745]: cluster 2022-04-28T19:59:13.085939+0000 mgr.y (mgr.24961) 465 : cluster [DBG] pgmap v810: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:14.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:14 smithi139 bash[43301]: cluster 2022-04-28T19:59:13.085939+0000 mgr.y (mgr.24961) 465 : cluster [DBG] pgmap v810: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:16 smithi059 bash[61152]: cluster 2022-04-28T19:59:15.086970+0000 mgr.y (mgr.24961) 466 : cluster [DBG] pgmap v811: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:16 smithi059 bash[59745]: cluster 2022-04-28T19:59:15.086970+0000 mgr.y (mgr.24961) 466 : cluster [DBG] pgmap v811: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:16 smithi139 bash[43301]: cluster 2022-04-28T19:59:15.086970+0000 mgr.y (mgr.24961) 466 : cluster [DBG] pgmap v811: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:17.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:17 smithi059 bash[61152]: audit 2022-04-28T19:59:16.947889+0000 mon.a (mon.0) 1548 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T19:59:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:17 smithi059 bash[61152]: audit 2022-04-28T19:59:16.948507+0000 mon.a (mon.0) 1549 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T19:59:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:17 smithi059 bash[61152]: audit 2022-04-28T19:59:16.948876+0000 mon.a (mon.0) 1550 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:59:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:17 smithi059 bash[59745]: audit 2022-04-28T19:59:16.947889+0000 mon.a (mon.0) 1548 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T19:59:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:17 smithi059 bash[59745]: audit 2022-04-28T19:59:16.948507+0000 mon.a (mon.0) 1549 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T19:59:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:17 smithi059 bash[59745]: audit 2022-04-28T19:59:16.948876+0000 mon.a (mon.0) 1550 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:59:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:17 smithi139 bash[43301]: audit 2022-04-28T19:59:16.947889+0000 mon.a (mon.0) 1548 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T19:59:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:17 smithi139 bash[43301]: audit 2022-04-28T19:59:16.948507+0000 mon.a (mon.0) 1549 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T19:59:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:17 smithi139 bash[43301]: audit 2022-04-28T19:59:16.948876+0000 mon.a (mon.0) 1550 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T19:59:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:18 smithi059 bash[61152]: cluster 2022-04-28T19:59:17.087868+0000 mgr.y (mgr.24961) 467 : cluster [DBG] pgmap v812: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:18 smithi059 bash[61152]: audit 2022-04-28T19:59:17.193781+0000 mon.a (mon.0) 1551 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:59:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:18 smithi059 bash[61152]: audit 2022-04-28T19:59:17.201152+0000 mon.a (mon.0) 1552 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:59:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:18 smithi059 bash[59745]: cluster 2022-04-28T19:59:17.087868+0000 mgr.y (mgr.24961) 467 : cluster [DBG] pgmap v812: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:18 smithi059 bash[59745]: audit 2022-04-28T19:59:17.193781+0000 mon.a (mon.0) 1551 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:59:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:18 smithi059 bash[59745]: audit 2022-04-28T19:59:17.201152+0000 mon.a (mon.0) 1552 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:59:18.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:18 smithi139 bash[43301]: cluster 2022-04-28T19:59:17.087868+0000 mgr.y (mgr.24961) 467 : cluster [DBG] pgmap v812: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:18 smithi139 bash[43301]: audit 2022-04-28T19:59:17.193781+0000 mon.a (mon.0) 1551 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T19:59:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:18 smithi139 bash[43301]: audit 2022-04-28T19:59:17.201152+0000 mon.a (mon.0) 1552 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T19:59:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:20 smithi059 bash[61152]: cluster 2022-04-28T19:59:19.088704+0000 mgr.y (mgr.24961) 468 : cluster [DBG] pgmap v813: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:20 smithi059 bash[59745]: cluster 2022-04-28T19:59:19.088704+0000 mgr.y (mgr.24961) 468 : cluster [DBG] pgmap v813: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:20.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:20 smithi139 bash[43301]: cluster 2022-04-28T19:59:19.088704+0000 mgr.y (mgr.24961) 468 : cluster [DBG] pgmap v813: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:21 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:21.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:22 smithi059 bash[61152]: cluster 2022-04-28T19:59:21.089696+0000 mgr.y (mgr.24961) 469 : cluster [DBG] pgmap v814: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:22.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:22 smithi059 bash[59745]: cluster 2022-04-28T19:59:21.089696+0000 mgr.y (mgr.24961) 469 : cluster [DBG] pgmap v814: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:22 smithi139 bash[43301]: cluster 2022-04-28T19:59:21.089696+0000 mgr.y (mgr.24961) 469 : cluster [DBG] pgmap v814: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:23 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:23.545Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:24 smithi059 bash[61152]: cluster 2022-04-28T19:59:23.090287+0000 mgr.y (mgr.24961) 470 : cluster [DBG] pgmap v815: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:24 smithi059 bash[59745]: cluster 2022-04-28T19:59:23.090287+0000 mgr.y (mgr.24961) 470 : cluster [DBG] pgmap v815: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:24 smithi139 bash[43301]: cluster 2022-04-28T19:59:23.090287+0000 mgr.y (mgr.24961) 470 : cluster [DBG] pgmap v815: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:26 smithi059 bash[61152]: cluster 2022-04-28T19:59:25.091358+0000 mgr.y (mgr.24961) 471 : cluster [DBG] pgmap v816: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:26 smithi059 bash[59745]: cluster 2022-04-28T19:59:25.091358+0000 mgr.y (mgr.24961) 471 : cluster [DBG] pgmap v816: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:26 smithi139 bash[43301]: cluster 2022-04-28T19:59:25.091358+0000 mgr.y (mgr.24961) 471 : cluster [DBG] pgmap v816: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:28.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:28 smithi059 bash[61152]: cluster 2022-04-28T19:59:27.092299+0000 mgr.y (mgr.24961) 472 : cluster [DBG] pgmap v817: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:28 smithi059 bash[59745]: cluster 2022-04-28T19:59:27.092299+0000 mgr.y (mgr.24961) 472 : cluster [DBG] pgmap v817: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:28 smithi139 bash[43301]: cluster 2022-04-28T19:59:27.092299+0000 mgr.y (mgr.24961) 472 : cluster [DBG] pgmap v817: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:30 smithi059 bash[61152]: cluster 2022-04-28T19:59:29.093169+0000 mgr.y (mgr.24961) 473 : cluster [DBG] pgmap v818: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:30 smithi059 bash[59745]: cluster 2022-04-28T19:59:29.093169+0000 mgr.y (mgr.24961) 473 : cluster [DBG] pgmap v818: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:30.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:30 smithi139 bash[43301]: cluster 2022-04-28T19:59:29.093169+0000 mgr.y (mgr.24961) 473 : cluster [DBG] pgmap v818: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:31 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:31.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:32.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:32 smithi059 bash[61152]: cluster 2022-04-28T19:59:31.094078+0000 mgr.y (mgr.24961) 474 : cluster [DBG] pgmap v819: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:32.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:32 smithi059 bash[59745]: cluster 2022-04-28T19:59:31.094078+0000 mgr.y (mgr.24961) 474 : cluster [DBG] pgmap v819: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:32 smithi139 bash[43301]: cluster 2022-04-28T19:59:31.094078+0000 mgr.y (mgr.24961) 474 : cluster [DBG] pgmap v819: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:33 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:33.546Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:34.575 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:34 smithi059 bash[61152]: cluster 2022-04-28T19:59:33.094824+0000 mgr.y (mgr.24961) 475 : cluster [DBG] pgmap v820: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:34.576 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:34 smithi059 bash[59745]: cluster 2022-04-28T19:59:33.094824+0000 mgr.y (mgr.24961) 475 : cluster [DBG] pgmap v820: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:34.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:34 smithi139 bash[43301]: cluster 2022-04-28T19:59:33.094824+0000 mgr.y (mgr.24961) 475 : cluster [DBG] pgmap v820: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:36.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:36 smithi059 bash[61152]: cluster 2022-04-28T19:59:35.095930+0000 mgr.y (mgr.24961) 476 : cluster [DBG] pgmap v821: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:36 smithi059 bash[59745]: cluster 2022-04-28T19:59:35.095930+0000 mgr.y (mgr.24961) 476 : cluster [DBG] pgmap v821: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:36.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:36 smithi139 bash[43301]: cluster 2022-04-28T19:59:35.095930+0000 mgr.y (mgr.24961) 476 : cluster [DBG] pgmap v821: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:38.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:38 smithi059 bash[61152]: cluster 2022-04-28T19:59:37.096798+0000 mgr.y (mgr.24961) 477 : cluster [DBG] pgmap v822: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:38 smithi059 bash[59745]: cluster 2022-04-28T19:59:37.096798+0000 mgr.y (mgr.24961) 477 : cluster [DBG] pgmap v822: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:38 smithi139 bash[43301]: cluster 2022-04-28T19:59:37.096798+0000 mgr.y (mgr.24961) 477 : cluster [DBG] pgmap v822: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:40 smithi059 bash[61152]: cluster 2022-04-28T19:59:39.097488+0000 mgr.y (mgr.24961) 478 : cluster [DBG] pgmap v823: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:40 smithi059 bash[59745]: cluster 2022-04-28T19:59:39.097488+0000 mgr.y (mgr.24961) 478 : cluster [DBG] pgmap v823: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:40 smithi139 bash[43301]: cluster 2022-04-28T19:59:39.097488+0000 mgr.y (mgr.24961) 478 : cluster [DBG] pgmap v823: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:41 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:41.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:42 smithi059 bash[61152]: cluster 2022-04-28T19:59:41.098446+0000 mgr.y (mgr.24961) 479 : cluster [DBG] pgmap v824: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:42 smithi059 bash[59745]: cluster 2022-04-28T19:59:41.098446+0000 mgr.y (mgr.24961) 479 : cluster [DBG] pgmap v824: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:42 smithi139 bash[43301]: cluster 2022-04-28T19:59:41.098446+0000 mgr.y (mgr.24961) 479 : cluster [DBG] pgmap v824: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:43 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:43.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:44.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:44 smithi059 bash[61152]: cluster 2022-04-28T19:59:43.099082+0000 mgr.y (mgr.24961) 480 : cluster [DBG] pgmap v825: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:44.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:44 smithi059 bash[59745]: cluster 2022-04-28T19:59:43.099082+0000 mgr.y (mgr.24961) 480 : cluster [DBG] pgmap v825: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:44.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:44 smithi139 bash[43301]: cluster 2022-04-28T19:59:43.099082+0000 mgr.y (mgr.24961) 480 : cluster [DBG] pgmap v825: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:46.554 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:46 smithi059 bash[61152]: cluster 2022-04-28T19:59:45.100130+0000 mgr.y (mgr.24961) 481 : cluster [DBG] pgmap v826: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:46.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:46 smithi059 bash[59745]: cluster 2022-04-28T19:59:45.100130+0000 mgr.y (mgr.24961) 481 : cluster [DBG] pgmap v826: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:46.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:46 smithi139 bash[43301]: cluster 2022-04-28T19:59:45.100130+0000 mgr.y (mgr.24961) 481 : cluster [DBG] pgmap v826: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:48 smithi139 bash[43301]: cluster 2022-04-28T19:59:47.100884+0000 mgr.y (mgr.24961) 482 : cluster [DBG] pgmap v827: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:48.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:48 smithi059 bash[59745]: cluster 2022-04-28T19:59:47.100884+0000 mgr.y (mgr.24961) 482 : cluster [DBG] pgmap v827: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:48 smithi059 bash[61152]: cluster 2022-04-28T19:59:47.100884+0000 mgr.y (mgr.24961) 482 : cluster [DBG] pgmap v827: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:50.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:50 smithi059 bash[61152]: cluster 2022-04-28T19:59:49.101691+0000 mgr.y (mgr.24961) 483 : cluster [DBG] pgmap v828: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:50 smithi059 bash[59745]: cluster 2022-04-28T19:59:49.101691+0000 mgr.y (mgr.24961) 483 : cluster [DBG] pgmap v828: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:50 smithi139 bash[43301]: cluster 2022-04-28T19:59:49.101691+0000 mgr.y (mgr.24961) 483 : cluster [DBG] pgmap v828: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:51 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:51.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:52.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[61152]: cluster 2022-04-28T19:59:51.102731+0000 mgr.y (mgr.24961) 484 : cluster [DBG] pgmap v829: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[61152]: audit 2022-04-28T19:59:51.916667+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:59:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[61152]: audit 2022-04-28T19:59:51.919371+0000 mon.a (mon.0) 1554 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:59:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[61152]: audit 2022-04-28T19:59:51.921029+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:59:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[59745]: cluster 2022-04-28T19:59:51.102731+0000 mgr.y (mgr.24961) 484 : cluster [DBG] pgmap v829: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[59745]: audit 2022-04-28T19:59:51.916667+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:59:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[59745]: audit 2022-04-28T19:59:51.919371+0000 mon.a (mon.0) 1554 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:59:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:52 smithi059 bash[59745]: audit 2022-04-28T19:59:51.921029+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:59:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:52 smithi139 bash[43301]: cluster 2022-04-28T19:59:51.102731+0000 mgr.y (mgr.24961) 484 : cluster [DBG] pgmap v829: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:52 smithi139 bash[43301]: audit 2022-04-28T19:59:51.916667+0000 mon.a (mon.0) 1553 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T19:59:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:52 smithi139 bash[43301]: audit 2022-04-28T19:59:51.919371+0000 mon.a (mon.0) 1554 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T19:59:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:52 smithi139 bash[43301]: audit 2022-04-28T19:59:51.921029+0000 mon.a (mon.0) 1555 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T19:59:53.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: cephadm 2022-04-28T19:59:51.922622+0000 mgr.y (mgr.24961) 485 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:59:53.548 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.236259+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.236960+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.237526+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.238066+0000 mon.a (mon.0) 1559 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.238616+0000 mon.a (mon.0) 1560 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.239203+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.239784+0000 mon.a (mon.0) 1562 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.240366+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.240975+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:59:53.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.241614+0000 mon.a (mon.0) 1565 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.246919+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.247513+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.550 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.248049+0000 mon.a (mon.0) 1568 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.551 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.248620+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.551 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: cephadm 2022-04-28T19:59:51.922622+0000 mgr.y (mgr.24961) 485 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:59:53.551 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.236259+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.551 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.236960+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.237526+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.238066+0000 mon.a (mon.0) 1559 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.238616+0000 mon.a (mon.0) 1560 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.239203+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.239784+0000 mon.a (mon.0) 1562 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.552 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.240366+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.553 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.240975+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:59:53.553 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.241614+0000 mon.a (mon.0) 1565 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.553 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.246919+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.553 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.247513+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.553 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.248049+0000 mon.a (mon.0) 1568 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.554 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.248620+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.249165+0000 mon.a (mon.0) 1570 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.249683+0000 mon.a (mon.0) 1571 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.250205+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.250715+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.251257+0000 mon.a (mon.0) 1574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:59:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.251829+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[59745]: audit 2022-04-28T19:59:52.257550+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:59:53.555 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.249165+0000 mon.a (mon.0) 1570 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.249683+0000 mon.a (mon.0) 1571 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.250205+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.250715+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.251257+0000 mon.a (mon.0) 1574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:59:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.251829+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[61152]: audit 2022-04-28T19:59:52.257550+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:59:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: cephadm 2022-04-28T19:59:51.922622+0000 mgr.y (mgr.24961) 485 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T19:59:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.236259+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.236960+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.237526+0000 mon.a (mon.0) 1558 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.238066+0000 mon.a (mon.0) 1559 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.238616+0000 mon.a (mon.0) 1560 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.239203+0000 mon.a (mon.0) 1561 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.239784+0000 mon.a (mon.0) 1562 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.240366+0000 mon.a (mon.0) 1563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.240975+0000 mon.a (mon.0) 1564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:59:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.241614+0000 mon.a (mon.0) 1565 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.246919+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.247513+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.248049+0000 mon.a (mon.0) 1568 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.248620+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.249165+0000 mon.a (mon.0) 1570 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.249683+0000 mon.a (mon.0) 1571 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.250205+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T19:59:53.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.250715+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.251257+0000 mon.a (mon.0) 1574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T19:59:53.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.251829+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T19:59:53.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:53 smithi139 bash[43301]: audit 2022-04-28T19:59:52.257550+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T19:59:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 19:59:53 smithi059 bash[72035]: level=error ts=2022-04-28T19:59:53.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T19:59:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:54 smithi059 bash[61152]: cluster 2022-04-28T19:59:53.103607+0000 mgr.y (mgr.24961) 486 : cluster [DBG] pgmap v830: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:54.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:54 smithi059 bash[59745]: cluster 2022-04-28T19:59:53.103607+0000 mgr.y (mgr.24961) 486 : cluster [DBG] pgmap v830: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:54 smithi139 bash[43301]: cluster 2022-04-28T19:59:53.103607+0000 mgr.y (mgr.24961) 486 : cluster [DBG] pgmap v830: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:56 smithi059 bash[61152]: cluster 2022-04-28T19:59:55.104629+0000 mgr.y (mgr.24961) 487 : cluster [DBG] pgmap v831: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:56.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:56 smithi059 bash[59745]: cluster 2022-04-28T19:59:55.104629+0000 mgr.y (mgr.24961) 487 : cluster [DBG] pgmap v831: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:56.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:56 smithi139 bash[43301]: cluster 2022-04-28T19:59:55.104629+0000 mgr.y (mgr.24961) 487 : cluster [DBG] pgmap v831: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 19:59:58 smithi139 bash[43301]: cluster 2022-04-28T19:59:57.105427+0000 mgr.y (mgr.24961) 488 : cluster [DBG] pgmap v832: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 19:59:58 smithi059 bash[61152]: cluster 2022-04-28T19:59:57.105427+0000 mgr.y (mgr.24961) 488 : cluster [DBG] pgmap v832: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T19:59:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 19:59:58 smithi059 bash[59745]: cluster 2022-04-28T19:59:57.105427+0000 mgr.y (mgr.24961) 488 : cluster [DBG] pgmap v832: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:00 smithi059 bash[61152]: cluster 2022-04-28T19:59:59.106261+0000 mgr.y (mgr.24961) 489 : cluster [DBG] pgmap v833: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:00 smithi059 bash[61152]: cluster 2022-04-28T20:00:00.000176+0000 mon.a (mon.0) 1577 : cluster [INF] overall HEALTH_OK 2022-04-28T20:00:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:00 smithi059 bash[59745]: cluster 2022-04-28T19:59:59.106261+0000 mgr.y (mgr.24961) 489 : cluster [DBG] pgmap v833: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:00 smithi059 bash[59745]: cluster 2022-04-28T20:00:00.000176+0000 mon.a (mon.0) 1577 : cluster [INF] overall HEALTH_OK 2022-04-28T20:00:00.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:00 smithi139 bash[43301]: cluster 2022-04-28T19:59:59.106261+0000 mgr.y (mgr.24961) 489 : cluster [DBG] pgmap v833: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:00 smithi139 bash[43301]: cluster 2022-04-28T20:00:00.000176+0000 mon.a (mon.0) 1577 : cluster [INF] overall HEALTH_OK 2022-04-28T20:00:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:01.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:02 smithi059 bash[61152]: cluster 2022-04-28T20:00:01.107267+0000 mgr.y (mgr.24961) 490 : cluster [DBG] pgmap v834: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:02 smithi059 bash[59745]: cluster 2022-04-28T20:00:01.107267+0000 mgr.y (mgr.24961) 490 : cluster [DBG] pgmap v834: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:02 smithi139 bash[43301]: cluster 2022-04-28T20:00:01.107267+0000 mgr.y (mgr.24961) 490 : cluster [DBG] pgmap v834: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:03.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:04.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:04 smithi059 bash[59745]: cluster 2022-04-28T20:00:03.107917+0000 mgr.y (mgr.24961) 491 : cluster [DBG] pgmap v835: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:04 smithi059 bash[61152]: cluster 2022-04-28T20:00:03.107917+0000 mgr.y (mgr.24961) 491 : cluster [DBG] pgmap v835: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:04.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:04 smithi139 bash[43301]: cluster 2022-04-28T20:00:03.107917+0000 mgr.y (mgr.24961) 491 : cluster [DBG] pgmap v835: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:06.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:06 smithi059 bash[59745]: cluster 2022-04-28T20:00:05.109004+0000 mgr.y (mgr.24961) 492 : cluster [DBG] pgmap v836: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:06 smithi059 bash[61152]: cluster 2022-04-28T20:00:05.109004+0000 mgr.y (mgr.24961) 492 : cluster [DBG] pgmap v836: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:06 smithi139 bash[43301]: cluster 2022-04-28T20:00:05.109004+0000 mgr.y (mgr.24961) 492 : cluster [DBG] pgmap v836: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:08 smithi139 bash[43301]: cluster 2022-04-28T20:00:07.109870+0000 mgr.y (mgr.24961) 493 : cluster [DBG] pgmap v837: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:08.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:08 smithi059 bash[59745]: cluster 2022-04-28T20:00:07.109870+0000 mgr.y (mgr.24961) 493 : cluster [DBG] pgmap v837: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:08 smithi059 bash[61152]: cluster 2022-04-28T20:00:07.109870+0000 mgr.y (mgr.24961) 493 : cluster [DBG] pgmap v837: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:10 smithi059 bash[61152]: cluster 2022-04-28T20:00:09.110671+0000 mgr.y (mgr.24961) 494 : cluster [DBG] pgmap v838: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:10 smithi059 bash[59745]: cluster 2022-04-28T20:00:09.110671+0000 mgr.y (mgr.24961) 494 : cluster [DBG] pgmap v838: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:10 smithi139 bash[43301]: cluster 2022-04-28T20:00:09.110671+0000 mgr.y (mgr.24961) 494 : cluster [DBG] pgmap v838: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:11.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:12 smithi059 bash[61152]: cluster 2022-04-28T20:00:11.111598+0000 mgr.y (mgr.24961) 495 : cluster [DBG] pgmap v839: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:12 smithi059 bash[59745]: cluster 2022-04-28T20:00:11.111598+0000 mgr.y (mgr.24961) 495 : cluster [DBG] pgmap v839: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:12 smithi139 bash[43301]: cluster 2022-04-28T20:00:11.111598+0000 mgr.y (mgr.24961) 495 : cluster [DBG] pgmap v839: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:13.549Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:14 smithi059 bash[61152]: cluster 2022-04-28T20:00:13.112212+0000 mgr.y (mgr.24961) 496 : cluster [DBG] pgmap v840: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:14 smithi059 bash[59745]: cluster 2022-04-28T20:00:13.112212+0000 mgr.y (mgr.24961) 496 : cluster [DBG] pgmap v840: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:14.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:14 smithi139 bash[43301]: cluster 2022-04-28T20:00:13.112212+0000 mgr.y (mgr.24961) 496 : cluster [DBG] pgmap v840: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:16 smithi059 bash[61152]: cluster 2022-04-28T20:00:15.113328+0000 mgr.y (mgr.24961) 497 : cluster [DBG] pgmap v841: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:16.615 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:16 smithi059 bash[59745]: cluster 2022-04-28T20:00:15.113328+0000 mgr.y (mgr.24961) 497 : cluster [DBG] pgmap v841: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:16 smithi139 bash[43301]: cluster 2022-04-28T20:00:15.113328+0000 mgr.y (mgr.24961) 497 : cluster [DBG] pgmap v841: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:17.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:17 smithi059 bash[59745]: audit 2022-04-28T20:00:16.955875+0000 mon.a (mon.0) 1578 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:17 smithi059 bash[59745]: audit 2022-04-28T20:00:16.956681+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:17 smithi059 bash[59745]: audit 2022-04-28T20:00:16.957407+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:17 smithi059 bash[61152]: audit 2022-04-28T20:00:16.955875+0000 mon.a (mon.0) 1578 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:17 smithi059 bash[61152]: audit 2022-04-28T20:00:16.956681+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:17 smithi059 bash[61152]: audit 2022-04-28T20:00:16.957407+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:17 smithi139 bash[43301]: audit 2022-04-28T20:00:16.955875+0000 mon.a (mon.0) 1578 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:17 smithi139 bash[43301]: audit 2022-04-28T20:00:16.956681+0000 mon.a (mon.0) 1579 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:00:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:17 smithi139 bash[43301]: audit 2022-04-28T20:00:16.957407+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:00:18.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:18 smithi059 bash[59745]: cluster 2022-04-28T20:00:17.114173+0000 mgr.y (mgr.24961) 498 : cluster [DBG] pgmap v842: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:18 smithi059 bash[59745]: audit 2022-04-28T20:00:17.204944+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:00:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:18 smithi059 bash[59745]: audit 2022-04-28T20:00:17.205589+0000 mon.a (mon.0) 1582 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:00:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:18 smithi059 bash[61152]: cluster 2022-04-28T20:00:17.114173+0000 mgr.y (mgr.24961) 498 : cluster [DBG] pgmap v842: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:18 smithi059 bash[61152]: audit 2022-04-28T20:00:17.204944+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:00:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:18 smithi059 bash[61152]: audit 2022-04-28T20:00:17.205589+0000 mon.a (mon.0) 1582 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:00:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:18 smithi139 bash[43301]: cluster 2022-04-28T20:00:17.114173+0000 mgr.y (mgr.24961) 498 : cluster [DBG] pgmap v842: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:18 smithi139 bash[43301]: audit 2022-04-28T20:00:17.204944+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:00:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:18 smithi139 bash[43301]: audit 2022-04-28T20:00:17.205589+0000 mon.a (mon.0) 1582 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:00:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:20 smithi059 bash[59745]: cluster 2022-04-28T20:00:19.114983+0000 mgr.y (mgr.24961) 499 : cluster [DBG] pgmap v843: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:20 smithi059 bash[61152]: cluster 2022-04-28T20:00:19.114983+0000 mgr.y (mgr.24961) 499 : cluster [DBG] pgmap v843: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:20.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:20 smithi139 bash[43301]: cluster 2022-04-28T20:00:19.114983+0000 mgr.y (mgr.24961) 499 : cluster [DBG] pgmap v843: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:21.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:22.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:22 smithi059 bash[59745]: cluster 2022-04-28T20:00:21.116018+0000 mgr.y (mgr.24961) 500 : cluster [DBG] pgmap v844: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:22 smithi059 bash[61152]: cluster 2022-04-28T20:00:21.116018+0000 mgr.y (mgr.24961) 500 : cluster [DBG] pgmap v844: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:22 smithi139 bash[43301]: cluster 2022-04-28T20:00:21.116018+0000 mgr.y (mgr.24961) 500 : cluster [DBG] pgmap v844: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:23.548 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:23 smithi059 bash[61152]: audit 2022-04-28T20:00:22.364363+0000 mon.a (mon.0) 1583 : audit [DBG] from='client.? 172.21.15.59:0/2752753947' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:00:23.548 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:23.548Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:23.549 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:23 smithi059 bash[59745]: audit 2022-04-28T20:00:22.364363+0000 mon.a (mon.0) 1583 : audit [DBG] from='client.? 172.21.15.59:0/2752753947' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:00:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:23 smithi139 bash[43301]: audit 2022-04-28T20:00:22.364363+0000 mon.a (mon.0) 1583 : audit [DBG] from='client.? 172.21.15.59:0/2752753947' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:00:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:24 smithi059 bash[61152]: cluster 2022-04-28T20:00:23.116681+0000 mgr.y (mgr.24961) 501 : cluster [DBG] pgmap v845: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:24 smithi059 bash[59745]: cluster 2022-04-28T20:00:23.116681+0000 mgr.y (mgr.24961) 501 : cluster [DBG] pgmap v845: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:24.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:24 smithi139 bash[43301]: cluster 2022-04-28T20:00:23.116681+0000 mgr.y (mgr.24961) 501 : cluster [DBG] pgmap v845: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:26 smithi059 bash[61152]: cluster 2022-04-28T20:00:25.117673+0000 mgr.y (mgr.24961) 502 : cluster [DBG] pgmap v846: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:26 smithi059 bash[59745]: cluster 2022-04-28T20:00:25.117673+0000 mgr.y (mgr.24961) 502 : cluster [DBG] pgmap v846: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:26 smithi139 bash[43301]: cluster 2022-04-28T20:00:25.117673+0000 mgr.y (mgr.24961) 502 : cluster [DBG] pgmap v846: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:28 smithi139 bash[43301]: cluster 2022-04-28T20:00:27.118457+0000 mgr.y (mgr.24961) 503 : cluster [DBG] pgmap v847: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:28.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:28 smithi059 bash[59745]: cluster 2022-04-28T20:00:27.118457+0000 mgr.y (mgr.24961) 503 : cluster [DBG] pgmap v847: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:28 smithi059 bash[61152]: cluster 2022-04-28T20:00:27.118457+0000 mgr.y (mgr.24961) 503 : cluster [DBG] pgmap v847: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:30.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:30 smithi059 bash[59745]: cluster 2022-04-28T20:00:29.119249+0000 mgr.y (mgr.24961) 504 : cluster [DBG] pgmap v848: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:30 smithi059 bash[61152]: cluster 2022-04-28T20:00:29.119249+0000 mgr.y (mgr.24961) 504 : cluster [DBG] pgmap v848: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:30 smithi139 bash[43301]: cluster 2022-04-28T20:00:29.119249+0000 mgr.y (mgr.24961) 504 : cluster [DBG] pgmap v848: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:31.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:32.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:32 smithi059 bash[61152]: cluster 2022-04-28T20:00:31.120340+0000 mgr.y (mgr.24961) 505 : cluster [DBG] pgmap v849: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:32 smithi059 bash[59745]: cluster 2022-04-28T20:00:31.120340+0000 mgr.y (mgr.24961) 505 : cluster [DBG] pgmap v849: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:32 smithi139 bash[43301]: cluster 2022-04-28T20:00:31.120340+0000 mgr.y (mgr.24961) 505 : cluster [DBG] pgmap v849: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:33.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:33.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:34.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:34 smithi059 bash[61152]: cluster 2022-04-28T20:00:33.121124+0000 mgr.y (mgr.24961) 506 : cluster [DBG] pgmap v850: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:34.665 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:34 smithi059 bash[59745]: cluster 2022-04-28T20:00:33.121124+0000 mgr.y (mgr.24961) 506 : cluster [DBG] pgmap v850: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:34.665 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:34 smithi139 bash[43301]: cluster 2022-04-28T20:00:33.121124+0000 mgr.y (mgr.24961) 506 : cluster [DBG] pgmap v850: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:36.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:36 smithi059 bash[61152]: cluster 2022-04-28T20:00:35.122102+0000 mgr.y (mgr.24961) 507 : cluster [DBG] pgmap v851: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:36 smithi059 bash[59745]: cluster 2022-04-28T20:00:35.122102+0000 mgr.y (mgr.24961) 507 : cluster [DBG] pgmap v851: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:36 smithi139 bash[43301]: cluster 2022-04-28T20:00:35.122102+0000 mgr.y (mgr.24961) 507 : cluster [DBG] pgmap v851: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:38 smithi139 bash[43301]: cluster 2022-04-28T20:00:37.122953+0000 mgr.y (mgr.24961) 508 : cluster [DBG] pgmap v852: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:38 smithi059 bash[61152]: cluster 2022-04-28T20:00:37.122953+0000 mgr.y (mgr.24961) 508 : cluster [DBG] pgmap v852: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:38 smithi059 bash[59745]: cluster 2022-04-28T20:00:37.122953+0000 mgr.y (mgr.24961) 508 : cluster [DBG] pgmap v852: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:40 smithi059 bash[61152]: cluster 2022-04-28T20:00:39.123891+0000 mgr.y (mgr.24961) 509 : cluster [DBG] pgmap v853: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:40 smithi059 bash[59745]: cluster 2022-04-28T20:00:39.123891+0000 mgr.y (mgr.24961) 509 : cluster [DBG] pgmap v853: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:40 smithi139 bash[43301]: cluster 2022-04-28T20:00:39.123891+0000 mgr.y (mgr.24961) 509 : cluster [DBG] pgmap v853: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:41.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:42 smithi059 bash[59745]: cluster 2022-04-28T20:00:41.124922+0000 mgr.y (mgr.24961) 510 : cluster [DBG] pgmap v854: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:42 smithi059 bash[61152]: cluster 2022-04-28T20:00:41.124922+0000 mgr.y (mgr.24961) 510 : cluster [DBG] pgmap v854: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:42.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:42 smithi139 bash[43301]: cluster 2022-04-28T20:00:41.124922+0000 mgr.y (mgr.24961) 510 : cluster [DBG] pgmap v854: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:43.551Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[59745]: cluster 2022-04-28T20:00:42.567036+0000 mon.a (mon.0) 1584 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2022-04-28T20:00:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[59745]: audit 2022-04-28T20:00:42.572919+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.59:0/988576420' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:00:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[59745]: audit 2022-04-28T20:00:42.574323+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:00:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[59745]: cluster 2022-04-28T20:00:43.125626+0000 mgr.y (mgr.24961) 511 : cluster [DBG] pgmap v856: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[61152]: cluster 2022-04-28T20:00:42.567036+0000 mon.a (mon.0) 1584 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2022-04-28T20:00:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[61152]: audit 2022-04-28T20:00:42.572919+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.59:0/988576420' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:00:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[61152]: audit 2022-04-28T20:00:42.574323+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:00:43.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:43 smithi059 bash[61152]: cluster 2022-04-28T20:00:43.125626+0000 mgr.y (mgr.24961) 511 : cluster [DBG] pgmap v856: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:43 smithi139 bash[43301]: cluster 2022-04-28T20:00:42.567036+0000 mon.a (mon.0) 1584 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2022-04-28T20:00:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:43 smithi139 bash[43301]: audit 2022-04-28T20:00:42.572919+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.59:0/988576420' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:00:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:43 smithi139 bash[43301]: audit 2022-04-28T20:00:42.574323+0000 mon.a (mon.0) 1585 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:00:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:43 smithi139 bash[43301]: cluster 2022-04-28T20:00:43.125626+0000 mgr.y (mgr.24961) 511 : cluster [DBG] pgmap v856: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:44.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:44 smithi059 bash[61152]: audit 2022-04-28T20:00:43.578065+0000 mon.a (mon.0) 1586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:00:44.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:44 smithi059 bash[61152]: cluster 2022-04-28T20:00:43.578223+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2022-04-28T20:00:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:44 smithi059 bash[59745]: audit 2022-04-28T20:00:43.578065+0000 mon.a (mon.0) 1586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:00:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:44 smithi059 bash[59745]: cluster 2022-04-28T20:00:43.578223+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2022-04-28T20:00:44.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:44 smithi139 bash[43301]: audit 2022-04-28T20:00:43.578065+0000 mon.a (mon.0) 1586 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74709-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:00:44.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:44 smithi139 bash[43301]: cluster 2022-04-28T20:00:43.578223+0000 mon.a (mon.0) 1587 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2022-04-28T20:00:45.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:45 smithi059 bash[61152]: cluster 2022-04-28T20:00:44.581599+0000 mon.a (mon.0) 1588 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2022-04-28T20:00:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:45 smithi059 bash[61152]: cluster 2022-04-28T20:00:45.126420+0000 mgr.y (mgr.24961) 512 : cluster [DBG] pgmap v859: 73 pgs: 73 active+clean; 24 KiB data, 700 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:45 smithi059 bash[59745]: cluster 2022-04-28T20:00:44.581599+0000 mon.a (mon.0) 1588 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2022-04-28T20:00:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:45 smithi059 bash[59745]: cluster 2022-04-28T20:00:45.126420+0000 mgr.y (mgr.24961) 512 : cluster [DBG] pgmap v859: 73 pgs: 73 active+clean; 24 KiB data, 700 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:45 smithi139 bash[43301]: cluster 2022-04-28T20:00:44.581599+0000 mon.a (mon.0) 1588 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2022-04-28T20:00:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:45 smithi139 bash[43301]: cluster 2022-04-28T20:00:45.126420+0000 mgr.y (mgr.24961) 512 : cluster [DBG] pgmap v859: 73 pgs: 73 active+clean; 24 KiB data, 700 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:48 smithi059 bash[61152]: cluster 2022-04-28T20:00:47.127241+0000 mgr.y (mgr.24961) 513 : cluster [DBG] pgmap v860: 73 pgs: 73 active+clean; 24 KiB data, 701 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:48 smithi059 bash[59745]: cluster 2022-04-28T20:00:47.127241+0000 mgr.y (mgr.24961) 513 : cluster [DBG] pgmap v860: 73 pgs: 73 active+clean; 24 KiB data, 701 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:48 smithi139 bash[43301]: cluster 2022-04-28T20:00:47.127241+0000 mgr.y (mgr.24961) 513 : cluster [DBG] pgmap v860: 73 pgs: 73 active+clean; 24 KiB data, 701 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:50.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:50 smithi059 bash[61152]: cluster 2022-04-28T20:00:49.128217+0000 mgr.y (mgr.24961) 514 : cluster [DBG] pgmap v861: 73 pgs: 73 active+clean; 24 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:50 smithi059 bash[59745]: cluster 2022-04-28T20:00:49.128217+0000 mgr.y (mgr.24961) 514 : cluster [DBG] pgmap v861: 73 pgs: 73 active+clean; 24 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:50.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:50 smithi139 bash[43301]: cluster 2022-04-28T20:00:49.128217+0000 mgr.y (mgr.24961) 514 : cluster [DBG] pgmap v861: 73 pgs: 73 active+clean; 24 KiB data, 706 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:51.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:52.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:52 smithi059 bash[61152]: cluster 2022-04-28T20:00:51.129313+0000 mgr.y (mgr.24961) 515 : cluster [DBG] pgmap v862: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:52 smithi059 bash[59745]: cluster 2022-04-28T20:00:51.129313+0000 mgr.y (mgr.24961) 515 : cluster [DBG] pgmap v862: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:52.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:52 smithi139 bash[43301]: cluster 2022-04-28T20:00:51.129313+0000 mgr.y (mgr.24961) 515 : cluster [DBG] pgmap v862: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:53.554 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[59745]: audit 2022-04-28T20:00:52.264637+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:00:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[59745]: audit 2022-04-28T20:00:52.267562+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:00:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[59745]: audit 2022-04-28T20:00:52.269503+0000 mon.a (mon.0) 1591 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:00:53.555 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[59745]: cephadm 2022-04-28T20:00:52.271264+0000 mgr.y (mgr.24961) 516 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:00:53.556 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[59745]: audit 2022-04-28T20:00:52.597329+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:00:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[61152]: audit 2022-04-28T20:00:52.264637+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:00:53.556 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[61152]: audit 2022-04-28T20:00:52.267562+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:00:53.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[61152]: audit 2022-04-28T20:00:52.269503+0000 mon.a (mon.0) 1591 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:00:53.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[61152]: cephadm 2022-04-28T20:00:52.271264+0000 mgr.y (mgr.24961) 516 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:00:53.557 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[61152]: audit 2022-04-28T20:00:52.597329+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:00:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:53 smithi139 bash[43301]: audit 2022-04-28T20:00:52.264637+0000 mon.a (mon.0) 1589 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:00:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:53 smithi139 bash[43301]: audit 2022-04-28T20:00:52.267562+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:00:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:53 smithi139 bash[43301]: audit 2022-04-28T20:00:52.269503+0000 mon.a (mon.0) 1591 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:00:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:53 smithi139 bash[43301]: cephadm 2022-04-28T20:00:52.271264+0000 mgr.y (mgr.24961) 516 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:00:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:53 smithi139 bash[43301]: audit 2022-04-28T20:00:52.597329+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:00:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:00:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:00:53.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:00:54.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:54 smithi059 bash[61152]: cluster 2022-04-28T20:00:53.129978+0000 mgr.y (mgr.24961) 517 : cluster [DBG] pgmap v863: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:54.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:54 smithi059 bash[59745]: cluster 2022-04-28T20:00:53.129978+0000 mgr.y (mgr.24961) 517 : cluster [DBG] pgmap v863: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:54.848 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:54 smithi139 bash[43301]: cluster 2022-04-28T20:00:53.129978+0000 mgr.y (mgr.24961) 517 : cluster [DBG] pgmap v863: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:56 smithi059 bash[61152]: cluster 2022-04-28T20:00:55.131071+0000 mgr.y (mgr.24961) 518 : cluster [DBG] pgmap v864: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:56 smithi059 bash[59745]: cluster 2022-04-28T20:00:55.131071+0000 mgr.y (mgr.24961) 518 : cluster [DBG] pgmap v864: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:56 smithi139 bash[43301]: cluster 2022-04-28T20:00:55.131071+0000 mgr.y (mgr.24961) 518 : cluster [DBG] pgmap v864: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:00:58 smithi059 bash[61152]: cluster 2022-04-28T20:00:57.131944+0000 mgr.y (mgr.24961) 519 : cluster [DBG] pgmap v865: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:00:58 smithi059 bash[59745]: cluster 2022-04-28T20:00:57.131944+0000 mgr.y (mgr.24961) 519 : cluster [DBG] pgmap v865: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:00:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:00:58 smithi139 bash[43301]: cluster 2022-04-28T20:00:57.131944+0000 mgr.y (mgr.24961) 519 : cluster [DBG] pgmap v865: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:00 smithi059 bash[61152]: cluster 2022-04-28T20:00:59.132845+0000 mgr.y (mgr.24961) 520 : cluster [DBG] pgmap v866: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:00 smithi059 bash[59745]: cluster 2022-04-28T20:00:59.132845+0000 mgr.y (mgr.24961) 520 : cluster [DBG] pgmap v866: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:00 smithi139 bash[43301]: cluster 2022-04-28T20:00:59.132845+0000 mgr.y (mgr.24961) 520 : cluster [DBG] pgmap v866: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:01.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:02 smithi059 bash[61152]: cluster 2022-04-28T20:01:01.133816+0000 mgr.y (mgr.24961) 521 : cluster [DBG] pgmap v867: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:02 smithi059 bash[59745]: cluster 2022-04-28T20:01:01.133816+0000 mgr.y (mgr.24961) 521 : cluster [DBG] pgmap v867: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:02 smithi139 bash[43301]: cluster 2022-04-28T20:01:01.133816+0000 mgr.y (mgr.24961) 521 : cluster [DBG] pgmap v867: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:03 smithi059 bash[61152]: cluster 2022-04-28T20:01:03.134454+0000 mgr.y (mgr.24961) 522 : cluster [DBG] pgmap v868: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:03.877 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:03.552Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:03.878 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:03 smithi059 bash[59745]: cluster 2022-04-28T20:01:03.134454+0000 mgr.y (mgr.24961) 522 : cluster [DBG] pgmap v868: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:03.878 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:03 smithi139 bash[43301]: cluster 2022-04-28T20:01:03.134454+0000 mgr.y (mgr.24961) 522 : cluster [DBG] pgmap v868: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:06.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:06 smithi059 bash[59745]: cluster 2022-04-28T20:01:05.135458+0000 mgr.y (mgr.24961) 523 : cluster [DBG] pgmap v869: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:06 smithi059 bash[61152]: cluster 2022-04-28T20:01:05.135458+0000 mgr.y (mgr.24961) 523 : cluster [DBG] pgmap v869: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:06 smithi139 bash[43301]: cluster 2022-04-28T20:01:05.135458+0000 mgr.y (mgr.24961) 523 : cluster [DBG] pgmap v869: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:07.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:07 smithi059 bash[61152]: cluster 2022-04-28T20:01:06.205518+0000 mon.a (mon.0) 1593 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2022-04-28T20:01:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:07 smithi059 bash[61152]: audit 2022-04-28T20:01:06.210591+0000 mon.c (mon.1) 21 : audit [INF] from='client.? 172.21.15.59:0/2352182682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:07 smithi059 bash[61152]: audit 2022-04-28T20:01:06.211546+0000 mon.a (mon.0) 1594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:07 smithi059 bash[59745]: cluster 2022-04-28T20:01:06.205518+0000 mon.a (mon.0) 1593 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2022-04-28T20:01:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:07 smithi059 bash[59745]: audit 2022-04-28T20:01:06.210591+0000 mon.c (mon.1) 21 : audit [INF] from='client.? 172.21.15.59:0/2352182682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:07 smithi059 bash[59745]: audit 2022-04-28T20:01:06.211546+0000 mon.a (mon.0) 1594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:07 smithi139 bash[43301]: cluster 2022-04-28T20:01:06.205518+0000 mon.a (mon.0) 1593 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2022-04-28T20:01:07.761 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:07 smithi139 bash[43301]: audit 2022-04-28T20:01:06.210591+0000 mon.c (mon.1) 21 : audit [INF] from='client.? 172.21.15.59:0/2352182682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:07.761 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:07 smithi139 bash[43301]: audit 2022-04-28T20:01:06.211546+0000 mon.a (mon.0) 1594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[61152]: cluster 2022-04-28T20:01:07.136144+0000 mgr.y (mgr.24961) 524 : cluster [DBG] pgmap v871: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:08.695 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[61152]: audit 2022-04-28T20:01:07.210164+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:08.695 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[61152]: cluster 2022-04-28T20:01:07.210621+0000 mon.a (mon.0) 1596 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2022-04-28T20:01:08.695 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[61152]: cluster 2022-04-28T20:01:08.211321+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2022-04-28T20:01:08.696 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[59745]: cluster 2022-04-28T20:01:07.136144+0000 mgr.y (mgr.24961) 524 : cluster [DBG] pgmap v871: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:08.698 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[59745]: audit 2022-04-28T20:01:07.210164+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:08.699 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[59745]: cluster 2022-04-28T20:01:07.210621+0000 mon.a (mon.0) 1596 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2022-04-28T20:01:08.699 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:08 smithi059 bash[59745]: cluster 2022-04-28T20:01:08.211321+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2022-04-28T20:01:08.700 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:08 smithi139 bash[43301]: cluster 2022-04-28T20:01:07.136144+0000 mgr.y (mgr.24961) 524 : cluster [DBG] pgmap v871: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:08.700 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:08 smithi139 bash[43301]: audit 2022-04-28T20:01:07.210164+0000 mon.a (mon.0) 1595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74739-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:08.701 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:08 smithi139 bash[43301]: cluster 2022-04-28T20:01:07.210621+0000 mon.a (mon.0) 1596 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2022-04-28T20:01:08.701 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:08 smithi139 bash[43301]: cluster 2022-04-28T20:01:08.211321+0000 mon.a (mon.0) 1597 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2022-04-28T20:01:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:10 smithi059 bash[61152]: cluster 2022-04-28T20:01:09.137036+0000 mgr.y (mgr.24961) 525 : cluster [DBG] pgmap v874: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:10 smithi059 bash[59745]: cluster 2022-04-28T20:01:09.137036+0000 mgr.y (mgr.24961) 525 : cluster [DBG] pgmap v874: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:10 smithi139 bash[43301]: cluster 2022-04-28T20:01:09.137036+0000 mgr.y (mgr.24961) 525 : cluster [DBG] pgmap v874: 73 pgs: 73 active+clean; 24 KiB data, 710 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:11.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:12 smithi059 bash[61152]: cluster 2022-04-28T20:01:11.137992+0000 mgr.y (mgr.24961) 526 : cluster [DBG] pgmap v875: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:12.640 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:12 smithi059 bash[59745]: cluster 2022-04-28T20:01:11.137992+0000 mgr.y (mgr.24961) 526 : cluster [DBG] pgmap v875: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:12.641 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:12 smithi139 bash[43301]: cluster 2022-04-28T20:01:11.137992+0000 mgr.y (mgr.24961) 526 : cluster [DBG] pgmap v875: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:13 smithi059 bash[61152]: cluster 2022-04-28T20:01:13.138601+0000 mgr.y (mgr.24961) 527 : cluster [DBG] pgmap v876: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:13.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:13 smithi059 bash[59745]: cluster 2022-04-28T20:01:13.138601+0000 mgr.y (mgr.24961) 527 : cluster [DBG] pgmap v876: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:13 smithi139 bash[43301]: cluster 2022-04-28T20:01:13.138601+0000 mgr.y (mgr.24961) 527 : cluster [DBG] pgmap v876: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:16 smithi059 bash[61152]: cluster 2022-04-28T20:01:15.139619+0000 mgr.y (mgr.24961) 528 : cluster [DBG] pgmap v877: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:16.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:16 smithi059 bash[59745]: cluster 2022-04-28T20:01:15.139619+0000 mgr.y (mgr.24961) 528 : cluster [DBG] pgmap v877: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:16 smithi139 bash[43301]: cluster 2022-04-28T20:01:15.139619+0000 mgr.y (mgr.24961) 528 : cluster [DBG] pgmap v877: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:17.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:17 smithi059 bash[61152]: audit 2022-04-28T20:01:16.964701+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:01:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:17 smithi059 bash[61152]: audit 2022-04-28T20:01:16.965418+0000 mon.a (mon.0) 1599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:01:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:17 smithi059 bash[61152]: audit 2022-04-28T20:01:16.965933+0000 mon.a (mon.0) 1600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:01:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:17 smithi059 bash[59745]: audit 2022-04-28T20:01:16.964701+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:01:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:17 smithi059 bash[59745]: audit 2022-04-28T20:01:16.965418+0000 mon.a (mon.0) 1599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:01:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:17 smithi059 bash[59745]: audit 2022-04-28T20:01:16.965933+0000 mon.a (mon.0) 1600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:01:17.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:17 smithi139 bash[43301]: audit 2022-04-28T20:01:16.964701+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:01:17.755 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:17 smithi139 bash[43301]: audit 2022-04-28T20:01:16.965418+0000 mon.a (mon.0) 1599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:01:17.755 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:17 smithi139 bash[43301]: audit 2022-04-28T20:01:16.965933+0000 mon.a (mon.0) 1600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:01:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:18 smithi059 bash[61152]: cluster 2022-04-28T20:01:17.140443+0000 mgr.y (mgr.24961) 529 : cluster [DBG] pgmap v878: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:18 smithi059 bash[61152]: audit 2022-04-28T20:01:17.215074+0000 mon.a (mon.0) 1601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:01:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:18 smithi059 bash[61152]: audit 2022-04-28T20:01:17.215446+0000 mon.a (mon.0) 1602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:01:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:18 smithi059 bash[59745]: cluster 2022-04-28T20:01:17.140443+0000 mgr.y (mgr.24961) 529 : cluster [DBG] pgmap v878: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:18 smithi059 bash[59745]: audit 2022-04-28T20:01:17.215074+0000 mon.a (mon.0) 1601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:01:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:18 smithi059 bash[59745]: audit 2022-04-28T20:01:17.215446+0000 mon.a (mon.0) 1602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:01:18.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:18 smithi139 bash[43301]: cluster 2022-04-28T20:01:17.140443+0000 mgr.y (mgr.24961) 529 : cluster [DBG] pgmap v878: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:18 smithi139 bash[43301]: audit 2022-04-28T20:01:17.215074+0000 mon.a (mon.0) 1601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:01:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:18 smithi139 bash[43301]: audit 2022-04-28T20:01:17.215446+0000 mon.a (mon.0) 1602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:01:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:20 smithi059 bash[61152]: cluster 2022-04-28T20:01:19.141352+0000 mgr.y (mgr.24961) 530 : cluster [DBG] pgmap v879: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:20.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:20 smithi059 bash[59745]: cluster 2022-04-28T20:01:19.141352+0000 mgr.y (mgr.24961) 530 : cluster [DBG] pgmap v879: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:20 smithi139 bash[43301]: cluster 2022-04-28T20:01:19.141352+0000 mgr.y (mgr.24961) 530 : cluster [DBG] pgmap v879: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:21.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:22 smithi059 bash[61152]: cluster 2022-04-28T20:01:21.142299+0000 mgr.y (mgr.24961) 531 : cluster [DBG] pgmap v880: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:22.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:22 smithi059 bash[59745]: cluster 2022-04-28T20:01:21.142299+0000 mgr.y (mgr.24961) 531 : cluster [DBG] pgmap v880: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:22 smithi139 bash[43301]: cluster 2022-04-28T20:01:21.142299+0000 mgr.y (mgr.24961) 531 : cluster [DBG] pgmap v880: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:23.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:23 smithi059 bash[59745]: cluster 2022-04-28T20:01:23.142973+0000 mgr.y (mgr.24961) 532 : cluster [DBG] pgmap v881: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:23 smithi059 bash[61152]: cluster 2022-04-28T20:01:23.142973+0000 mgr.y (mgr.24961) 532 : cluster [DBG] pgmap v881: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:23.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:23 smithi139 bash[43301]: cluster 2022-04-28T20:01:23.142973+0000 mgr.y (mgr.24961) 532 : cluster [DBG] pgmap v881: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:26 smithi059 bash[61152]: cluster 2022-04-28T20:01:25.143993+0000 mgr.y (mgr.24961) 533 : cluster [DBG] pgmap v882: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:26 smithi059 bash[59745]: cluster 2022-04-28T20:01:25.143993+0000 mgr.y (mgr.24961) 533 : cluster [DBG] pgmap v882: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:26 smithi139 bash[43301]: cluster 2022-04-28T20:01:25.143993+0000 mgr.y (mgr.24961) 533 : cluster [DBG] pgmap v882: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:28.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:28 smithi059 bash[59745]: cluster 2022-04-28T20:01:27.144928+0000 mgr.y (mgr.24961) 534 : cluster [DBG] pgmap v883: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:28 smithi059 bash[61152]: cluster 2022-04-28T20:01:27.144928+0000 mgr.y (mgr.24961) 534 : cluster [DBG] pgmap v883: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:28 smithi139 bash[43301]: cluster 2022-04-28T20:01:27.144928+0000 mgr.y (mgr.24961) 534 : cluster [DBG] pgmap v883: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:30 smithi059 bash[61152]: cluster 2022-04-28T20:01:29.145849+0000 mgr.y (mgr.24961) 535 : cluster [DBG] pgmap v884: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:30.677 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:30 smithi059 bash[59745]: cluster 2022-04-28T20:01:29.145849+0000 mgr.y (mgr.24961) 535 : cluster [DBG] pgmap v884: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:30.677 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:30 smithi139 bash[43301]: cluster 2022-04-28T20:01:29.145849+0000 mgr.y (mgr.24961) 535 : cluster [DBG] pgmap v884: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:31.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:32.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:32 smithi059 bash[59745]: cluster 2022-04-28T20:01:31.146871+0000 mgr.y (mgr.24961) 536 : cluster [DBG] pgmap v885: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:32.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:32 smithi059 bash[61152]: cluster 2022-04-28T20:01:31.146871+0000 mgr.y (mgr.24961) 536 : cluster [DBG] pgmap v885: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:32.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:32 smithi139 bash[43301]: cluster 2022-04-28T20:01:31.146871+0000 mgr.y (mgr.24961) 536 : cluster [DBG] pgmap v885: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:33 smithi059 bash[61152]: cluster 2022-04-28T20:01:33.147687+0000 mgr.y (mgr.24961) 537 : cluster [DBG] pgmap v886: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:33.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:33 smithi059 bash[59745]: cluster 2022-04-28T20:01:33.147687+0000 mgr.y (mgr.24961) 537 : cluster [DBG] pgmap v886: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:33.553Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:33.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:33 smithi139 bash[43301]: cluster 2022-04-28T20:01:33.147687+0000 mgr.y (mgr.24961) 537 : cluster [DBG] pgmap v886: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:35.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:35 smithi059 bash[61152]: cluster 2022-04-28T20:01:34.555778+0000 mon.a (mon.0) 1603 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2022-04-28T20:01:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:35 smithi059 bash[61152]: audit 2022-04-28T20:01:34.558827+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.59:0/3527758401' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74759-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:35 smithi059 bash[61152]: cluster 2022-04-28T20:01:35.148664+0000 mgr.y (mgr.24961) 538 : cluster [DBG] pgmap v888: 105 pgs: 16 creating+peering, 16 unknown, 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:35 smithi059 bash[59745]: cluster 2022-04-28T20:01:34.555778+0000 mon.a (mon.0) 1603 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2022-04-28T20:01:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:35 smithi059 bash[59745]: audit 2022-04-28T20:01:34.558827+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.59:0/3527758401' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74759-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:35.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:35 smithi059 bash[59745]: cluster 2022-04-28T20:01:35.148664+0000 mgr.y (mgr.24961) 538 : cluster [DBG] pgmap v888: 105 pgs: 16 creating+peering, 16 unknown, 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:35 smithi139 bash[43301]: cluster 2022-04-28T20:01:34.555778+0000 mon.a (mon.0) 1603 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2022-04-28T20:01:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:35 smithi139 bash[43301]: audit 2022-04-28T20:01:34.558827+0000 mon.a (mon.0) 1604 : audit [INF] from='client.? 172.21.15.59:0/3527758401' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74759-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:35 smithi139 bash[43301]: cluster 2022-04-28T20:01:35.148664+0000 mgr.y (mgr.24961) 538 : cluster [DBG] pgmap v888: 105 pgs: 16 creating+peering, 16 unknown, 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:36.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:36 smithi059 bash[59745]: audit 2022-04-28T20:01:35.555747+0000 mon.a (mon.0) 1605 : audit [INF] from='client.? 172.21.15.59:0/3527758401' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74759-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:36.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:36 smithi059 bash[59745]: cluster 2022-04-28T20:01:35.556156+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2022-04-28T20:01:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:36 smithi059 bash[61152]: audit 2022-04-28T20:01:35.555747+0000 mon.a (mon.0) 1605 : audit [INF] from='client.? 172.21.15.59:0/3527758401' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74759-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:36 smithi059 bash[61152]: cluster 2022-04-28T20:01:35.556156+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2022-04-28T20:01:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:36 smithi139 bash[43301]: audit 2022-04-28T20:01:35.555747+0000 mon.a (mon.0) 1605 : audit [INF] from='client.? 172.21.15.59:0/3527758401' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74759-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:36 smithi139 bash[43301]: cluster 2022-04-28T20:01:35.556156+0000 mon.a (mon.0) 1606 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2022-04-28T20:01:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:37 smithi059 bash[61152]: cluster 2022-04-28T20:01:36.570126+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2022-04-28T20:01:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:37 smithi059 bash[61152]: cluster 2022-04-28T20:01:37.149335+0000 mgr.y (mgr.24961) 539 : cluster [DBG] pgmap v891: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:37 smithi059 bash[59745]: cluster 2022-04-28T20:01:36.570126+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2022-04-28T20:01:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:37 smithi059 bash[59745]: cluster 2022-04-28T20:01:37.149335+0000 mgr.y (mgr.24961) 539 : cluster [DBG] pgmap v891: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:37 smithi139 bash[43301]: cluster 2022-04-28T20:01:36.570126+0000 mon.a (mon.0) 1607 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2022-04-28T20:01:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:37 smithi139 bash[43301]: cluster 2022-04-28T20:01:37.149335+0000 mgr.y (mgr.24961) 539 : cluster [DBG] pgmap v891: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:40 smithi059 bash[61152]: cluster 2022-04-28T20:01:39.150230+0000 mgr.y (mgr.24961) 540 : cluster [DBG] pgmap v892: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:40.655 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:40 smithi059 bash[59745]: cluster 2022-04-28T20:01:39.150230+0000 mgr.y (mgr.24961) 540 : cluster [DBG] pgmap v892: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:40.655 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:40 smithi139 bash[43301]: cluster 2022-04-28T20:01:39.150230+0000 mgr.y (mgr.24961) 540 : cluster [DBG] pgmap v892: 73 pgs: 73 active+clean; 24 KiB data, 711 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:41.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:42 smithi059 bash[61152]: cluster 2022-04-28T20:01:41.151265+0000 mgr.y (mgr.24961) 541 : cluster [DBG] pgmap v893: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:42 smithi059 bash[59745]: cluster 2022-04-28T20:01:41.151265+0000 mgr.y (mgr.24961) 541 : cluster [DBG] pgmap v893: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:42 smithi139 bash[43301]: cluster 2022-04-28T20:01:41.151265+0000 mgr.y (mgr.24961) 541 : cluster [DBG] pgmap v893: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:43 smithi059 bash[61152]: cluster 2022-04-28T20:01:43.151955+0000 mgr.y (mgr.24961) 542 : cluster [DBG] pgmap v894: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:43.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:43 smithi059 bash[59745]: cluster 2022-04-28T20:01:43.151955+0000 mgr.y (mgr.24961) 542 : cluster [DBG] pgmap v894: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:43 smithi139 bash[43301]: cluster 2022-04-28T20:01:43.151955+0000 mgr.y (mgr.24961) 542 : cluster [DBG] pgmap v894: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:45 smithi059 bash[59745]: cluster 2022-04-28T20:01:45.152956+0000 mgr.y (mgr.24961) 543 : cluster [DBG] pgmap v895: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:46.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:45 smithi059 bash[61152]: cluster 2022-04-28T20:01:45.152956+0000 mgr.y (mgr.24961) 543 : cluster [DBG] pgmap v895: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:45 smithi139 bash[43301]: cluster 2022-04-28T20:01:45.152956+0000 mgr.y (mgr.24961) 543 : cluster [DBG] pgmap v895: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:47.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[61152]: cluster 2022-04-28T20:01:45.698775+0000 mon.a (mon.0) 1608 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2022-04-28T20:01:47.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[61152]: audit 2022-04-28T20:01:45.700212+0000 mon.c (mon.1) 22 : audit [INF] from='client.? 172.21.15.59:0/1064811987' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[61152]: audit 2022-04-28T20:01:45.704033+0000 mon.a (mon.0) 1609 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[61152]: audit 2022-04-28T20:01:46.704869+0000 mon.a (mon.0) 1610 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[61152]: cluster 2022-04-28T20:01:46.705074+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2022-04-28T20:01:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[59745]: cluster 2022-04-28T20:01:45.698775+0000 mon.a (mon.0) 1608 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2022-04-28T20:01:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[59745]: audit 2022-04-28T20:01:45.700212+0000 mon.c (mon.1) 22 : audit [INF] from='client.? 172.21.15.59:0/1064811987' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[59745]: audit 2022-04-28T20:01:45.704033+0000 mon.a (mon.0) 1609 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[59745]: audit 2022-04-28T20:01:46.704869+0000 mon.a (mon.0) 1610 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:46 smithi059 bash[59745]: cluster 2022-04-28T20:01:46.705074+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2022-04-28T20:01:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:46 smithi139 bash[43301]: cluster 2022-04-28T20:01:45.698775+0000 mon.a (mon.0) 1608 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2022-04-28T20:01:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:46 smithi139 bash[43301]: audit 2022-04-28T20:01:45.700212+0000 mon.c (mon.1) 22 : audit [INF] from='client.? 172.21.15.59:0/1064811987' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:46 smithi139 bash[43301]: audit 2022-04-28T20:01:45.704033+0000 mon.a (mon.0) 1609 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:46 smithi139 bash[43301]: audit 2022-04-28T20:01:46.704869+0000 mon.a (mon.0) 1610 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74779-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:46 smithi139 bash[43301]: cluster 2022-04-28T20:01:46.705074+0000 mon.a (mon.0) 1611 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2022-04-28T20:01:48.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:47 smithi059 bash[61152]: cluster 2022-04-28T20:01:47.153621+0000 mgr.y (mgr.24961) 544 : cluster [DBG] pgmap v898: 105 pgs: 10 creating+peering, 22 unknown, 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:47 smithi059 bash[59745]: cluster 2022-04-28T20:01:47.153621+0000 mgr.y (mgr.24961) 544 : cluster [DBG] pgmap v898: 105 pgs: 10 creating+peering, 22 unknown, 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:48.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:47 smithi139 bash[43301]: cluster 2022-04-28T20:01:47.153621+0000 mgr.y (mgr.24961) 544 : cluster [DBG] pgmap v898: 105 pgs: 10 creating+peering, 22 unknown, 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:49.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:48 smithi059 bash[61152]: cluster 2022-04-28T20:01:47.719634+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2022-04-28T20:01:49.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:48 smithi059 bash[59745]: cluster 2022-04-28T20:01:47.719634+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2022-04-28T20:01:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:48 smithi139 bash[43301]: cluster 2022-04-28T20:01:47.719634+0000 mon.a (mon.0) 1612 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2022-04-28T20:01:50.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:49 smithi059 bash[59745]: cluster 2022-04-28T20:01:49.154513+0000 mgr.y (mgr.24961) 545 : cluster [DBG] pgmap v900: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:49 smithi059 bash[61152]: cluster 2022-04-28T20:01:49.154513+0000 mgr.y (mgr.24961) 545 : cluster [DBG] pgmap v900: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:49 smithi139 bash[43301]: cluster 2022-04-28T20:01:49.154513+0000 mgr.y (mgr.24961) 545 : cluster [DBG] pgmap v900: 73 pgs: 73 active+clean; 24 KiB data, 712 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:51.661Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:52.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:52 smithi059 bash[61152]: cluster 2022-04-28T20:01:51.155474+0000 mgr.y (mgr.24961) 546 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:52 smithi059 bash[59745]: cluster 2022-04-28T20:01:51.155474+0000 mgr.y (mgr.24961) 546 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:52 smithi139 bash[43301]: cluster 2022-04-28T20:01:51.155474+0000 mgr.y (mgr.24961) 546 : cluster [DBG] pgmap v901: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[61152]: audit 2022-04-28T20:01:52.606027+0000 mon.a (mon.0) 1613 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:01:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[61152]: audit 2022-04-28T20:01:52.608969+0000 mon.a (mon.0) 1614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:01:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[61152]: audit 2022-04-28T20:01:52.610850+0000 mon.a (mon.0) 1615 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:01:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[61152]: cephadm 2022-04-28T20:01:52.612567+0000 mgr.y (mgr.24961) 547 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:01:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[61152]: audit 2022-04-28T20:01:52.945373+0000 mon.a (mon.0) 1616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:01:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[61152]: cluster 2022-04-28T20:01:53.156162+0000 mgr.y (mgr.24961) 548 : cluster [DBG] pgmap v902: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[59745]: audit 2022-04-28T20:01:52.606027+0000 mon.a (mon.0) 1613 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:01:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[59745]: audit 2022-04-28T20:01:52.608969+0000 mon.a (mon.0) 1614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:01:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[59745]: audit 2022-04-28T20:01:52.610850+0000 mon.a (mon.0) 1615 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:01:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[59745]: cephadm 2022-04-28T20:01:52.612567+0000 mgr.y (mgr.24961) 547 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:01:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[59745]: audit 2022-04-28T20:01:52.945373+0000 mon.a (mon.0) 1616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:01:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[59745]: cluster 2022-04-28T20:01:53.156162+0000 mgr.y (mgr.24961) 548 : cluster [DBG] pgmap v902: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:53.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:01:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:01:53.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:01:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:53 smithi139 bash[43301]: audit 2022-04-28T20:01:52.606027+0000 mon.a (mon.0) 1613 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:01:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:53 smithi139 bash[43301]: audit 2022-04-28T20:01:52.608969+0000 mon.a (mon.0) 1614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:01:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:53 smithi139 bash[43301]: audit 2022-04-28T20:01:52.610850+0000 mon.a (mon.0) 1615 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:01:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:53 smithi139 bash[43301]: cephadm 2022-04-28T20:01:52.612567+0000 mgr.y (mgr.24961) 547 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:01:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:53 smithi139 bash[43301]: audit 2022-04-28T20:01:52.945373+0000 mon.a (mon.0) 1616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:01:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:53 smithi139 bash[43301]: cluster 2022-04-28T20:01:53.156162+0000 mgr.y (mgr.24961) 548 : cluster [DBG] pgmap v902: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:56.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:56 smithi059 bash[59745]: cluster 2022-04-28T20:01:55.076323+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2022-04-28T20:01:56.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:56 smithi059 bash[59745]: audit 2022-04-28T20:01:55.080983+0000 mon.a (mon.0) 1618 : audit [INF] from='client.? 172.21.15.59:0/2394258810' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74811-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:56 smithi059 bash[59745]: cluster 2022-04-28T20:01:55.156962+0000 mgr.y (mgr.24961) 549 : cluster [DBG] pgmap v904: 105 pgs: 6 creating+peering, 26 unknown, 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:56 smithi059 bash[61152]: cluster 2022-04-28T20:01:55.076323+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2022-04-28T20:01:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:56 smithi059 bash[61152]: audit 2022-04-28T20:01:55.080983+0000 mon.a (mon.0) 1618 : audit [INF] from='client.? 172.21.15.59:0/2394258810' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74811-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:56 smithi059 bash[61152]: cluster 2022-04-28T20:01:55.156962+0000 mgr.y (mgr.24961) 549 : cluster [DBG] pgmap v904: 105 pgs: 6 creating+peering, 26 unknown, 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:56 smithi139 bash[43301]: cluster 2022-04-28T20:01:55.076323+0000 mon.a (mon.0) 1617 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2022-04-28T20:01:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:56 smithi139 bash[43301]: audit 2022-04-28T20:01:55.080983+0000 mon.a (mon.0) 1618 : audit [INF] from='client.? 172.21.15.59:0/2394258810' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74811-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:01:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:56 smithi139 bash[43301]: cluster 2022-04-28T20:01:55.156962+0000 mgr.y (mgr.24961) 549 : cluster [DBG] pgmap v904: 105 pgs: 6 creating+peering, 26 unknown, 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:57.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:57 smithi059 bash[59745]: audit 2022-04-28T20:01:56.078529+0000 mon.a (mon.0) 1619 : audit [INF] from='client.? 172.21.15.59:0/2394258810' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74811-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:57.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:57 smithi059 bash[59745]: cluster 2022-04-28T20:01:56.078852+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2022-04-28T20:01:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:57 smithi059 bash[61152]: audit 2022-04-28T20:01:56.078529+0000 mon.a (mon.0) 1619 : audit [INF] from='client.? 172.21.15.59:0/2394258810' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74811-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:57 smithi059 bash[61152]: cluster 2022-04-28T20:01:56.078852+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2022-04-28T20:01:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:57 smithi139 bash[43301]: audit 2022-04-28T20:01:56.078529+0000 mon.a (mon.0) 1619 : audit [INF] from='client.? 172.21.15.59:0/2394258810' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74811-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:01:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:57 smithi139 bash[43301]: cluster 2022-04-28T20:01:56.078852+0000 mon.a (mon.0) 1620 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2022-04-28T20:01:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:58 smithi139 bash[43301]: cluster 2022-04-28T20:01:57.102642+0000 mon.a (mon.0) 1621 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2022-04-28T20:01:58.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:01:58 smithi139 bash[43301]: cluster 2022-04-28T20:01:57.157620+0000 mgr.y (mgr.24961) 550 : cluster [DBG] pgmap v907: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:58 smithi059 bash[61152]: cluster 2022-04-28T20:01:57.102642+0000 mon.a (mon.0) 1621 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2022-04-28T20:01:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:01:58 smithi059 bash[61152]: cluster 2022-04-28T20:01:57.157620+0000 mgr.y (mgr.24961) 550 : cluster [DBG] pgmap v907: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:01:58.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:58 smithi059 bash[59745]: cluster 2022-04-28T20:01:57.102642+0000 mon.a (mon.0) 1621 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2022-04-28T20:01:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:01:58 smithi059 bash[59745]: cluster 2022-04-28T20:01:57.157620+0000 mgr.y (mgr.24961) 550 : cluster [DBG] pgmap v907: 73 pgs: 73 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:00 smithi059 bash[61152]: cluster 2022-04-28T20:01:59.158488+0000 mgr.y (mgr.24961) 551 : cluster [DBG] pgmap v908: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:00.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:00 smithi059 bash[59745]: cluster 2022-04-28T20:01:59.158488+0000 mgr.y (mgr.24961) 551 : cluster [DBG] pgmap v908: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:00 smithi139 bash[43301]: cluster 2022-04-28T20:01:59.158488+0000 mgr.y (mgr.24961) 551 : cluster [DBG] pgmap v908: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:01.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:02.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:02 smithi059 bash[59745]: cluster 2022-04-28T20:02:01.159567+0000 mgr.y (mgr.24961) 552 : cluster [DBG] pgmap v909: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:02 smithi059 bash[61152]: cluster 2022-04-28T20:02:01.159567+0000 mgr.y (mgr.24961) 552 : cluster [DBG] pgmap v909: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:02 smithi139 bash[43301]: cluster 2022-04-28T20:02:01.159567+0000 mgr.y (mgr.24961) 552 : cluster [DBG] pgmap v909: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:03.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:03 smithi059 bash[59745]: cluster 2022-04-28T20:02:03.160183+0000 mgr.y (mgr.24961) 553 : cluster [DBG] pgmap v910: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:03.980 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:03 smithi059 bash[61152]: cluster 2022-04-28T20:02:03.160183+0000 mgr.y (mgr.24961) 553 : cluster [DBG] pgmap v910: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:03.981 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:03.554Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:03.981 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:03 smithi139 bash[43301]: cluster 2022-04-28T20:02:03.160183+0000 mgr.y (mgr.24961) 553 : cluster [DBG] pgmap v910: 73 pgs: 73 active+clean; 24 KiB data, 722 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:04.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:04 smithi059 bash[59745]: cluster 2022-04-28T20:02:03.557669+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2022-04-28T20:02:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:04 smithi059 bash[59745]: audit 2022-04-28T20:02:03.558908+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.59:0/1606410115' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:04 smithi059 bash[59745]: audit 2022-04-28T20:02:03.559904+0000 mon.a (mon.0) 1623 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:04 smithi059 bash[61152]: cluster 2022-04-28T20:02:03.557669+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2022-04-28T20:02:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:04 smithi059 bash[61152]: audit 2022-04-28T20:02:03.558908+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.59:0/1606410115' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:04 smithi059 bash[61152]: audit 2022-04-28T20:02:03.559904+0000 mon.a (mon.0) 1623 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:04 smithi139 bash[43301]: cluster 2022-04-28T20:02:03.557669+0000 mon.a (mon.0) 1622 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2022-04-28T20:02:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:04 smithi139 bash[43301]: audit 2022-04-28T20:02:03.558908+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.59:0/1606410115' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:04 smithi139 bash[43301]: audit 2022-04-28T20:02:03.559904+0000 mon.a (mon.0) 1623 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:05.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:05 smithi059 bash[61152]: audit 2022-04-28T20:02:04.560187+0000 mon.a (mon.0) 1624 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:05.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:05 smithi059 bash[61152]: cluster 2022-04-28T20:02:04.560396+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2022-04-28T20:02:05.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:05 smithi059 bash[61152]: cluster 2022-04-28T20:02:05.161158+0000 mgr.y (mgr.24961) 554 : cluster [DBG] pgmap v913: 105 pgs: 12 creating+peering, 11 unknown, 82 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:05 smithi059 bash[59745]: audit 2022-04-28T20:02:04.560187+0000 mon.a (mon.0) 1624 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:05 smithi059 bash[59745]: cluster 2022-04-28T20:02:04.560396+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2022-04-28T20:02:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:05 smithi059 bash[59745]: cluster 2022-04-28T20:02:05.161158+0000 mgr.y (mgr.24961) 554 : cluster [DBG] pgmap v913: 105 pgs: 12 creating+peering, 11 unknown, 82 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:05 smithi139 bash[43301]: audit 2022-04-28T20:02:04.560187+0000 mon.a (mon.0) 1624 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74831-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:05 smithi139 bash[43301]: cluster 2022-04-28T20:02:04.560396+0000 mon.a (mon.0) 1625 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2022-04-28T20:02:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:05 smithi139 bash[43301]: cluster 2022-04-28T20:02:05.161158+0000 mgr.y (mgr.24961) 554 : cluster [DBG] pgmap v913: 105 pgs: 12 creating+peering, 11 unknown, 82 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:06.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:06 smithi059 bash[61152]: cluster 2022-04-28T20:02:05.570353+0000 mon.a (mon.0) 1626 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2022-04-28T20:02:06.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:06 smithi059 bash[59745]: cluster 2022-04-28T20:02:05.570353+0000 mon.a (mon.0) 1626 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2022-04-28T20:02:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:06 smithi139 bash[43301]: cluster 2022-04-28T20:02:05.570353+0000 mon.a (mon.0) 1626 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2022-04-28T20:02:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:07 smithi059 bash[61152]: cluster 2022-04-28T20:02:07.162064+0000 mgr.y (mgr.24961) 555 : cluster [DBG] pgmap v915: 73 pgs: 73 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:07.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:07 smithi059 bash[59745]: cluster 2022-04-28T20:02:07.162064+0000 mgr.y (mgr.24961) 555 : cluster [DBG] pgmap v915: 73 pgs: 73 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:07 smithi139 bash[43301]: cluster 2022-04-28T20:02:07.162064+0000 mgr.y (mgr.24961) 555 : cluster [DBG] pgmap v915: 73 pgs: 73 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:10 smithi059 bash[61152]: cluster 2022-04-28T20:02:09.162953+0000 mgr.y (mgr.24961) 556 : cluster [DBG] pgmap v916: 73 pgs: 73 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:10 smithi059 bash[59745]: cluster 2022-04-28T20:02:09.162953+0000 mgr.y (mgr.24961) 556 : cluster [DBG] pgmap v916: 73 pgs: 73 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:10 smithi139 bash[43301]: cluster 2022-04-28T20:02:09.162953+0000 mgr.y (mgr.24961) 556 : cluster [DBG] pgmap v916: 73 pgs: 73 active+clean; 24 KiB data, 723 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:11.662Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:12 smithi059 bash[61152]: cluster 2022-04-28T20:02:11.163938+0000 mgr.y (mgr.24961) 557 : cluster [DBG] pgmap v917: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:12 smithi059 bash[59745]: cluster 2022-04-28T20:02:11.163938+0000 mgr.y (mgr.24961) 557 : cluster [DBG] pgmap v917: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:12 smithi139 bash[43301]: cluster 2022-04-28T20:02:11.163938+0000 mgr.y (mgr.24961) 557 : cluster [DBG] pgmap v917: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:13 smithi059 bash[61152]: cluster 2022-04-28T20:02:13.164637+0000 mgr.y (mgr.24961) 558 : cluster [DBG] pgmap v918: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:13.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:13 smithi059 bash[59745]: cluster 2022-04-28T20:02:13.164637+0000 mgr.y (mgr.24961) 558 : cluster [DBG] pgmap v918: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:13 smithi139 bash[43301]: cluster 2022-04-28T20:02:13.164637+0000 mgr.y (mgr.24961) 558 : cluster [DBG] pgmap v918: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:16 smithi059 bash[61152]: cluster 2022-04-28T20:02:15.165651+0000 mgr.y (mgr.24961) 559 : cluster [DBG] pgmap v919: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:16.721 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:16 smithi059 bash[59745]: cluster 2022-04-28T20:02:15.165651+0000 mgr.y (mgr.24961) 559 : cluster [DBG] pgmap v919: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:16.722 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:16 smithi139 bash[43301]: cluster 2022-04-28T20:02:15.165651+0000 mgr.y (mgr.24961) 559 : cluster [DBG] pgmap v919: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:17.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:17 smithi059 bash[59745]: audit 2022-04-28T20:02:16.973488+0000 mon.a (mon.0) 1627 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:02:17.706 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:17 smithi059 bash[59745]: audit 2022-04-28T20:02:16.974058+0000 mon.a (mon.0) 1628 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:02:17.706 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:17 smithi059 bash[59745]: audit 2022-04-28T20:02:16.974644+0000 mon.a (mon.0) 1629 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:02:17.706 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:17 smithi059 bash[61152]: audit 2022-04-28T20:02:16.973488+0000 mon.a (mon.0) 1627 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:02:17.707 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:17 smithi059 bash[61152]: audit 2022-04-28T20:02:16.974058+0000 mon.a (mon.0) 1628 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:02:17.707 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:17 smithi059 bash[61152]: audit 2022-04-28T20:02:16.974644+0000 mon.a (mon.0) 1629 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:02:17.707 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:17 smithi139 bash[43301]: audit 2022-04-28T20:02:16.973488+0000 mon.a (mon.0) 1627 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:02:17.707 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:17 smithi139 bash[43301]: audit 2022-04-28T20:02:16.974058+0000 mon.a (mon.0) 1628 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:02:17.708 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:17 smithi139 bash[43301]: audit 2022-04-28T20:02:16.974644+0000 mon.a (mon.0) 1629 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:02:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:18 smithi059 bash[61152]: cluster 2022-04-28T20:02:17.166444+0000 mgr.y (mgr.24961) 560 : cluster [DBG] pgmap v920: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:18 smithi059 bash[61152]: audit 2022-04-28T20:02:17.236557+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:02:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:18 smithi059 bash[61152]: audit 2022-04-28T20:02:17.237028+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:02:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:18 smithi059 bash[59745]: cluster 2022-04-28T20:02:17.166444+0000 mgr.y (mgr.24961) 560 : cluster [DBG] pgmap v920: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:18 smithi059 bash[59745]: audit 2022-04-28T20:02:17.236557+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:02:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:18 smithi059 bash[59745]: audit 2022-04-28T20:02:17.237028+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:02:18.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:18 smithi139 bash[43301]: cluster 2022-04-28T20:02:17.166444+0000 mgr.y (mgr.24961) 560 : cluster [DBG] pgmap v920: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:18 smithi139 bash[43301]: audit 2022-04-28T20:02:17.236557+0000 mon.a (mon.0) 1630 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:02:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:18 smithi139 bash[43301]: audit 2022-04-28T20:02:17.237028+0000 mon.a (mon.0) 1631 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:02:20.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:20 smithi059 bash[61152]: cluster 2022-04-28T20:02:19.167328+0000 mgr.y (mgr.24961) 561 : cluster [DBG] pgmap v921: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:20.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:20 smithi059 bash[59745]: cluster 2022-04-28T20:02:19.167328+0000 mgr.y (mgr.24961) 561 : cluster [DBG] pgmap v921: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:20.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:20 smithi139 bash[43301]: cluster 2022-04-28T20:02:19.167328+0000 mgr.y (mgr.24961) 561 : cluster [DBG] pgmap v921: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:21.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:22 smithi059 bash[61152]: cluster 2022-04-28T20:02:21.168364+0000 mgr.y (mgr.24961) 562 : cluster [DBG] pgmap v922: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:22.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:22 smithi059 bash[59745]: cluster 2022-04-28T20:02:21.168364+0000 mgr.y (mgr.24961) 562 : cluster [DBG] pgmap v922: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:22 smithi139 bash[43301]: cluster 2022-04-28T20:02:21.168364+0000 mgr.y (mgr.24961) 562 : cluster [DBG] pgmap v922: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:23.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:23 smithi059 bash[61152]: cluster 2022-04-28T20:02:23.169059+0000 mgr.y (mgr.24961) 563 : cluster [DBG] pgmap v923: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:23.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:23 smithi059 bash[59745]: cluster 2022-04-28T20:02:23.169059+0000 mgr.y (mgr.24961) 563 : cluster [DBG] pgmap v923: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:23 smithi139 bash[43301]: cluster 2022-04-28T20:02:23.169059+0000 mgr.y (mgr.24961) 563 : cluster [DBG] pgmap v923: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:26 smithi059 bash[61152]: cluster 2022-04-28T20:02:25.170059+0000 mgr.y (mgr.24961) 564 : cluster [DBG] pgmap v924: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:26.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:26 smithi059 bash[59745]: cluster 2022-04-28T20:02:25.170059+0000 mgr.y (mgr.24961) 564 : cluster [DBG] pgmap v924: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:26 smithi139 bash[43301]: cluster 2022-04-28T20:02:25.170059+0000 mgr.y (mgr.24961) 564 : cluster [DBG] pgmap v924: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:28 smithi059 bash[61152]: cluster 2022-04-28T20:02:27.170971+0000 mgr.y (mgr.24961) 565 : cluster [DBG] pgmap v925: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:28.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:28 smithi059 bash[59745]: cluster 2022-04-28T20:02:27.170971+0000 mgr.y (mgr.24961) 565 : cluster [DBG] pgmap v925: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:28 smithi139 bash[43301]: cluster 2022-04-28T20:02:27.170971+0000 mgr.y (mgr.24961) 565 : cluster [DBG] pgmap v925: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:30 smithi059 bash[61152]: cluster 2022-04-28T20:02:29.171869+0000 mgr.y (mgr.24961) 566 : cluster [DBG] pgmap v926: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:30 smithi059 bash[59745]: cluster 2022-04-28T20:02:29.171869+0000 mgr.y (mgr.24961) 566 : cluster [DBG] pgmap v926: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:30 smithi139 bash[43301]: cluster 2022-04-28T20:02:29.171869+0000 mgr.y (mgr.24961) 566 : cluster [DBG] pgmap v926: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:31.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:32.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:32 smithi059 bash[61152]: cluster 2022-04-28T20:02:31.172826+0000 mgr.y (mgr.24961) 567 : cluster [DBG] pgmap v927: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:32.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:32 smithi059 bash[59745]: cluster 2022-04-28T20:02:31.172826+0000 mgr.y (mgr.24961) 567 : cluster [DBG] pgmap v927: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:32 smithi139 bash[43301]: cluster 2022-04-28T20:02:31.172826+0000 mgr.y (mgr.24961) 567 : cluster [DBG] pgmap v927: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:33 smithi059 bash[61152]: cluster 2022-04-28T20:02:33.173462+0000 mgr.y (mgr.24961) 568 : cluster [DBG] pgmap v928: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:33.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:33.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:33 smithi059 bash[59745]: cluster 2022-04-28T20:02:33.173462+0000 mgr.y (mgr.24961) 568 : cluster [DBG] pgmap v928: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:33 smithi139 bash[43301]: cluster 2022-04-28T20:02:33.173462+0000 mgr.y (mgr.24961) 568 : cluster [DBG] pgmap v928: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:36.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:36 smithi059 bash[59745]: cluster 2022-04-28T20:02:35.174478+0000 mgr.y (mgr.24961) 569 : cluster [DBG] pgmap v929: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:36.608 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:36 smithi059 bash[61152]: cluster 2022-04-28T20:02:35.174478+0000 mgr.y (mgr.24961) 569 : cluster [DBG] pgmap v929: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:36 smithi139 bash[43301]: cluster 2022-04-28T20:02:35.174478+0000 mgr.y (mgr.24961) 569 : cluster [DBG] pgmap v929: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:37 smithi059 bash[61152]: cluster 2022-04-28T20:02:36.245277+0000 mon.a (mon.0) 1632 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2022-04-28T20:02:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:37 smithi059 bash[61152]: audit 2022-04-28T20:02:36.249407+0000 mon.a (mon.0) 1633 : audit [INF] from='client.? 172.21.15.59:0/443428559' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74851-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:37.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:37 smithi059 bash[59745]: cluster 2022-04-28T20:02:36.245277+0000 mon.a (mon.0) 1632 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2022-04-28T20:02:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:37 smithi059 bash[59745]: audit 2022-04-28T20:02:36.249407+0000 mon.a (mon.0) 1633 : audit [INF] from='client.? 172.21.15.59:0/443428559' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74851-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:37 smithi139 bash[43301]: cluster 2022-04-28T20:02:36.245277+0000 mon.a (mon.0) 1632 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2022-04-28T20:02:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:37 smithi139 bash[43301]: audit 2022-04-28T20:02:36.249407+0000 mon.a (mon.0) 1633 : audit [INF] from='client.? 172.21.15.59:0/443428559' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74851-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:38 smithi059 bash[61152]: cluster 2022-04-28T20:02:37.175185+0000 mgr.y (mgr.24961) 570 : cluster [DBG] pgmap v931: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:38 smithi059 bash[61152]: audit 2022-04-28T20:02:37.248715+0000 mon.a (mon.0) 1634 : audit [INF] from='client.? 172.21.15.59:0/443428559' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74851-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:38 smithi059 bash[61152]: cluster 2022-04-28T20:02:37.248913+0000 mon.a (mon.0) 1635 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2022-04-28T20:02:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:38 smithi059 bash[59745]: cluster 2022-04-28T20:02:37.175185+0000 mgr.y (mgr.24961) 570 : cluster [DBG] pgmap v931: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:38 smithi059 bash[59745]: audit 2022-04-28T20:02:37.248715+0000 mon.a (mon.0) 1634 : audit [INF] from='client.? 172.21.15.59:0/443428559' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74851-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:38 smithi059 bash[59745]: cluster 2022-04-28T20:02:37.248913+0000 mon.a (mon.0) 1635 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2022-04-28T20:02:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:38 smithi139 bash[43301]: cluster 2022-04-28T20:02:37.175185+0000 mgr.y (mgr.24961) 570 : cluster [DBG] pgmap v931: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:38 smithi139 bash[43301]: audit 2022-04-28T20:02:37.248715+0000 mon.a (mon.0) 1634 : audit [INF] from='client.? 172.21.15.59:0/443428559' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74851-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:38 smithi139 bash[43301]: cluster 2022-04-28T20:02:37.248913+0000 mon.a (mon.0) 1635 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2022-04-28T20:02:39.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:39 smithi059 bash[61152]: cluster 2022-04-28T20:02:38.260238+0000 mon.a (mon.0) 1636 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2022-04-28T20:02:39.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:39 smithi059 bash[59745]: cluster 2022-04-28T20:02:38.260238+0000 mon.a (mon.0) 1636 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2022-04-28T20:02:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:39 smithi139 bash[43301]: cluster 2022-04-28T20:02:38.260238+0000 mon.a (mon.0) 1636 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2022-04-28T20:02:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:40 smithi059 bash[61152]: cluster 2022-04-28T20:02:39.176089+0000 mgr.y (mgr.24961) 571 : cluster [DBG] pgmap v934: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:40.648 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:40 smithi059 bash[59745]: cluster 2022-04-28T20:02:39.176089+0000 mgr.y (mgr.24961) 571 : cluster [DBG] pgmap v934: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:40.649 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:40 smithi139 bash[43301]: cluster 2022-04-28T20:02:39.176089+0000 mgr.y (mgr.24961) 571 : cluster [DBG] pgmap v934: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:41.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:42 smithi059 bash[61152]: cluster 2022-04-28T20:02:41.177100+0000 mgr.y (mgr.24961) 572 : cluster [DBG] pgmap v935: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:42.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:42 smithi059 bash[59745]: cluster 2022-04-28T20:02:41.177100+0000 mgr.y (mgr.24961) 572 : cluster [DBG] pgmap v935: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:42 smithi139 bash[43301]: cluster 2022-04-28T20:02:41.177100+0000 mgr.y (mgr.24961) 572 : cluster [DBG] pgmap v935: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:43 smithi059 bash[61152]: cluster 2022-04-28T20:02:43.177753+0000 mgr.y (mgr.24961) 573 : cluster [DBG] pgmap v936: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:43.558Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:43 smithi059 bash[59745]: cluster 2022-04-28T20:02:43.177753+0000 mgr.y (mgr.24961) 573 : cluster [DBG] pgmap v936: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:43.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:43 smithi139 bash[43301]: cluster 2022-04-28T20:02:43.177753+0000 mgr.y (mgr.24961) 573 : cluster [DBG] pgmap v936: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:46.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:46 smithi059 bash[61152]: cluster 2022-04-28T20:02:45.178750+0000 mgr.y (mgr.24961) 574 : cluster [DBG] pgmap v937: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:46.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:46 smithi059 bash[59745]: cluster 2022-04-28T20:02:45.178750+0000 mgr.y (mgr.24961) 574 : cluster [DBG] pgmap v937: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:46 smithi139 bash[43301]: cluster 2022-04-28T20:02:45.178750+0000 mgr.y (mgr.24961) 574 : cluster [DBG] pgmap v937: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:48.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[59745]: cluster 2022-04-28T20:02:47.179589+0000 mgr.y (mgr.24961) 575 : cluster [DBG] pgmap v938: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:48.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[59745]: cluster 2022-04-28T20:02:47.253252+0000 mon.a (mon.0) 1637 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2022-04-28T20:02:48.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[59745]: audit 2022-04-28T20:02:47.256172+0000 mon.c (mon.1) 24 : audit [INF] from='client.? 172.21.15.59:0/2182481511' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[59745]: audit 2022-04-28T20:02:47.256817+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[61152]: cluster 2022-04-28T20:02:47.179589+0000 mgr.y (mgr.24961) 575 : cluster [DBG] pgmap v938: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[61152]: cluster 2022-04-28T20:02:47.253252+0000 mon.a (mon.0) 1637 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2022-04-28T20:02:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[61152]: audit 2022-04-28T20:02:47.256172+0000 mon.c (mon.1) 24 : audit [INF] from='client.? 172.21.15.59:0/2182481511' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:48 smithi059 bash[61152]: audit 2022-04-28T20:02:47.256817+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:48 smithi139 bash[43301]: cluster 2022-04-28T20:02:47.179589+0000 mgr.y (mgr.24961) 575 : cluster [DBG] pgmap v938: 73 pgs: 73 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:48 smithi139 bash[43301]: cluster 2022-04-28T20:02:47.253252+0000 mon.a (mon.0) 1637 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2022-04-28T20:02:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:48 smithi139 bash[43301]: audit 2022-04-28T20:02:47.256172+0000 mon.c (mon.1) 24 : audit [INF] from='client.? 172.21.15.59:0/2182481511' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:48 smithi139 bash[43301]: audit 2022-04-28T20:02:47.256817+0000 mon.a (mon.0) 1638 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:02:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:49 smithi059 bash[61152]: audit 2022-04-28T20:02:48.255765+0000 mon.a (mon.0) 1639 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:49 smithi059 bash[61152]: cluster 2022-04-28T20:02:48.256059+0000 mon.a (mon.0) 1640 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2022-04-28T20:02:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:49 smithi059 bash[59745]: audit 2022-04-28T20:02:48.255765+0000 mon.a (mon.0) 1639 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:49 smithi059 bash[59745]: cluster 2022-04-28T20:02:48.256059+0000 mon.a (mon.0) 1640 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2022-04-28T20:02:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:49 smithi139 bash[43301]: audit 2022-04-28T20:02:48.255765+0000 mon.a (mon.0) 1639 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74871-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:02:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:49 smithi139 bash[43301]: cluster 2022-04-28T20:02:48.256059+0000 mon.a (mon.0) 1640 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2022-04-28T20:02:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:50 smithi059 bash[59745]: cluster 2022-04-28T20:02:49.180517+0000 mgr.y (mgr.24961) 576 : cluster [DBG] pgmap v941: 105 pgs: 21 unknown, 84 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:02:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:50 smithi059 bash[59745]: cluster 2022-04-28T20:02:49.276492+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2022-04-28T20:02:50.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:50 smithi059 bash[61152]: cluster 2022-04-28T20:02:49.180517+0000 mgr.y (mgr.24961) 576 : cluster [DBG] pgmap v941: 105 pgs: 21 unknown, 84 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:02:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:50 smithi059 bash[61152]: cluster 2022-04-28T20:02:49.276492+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2022-04-28T20:02:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:50 smithi139 bash[43301]: cluster 2022-04-28T20:02:49.180517+0000 mgr.y (mgr.24961) 576 : cluster [DBG] pgmap v941: 105 pgs: 21 unknown, 84 active+clean; 24 KiB data, 733 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:02:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:50 smithi139 bash[43301]: cluster 2022-04-28T20:02:49.276492+0000 mon.a (mon.0) 1641 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2022-04-28T20:02:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:51.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:52.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:52 smithi059 bash[61152]: cluster 2022-04-28T20:02:51.181486+0000 mgr.y (mgr.24961) 577 : cluster [DBG] pgmap v943: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:52 smithi059 bash[59745]: cluster 2022-04-28T20:02:51.181486+0000 mgr.y (mgr.24961) 577 : cluster [DBG] pgmap v943: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:52 smithi139 bash[43301]: cluster 2022-04-28T20:02:51.181486+0000 mgr.y (mgr.24961) 577 : cluster [DBG] pgmap v943: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:53.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:02:53.557Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:02:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[61152]: audit 2022-04-28T20:02:52.953221+0000 mon.a (mon.0) 1642 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:02:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[61152]: audit 2022-04-28T20:02:52.956210+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:02:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[61152]: audit 2022-04-28T20:02:52.958124+0000 mon.a (mon.0) 1644 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:02:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[61152]: cephadm 2022-04-28T20:02:52.959859+0000 mgr.y (mgr.24961) 578 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:02:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[61152]: cluster 2022-04-28T20:02:53.182187+0000 mgr.y (mgr.24961) 579 : cluster [DBG] pgmap v944: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[61152]: audit 2022-04-28T20:02:53.282172+0000 mon.a (mon.0) 1645 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:02:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[59745]: audit 2022-04-28T20:02:52.953221+0000 mon.a (mon.0) 1642 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:02:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[59745]: audit 2022-04-28T20:02:52.956210+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:02:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[59745]: audit 2022-04-28T20:02:52.958124+0000 mon.a (mon.0) 1644 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:02:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[59745]: cephadm 2022-04-28T20:02:52.959859+0000 mgr.y (mgr.24961) 578 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:02:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[59745]: cluster 2022-04-28T20:02:53.182187+0000 mgr.y (mgr.24961) 579 : cluster [DBG] pgmap v944: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:53 smithi059 bash[59745]: audit 2022-04-28T20:02:53.282172+0000 mon.a (mon.0) 1645 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:02:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:53 smithi139 bash[43301]: audit 2022-04-28T20:02:52.953221+0000 mon.a (mon.0) 1642 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:02:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:53 smithi139 bash[43301]: audit 2022-04-28T20:02:52.956210+0000 mon.a (mon.0) 1643 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:02:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:53 smithi139 bash[43301]: audit 2022-04-28T20:02:52.958124+0000 mon.a (mon.0) 1644 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:02:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:53 smithi139 bash[43301]: cephadm 2022-04-28T20:02:52.959859+0000 mgr.y (mgr.24961) 578 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:02:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:53 smithi139 bash[43301]: cluster 2022-04-28T20:02:53.182187+0000 mgr.y (mgr.24961) 579 : cluster [DBG] pgmap v944: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:53 smithi139 bash[43301]: audit 2022-04-28T20:02:53.282172+0000 mon.a (mon.0) 1645 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:02:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:56 smithi059 bash[61152]: cluster 2022-04-28T20:02:55.183161+0000 mgr.y (mgr.24961) 580 : cluster [DBG] pgmap v945: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:56.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:56 smithi059 bash[59745]: cluster 2022-04-28T20:02:55.183161+0000 mgr.y (mgr.24961) 580 : cluster [DBG] pgmap v945: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:56 smithi139 bash[43301]: cluster 2022-04-28T20:02:55.183161+0000 mgr.y (mgr.24961) 580 : cluster [DBG] pgmap v945: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:02:58 smithi059 bash[61152]: cluster 2022-04-28T20:02:57.184002+0000 mgr.y (mgr.24961) 581 : cluster [DBG] pgmap v946: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:58.616 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:02:58 smithi059 bash[59745]: cluster 2022-04-28T20:02:57.184002+0000 mgr.y (mgr.24961) 581 : cluster [DBG] pgmap v946: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:02:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:02:58 smithi139 bash[43301]: cluster 2022-04-28T20:02:57.184002+0000 mgr.y (mgr.24961) 581 : cluster [DBG] pgmap v946: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:00 smithi059 bash[61152]: cluster 2022-04-28T20:02:59.184873+0000 mgr.y (mgr.24961) 582 : cluster [DBG] pgmap v947: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:00.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:00 smithi059 bash[59745]: cluster 2022-04-28T20:02:59.184873+0000 mgr.y (mgr.24961) 582 : cluster [DBG] pgmap v947: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:00.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:00 smithi139 bash[43301]: cluster 2022-04-28T20:02:59.184873+0000 mgr.y (mgr.24961) 582 : cluster [DBG] pgmap v947: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:01.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:02 smithi059 bash[61152]: cluster 2022-04-28T20:03:01.185822+0000 mgr.y (mgr.24961) 583 : cluster [DBG] pgmap v948: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:02.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:02 smithi059 bash[59745]: cluster 2022-04-28T20:03:01.185822+0000 mgr.y (mgr.24961) 583 : cluster [DBG] pgmap v948: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:02 smithi139 bash[43301]: cluster 2022-04-28T20:03:01.185822+0000 mgr.y (mgr.24961) 583 : cluster [DBG] pgmap v948: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:03 smithi059 bash[61152]: cluster 2022-04-28T20:03:03.186582+0000 mgr.y (mgr.24961) 584 : cluster [DBG] pgmap v949: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:03.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:03 smithi059 bash[59745]: cluster 2022-04-28T20:03:03.186582+0000 mgr.y (mgr.24961) 584 : cluster [DBG] pgmap v949: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:03 smithi139 bash[43301]: cluster 2022-04-28T20:03:03.186582+0000 mgr.y (mgr.24961) 584 : cluster [DBG] pgmap v949: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:06.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:06 smithi059 bash[61152]: cluster 2022-04-28T20:03:05.187638+0000 mgr.y (mgr.24961) 585 : cluster [DBG] pgmap v950: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:06.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:06 smithi059 bash[59745]: cluster 2022-04-28T20:03:05.187638+0000 mgr.y (mgr.24961) 585 : cluster [DBG] pgmap v950: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:06 smithi139 bash[43301]: cluster 2022-04-28T20:03:05.187638+0000 mgr.y (mgr.24961) 585 : cluster [DBG] pgmap v950: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:08 smithi059 bash[61152]: cluster 2022-04-28T20:03:07.188481+0000 mgr.y (mgr.24961) 586 : cluster [DBG] pgmap v951: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:08.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:08 smithi059 bash[59745]: cluster 2022-04-28T20:03:07.188481+0000 mgr.y (mgr.24961) 586 : cluster [DBG] pgmap v951: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:08 smithi139 bash[43301]: cluster 2022-04-28T20:03:07.188481+0000 mgr.y (mgr.24961) 586 : cluster [DBG] pgmap v951: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:10 smithi059 bash[61152]: cluster 2022-04-28T20:03:09.189336+0000 mgr.y (mgr.24961) 587 : cluster [DBG] pgmap v952: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:10 smithi059 bash[59745]: cluster 2022-04-28T20:03:09.189336+0000 mgr.y (mgr.24961) 587 : cluster [DBG] pgmap v952: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:10 smithi139 bash[43301]: cluster 2022-04-28T20:03:09.189336+0000 mgr.y (mgr.24961) 587 : cluster [DBG] pgmap v952: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:11.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:12 smithi059 bash[61152]: cluster 2022-04-28T20:03:11.190280+0000 mgr.y (mgr.24961) 588 : cluster [DBG] pgmap v953: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:12.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:12 smithi059 bash[59745]: cluster 2022-04-28T20:03:11.190280+0000 mgr.y (mgr.24961) 588 : cluster [DBG] pgmap v953: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:12 smithi139 bash[43301]: cluster 2022-04-28T20:03:11.190280+0000 mgr.y (mgr.24961) 588 : cluster [DBG] pgmap v953: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:13 smithi059 bash[61152]: cluster 2022-04-28T20:03:13.190874+0000 mgr.y (mgr.24961) 589 : cluster [DBG] pgmap v954: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:13.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:13.559Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:13 smithi059 bash[59745]: cluster 2022-04-28T20:03:13.190874+0000 mgr.y (mgr.24961) 589 : cluster [DBG] pgmap v954: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:13 smithi139 bash[43301]: cluster 2022-04-28T20:03:13.190874+0000 mgr.y (mgr.24961) 589 : cluster [DBG] pgmap v954: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:16.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:16 smithi059 bash[59745]: cluster 2022-04-28T20:03:15.191872+0000 mgr.y (mgr.24961) 590 : cluster [DBG] pgmap v955: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:16.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:16 smithi059 bash[61152]: cluster 2022-04-28T20:03:15.191872+0000 mgr.y (mgr.24961) 590 : cluster [DBG] pgmap v955: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:16.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:16 smithi139 bash[43301]: cluster 2022-04-28T20:03:15.191872+0000 mgr.y (mgr.24961) 590 : cluster [DBG] pgmap v955: 73 pgs: 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[61152]: cluster 2022-04-28T20:03:16.474705+0000 mon.a (mon.0) 1646 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2022-04-28T20:03:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[61152]: audit 2022-04-28T20:03:16.477339+0000 mon.a (mon.0) 1647 : audit [INF] from='client.? 172.21.15.59:0/408815113' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74901-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:03:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[61152]: cluster 2022-04-28T20:03:17.192538+0000 mgr.y (mgr.24961) 591 : cluster [DBG] pgmap v957: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[61152]: audit 2022-04-28T20:03:17.247626+0000 mon.a (mon.0) 1648 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:03:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[61152]: audit 2022-04-28T20:03:17.248574+0000 mon.a (mon.0) 1649 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:03:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[59745]: cluster 2022-04-28T20:03:16.474705+0000 mon.a (mon.0) 1646 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2022-04-28T20:03:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[59745]: audit 2022-04-28T20:03:16.477339+0000 mon.a (mon.0) 1647 : audit [INF] from='client.? 172.21.15.59:0/408815113' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74901-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:03:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[59745]: cluster 2022-04-28T20:03:17.192538+0000 mgr.y (mgr.24961) 591 : cluster [DBG] pgmap v957: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[59745]: audit 2022-04-28T20:03:17.247626+0000 mon.a (mon.0) 1648 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:03:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:17 smithi059 bash[59745]: audit 2022-04-28T20:03:17.248574+0000 mon.a (mon.0) 1649 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:03:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:17 smithi139 bash[43301]: cluster 2022-04-28T20:03:16.474705+0000 mon.a (mon.0) 1646 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2022-04-28T20:03:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:17 smithi139 bash[43301]: audit 2022-04-28T20:03:16.477339+0000 mon.a (mon.0) 1647 : audit [INF] from='client.? 172.21.15.59:0/408815113' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74901-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:03:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:17 smithi139 bash[43301]: cluster 2022-04-28T20:03:17.192538+0000 mgr.y (mgr.24961) 591 : cluster [DBG] pgmap v957: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:03:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:17 smithi139 bash[43301]: audit 2022-04-28T20:03:17.247626+0000 mon.a (mon.0) 1648 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:03:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:17 smithi139 bash[43301]: audit 2022-04-28T20:03:17.248574+0000 mon.a (mon.0) 1649 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:03:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:18 smithi059 bash[61152]: audit 2022-04-28T20:03:17.480263+0000 mon.a (mon.0) 1650 : audit [INF] from='client.? 172.21.15.59:0/408815113' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74901-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:03:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:18 smithi059 bash[61152]: cluster 2022-04-28T20:03:17.480482+0000 mon.a (mon.0) 1651 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2022-04-28T20:03:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:18 smithi059 bash[59745]: audit 2022-04-28T20:03:17.480263+0000 mon.a (mon.0) 1650 : audit [INF] from='client.? 172.21.15.59:0/408815113' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74901-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:03:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:18 smithi059 bash[59745]: cluster 2022-04-28T20:03:17.480482+0000 mon.a (mon.0) 1651 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2022-04-28T20:03:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:18 smithi139 bash[43301]: audit 2022-04-28T20:03:17.480263+0000 mon.a (mon.0) 1650 : audit [INF] from='client.? 172.21.15.59:0/408815113' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74901-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:03:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:18 smithi139 bash[43301]: cluster 2022-04-28T20:03:17.480482+0000 mon.a (mon.0) 1651 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2022-04-28T20:03:19.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:19 smithi059 bash[61152]: cluster 2022-04-28T20:03:18.500366+0000 mon.a (mon.0) 1652 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2022-04-28T20:03:19.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:19 smithi059 bash[61152]: cluster 2022-04-28T20:03:19.193104+0000 mgr.y (mgr.24961) 592 : cluster [DBG] pgmap v960: 73 pgs: 73 active+clean; 24 KiB data, 738 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:19.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:19 smithi059 bash[59745]: cluster 2022-04-28T20:03:18.500366+0000 mon.a (mon.0) 1652 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2022-04-28T20:03:19.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:19 smithi059 bash[59745]: cluster 2022-04-28T20:03:19.193104+0000 mgr.y (mgr.24961) 592 : cluster [DBG] pgmap v960: 73 pgs: 73 active+clean; 24 KiB data, 738 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:19.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:19 smithi139 bash[43301]: cluster 2022-04-28T20:03:18.500366+0000 mon.a (mon.0) 1652 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2022-04-28T20:03:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:19 smithi139 bash[43301]: cluster 2022-04-28T20:03:19.193104+0000 mgr.y (mgr.24961) 592 : cluster [DBG] pgmap v960: 73 pgs: 73 active+clean; 24 KiB data, 738 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:21.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:22.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:22 smithi059 bash[59745]: cluster 2022-04-28T20:03:21.193976+0000 mgr.y (mgr.24961) 593 : cluster [DBG] pgmap v961: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:22 smithi059 bash[61152]: cluster 2022-04-28T20:03:21.193976+0000 mgr.y (mgr.24961) 593 : cluster [DBG] pgmap v961: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:22 smithi139 bash[43301]: cluster 2022-04-28T20:03:21.193976+0000 mgr.y (mgr.24961) 593 : cluster [DBG] pgmap v961: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:23.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:23 smithi059 bash[61152]: cluster 2022-04-28T20:03:23.194642+0000 mgr.y (mgr.24961) 594 : cluster [DBG] pgmap v962: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:23.560Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:23 smithi059 bash[59745]: cluster 2022-04-28T20:03:23.194642+0000 mgr.y (mgr.24961) 594 : cluster [DBG] pgmap v962: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:23 smithi139 bash[43301]: cluster 2022-04-28T20:03:23.194642+0000 mgr.y (mgr.24961) 594 : cluster [DBG] pgmap v962: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:26.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:26 smithi059 bash[59745]: cluster 2022-04-28T20:03:25.195646+0000 mgr.y (mgr.24961) 595 : cluster [DBG] pgmap v963: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:26 smithi059 bash[61152]: cluster 2022-04-28T20:03:25.195646+0000 mgr.y (mgr.24961) 595 : cluster [DBG] pgmap v963: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:26 smithi139 bash[43301]: cluster 2022-04-28T20:03:25.195646+0000 mgr.y (mgr.24961) 595 : cluster [DBG] pgmap v963: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:28 smithi059 bash[61152]: cluster 2022-04-28T20:03:27.196469+0000 mgr.y (mgr.24961) 596 : cluster [DBG] pgmap v964: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:28.700 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:28 smithi059 bash[59745]: cluster 2022-04-28T20:03:27.196469+0000 mgr.y (mgr.24961) 596 : cluster [DBG] pgmap v964: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:28.701 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:28 smithi139 bash[43301]: cluster 2022-04-28T20:03:27.196469+0000 mgr.y (mgr.24961) 596 : cluster [DBG] pgmap v964: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:30 smithi059 bash[61152]: cluster 2022-04-28T20:03:29.197325+0000 mgr.y (mgr.24961) 597 : cluster [DBG] pgmap v965: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:30 smithi059 bash[59745]: cluster 2022-04-28T20:03:29.197325+0000 mgr.y (mgr.24961) 597 : cluster [DBG] pgmap v965: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:30 smithi139 bash[43301]: cluster 2022-04-28T20:03:29.197325+0000 mgr.y (mgr.24961) 597 : cluster [DBG] pgmap v965: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:31.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:32.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:32 smithi059 bash[61152]: cluster 2022-04-28T20:03:31.198276+0000 mgr.y (mgr.24961) 598 : cluster [DBG] pgmap v966: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:32.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:32 smithi059 bash[59745]: cluster 2022-04-28T20:03:31.198276+0000 mgr.y (mgr.24961) 598 : cluster [DBG] pgmap v966: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:32 smithi139 bash[43301]: cluster 2022-04-28T20:03:31.198276+0000 mgr.y (mgr.24961) 598 : cluster [DBG] pgmap v966: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:33 smithi059 bash[61152]: cluster 2022-04-28T20:03:33.198943+0000 mgr.y (mgr.24961) 599 : cluster [DBG] pgmap v967: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:33.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:33 smithi059 bash[59745]: cluster 2022-04-28T20:03:33.198943+0000 mgr.y (mgr.24961) 599 : cluster [DBG] pgmap v967: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:33 smithi139 bash[43301]: cluster 2022-04-28T20:03:33.198943+0000 mgr.y (mgr.24961) 599 : cluster [DBG] pgmap v967: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:36.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:36 smithi059 bash[61152]: cluster 2022-04-28T20:03:35.200091+0000 mgr.y (mgr.24961) 600 : cluster [DBG] pgmap v968: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:36.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:36 smithi059 bash[59745]: cluster 2022-04-28T20:03:35.200091+0000 mgr.y (mgr.24961) 600 : cluster [DBG] pgmap v968: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:36 smithi139 bash[43301]: cluster 2022-04-28T20:03:35.200091+0000 mgr.y (mgr.24961) 600 : cluster [DBG] pgmap v968: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:38 smithi059 bash[61152]: cluster 2022-04-28T20:03:37.200946+0000 mgr.y (mgr.24961) 601 : cluster [DBG] pgmap v969: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:38.660 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:38 smithi059 bash[59745]: cluster 2022-04-28T20:03:37.200946+0000 mgr.y (mgr.24961) 601 : cluster [DBG] pgmap v969: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:38.660 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:38 smithi139 bash[43301]: cluster 2022-04-28T20:03:37.200946+0000 mgr.y (mgr.24961) 601 : cluster [DBG] pgmap v969: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:40 smithi059 bash[61152]: cluster 2022-04-28T20:03:39.201825+0000 mgr.y (mgr.24961) 602 : cluster [DBG] pgmap v970: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:40 smithi059 bash[59745]: cluster 2022-04-28T20:03:39.201825+0000 mgr.y (mgr.24961) 602 : cluster [DBG] pgmap v970: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:40 smithi139 bash[43301]: cluster 2022-04-28T20:03:39.201825+0000 mgr.y (mgr.24961) 602 : cluster [DBG] pgmap v970: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:41.669Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:42 smithi059 bash[61152]: cluster 2022-04-28T20:03:41.202764+0000 mgr.y (mgr.24961) 603 : cluster [DBG] pgmap v971: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:42.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:42 smithi059 bash[59745]: cluster 2022-04-28T20:03:41.202764+0000 mgr.y (mgr.24961) 603 : cluster [DBG] pgmap v971: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:42 smithi139 bash[43301]: cluster 2022-04-28T20:03:41.202764+0000 mgr.y (mgr.24961) 603 : cluster [DBG] pgmap v971: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:43 smithi059 bash[61152]: cluster 2022-04-28T20:03:43.203424+0000 mgr.y (mgr.24961) 604 : cluster [DBG] pgmap v972: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:43.561Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:43 smithi059 bash[59745]: cluster 2022-04-28T20:03:43.203424+0000 mgr.y (mgr.24961) 604 : cluster [DBG] pgmap v972: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:43 smithi139 bash[43301]: cluster 2022-04-28T20:03:43.203424+0000 mgr.y (mgr.24961) 604 : cluster [DBG] pgmap v972: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:46.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:46 smithi059 bash[61152]: cluster 2022-04-28T20:03:45.204384+0000 mgr.y (mgr.24961) 605 : cluster [DBG] pgmap v973: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:46.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:46 smithi059 bash[59745]: cluster 2022-04-28T20:03:45.204384+0000 mgr.y (mgr.24961) 605 : cluster [DBG] pgmap v973: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:46 smithi139 bash[43301]: cluster 2022-04-28T20:03:45.204384+0000 mgr.y (mgr.24961) 605 : cluster [DBG] pgmap v973: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:48 smithi059 bash[61152]: cluster 2022-04-28T20:03:47.204931+0000 mgr.y (mgr.24961) 606 : cluster [DBG] pgmap v974: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:48.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:48 smithi059 bash[59745]: cluster 2022-04-28T20:03:47.204931+0000 mgr.y (mgr.24961) 606 : cluster [DBG] pgmap v974: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:48.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:48 smithi139 bash[43301]: cluster 2022-04-28T20:03:47.204931+0000 mgr.y (mgr.24961) 606 : cluster [DBG] pgmap v974: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:50.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:50 smithi059 bash[61152]: cluster 2022-04-28T20:03:49.205829+0000 mgr.y (mgr.24961) 607 : cluster [DBG] pgmap v975: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:50 smithi059 bash[59745]: cluster 2022-04-28T20:03:49.205829+0000 mgr.y (mgr.24961) 607 : cluster [DBG] pgmap v975: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:50 smithi139 bash[43301]: cluster 2022-04-28T20:03:49.205829+0000 mgr.y (mgr.24961) 607 : cluster [DBG] pgmap v975: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:51.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:52.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:52 smithi059 bash[61152]: cluster 2022-04-28T20:03:51.206775+0000 mgr.y (mgr.24961) 608 : cluster [DBG] pgmap v976: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:52.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:52 smithi059 bash[59745]: cluster 2022-04-28T20:03:51.206775+0000 mgr.y (mgr.24961) 608 : cluster [DBG] pgmap v976: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:52 smithi139 bash[43301]: cluster 2022-04-28T20:03:51.206775+0000 mgr.y (mgr.24961) 608 : cluster [DBG] pgmap v976: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:53.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[59745]: cluster 2022-04-28T20:03:53.207417+0000 mgr.y (mgr.24961) 609 : cluster [DBG] pgmap v977: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:53.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[59745]: audit 2022-04-28T20:03:53.292152+0000 mon.a (mon.0) 1653 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:03:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[59745]: audit 2022-04-28T20:03:53.294997+0000 mon.a (mon.0) 1654 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:03:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[59745]: audit 2022-04-28T20:03:53.296876+0000 mon.a (mon.0) 1655 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:03:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[59745]: cephadm 2022-04-28T20:03:53.298483+0000 mgr.y (mgr.24961) 610 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:03:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[61152]: cluster 2022-04-28T20:03:53.207417+0000 mgr.y (mgr.24961) 609 : cluster [DBG] pgmap v977: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[61152]: audit 2022-04-28T20:03:53.292152+0000 mon.a (mon.0) 1653 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:03:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[61152]: audit 2022-04-28T20:03:53.294997+0000 mon.a (mon.0) 1654 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:03:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[61152]: audit 2022-04-28T20:03:53.296876+0000 mon.a (mon.0) 1655 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:03:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[61152]: cephadm 2022-04-28T20:03:53.298483+0000 mgr.y (mgr.24961) 610 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:03:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:03:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:03:53.556Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:03:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:53 smithi139 bash[43301]: cluster 2022-04-28T20:03:53.207417+0000 mgr.y (mgr.24961) 609 : cluster [DBG] pgmap v977: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:53 smithi139 bash[43301]: audit 2022-04-28T20:03:53.292152+0000 mon.a (mon.0) 1653 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:03:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:53 smithi139 bash[43301]: audit 2022-04-28T20:03:53.294997+0000 mon.a (mon.0) 1654 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:03:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:53 smithi139 bash[43301]: audit 2022-04-28T20:03:53.296876+0000 mon.a (mon.0) 1655 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:03:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:53 smithi139 bash[43301]: cephadm 2022-04-28T20:03:53.298483+0000 mgr.y (mgr.24961) 610 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:03:54.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:54 smithi139 bash[43301]: audit 2022-04-28T20:03:53.617585+0000 mon.a (mon.0) 1656 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:03:55.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:54 smithi059 bash[61152]: audit 2022-04-28T20:03:53.617585+0000 mon.a (mon.0) 1656 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:03:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:54 smithi059 bash[59745]: audit 2022-04-28T20:03:53.617585+0000 mon.a (mon.0) 1656 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:03:56.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:55 smithi059 bash[59745]: cluster 2022-04-28T20:03:55.208432+0000 mgr.y (mgr.24961) 611 : cluster [DBG] pgmap v978: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:56.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:55 smithi059 bash[59745]: audit 2022-04-28T20:03:55.614997+0000 mon.a (mon.0) 1657 : audit [DBG] from='client.? 172.21.15.59:0/3806367574' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:03:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:55 smithi059 bash[61152]: cluster 2022-04-28T20:03:55.208432+0000 mgr.y (mgr.24961) 611 : cluster [DBG] pgmap v978: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:55 smithi059 bash[61152]: audit 2022-04-28T20:03:55.614997+0000 mon.a (mon.0) 1657 : audit [DBG] from='client.? 172.21.15.59:0/3806367574' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:03:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:55 smithi139 bash[43301]: cluster 2022-04-28T20:03:55.208432+0000 mgr.y (mgr.24961) 611 : cluster [DBG] pgmap v978: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:55 smithi139 bash[43301]: audit 2022-04-28T20:03:55.614997+0000 mon.a (mon.0) 1657 : audit [DBG] from='client.? 172.21.15.59:0/3806367574' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:03:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:03:58 smithi059 bash[61152]: cluster 2022-04-28T20:03:57.209258+0000 mgr.y (mgr.24961) 612 : cluster [DBG] pgmap v979: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:03:58 smithi059 bash[59745]: cluster 2022-04-28T20:03:57.209258+0000 mgr.y (mgr.24961) 612 : cluster [DBG] pgmap v979: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:03:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:03:58 smithi139 bash[43301]: cluster 2022-04-28T20:03:57.209258+0000 mgr.y (mgr.24961) 612 : cluster [DBG] pgmap v979: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:00.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:00 smithi059 bash[61152]: cluster 2022-04-28T20:03:59.210147+0000 mgr.y (mgr.24961) 613 : cluster [DBG] pgmap v980: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:00.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:00 smithi059 bash[59745]: cluster 2022-04-28T20:03:59.210147+0000 mgr.y (mgr.24961) 613 : cluster [DBG] pgmap v980: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:00 smithi139 bash[43301]: cluster 2022-04-28T20:03:59.210147+0000 mgr.y (mgr.24961) 613 : cluster [DBG] pgmap v980: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:01.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:02.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:02 smithi059 bash[59745]: cluster 2022-04-28T20:04:01.211177+0000 mgr.y (mgr.24961) 614 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:02 smithi059 bash[61152]: cluster 2022-04-28T20:04:01.211177+0000 mgr.y (mgr.24961) 614 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:02 smithi139 bash[43301]: cluster 2022-04-28T20:04:01.211177+0000 mgr.y (mgr.24961) 614 : cluster [DBG] pgmap v981: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:03.562 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:03 smithi059 bash[61152]: audit 2022-04-28T20:04:02.424725+0000 mon.a (mon.0) 1658 : audit [DBG] from='client.? 172.21.15.59:0/2820212101' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:04:03.563 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:03 smithi059 bash[59745]: audit 2022-04-28T20:04:02.424725+0000 mon.a (mon.0) 1658 : audit [DBG] from='client.? 172.21.15.59:0/2820212101' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:04:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:03 smithi139 bash[43301]: audit 2022-04-28T20:04:02.424725+0000 mon.a (mon.0) 1658 : audit [DBG] from='client.? 172.21.15.59:0/2820212101' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:04:03.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:03.562Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:04.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:04 smithi059 bash[61152]: cluster 2022-04-28T20:04:03.211888+0000 mgr.y (mgr.24961) 615 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:04 smithi059 bash[59745]: cluster 2022-04-28T20:04:03.211888+0000 mgr.y (mgr.24961) 615 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:04 smithi139 bash[43301]: cluster 2022-04-28T20:04:03.211888+0000 mgr.y (mgr.24961) 615 : cluster [DBG] pgmap v982: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:06.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:06 smithi059 bash[61152]: cluster 2022-04-28T20:04:05.212907+0000 mgr.y (mgr.24961) 616 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:06 smithi059 bash[59745]: cluster 2022-04-28T20:04:05.212907+0000 mgr.y (mgr.24961) 616 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:06 smithi139 bash[43301]: cluster 2022-04-28T20:04:05.212907+0000 mgr.y (mgr.24961) 616 : cluster [DBG] pgmap v983: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:08 smithi059 bash[61152]: cluster 2022-04-28T20:04:07.213723+0000 mgr.y (mgr.24961) 617 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:08.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:08 smithi059 bash[59745]: cluster 2022-04-28T20:04:07.213723+0000 mgr.y (mgr.24961) 617 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:08 smithi139 bash[43301]: cluster 2022-04-28T20:04:07.213723+0000 mgr.y (mgr.24961) 617 : cluster [DBG] pgmap v984: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:10 smithi059 bash[61152]: cluster 2022-04-28T20:04:09.214599+0000 mgr.y (mgr.24961) 618 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:10 smithi059 bash[59745]: cluster 2022-04-28T20:04:09.214599+0000 mgr.y (mgr.24961) 618 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:10.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:10 smithi139 bash[43301]: cluster 2022-04-28T20:04:09.214599+0000 mgr.y (mgr.24961) 618 : cluster [DBG] pgmap v985: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:11.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:12 smithi059 bash[61152]: cluster 2022-04-28T20:04:11.215570+0000 mgr.y (mgr.24961) 619 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:12.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:12 smithi059 bash[59745]: cluster 2022-04-28T20:04:11.215570+0000 mgr.y (mgr.24961) 619 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:12 smithi139 bash[43301]: cluster 2022-04-28T20:04:11.215570+0000 mgr.y (mgr.24961) 619 : cluster [DBG] pgmap v986: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:13 smithi059 bash[61152]: cluster 2022-04-28T20:04:13.216249+0000 mgr.y (mgr.24961) 620 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:13.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:13 smithi059 bash[59745]: cluster 2022-04-28T20:04:13.216249+0000 mgr.y (mgr.24961) 620 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:13 smithi139 bash[43301]: cluster 2022-04-28T20:04:13.216249+0000 mgr.y (mgr.24961) 620 : cluster [DBG] pgmap v987: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:16 smithi059 bash[61152]: cluster 2022-04-28T20:04:15.217201+0000 mgr.y (mgr.24961) 621 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:16 smithi059 bash[59745]: cluster 2022-04-28T20:04:15.217201+0000 mgr.y (mgr.24961) 621 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:16 smithi139 bash[43301]: cluster 2022-04-28T20:04:15.217201+0000 mgr.y (mgr.24961) 621 : cluster [DBG] pgmap v988: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:17.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[59745]: audit 2022-04-28T20:04:17.391989+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:04:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[59745]: audit 2022-04-28T20:04:17.392435+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:04:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[59745]: audit 2022-04-28T20:04:17.392858+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:04:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[59745]: audit 2022-04-28T20:04:17.393751+0000 mon.a (mon.0) 1662 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:04:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[59745]: audit 2022-04-28T20:04:17.393983+0000 mon.a (mon.0) 1663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:04:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[61152]: audit 2022-04-28T20:04:17.391989+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:04:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[61152]: audit 2022-04-28T20:04:17.392435+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:04:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[61152]: audit 2022-04-28T20:04:17.392858+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:04:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[61152]: audit 2022-04-28T20:04:17.393751+0000 mon.a (mon.0) 1662 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:04:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:17 smithi059 bash[61152]: audit 2022-04-28T20:04:17.393983+0000 mon.a (mon.0) 1663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:04:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:17 smithi139 bash[43301]: audit 2022-04-28T20:04:17.391989+0000 mon.a (mon.0) 1659 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:04:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:17 smithi139 bash[43301]: audit 2022-04-28T20:04:17.392435+0000 mon.a (mon.0) 1660 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:04:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:17 smithi139 bash[43301]: audit 2022-04-28T20:04:17.392858+0000 mon.a (mon.0) 1661 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:04:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:17 smithi139 bash[43301]: audit 2022-04-28T20:04:17.393751+0000 mon.a (mon.0) 1662 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:04:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:17 smithi139 bash[43301]: audit 2022-04-28T20:04:17.393983+0000 mon.a (mon.0) 1663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:04:18.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[61152]: cluster 2022-04-28T20:04:17.217978+0000 mgr.y (mgr.24961) 622 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[61152]: cluster 2022-04-28T20:04:17.453753+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2022-04-28T20:04:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[61152]: audit 2022-04-28T20:04:17.456760+0000 mon.a (mon.0) 1665 : audit [INF] from='client.? 172.21.15.59:0/4104210596' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74999-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:04:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[61152]: audit 2022-04-28T20:04:18.453500+0000 mon.a (mon.0) 1666 : audit [INF] from='client.? 172.21.15.59:0/4104210596' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74999-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:04:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[61152]: cluster 2022-04-28T20:04:18.453673+0000 mon.a (mon.0) 1667 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2022-04-28T20:04:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[59745]: cluster 2022-04-28T20:04:17.217978+0000 mgr.y (mgr.24961) 622 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[59745]: cluster 2022-04-28T20:04:17.453753+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2022-04-28T20:04:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[59745]: audit 2022-04-28T20:04:17.456760+0000 mon.a (mon.0) 1665 : audit [INF] from='client.? 172.21.15.59:0/4104210596' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74999-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:04:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[59745]: audit 2022-04-28T20:04:18.453500+0000 mon.a (mon.0) 1666 : audit [INF] from='client.? 172.21.15.59:0/4104210596' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74999-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:04:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:18 smithi059 bash[59745]: cluster 2022-04-28T20:04:18.453673+0000 mon.a (mon.0) 1667 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2022-04-28T20:04:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:18 smithi139 bash[43301]: cluster 2022-04-28T20:04:17.217978+0000 mgr.y (mgr.24961) 622 : cluster [DBG] pgmap v989: 73 pgs: 73 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:18 smithi139 bash[43301]: cluster 2022-04-28T20:04:17.453753+0000 mon.a (mon.0) 1664 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2022-04-28T20:04:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:18 smithi139 bash[43301]: audit 2022-04-28T20:04:17.456760+0000 mon.a (mon.0) 1665 : audit [INF] from='client.? 172.21.15.59:0/4104210596' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74999-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:04:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:18 smithi139 bash[43301]: audit 2022-04-28T20:04:18.453500+0000 mon.a (mon.0) 1666 : audit [INF] from='client.? 172.21.15.59:0/4104210596' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-74999-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:04:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:18 smithi139 bash[43301]: cluster 2022-04-28T20:04:18.453673+0000 mon.a (mon.0) 1667 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2022-04-28T20:04:20.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:20 smithi059 bash[61152]: cluster 2022-04-28T20:04:19.218880+0000 mgr.y (mgr.24961) 623 : cluster [DBG] pgmap v992: 105 pgs: 17 unknown, 88 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:20 smithi059 bash[61152]: cluster 2022-04-28T20:04:19.468692+0000 mon.a (mon.0) 1668 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2022-04-28T20:04:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:20 smithi059 bash[59745]: cluster 2022-04-28T20:04:19.218880+0000 mgr.y (mgr.24961) 623 : cluster [DBG] pgmap v992: 105 pgs: 17 unknown, 88 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:20 smithi059 bash[59745]: cluster 2022-04-28T20:04:19.468692+0000 mon.a (mon.0) 1668 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2022-04-28T20:04:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:20 smithi139 bash[43301]: cluster 2022-04-28T20:04:19.218880+0000 mgr.y (mgr.24961) 623 : cluster [DBG] pgmap v992: 105 pgs: 17 unknown, 88 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T20:04:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:20 smithi139 bash[43301]: cluster 2022-04-28T20:04:19.468692+0000 mon.a (mon.0) 1668 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2022-04-28T20:04:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:21.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:22.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:22 smithi059 bash[61152]: cluster 2022-04-28T20:04:21.219878+0000 mgr.y (mgr.24961) 624 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:22 smithi059 bash[59745]: cluster 2022-04-28T20:04:21.219878+0000 mgr.y (mgr.24961) 624 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:22.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:22 smithi139 bash[43301]: cluster 2022-04-28T20:04:21.219878+0000 mgr.y (mgr.24961) 624 : cluster [DBG] pgmap v994: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:23.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:23 smithi059 bash[61152]: cluster 2022-04-28T20:04:23.220634+0000 mgr.y (mgr.24961) 625 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:23.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:23 smithi059 bash[59745]: cluster 2022-04-28T20:04:23.220634+0000 mgr.y (mgr.24961) 625 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:23.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:23 smithi139 bash[43301]: cluster 2022-04-28T20:04:23.220634+0000 mgr.y (mgr.24961) 625 : cluster [DBG] pgmap v995: 73 pgs: 73 active+clean; 24 KiB data, 724 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:26 smithi059 bash[61152]: cluster 2022-04-28T20:04:25.221595+0000 mgr.y (mgr.24961) 626 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:26 smithi059 bash[59745]: cluster 2022-04-28T20:04:25.221595+0000 mgr.y (mgr.24961) 626 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:26.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:26 smithi139 bash[43301]: cluster 2022-04-28T20:04:25.221595+0000 mgr.y (mgr.24961) 626 : cluster [DBG] pgmap v996: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:28 smithi059 bash[61152]: cluster 2022-04-28T20:04:27.222369+0000 mgr.y (mgr.24961) 627 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:28 smithi059 bash[59745]: cluster 2022-04-28T20:04:27.222369+0000 mgr.y (mgr.24961) 627 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:28 smithi139 bash[43301]: cluster 2022-04-28T20:04:27.222369+0000 mgr.y (mgr.24961) 627 : cluster [DBG] pgmap v997: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:30.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:30 smithi059 bash[59745]: cluster 2022-04-28T20:04:29.223227+0000 mgr.y (mgr.24961) 628 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:30 smithi059 bash[61152]: cluster 2022-04-28T20:04:29.223227+0000 mgr.y (mgr.24961) 628 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:30 smithi139 bash[43301]: cluster 2022-04-28T20:04:29.223227+0000 mgr.y (mgr.24961) 628 : cluster [DBG] pgmap v998: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:31.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:32.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:32 smithi059 bash[61152]: cluster 2022-04-28T20:04:31.224177+0000 mgr.y (mgr.24961) 629 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:32 smithi059 bash[59745]: cluster 2022-04-28T20:04:31.224177+0000 mgr.y (mgr.24961) 629 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:32 smithi139 bash[43301]: cluster 2022-04-28T20:04:31.224177+0000 mgr.y (mgr.24961) 629 : cluster [DBG] pgmap v999: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:33 smithi059 bash[61152]: cluster 2022-04-28T20:04:33.224791+0000 mgr.y (mgr.24961) 630 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:33.564Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:33 smithi059 bash[59745]: cluster 2022-04-28T20:04:33.224791+0000 mgr.y (mgr.24961) 630 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:33.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:33 smithi139 bash[43301]: cluster 2022-04-28T20:04:33.224791+0000 mgr.y (mgr.24961) 630 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:36.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:36 smithi059 bash[61152]: cluster 2022-04-28T20:04:35.225789+0000 mgr.y (mgr.24961) 631 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:36 smithi059 bash[59745]: cluster 2022-04-28T20:04:35.225789+0000 mgr.y (mgr.24961) 631 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:36.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:36 smithi139 bash[43301]: cluster 2022-04-28T20:04:35.225789+0000 mgr.y (mgr.24961) 631 : cluster [DBG] pgmap v1001: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:38.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:38 smithi059 bash[59745]: cluster 2022-04-28T20:04:37.226605+0000 mgr.y (mgr.24961) 632 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:38.707 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:38 smithi059 bash[61152]: cluster 2022-04-28T20:04:37.226605+0000 mgr.y (mgr.24961) 632 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:38.708 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:38 smithi139 bash[43301]: cluster 2022-04-28T20:04:37.226605+0000 mgr.y (mgr.24961) 632 : cluster [DBG] pgmap v1002: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:40.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:40 smithi059 bash[59745]: cluster 2022-04-28T20:04:39.227520+0000 mgr.y (mgr.24961) 633 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:40 smithi059 bash[61152]: cluster 2022-04-28T20:04:39.227520+0000 mgr.y (mgr.24961) 633 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:40 smithi139 bash[43301]: cluster 2022-04-28T20:04:39.227520+0000 mgr.y (mgr.24961) 633 : cluster [DBG] pgmap v1003: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:41.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:42 smithi059 bash[61152]: cluster 2022-04-28T20:04:41.229003+0000 mgr.y (mgr.24961) 634 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:42.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:42 smithi059 bash[59745]: cluster 2022-04-28T20:04:41.229003+0000 mgr.y (mgr.24961) 634 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:42.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:42 smithi139 bash[43301]: cluster 2022-04-28T20:04:41.229003+0000 mgr.y (mgr.24961) 634 : cluster [DBG] pgmap v1004: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:43 smithi059 bash[61152]: cluster 2022-04-28T20:04:43.229715+0000 mgr.y (mgr.24961) 635 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:43.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:43 smithi059 bash[59745]: cluster 2022-04-28T20:04:43.229715+0000 mgr.y (mgr.24961) 635 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:43.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:43 smithi139 bash[43301]: cluster 2022-04-28T20:04:43.229715+0000 mgr.y (mgr.24961) 635 : cluster [DBG] pgmap v1005: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:46.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:46 smithi059 bash[59745]: cluster 2022-04-28T20:04:45.230758+0000 mgr.y (mgr.24961) 636 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:46 smithi059 bash[61152]: cluster 2022-04-28T20:04:45.230758+0000 mgr.y (mgr.24961) 636 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:46 smithi139 bash[43301]: cluster 2022-04-28T20:04:45.230758+0000 mgr.y (mgr.24961) 636 : cluster [DBG] pgmap v1006: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:48.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:48 smithi059 bash[61152]: cluster 2022-04-28T20:04:47.231597+0000 mgr.y (mgr.24961) 637 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:48 smithi059 bash[59745]: cluster 2022-04-28T20:04:47.231597+0000 mgr.y (mgr.24961) 637 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:48 smithi139 bash[43301]: cluster 2022-04-28T20:04:47.231597+0000 mgr.y (mgr.24961) 637 : cluster [DBG] pgmap v1007: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:50.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:50 smithi059 bash[59745]: cluster 2022-04-28T20:04:49.232479+0000 mgr.y (mgr.24961) 638 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:50 smithi059 bash[61152]: cluster 2022-04-28T20:04:49.232479+0000 mgr.y (mgr.24961) 638 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:50 smithi139 bash[43301]: cluster 2022-04-28T20:04:49.232479+0000 mgr.y (mgr.24961) 638 : cluster [DBG] pgmap v1008: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:51.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:52.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:52 smithi059 bash[59745]: cluster 2022-04-28T20:04:51.233541+0000 mgr.y (mgr.24961) 639 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:52 smithi059 bash[61152]: cluster 2022-04-28T20:04:51.233541+0000 mgr.y (mgr.24961) 639 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:52 smithi139 bash[43301]: cluster 2022-04-28T20:04:51.233541+0000 mgr.y (mgr.24961) 639 : cluster [DBG] pgmap v1009: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:53 smithi059 bash[61152]: cluster 2022-04-28T20:04:53.234225+0000 mgr.y (mgr.24961) 640 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:53 smithi059 bash[59745]: cluster 2022-04-28T20:04:53.234225+0000 mgr.y (mgr.24961) 640 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:04:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:04:53.565Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:04:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:53 smithi139 bash[43301]: cluster 2022-04-28T20:04:53.234225+0000 mgr.y (mgr.24961) 640 : cluster [DBG] pgmap v1010: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[59745]: audit 2022-04-28T20:04:53.624537+0000 mon.a (mon.0) 1669 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:04:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[59745]: audit 2022-04-28T20:04:53.626957+0000 mon.a (mon.0) 1670 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:04:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[59745]: audit 2022-04-28T20:04:53.628562+0000 mon.a (mon.0) 1671 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:04:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[59745]: cephadm 2022-04-28T20:04:53.630051+0000 mgr.y (mgr.24961) 641 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:04:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[59745]: audit 2022-04-28T20:04:53.936445+0000 mon.a (mon.0) 1672 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:04:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[61152]: audit 2022-04-28T20:04:53.624537+0000 mon.a (mon.0) 1669 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:04:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[61152]: audit 2022-04-28T20:04:53.626957+0000 mon.a (mon.0) 1670 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:04:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[61152]: audit 2022-04-28T20:04:53.628562+0000 mon.a (mon.0) 1671 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:04:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[61152]: cephadm 2022-04-28T20:04:53.630051+0000 mgr.y (mgr.24961) 641 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:04:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:54 smithi059 bash[61152]: audit 2022-04-28T20:04:53.936445+0000 mon.a (mon.0) 1672 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:04:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:54 smithi139 bash[43301]: audit 2022-04-28T20:04:53.624537+0000 mon.a (mon.0) 1669 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:04:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:54 smithi139 bash[43301]: audit 2022-04-28T20:04:53.626957+0000 mon.a (mon.0) 1670 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:04:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:54 smithi139 bash[43301]: audit 2022-04-28T20:04:53.628562+0000 mon.a (mon.0) 1671 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:04:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:54 smithi139 bash[43301]: cephadm 2022-04-28T20:04:53.630051+0000 mgr.y (mgr.24961) 641 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:04:55.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:54 smithi139 bash[43301]: audit 2022-04-28T20:04:53.936445+0000 mon.a (mon.0) 1672 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:04:56.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:55 smithi059 bash[59745]: cluster 2022-04-28T20:04:55.235194+0000 mgr.y (mgr.24961) 642 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:55 smithi059 bash[59745]: audit 2022-04-28T20:04:55.355694+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.59:0/3506188126' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:04:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:55 smithi059 bash[61152]: cluster 2022-04-28T20:04:55.235194+0000 mgr.y (mgr.24961) 642 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:56.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:55 smithi059 bash[61152]: audit 2022-04-28T20:04:55.355694+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.59:0/3506188126' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:04:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:55 smithi139 bash[43301]: cluster 2022-04-28T20:04:55.235194+0000 mgr.y (mgr.24961) 642 : cluster [DBG] pgmap v1011: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:55 smithi139 bash[43301]: audit 2022-04-28T20:04:55.355694+0000 mon.a (mon.0) 1673 : audit [DBG] from='client.? 172.21.15.59:0/3506188126' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:04:58.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:04:58 smithi059 bash[61152]: cluster 2022-04-28T20:04:57.236027+0000 mgr.y (mgr.24961) 643 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:04:58 smithi059 bash[59745]: cluster 2022-04-28T20:04:57.236027+0000 mgr.y (mgr.24961) 643 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:04:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:04:58 smithi139 bash[43301]: cluster 2022-04-28T20:04:57.236027+0000 mgr.y (mgr.24961) 643 : cluster [DBG] pgmap v1012: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:00.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:00 smithi059 bash[59745]: cluster 2022-04-28T20:04:59.236856+0000 mgr.y (mgr.24961) 644 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:00 smithi059 bash[59745]: cluster 2022-04-28T20:04:59.311932+0000 mon.a (mon.0) 1674 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2022-04-28T20:05:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:00 smithi059 bash[59745]: audit 2022-04-28T20:04:59.317386+0000 mon.a (mon.0) 1675 : audit [INF] from='client.? 172.21.15.59:0/2499574263' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75087-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:00 smithi059 bash[61152]: cluster 2022-04-28T20:04:59.236856+0000 mgr.y (mgr.24961) 644 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:00 smithi059 bash[61152]: cluster 2022-04-28T20:04:59.311932+0000 mon.a (mon.0) 1674 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2022-04-28T20:05:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:00 smithi059 bash[61152]: audit 2022-04-28T20:04:59.317386+0000 mon.a (mon.0) 1675 : audit [INF] from='client.? 172.21.15.59:0/2499574263' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75087-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:00 smithi139 bash[43301]: cluster 2022-04-28T20:04:59.236856+0000 mgr.y (mgr.24961) 644 : cluster [DBG] pgmap v1013: 73 pgs: 73 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:00 smithi139 bash[43301]: cluster 2022-04-28T20:04:59.311932+0000 mon.a (mon.0) 1674 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2022-04-28T20:05:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:00 smithi139 bash[43301]: audit 2022-04-28T20:04:59.317386+0000 mon.a (mon.0) 1675 : audit [INF] from='client.? 172.21.15.59:0/2499574263' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75087-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:01.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:01 smithi059 bash[61152]: audit 2022-04-28T20:05:00.310883+0000 mon.a (mon.0) 1676 : audit [INF] from='client.? 172.21.15.59:0/2499574263' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75087-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:01 smithi059 bash[61152]: cluster 2022-04-28T20:05:00.311009+0000 mon.a (mon.0) 1677 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2022-04-28T20:05:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:01 smithi059 bash[59745]: audit 2022-04-28T20:05:00.310883+0000 mon.a (mon.0) 1676 : audit [INF] from='client.? 172.21.15.59:0/2499574263' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75087-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:01 smithi059 bash[59745]: cluster 2022-04-28T20:05:00.311009+0000 mon.a (mon.0) 1677 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2022-04-28T20:05:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:01 smithi139 bash[43301]: audit 2022-04-28T20:05:00.310883+0000 mon.a (mon.0) 1676 : audit [INF] from='client.? 172.21.15.59:0/2499574263' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75087-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:01 smithi139 bash[43301]: cluster 2022-04-28T20:05:00.311009+0000 mon.a (mon.0) 1677 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2022-04-28T20:05:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:01.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:02.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:02 smithi059 bash[61152]: cluster 2022-04-28T20:05:01.237777+0000 mgr.y (mgr.24961) 645 : cluster [DBG] pgmap v1016: 105 pgs: 12 creating+peering, 8 unknown, 85 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:02 smithi059 bash[61152]: cluster 2022-04-28T20:05:01.327328+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2022-04-28T20:05:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:02 smithi059 bash[59745]: cluster 2022-04-28T20:05:01.237777+0000 mgr.y (mgr.24961) 645 : cluster [DBG] pgmap v1016: 105 pgs: 12 creating+peering, 8 unknown, 85 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:02 smithi059 bash[59745]: cluster 2022-04-28T20:05:01.327328+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2022-04-28T20:05:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:02 smithi139 bash[43301]: cluster 2022-04-28T20:05:01.237777+0000 mgr.y (mgr.24961) 645 : cluster [DBG] pgmap v1016: 105 pgs: 12 creating+peering, 8 unknown, 85 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:02 smithi139 bash[43301]: cluster 2022-04-28T20:05:01.327328+0000 mon.a (mon.0) 1678 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2022-04-28T20:05:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:03 smithi059 bash[61152]: cluster 2022-04-28T20:05:03.238242+0000 mgr.y (mgr.24961) 646 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:03.566Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:03 smithi059 bash[59745]: cluster 2022-04-28T20:05:03.238242+0000 mgr.y (mgr.24961) 646 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:03 smithi139 bash[43301]: cluster 2022-04-28T20:05:03.238242+0000 mgr.y (mgr.24961) 646 : cluster [DBG] pgmap v1018: 73 pgs: 73 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:06.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:06 smithi059 bash[59745]: cluster 2022-04-28T20:05:05.239179+0000 mgr.y (mgr.24961) 647 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:06 smithi059 bash[61152]: cluster 2022-04-28T20:05:05.239179+0000 mgr.y (mgr.24961) 647 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:06.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:06 smithi139 bash[43301]: cluster 2022-04-28T20:05:05.239179+0000 mgr.y (mgr.24961) 647 : cluster [DBG] pgmap v1019: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:08 smithi059 bash[61152]: cluster 2022-04-28T20:05:07.240050+0000 mgr.y (mgr.24961) 648 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:08 smithi059 bash[59745]: cluster 2022-04-28T20:05:07.240050+0000 mgr.y (mgr.24961) 648 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:08 smithi139 bash[43301]: cluster 2022-04-28T20:05:07.240050+0000 mgr.y (mgr.24961) 648 : cluster [DBG] pgmap v1020: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:10.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:10 smithi059 bash[59745]: cluster 2022-04-28T20:05:09.240935+0000 mgr.y (mgr.24961) 649 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:10 smithi059 bash[61152]: cluster 2022-04-28T20:05:09.240935+0000 mgr.y (mgr.24961) 649 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:10 smithi139 bash[43301]: cluster 2022-04-28T20:05:09.240935+0000 mgr.y (mgr.24961) 649 : cluster [DBG] pgmap v1021: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:11.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:12 smithi059 bash[61152]: cluster 2022-04-28T20:05:11.241926+0000 mgr.y (mgr.24961) 650 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:12 smithi059 bash[59745]: cluster 2022-04-28T20:05:11.241926+0000 mgr.y (mgr.24961) 650 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:12.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:12 smithi139 bash[43301]: cluster 2022-04-28T20:05:11.241926+0000 mgr.y (mgr.24961) 650 : cluster [DBG] pgmap v1022: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:13 smithi059 bash[61152]: cluster 2022-04-28T20:05:13.242577+0000 mgr.y (mgr.24961) 651 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:13.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:13 smithi059 bash[59745]: cluster 2022-04-28T20:05:13.242577+0000 mgr.y (mgr.24961) 651 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:13 smithi139 bash[43301]: cluster 2022-04-28T20:05:13.242577+0000 mgr.y (mgr.24961) 651 : cluster [DBG] pgmap v1023: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:16 smithi059 bash[61152]: cluster 2022-04-28T20:05:15.243627+0000 mgr.y (mgr.24961) 652 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:16 smithi059 bash[59745]: cluster 2022-04-28T20:05:15.243627+0000 mgr.y (mgr.24961) 652 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:16.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:16 smithi139 bash[43301]: cluster 2022-04-28T20:05:15.243627+0000 mgr.y (mgr.24961) 652 : cluster [DBG] pgmap v1024: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:18.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[59745]: cluster 2022-04-28T20:05:17.244465+0000 mgr.y (mgr.24961) 653 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[59745]: audit 2022-04-28T20:05:17.401549+0000 mon.a (mon.0) 1679 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:05:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[59745]: audit 2022-04-28T20:05:17.402423+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:05:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[59745]: audit 2022-04-28T20:05:17.407726+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:05:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[59745]: audit 2022-04-28T20:05:17.408435+0000 mon.a (mon.0) 1682 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:05:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[59745]: audit 2022-04-28T20:05:17.408823+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T20:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[61152]: cluster 2022-04-28T20:05:17.244465+0000 mgr.y (mgr.24961) 653 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[61152]: audit 2022-04-28T20:05:17.401549+0000 mon.a (mon.0) 1679 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[61152]: audit 2022-04-28T20:05:17.402423+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[61152]: audit 2022-04-28T20:05:17.407726+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[61152]: audit 2022-04-28T20:05:17.408435+0000 mon.a (mon.0) 1682 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:05:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:18 smithi059 bash[61152]: audit 2022-04-28T20:05:17.408823+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T20:05:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:18 smithi139 bash[43301]: cluster 2022-04-28T20:05:17.244465+0000 mgr.y (mgr.24961) 653 : cluster [DBG] pgmap v1025: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:18 smithi139 bash[43301]: audit 2022-04-28T20:05:17.401549+0000 mon.a (mon.0) 1679 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:05:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:18 smithi139 bash[43301]: audit 2022-04-28T20:05:17.402423+0000 mon.a (mon.0) 1680 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:05:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:18 smithi139 bash[43301]: audit 2022-04-28T20:05:17.407726+0000 mon.a (mon.0) 1681 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:05:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:18 smithi139 bash[43301]: audit 2022-04-28T20:05:17.408435+0000 mon.a (mon.0) 1682 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:05:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:18 smithi139 bash[43301]: audit 2022-04-28T20:05:17.408823+0000 mon.a (mon.0) 1683 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T20:05:20.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:20 smithi059 bash[59745]: cluster 2022-04-28T20:05:19.245326+0000 mgr.y (mgr.24961) 654 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:20.693 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:20 smithi059 bash[61152]: cluster 2022-04-28T20:05:19.245326+0000 mgr.y (mgr.24961) 654 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:20.694 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:20 smithi139 bash[43301]: cluster 2022-04-28T20:05:19.245326+0000 mgr.y (mgr.24961) 654 : cluster [DBG] pgmap v1026: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:21.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:22 smithi059 bash[61152]: cluster 2022-04-28T20:05:21.246289+0000 mgr.y (mgr.24961) 655 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:22.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:22 smithi059 bash[59745]: cluster 2022-04-28T20:05:21.246289+0000 mgr.y (mgr.24961) 655 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:22.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:22 smithi139 bash[43301]: cluster 2022-04-28T20:05:21.246289+0000 mgr.y (mgr.24961) 655 : cluster [DBG] pgmap v1027: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:23 smithi059 bash[61152]: cluster 2022-04-28T20:05:23.246983+0000 mgr.y (mgr.24961) 656 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:23.844 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:23.568Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:23.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:23 smithi059 bash[59745]: cluster 2022-04-28T20:05:23.246983+0000 mgr.y (mgr.24961) 656 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:23 smithi139 bash[43301]: cluster 2022-04-28T20:05:23.246983+0000 mgr.y (mgr.24961) 656 : cluster [DBG] pgmap v1028: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:26 smithi059 bash[61152]: cluster 2022-04-28T20:05:25.248039+0000 mgr.y (mgr.24961) 657 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:26 smithi059 bash[59745]: cluster 2022-04-28T20:05:25.248039+0000 mgr.y (mgr.24961) 657 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:26 smithi139 bash[43301]: cluster 2022-04-28T20:05:25.248039+0000 mgr.y (mgr.24961) 657 : cluster [DBG] pgmap v1029: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:29.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:28 smithi059 bash[59745]: cluster 2022-04-28T20:05:27.248983+0000 mgr.y (mgr.24961) 658 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:28 smithi059 bash[61152]: cluster 2022-04-28T20:05:27.248983+0000 mgr.y (mgr.24961) 658 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:28 smithi139 bash[43301]: cluster 2022-04-28T20:05:27.248983+0000 mgr.y (mgr.24961) 658 : cluster [DBG] pgmap v1030: 73 pgs: 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:30.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[61152]: cluster 2022-04-28T20:05:28.754658+0000 mon.a (mon.0) 1684 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2022-04-28T20:05:30.185 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[61152]: audit 2022-04-28T20:05:28.757300+0000 mon.c (mon.1) 25 : audit [INF] from='client.? 172.21.15.59:0/577739253' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:30.186 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[61152]: audit 2022-04-28T20:05:28.758289+0000 mon.a (mon.0) 1685 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:30.186 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[61152]: cluster 2022-04-28T20:05:29.250128+0000 mgr.y (mgr.24961) 659 : cluster [DBG] pgmap v1032: 105 pgs: 12 creating+peering, 20 unknown, 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:30.186 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[59745]: cluster 2022-04-28T20:05:28.754658+0000 mon.a (mon.0) 1684 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2022-04-28T20:05:30.186 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[59745]: audit 2022-04-28T20:05:28.757300+0000 mon.c (mon.1) 25 : audit [INF] from='client.? 172.21.15.59:0/577739253' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:30.187 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[59745]: audit 2022-04-28T20:05:28.758289+0000 mon.a (mon.0) 1685 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:30.187 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:29 smithi059 bash[59745]: cluster 2022-04-28T20:05:29.250128+0000 mgr.y (mgr.24961) 659 : cluster [DBG] pgmap v1032: 105 pgs: 12 creating+peering, 20 unknown, 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:30.187 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:29 smithi139 bash[43301]: cluster 2022-04-28T20:05:28.754658+0000 mon.a (mon.0) 1684 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2022-04-28T20:05:30.187 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:29 smithi139 bash[43301]: audit 2022-04-28T20:05:28.757300+0000 mon.c (mon.1) 25 : audit [INF] from='client.? 172.21.15.59:0/577739253' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:30.188 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:29 smithi139 bash[43301]: audit 2022-04-28T20:05:28.758289+0000 mon.a (mon.0) 1685 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:30.188 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:29 smithi139 bash[43301]: cluster 2022-04-28T20:05:29.250128+0000 mgr.y (mgr.24961) 659 : cluster [DBG] pgmap v1032: 105 pgs: 12 creating+peering, 20 unknown, 73 active+clean; 24 KiB data, 693 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:31.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:30 smithi059 bash[61152]: audit 2022-04-28T20:05:29.762991+0000 mon.a (mon.0) 1686 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:30 smithi059 bash[61152]: cluster 2022-04-28T20:05:29.763555+0000 mon.a (mon.0) 1687 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2022-04-28T20:05:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:30 smithi059 bash[59745]: audit 2022-04-28T20:05:29.762991+0000 mon.a (mon.0) 1686 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:30 smithi059 bash[59745]: cluster 2022-04-28T20:05:29.763555+0000 mon.a (mon.0) 1687 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2022-04-28T20:05:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:30 smithi139 bash[43301]: audit 2022-04-28T20:05:29.762991+0000 mon.a (mon.0) 1686 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75152-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:30 smithi139 bash[43301]: cluster 2022-04-28T20:05:29.763555+0000 mon.a (mon.0) 1687 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2022-04-28T20:05:32.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:31 smithi059 bash[61152]: cluster 2022-04-28T20:05:30.766397+0000 mon.a (mon.0) 1688 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2022-04-28T20:05:32.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:31 smithi059 bash[61152]: cluster 2022-04-28T20:05:31.251018+0000 mgr.y (mgr.24961) 660 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:31.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:31 smithi059 bash[59745]: cluster 2022-04-28T20:05:30.766397+0000 mon.a (mon.0) 1688 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2022-04-28T20:05:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:31 smithi059 bash[59745]: cluster 2022-04-28T20:05:31.251018+0000 mgr.y (mgr.24961) 660 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:32.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:31 smithi139 bash[43301]: cluster 2022-04-28T20:05:30.766397+0000 mon.a (mon.0) 1688 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2022-04-28T20:05:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:31 smithi139 bash[43301]: cluster 2022-04-28T20:05:31.251018+0000 mgr.y (mgr.24961) 660 : cluster [DBG] pgmap v1035: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:33.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:33 smithi059 bash[59745]: cluster 2022-04-28T20:05:33.251692+0000 mgr.y (mgr.24961) 661 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:33 smithi059 bash[61152]: cluster 2022-04-28T20:05:33.251692+0000 mgr.y (mgr.24961) 661 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:33.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:33 smithi139 bash[43301]: cluster 2022-04-28T20:05:33.251692+0000 mgr.y (mgr.24961) 661 : cluster [DBG] pgmap v1036: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:36.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:36 smithi059 bash[59745]: cluster 2022-04-28T20:05:35.252689+0000 mgr.y (mgr.24961) 662 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:36 smithi059 bash[61152]: cluster 2022-04-28T20:05:35.252689+0000 mgr.y (mgr.24961) 662 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:36 smithi139 bash[43301]: cluster 2022-04-28T20:05:35.252689+0000 mgr.y (mgr.24961) 662 : cluster [DBG] pgmap v1037: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:38 smithi059 bash[61152]: cluster 2022-04-28T20:05:37.253516+0000 mgr.y (mgr.24961) 663 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:38 smithi059 bash[59745]: cluster 2022-04-28T20:05:37.253516+0000 mgr.y (mgr.24961) 663 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:38.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:38 smithi139 bash[43301]: cluster 2022-04-28T20:05:37.253516+0000 mgr.y (mgr.24961) 663 : cluster [DBG] pgmap v1038: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:40.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:40 smithi059 bash[61152]: cluster 2022-04-28T20:05:39.254392+0000 mgr.y (mgr.24961) 664 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:40 smithi059 bash[59745]: cluster 2022-04-28T20:05:39.254392+0000 mgr.y (mgr.24961) 664 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:40 smithi139 bash[43301]: cluster 2022-04-28T20:05:39.254392+0000 mgr.y (mgr.24961) 664 : cluster [DBG] pgmap v1039: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:41.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:42.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:42 smithi059 bash[59745]: cluster 2022-04-28T20:05:41.255260+0000 mgr.y (mgr.24961) 665 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:42.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:42 smithi059 bash[59745]: cluster 2022-04-28T20:05:41.335747+0000 mon.a (mon.0) 1689 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2022-04-28T20:05:42.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:42 smithi059 bash[59745]: audit 2022-04-28T20:05:41.338431+0000 mon.a (mon.0) 1690 : audit [INF] from='client.? 172.21.15.59:0/2579026385' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75183-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:42.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:42 smithi059 bash[61152]: cluster 2022-04-28T20:05:41.255260+0000 mgr.y (mgr.24961) 665 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:42.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:42 smithi059 bash[61152]: cluster 2022-04-28T20:05:41.335747+0000 mon.a (mon.0) 1689 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2022-04-28T20:05:42.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:42 smithi059 bash[61152]: audit 2022-04-28T20:05:41.338431+0000 mon.a (mon.0) 1690 : audit [INF] from='client.? 172.21.15.59:0/2579026385' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75183-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:42 smithi139 bash[43301]: cluster 2022-04-28T20:05:41.255260+0000 mgr.y (mgr.24961) 665 : cluster [DBG] pgmap v1040: 73 pgs: 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:42 smithi139 bash[43301]: cluster 2022-04-28T20:05:41.335747+0000 mon.a (mon.0) 1689 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2022-04-28T20:05:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:42 smithi139 bash[43301]: audit 2022-04-28T20:05:41.338431+0000 mon.a (mon.0) 1690 : audit [INF] from='client.? 172.21.15.59:0/2579026385' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75183-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:05:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:43 smithi139 bash[43301]: audit 2022-04-28T20:05:42.350015+0000 mon.a (mon.0) 1691 : audit [INF] from='client.? 172.21.15.59:0/2579026385' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75183-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:43 smithi139 bash[43301]: cluster 2022-04-28T20:05:42.350246+0000 mon.a (mon.0) 1692 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2022-04-28T20:05:43.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:43.569Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:43 smithi059 bash[61152]: audit 2022-04-28T20:05:42.350015+0000 mon.a (mon.0) 1691 : audit [INF] from='client.? 172.21.15.59:0/2579026385' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75183-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:43 smithi059 bash[61152]: cluster 2022-04-28T20:05:42.350246+0000 mon.a (mon.0) 1692 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2022-04-28T20:05:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:43 smithi059 bash[59745]: audit 2022-04-28T20:05:42.350015+0000 mon.a (mon.0) 1691 : audit [INF] from='client.? 172.21.15.59:0/2579026385' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75183-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:05:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:43 smithi059 bash[59745]: cluster 2022-04-28T20:05:42.350246+0000 mon.a (mon.0) 1692 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2022-04-28T20:05:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:44 smithi139 bash[43301]: cluster 2022-04-28T20:05:43.255946+0000 mgr.y (mgr.24961) 666 : cluster [DBG] pgmap v1043: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:44 smithi139 bash[43301]: cluster 2022-04-28T20:05:43.349698+0000 mon.a (mon.0) 1693 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2022-04-28T20:05:44.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:44 smithi059 bash[61152]: cluster 2022-04-28T20:05:43.255946+0000 mgr.y (mgr.24961) 666 : cluster [DBG] pgmap v1043: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:44 smithi059 bash[61152]: cluster 2022-04-28T20:05:43.349698+0000 mon.a (mon.0) 1693 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2022-04-28T20:05:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:44 smithi059 bash[59745]: cluster 2022-04-28T20:05:43.255946+0000 mgr.y (mgr.24961) 666 : cluster [DBG] pgmap v1043: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 694 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:44 smithi059 bash[59745]: cluster 2022-04-28T20:05:43.349698+0000 mon.a (mon.0) 1693 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2022-04-28T20:05:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:46 smithi139 bash[43301]: cluster 2022-04-28T20:05:45.256929+0000 mgr.y (mgr.24961) 667 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:46.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:46 smithi059 bash[59745]: cluster 2022-04-28T20:05:45.256929+0000 mgr.y (mgr.24961) 667 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:46 smithi059 bash[61152]: cluster 2022-04-28T20:05:45.256929+0000 mgr.y (mgr.24961) 667 : cluster [DBG] pgmap v1045: 73 pgs: 73 active+clean; 24 KiB data, 695 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:48.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:48 smithi059 bash[61152]: cluster 2022-04-28T20:05:47.257794+0000 mgr.y (mgr.24961) 668 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:48.935 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:48 smithi059 bash[59745]: cluster 2022-04-28T20:05:47.257794+0000 mgr.y (mgr.24961) 668 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:48.936 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:48 smithi139 bash[43301]: cluster 2022-04-28T20:05:47.257794+0000 mgr.y (mgr.24961) 668 : cluster [DBG] pgmap v1046: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:50.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:50 smithi059 bash[61152]: cluster 2022-04-28T20:05:49.258686+0000 mgr.y (mgr.24961) 669 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:50 smithi059 bash[59745]: cluster 2022-04-28T20:05:49.258686+0000 mgr.y (mgr.24961) 669 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:50.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:50 smithi139 bash[43301]: cluster 2022-04-28T20:05:49.258686+0000 mgr.y (mgr.24961) 669 : cluster [DBG] pgmap v1047: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:51.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:52 smithi059 bash[61152]: cluster 2022-04-28T20:05:51.259655+0000 mgr.y (mgr.24961) 670 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:52.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:52 smithi059 bash[59745]: cluster 2022-04-28T20:05:51.259655+0000 mgr.y (mgr.24961) 670 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:52.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:52 smithi139 bash[43301]: cluster 2022-04-28T20:05:51.259655+0000 mgr.y (mgr.24961) 670 : cluster [DBG] pgmap v1048: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:53 smithi059 bash[61152]: cluster 2022-04-28T20:05:53.260314+0000 mgr.y (mgr.24961) 671 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:05:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:05:53.570Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:05:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:53 smithi059 bash[59745]: cluster 2022-04-28T20:05:53.260314+0000 mgr.y (mgr.24961) 671 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:54.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:53 smithi139 bash[43301]: cluster 2022-04-28T20:05:53.260314+0000 mgr.y (mgr.24961) 671 : cluster [DBG] pgmap v1049: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:55.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[59745]: audit 2022-04-28T20:05:53.946500+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:05:55.232 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[59745]: audit 2022-04-28T20:05:53.949402+0000 mon.a (mon.0) 1695 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:05:55.232 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[59745]: audit 2022-04-28T20:05:53.951190+0000 mon.a (mon.0) 1696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:05:55.232 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[59745]: cephadm 2022-04-28T20:05:53.952907+0000 mgr.y (mgr.24961) 672 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:05:55.233 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[59745]: audit 2022-04-28T20:05:53.954503+0000 mon.a (mon.0) 1697 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:05:55.233 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[59745]: audit 2022-04-28T20:05:53.955123+0000 mon.a (mon.0) 1698 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:05:55.233 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[61152]: audit 2022-04-28T20:05:53.946500+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:05:55.233 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[61152]: audit 2022-04-28T20:05:53.949402+0000 mon.a (mon.0) 1695 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:05:55.233 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[61152]: audit 2022-04-28T20:05:53.951190+0000 mon.a (mon.0) 1696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:05:55.234 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[61152]: cephadm 2022-04-28T20:05:53.952907+0000 mgr.y (mgr.24961) 672 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:05:55.234 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[61152]: audit 2022-04-28T20:05:53.954503+0000 mon.a (mon.0) 1697 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:05:55.234 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:54 smithi059 bash[61152]: audit 2022-04-28T20:05:53.955123+0000 mon.a (mon.0) 1698 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:05:55.235 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:54 smithi139 bash[43301]: audit 2022-04-28T20:05:53.946500+0000 mon.a (mon.0) 1694 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:05:55.235 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:54 smithi139 bash[43301]: audit 2022-04-28T20:05:53.949402+0000 mon.a (mon.0) 1695 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:05:55.236 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:54 smithi139 bash[43301]: audit 2022-04-28T20:05:53.951190+0000 mon.a (mon.0) 1696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:05:55.236 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:54 smithi139 bash[43301]: cephadm 2022-04-28T20:05:53.952907+0000 mgr.y (mgr.24961) 672 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:05:55.236 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:54 smithi139 bash[43301]: audit 2022-04-28T20:05:53.954503+0000 mon.a (mon.0) 1697 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:05:55.236 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:54 smithi139 bash[43301]: audit 2022-04-28T20:05:53.955123+0000 mon.a (mon.0) 1698 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:05:56.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:55 smithi059 bash[61152]: cluster 2022-04-28T20:05:55.261438+0000 mgr.y (mgr.24961) 673 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:55 smithi059 bash[59745]: cluster 2022-04-28T20:05:55.261438+0000 mgr.y (mgr.24961) 673 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:55 smithi139 bash[43301]: cluster 2022-04-28T20:05:55.261438+0000 mgr.y (mgr.24961) 673 : cluster [DBG] pgmap v1050: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:58.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:58 smithi059 bash[59745]: cluster 2022-04-28T20:05:57.262230+0000 mgr.y (mgr.24961) 674 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:58 smithi059 bash[59745]: audit 2022-04-28T20:05:58.111971+0000 mon.a (mon.0) 1699 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:58 smithi059 bash[59745]: audit 2022-04-28T20:05:58.218549+0000 mon.a (mon.0) 1700 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:58 smithi059 bash[61152]: cluster 2022-04-28T20:05:57.262230+0000 mgr.y (mgr.24961) 674 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:58 smithi059 bash[61152]: audit 2022-04-28T20:05:58.111971+0000 mon.a (mon.0) 1699 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:58 smithi059 bash[61152]: audit 2022-04-28T20:05:58.218549+0000 mon.a (mon.0) 1700 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:58 smithi139 bash[43301]: cluster 2022-04-28T20:05:57.262230+0000 mgr.y (mgr.24961) 674 : cluster [DBG] pgmap v1051: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:58 smithi139 bash[43301]: audit 2022-04-28T20:05:58.111971+0000 mon.a (mon.0) 1699 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:58 smithi139 bash[43301]: audit 2022-04-28T20:05:58.218549+0000 mon.a (mon.0) 1700 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:59 smithi059 bash[61152]: audit 2022-04-28T20:05:58.507462+0000 mon.a (mon.0) 1701 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:05:59 smithi059 bash[61152]: cluster 2022-04-28T20:05:59.263098+0000 mgr.y (mgr.24961) 675 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:59 smithi059 bash[59745]: audit 2022-04-28T20:05:58.507462+0000 mon.a (mon.0) 1701 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:05:59 smithi059 bash[59745]: cluster 2022-04-28T20:05:59.263098+0000 mgr.y (mgr.24961) 675 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:05:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:59 smithi139 bash[43301]: audit 2022-04-28T20:05:58.507462+0000 mon.a (mon.0) 1701 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:05:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:05:59 smithi139 bash[43301]: cluster 2022-04-28T20:05:59.263098+0000 mgr.y (mgr.24961) 675 : cluster [DBG] pgmap v1052: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:01.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:02.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:02 smithi059 bash[59745]: cluster 2022-04-28T20:06:01.264111+0000 mgr.y (mgr.24961) 676 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:02 smithi059 bash[61152]: cluster 2022-04-28T20:06:01.264111+0000 mgr.y (mgr.24961) 676 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:02.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:02 smithi139 bash[43301]: cluster 2022-04-28T20:06:01.264111+0000 mgr.y (mgr.24961) 676 : cluster [DBG] pgmap v1053: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:03.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:03 smithi059 bash[59745]: cluster 2022-04-28T20:06:03.264766+0000 mgr.y (mgr.24961) 677 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:03 smithi059 bash[61152]: cluster 2022-04-28T20:06:03.264766+0000 mgr.y (mgr.24961) 677 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:03.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:03 smithi139 bash[43301]: cluster 2022-04-28T20:06:03.264766+0000 mgr.y (mgr.24961) 677 : cluster [DBG] pgmap v1054: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:06.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:06 smithi059 bash[59745]: cluster 2022-04-28T20:06:05.265704+0000 mgr.y (mgr.24961) 678 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:06 smithi059 bash[61152]: cluster 2022-04-28T20:06:05.265704+0000 mgr.y (mgr.24961) 678 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:06 smithi139 bash[43301]: cluster 2022-04-28T20:06:05.265704+0000 mgr.y (mgr.24961) 678 : cluster [DBG] pgmap v1055: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:08.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:08 smithi059 bash[61152]: cluster 2022-04-28T20:06:07.266536+0000 mgr.y (mgr.24961) 679 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:08.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:08 smithi059 bash[59745]: cluster 2022-04-28T20:06:07.266536+0000 mgr.y (mgr.24961) 679 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:08 smithi139 bash[43301]: cluster 2022-04-28T20:06:07.266536+0000 mgr.y (mgr.24961) 679 : cluster [DBG] pgmap v1056: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:10.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:10 smithi059 bash[61152]: cluster 2022-04-28T20:06:09.267411+0000 mgr.y (mgr.24961) 680 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:10 smithi059 bash[59745]: cluster 2022-04-28T20:06:09.267411+0000 mgr.y (mgr.24961) 680 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:10 smithi139 bash[43301]: cluster 2022-04-28T20:06:09.267411+0000 mgr.y (mgr.24961) 680 : cluster [DBG] pgmap v1057: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:11.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:12.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:12 smithi059 bash[61152]: cluster 2022-04-28T20:06:11.268399+0000 mgr.y (mgr.24961) 681 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:12.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:12 smithi059 bash[59745]: cluster 2022-04-28T20:06:11.268399+0000 mgr.y (mgr.24961) 681 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:12 smithi139 bash[43301]: cluster 2022-04-28T20:06:11.268399+0000 mgr.y (mgr.24961) 681 : cluster [DBG] pgmap v1058: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:13 smithi059 bash[61152]: cluster 2022-04-28T20:06:13.269041+0000 mgr.y (mgr.24961) 682 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:13.571Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:13 smithi059 bash[59745]: cluster 2022-04-28T20:06:13.269041+0000 mgr.y (mgr.24961) 682 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:13 smithi139 bash[43301]: cluster 2022-04-28T20:06:13.269041+0000 mgr.y (mgr.24961) 682 : cluster [DBG] pgmap v1059: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:16 smithi059 bash[61152]: cluster 2022-04-28T20:06:15.270053+0000 mgr.y (mgr.24961) 683 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:16 smithi059 bash[59745]: cluster 2022-04-28T20:06:15.270053+0000 mgr.y (mgr.24961) 683 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:16 smithi139 bash[43301]: cluster 2022-04-28T20:06:15.270053+0000 mgr.y (mgr.24961) 683 : cluster [DBG] pgmap v1060: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[61152]: cluster 2022-04-28T20:06:17.270999+0000 mgr.y (mgr.24961) 684 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[61152]: audit 2022-04-28T20:06:17.411995+0000 mon.a (mon.0) 1702 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:06:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[61152]: audit 2022-04-28T20:06:17.412530+0000 mon.a (mon.0) 1703 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:06:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[61152]: audit 2022-04-28T20:06:17.422562+0000 mon.a (mon.0) 1704 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:06:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[61152]: audit 2022-04-28T20:06:17.423137+0000 mon.a (mon.0) 1705 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:06:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[61152]: audit 2022-04-28T20:06:17.423729+0000 mon.a (mon.0) 1706 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:06:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[59745]: cluster 2022-04-28T20:06:17.270999+0000 mgr.y (mgr.24961) 684 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[59745]: audit 2022-04-28T20:06:17.411995+0000 mon.a (mon.0) 1702 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:06:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[59745]: audit 2022-04-28T20:06:17.412530+0000 mon.a (mon.0) 1703 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:06:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[59745]: audit 2022-04-28T20:06:17.422562+0000 mon.a (mon.0) 1704 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:06:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[59745]: audit 2022-04-28T20:06:17.423137+0000 mon.a (mon.0) 1705 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:06:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:18 smithi059 bash[59745]: audit 2022-04-28T20:06:17.423729+0000 mon.a (mon.0) 1706 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:06:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:18 smithi139 bash[43301]: cluster 2022-04-28T20:06:17.270999+0000 mgr.y (mgr.24961) 684 : cluster [DBG] pgmap v1061: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:18 smithi139 bash[43301]: audit 2022-04-28T20:06:17.411995+0000 mon.a (mon.0) 1702 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:06:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:18 smithi139 bash[43301]: audit 2022-04-28T20:06:17.412530+0000 mon.a (mon.0) 1703 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:06:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:18 smithi139 bash[43301]: audit 2022-04-28T20:06:17.422562+0000 mon.a (mon.0) 1704 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:06:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:18 smithi139 bash[43301]: audit 2022-04-28T20:06:17.423137+0000 mon.a (mon.0) 1705 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:06:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:18 smithi139 bash[43301]: audit 2022-04-28T20:06:17.423729+0000 mon.a (mon.0) 1706 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:06:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:20 smithi139 bash[43301]: cluster 2022-04-28T20:06:19.271917+0000 mgr.y (mgr.24961) 685 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:20.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:20 smithi059 bash[61152]: cluster 2022-04-28T20:06:19.271917+0000 mgr.y (mgr.24961) 685 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:20 smithi059 bash[59745]: cluster 2022-04-28T20:06:19.271917+0000 mgr.y (mgr.24961) 685 : cluster [DBG] pgmap v1062: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:21.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:22 smithi139 bash[43301]: cluster 2022-04-28T20:06:21.272877+0000 mgr.y (mgr.24961) 686 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:22.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:22 smithi059 bash[61152]: cluster 2022-04-28T20:06:21.272877+0000 mgr.y (mgr.24961) 686 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:22 smithi059 bash[59745]: cluster 2022-04-28T20:06:21.272877+0000 mgr.y (mgr.24961) 686 : cluster [DBG] pgmap v1063: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:23.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:23 smithi059 bash[61152]: cluster 2022-04-28T20:06:23.273532+0000 mgr.y (mgr.24961) 687 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:23.572Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:23 smithi059 bash[59745]: cluster 2022-04-28T20:06:23.273532+0000 mgr.y (mgr.24961) 687 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:23 smithi139 bash[43301]: cluster 2022-04-28T20:06:23.273532+0000 mgr.y (mgr.24961) 687 : cluster [DBG] pgmap v1064: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:26 smithi059 bash[61152]: cluster 2022-04-28T20:06:25.274543+0000 mgr.y (mgr.24961) 688 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:26 smithi059 bash[59745]: cluster 2022-04-28T20:06:25.274543+0000 mgr.y (mgr.24961) 688 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:26 smithi139 bash[43301]: cluster 2022-04-28T20:06:25.274543+0000 mgr.y (mgr.24961) 688 : cluster [DBG] pgmap v1065: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:28.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:28 smithi059 bash[61152]: cluster 2022-04-28T20:06:27.275358+0000 mgr.y (mgr.24961) 689 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:28 smithi059 bash[59745]: cluster 2022-04-28T20:06:27.275358+0000 mgr.y (mgr.24961) 689 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:28.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:28 smithi139 bash[43301]: cluster 2022-04-28T20:06:27.275358+0000 mgr.y (mgr.24961) 689 : cluster [DBG] pgmap v1066: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:30 smithi139 bash[43301]: cluster 2022-04-28T20:06:29.276282+0000 mgr.y (mgr.24961) 690 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:30.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:30 smithi059 bash[61152]: cluster 2022-04-28T20:06:29.276282+0000 mgr.y (mgr.24961) 690 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:30 smithi059 bash[59745]: cluster 2022-04-28T20:06:29.276282+0000 mgr.y (mgr.24961) 690 : cluster [DBG] pgmap v1067: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:31.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:32 smithi139 bash[43301]: cluster 2022-04-28T20:06:31.277321+0000 mgr.y (mgr.24961) 691 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:32.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:32 smithi059 bash[61152]: cluster 2022-04-28T20:06:31.277321+0000 mgr.y (mgr.24961) 691 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:32 smithi059 bash[59745]: cluster 2022-04-28T20:06:31.277321+0000 mgr.y (mgr.24961) 691 : cluster [DBG] pgmap v1068: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:33 smithi059 bash[61152]: cluster 2022-04-28T20:06:33.278003+0000 mgr.y (mgr.24961) 692 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:33.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:33 smithi059 bash[59745]: cluster 2022-04-28T20:06:33.278003+0000 mgr.y (mgr.24961) 692 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:33 smithi139 bash[43301]: cluster 2022-04-28T20:06:33.278003+0000 mgr.y (mgr.24961) 692 : cluster [DBG] pgmap v1069: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:36.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:36 smithi059 bash[59745]: cluster 2022-04-28T20:06:35.279011+0000 mgr.y (mgr.24961) 693 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:36 smithi059 bash[61152]: cluster 2022-04-28T20:06:35.279011+0000 mgr.y (mgr.24961) 693 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:36 smithi139 bash[43301]: cluster 2022-04-28T20:06:35.279011+0000 mgr.y (mgr.24961) 693 : cluster [DBG] pgmap v1070: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:38 smithi139 bash[43301]: cluster 2022-04-28T20:06:37.279878+0000 mgr.y (mgr.24961) 694 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:38.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:38 smithi059 bash[61152]: cluster 2022-04-28T20:06:37.279878+0000 mgr.y (mgr.24961) 694 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:38.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:38 smithi059 bash[59745]: cluster 2022-04-28T20:06:37.279878+0000 mgr.y (mgr.24961) 694 : cluster [DBG] pgmap v1071: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:40 smithi139 bash[43301]: cluster 2022-04-28T20:06:39.280764+0000 mgr.y (mgr.24961) 695 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:40.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:40 smithi059 bash[61152]: cluster 2022-04-28T20:06:39.280764+0000 mgr.y (mgr.24961) 695 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:40 smithi059 bash[59745]: cluster 2022-04-28T20:06:39.280764+0000 mgr.y (mgr.24961) 695 : cluster [DBG] pgmap v1072: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:41.681Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:42 smithi139 bash[43301]: cluster 2022-04-28T20:06:41.281757+0000 mgr.y (mgr.24961) 696 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:42 smithi059 bash[61152]: cluster 2022-04-28T20:06:41.281757+0000 mgr.y (mgr.24961) 696 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:42 smithi059 bash[59745]: cluster 2022-04-28T20:06:41.281757+0000 mgr.y (mgr.24961) 696 : cluster [DBG] pgmap v1073: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:43 smithi059 bash[59745]: cluster 2022-04-28T20:06:43.282417+0000 mgr.y (mgr.24961) 697 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:43 smithi059 bash[61152]: cluster 2022-04-28T20:06:43.282417+0000 mgr.y (mgr.24961) 697 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:43.573Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:43 smithi139 bash[43301]: cluster 2022-04-28T20:06:43.282417+0000 mgr.y (mgr.24961) 697 : cluster [DBG] pgmap v1074: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:46.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:46 smithi059 bash[61152]: cluster 2022-04-28T20:06:45.283395+0000 mgr.y (mgr.24961) 698 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:46 smithi059 bash[59745]: cluster 2022-04-28T20:06:45.283395+0000 mgr.y (mgr.24961) 698 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:46.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:46 smithi139 bash[43301]: cluster 2022-04-28T20:06:45.283395+0000 mgr.y (mgr.24961) 698 : cluster [DBG] pgmap v1075: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:48 smithi139 bash[43301]: cluster 2022-04-28T20:06:47.284253+0000 mgr.y (mgr.24961) 699 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:48 smithi059 bash[61152]: cluster 2022-04-28T20:06:47.284253+0000 mgr.y (mgr.24961) 699 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:48 smithi059 bash[59745]: cluster 2022-04-28T20:06:47.284253+0000 mgr.y (mgr.24961) 699 : cluster [DBG] pgmap v1076: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:50 smithi139 bash[43301]: cluster 2022-04-28T20:06:49.285143+0000 mgr.y (mgr.24961) 700 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:50.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:50 smithi059 bash[61152]: cluster 2022-04-28T20:06:49.285143+0000 mgr.y (mgr.24961) 700 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:51.011 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:50 smithi059 bash[59745]: cluster 2022-04-28T20:06:49.285143+0000 mgr.y (mgr.24961) 700 : cluster [DBG] pgmap v1077: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:51.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:52 smithi139 bash[43301]: cluster 2022-04-28T20:06:51.286077+0000 mgr.y (mgr.24961) 701 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:52.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:52 smithi059 bash[61152]: cluster 2022-04-28T20:06:51.286077+0000 mgr.y (mgr.24961) 701 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:52.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:52 smithi059 bash[59745]: cluster 2022-04-28T20:06:51.286077+0000 mgr.y (mgr.24961) 701 : cluster [DBG] pgmap v1078: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:53 smithi059 bash[61152]: cluster 2022-04-28T20:06:53.286737+0000 mgr.y (mgr.24961) 702 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:06:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:06:53.574Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:06:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:53 smithi059 bash[59745]: cluster 2022-04-28T20:06:53.286737+0000 mgr.y (mgr.24961) 702 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:53 smithi139 bash[43301]: cluster 2022-04-28T20:06:53.286737+0000 mgr.y (mgr.24961) 702 : cluster [DBG] pgmap v1079: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:56 smithi139 bash[43301]: cluster 2022-04-28T20:06:55.287817+0000 mgr.y (mgr.24961) 703 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:56.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:56 smithi059 bash[61152]: cluster 2022-04-28T20:06:55.287817+0000 mgr.y (mgr.24961) 703 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:56 smithi059 bash[59745]: cluster 2022-04-28T20:06:55.287817+0000 mgr.y (mgr.24961) 703 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:58 smithi139 bash[43301]: cluster 2022-04-28T20:06:57.288700+0000 mgr.y (mgr.24961) 704 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:58 smithi059 bash[61152]: cluster 2022-04-28T20:06:57.288700+0000 mgr.y (mgr.24961) 704 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:58 smithi059 bash[59745]: cluster 2022-04-28T20:06:57.288700+0000 mgr.y (mgr.24961) 704 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:06:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:59 smithi139 bash[43301]: audit 2022-04-28T20:06:58.517140+0000 mon.a (mon.0) 1707 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:06:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:59 smithi139 bash[43301]: audit 2022-04-28T20:06:58.520071+0000 mon.a (mon.0) 1708 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:06:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:59 smithi139 bash[43301]: audit 2022-04-28T20:06:58.521979+0000 mon.a (mon.0) 1709 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:06:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:06:59 smithi139 bash[43301]: audit 2022-04-28T20:06:58.829256+0000 mon.a (mon.0) 1710 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:06:59.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[59745]: audit 2022-04-28T20:06:58.517140+0000 mon.a (mon.0) 1707 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:06:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[59745]: audit 2022-04-28T20:06:58.520071+0000 mon.a (mon.0) 1708 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:06:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[59745]: audit 2022-04-28T20:06:58.521979+0000 mon.a (mon.0) 1709 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:06:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[59745]: audit 2022-04-28T20:06:58.829256+0000 mon.a (mon.0) 1710 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:06:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[61152]: audit 2022-04-28T20:06:58.517140+0000 mon.a (mon.0) 1707 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:06:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[61152]: audit 2022-04-28T20:06:58.520071+0000 mon.a (mon.0) 1708 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:06:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[61152]: audit 2022-04-28T20:06:58.521979+0000 mon.a (mon.0) 1709 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:06:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:06:59 smithi059 bash[61152]: audit 2022-04-28T20:06:58.829256+0000 mon.a (mon.0) 1710 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:07:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:00 smithi139 bash[43301]: cephadm 2022-04-28T20:06:58.523678+0000 mgr.y (mgr.24961) 705 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:07:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:00 smithi139 bash[43301]: cluster 2022-04-28T20:06:59.289544+0000 mgr.y (mgr.24961) 706 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:00.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:00 smithi059 bash[61152]: cephadm 2022-04-28T20:06:58.523678+0000 mgr.y (mgr.24961) 705 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:07:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:00 smithi059 bash[61152]: cluster 2022-04-28T20:06:59.289544+0000 mgr.y (mgr.24961) 706 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:00 smithi059 bash[59745]: cephadm 2022-04-28T20:06:58.523678+0000 mgr.y (mgr.24961) 705 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:07:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:00 smithi059 bash[59745]: cluster 2022-04-28T20:06:59.289544+0000 mgr.y (mgr.24961) 706 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:01.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:02 smithi139 bash[43301]: cluster 2022-04-28T20:07:01.290483+0000 mgr.y (mgr.24961) 707 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:02 smithi059 bash[61152]: cluster 2022-04-28T20:07:01.290483+0000 mgr.y (mgr.24961) 707 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:02.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:02 smithi059 bash[59745]: cluster 2022-04-28T20:07:01.290483+0000 mgr.y (mgr.24961) 707 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:03 smithi059 bash[61152]: cluster 2022-04-28T20:07:03.291146+0000 mgr.y (mgr.24961) 708 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:03.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:03 smithi059 bash[59745]: cluster 2022-04-28T20:07:03.291146+0000 mgr.y (mgr.24961) 708 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:03 smithi139 bash[43301]: cluster 2022-04-28T20:07:03.291146+0000 mgr.y (mgr.24961) 708 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:06 smithi139 bash[43301]: cluster 2022-04-28T20:07:05.292165+0000 mgr.y (mgr.24961) 709 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:06.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:06 smithi059 bash[61152]: cluster 2022-04-28T20:07:05.292165+0000 mgr.y (mgr.24961) 709 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:06 smithi059 bash[59745]: cluster 2022-04-28T20:07:05.292165+0000 mgr.y (mgr.24961) 709 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:08 smithi139 bash[43301]: cluster 2022-04-28T20:07:07.292997+0000 mgr.y (mgr.24961) 710 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:08.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:08 smithi059 bash[59745]: cluster 2022-04-28T20:07:07.292997+0000 mgr.y (mgr.24961) 710 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:08 smithi059 bash[61152]: cluster 2022-04-28T20:07:07.292997+0000 mgr.y (mgr.24961) 710 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:10 smithi139 bash[43301]: cluster 2022-04-28T20:07:09.293901+0000 mgr.y (mgr.24961) 711 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:10.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:10 smithi059 bash[61152]: cluster 2022-04-28T20:07:09.293901+0000 mgr.y (mgr.24961) 711 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:10 smithi059 bash[59745]: cluster 2022-04-28T20:07:09.293901+0000 mgr.y (mgr.24961) 711 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:11.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:12 smithi139 bash[43301]: cluster 2022-04-28T20:07:11.294880+0000 mgr.y (mgr.24961) 712 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:12.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:12 smithi059 bash[59745]: cluster 2022-04-28T20:07:11.294880+0000 mgr.y (mgr.24961) 712 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:12 smithi059 bash[61152]: cluster 2022-04-28T20:07:11.294880+0000 mgr.y (mgr.24961) 712 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:13.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:13 smithi059 bash[59745]: cluster 2022-04-28T20:07:13.295676+0000 mgr.y (mgr.24961) 713 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:13 smithi059 bash[61152]: cluster 2022-04-28T20:07:13.295676+0000 mgr.y (mgr.24961) 713 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:13.575Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:13 smithi139 bash[43301]: cluster 2022-04-28T20:07:13.295676+0000 mgr.y (mgr.24961) 713 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:16.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:16 smithi139 bash[43301]: cluster 2022-04-28T20:07:15.296794+0000 mgr.y (mgr.24961) 714 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:16.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:16 smithi059 bash[61152]: cluster 2022-04-28T20:07:15.296794+0000 mgr.y (mgr.24961) 714 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:16 smithi059 bash[59745]: cluster 2022-04-28T20:07:15.296794+0000 mgr.y (mgr.24961) 714 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:18 smithi139 bash[43301]: cluster 2022-04-28T20:07:17.297680+0000 mgr.y (mgr.24961) 715 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:18 smithi139 bash[43301]: audit 2022-04-28T20:07:17.421715+0000 mon.a (mon.0) 1711 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:18 smithi139 bash[43301]: audit 2022-04-28T20:07:17.425085+0000 mon.a (mon.0) 1712 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:07:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:18 smithi139 bash[43301]: audit 2022-04-28T20:07:17.434196+0000 mon.a (mon.0) 1713 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:07:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:18 smithi139 bash[43301]: audit 2022-04-28T20:07:17.434820+0000 mon.a (mon.0) 1714 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:07:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:18 smithi139 bash[43301]: audit 2022-04-28T20:07:17.435536+0000 mon.a (mon.0) 1715 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:07:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[59745]: cluster 2022-04-28T20:07:17.297680+0000 mgr.y (mgr.24961) 715 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[59745]: audit 2022-04-28T20:07:17.421715+0000 mon.a (mon.0) 1711 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:07:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[59745]: audit 2022-04-28T20:07:17.425085+0000 mon.a (mon.0) 1712 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:07:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[59745]: audit 2022-04-28T20:07:17.434196+0000 mon.a (mon.0) 1713 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:07:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[59745]: audit 2022-04-28T20:07:17.434820+0000 mon.a (mon.0) 1714 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:07:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[59745]: audit 2022-04-28T20:07:17.435536+0000 mon.a (mon.0) 1715 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:07:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[61152]: cluster 2022-04-28T20:07:17.297680+0000 mgr.y (mgr.24961) 715 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[61152]: audit 2022-04-28T20:07:17.421715+0000 mon.a (mon.0) 1711 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:07:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[61152]: audit 2022-04-28T20:07:17.425085+0000 mon.a (mon.0) 1712 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:07:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[61152]: audit 2022-04-28T20:07:17.434196+0000 mon.a (mon.0) 1713 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:07:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[61152]: audit 2022-04-28T20:07:17.434820+0000 mon.a (mon.0) 1714 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:07:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:18 smithi059 bash[61152]: audit 2022-04-28T20:07:17.435536+0000 mon.a (mon.0) 1715 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:07:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:20 smithi139 bash[43301]: cluster 2022-04-28T20:07:19.298516+0000 mgr.y (mgr.24961) 716 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:20.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:20 smithi059 bash[61152]: cluster 2022-04-28T20:07:19.298516+0000 mgr.y (mgr.24961) 716 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:20 smithi059 bash[59745]: cluster 2022-04-28T20:07:19.298516+0000 mgr.y (mgr.24961) 716 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:21.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:22 smithi059 bash[61152]: cluster 2022-04-28T20:07:21.299629+0000 mgr.y (mgr.24961) 717 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:22 smithi059 bash[59745]: cluster 2022-04-28T20:07:21.299629+0000 mgr.y (mgr.24961) 717 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:22.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:22 smithi139 bash[43301]: cluster 2022-04-28T20:07:21.299629+0000 mgr.y (mgr.24961) 717 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:23 smithi059 bash[61152]: cluster 2022-04-28T20:07:23.300429+0000 mgr.y (mgr.24961) 718 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:23.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:23 smithi059 bash[59745]: cluster 2022-04-28T20:07:23.300429+0000 mgr.y (mgr.24961) 718 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:23 smithi139 bash[43301]: cluster 2022-04-28T20:07:23.300429+0000 mgr.y (mgr.24961) 718 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:26 smithi139 bash[43301]: cluster 2022-04-28T20:07:25.301450+0000 mgr.y (mgr.24961) 719 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:26 smithi059 bash[61152]: cluster 2022-04-28T20:07:25.301450+0000 mgr.y (mgr.24961) 719 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:26 smithi059 bash[59745]: cluster 2022-04-28T20:07:25.301450+0000 mgr.y (mgr.24961) 719 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:28 smithi139 bash[43301]: cluster 2022-04-28T20:07:27.302286+0000 mgr.y (mgr.24961) 720 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:28 smithi059 bash[61152]: cluster 2022-04-28T20:07:27.302286+0000 mgr.y (mgr.24961) 720 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:28.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:28 smithi059 bash[59745]: cluster 2022-04-28T20:07:27.302286+0000 mgr.y (mgr.24961) 720 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:30 smithi059 bash[61152]: cluster 2022-04-28T20:07:29.303147+0000 mgr.y (mgr.24961) 721 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:30 smithi059 bash[59745]: cluster 2022-04-28T20:07:29.303147+0000 mgr.y (mgr.24961) 721 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:30 smithi139 bash[43301]: cluster 2022-04-28T20:07:29.303147+0000 mgr.y (mgr.24961) 721 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:31.685Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:32 smithi059 bash[61152]: cluster 2022-04-28T20:07:31.304108+0000 mgr.y (mgr.24961) 722 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:32 smithi059 bash[59745]: cluster 2022-04-28T20:07:31.304108+0000 mgr.y (mgr.24961) 722 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:32 smithi139 bash[43301]: cluster 2022-04-28T20:07:31.304108+0000 mgr.y (mgr.24961) 722 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:33 smithi059 bash[61152]: cluster 2022-04-28T20:07:33.304760+0000 mgr.y (mgr.24961) 723 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:33.576Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:33 smithi059 bash[59745]: cluster 2022-04-28T20:07:33.304760+0000 mgr.y (mgr.24961) 723 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:33 smithi139 bash[43301]: cluster 2022-04-28T20:07:33.304760+0000 mgr.y (mgr.24961) 723 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:36 smithi139 bash[43301]: cluster 2022-04-28T20:07:35.305737+0000 mgr.y (mgr.24961) 724 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:36 smithi059 bash[61152]: cluster 2022-04-28T20:07:35.305737+0000 mgr.y (mgr.24961) 724 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:36 smithi059 bash[59745]: cluster 2022-04-28T20:07:35.305737+0000 mgr.y (mgr.24961) 724 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:38 smithi139 bash[43301]: cluster 2022-04-28T20:07:37.306513+0000 mgr.y (mgr.24961) 725 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:38.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:38 smithi059 bash[59745]: cluster 2022-04-28T20:07:37.306513+0000 mgr.y (mgr.24961) 725 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:38 smithi059 bash[61152]: cluster 2022-04-28T20:07:37.306513+0000 mgr.y (mgr.24961) 725 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:40 smithi059 bash[61152]: cluster 2022-04-28T20:07:39.307518+0000 mgr.y (mgr.24961) 726 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:40.886 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:40 smithi059 bash[59745]: cluster 2022-04-28T20:07:39.307518+0000 mgr.y (mgr.24961) 726 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:40.887 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:40 smithi139 bash[43301]: cluster 2022-04-28T20:07:39.307518+0000 mgr.y (mgr.24961) 726 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:41.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:42 smithi059 bash[61152]: cluster 2022-04-28T20:07:41.308600+0000 mgr.y (mgr.24961) 727 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:42 smithi059 bash[59745]: cluster 2022-04-28T20:07:41.308600+0000 mgr.y (mgr.24961) 727 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:42 smithi139 bash[43301]: cluster 2022-04-28T20:07:41.308600+0000 mgr.y (mgr.24961) 727 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:43 smithi059 bash[61152]: cluster 2022-04-28T20:07:43.309294+0000 mgr.y (mgr.24961) 728 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:43.867 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:43.578Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:43.867 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:43 smithi059 bash[59745]: cluster 2022-04-28T20:07:43.309294+0000 mgr.y (mgr.24961) 728 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:43 smithi139 bash[43301]: cluster 2022-04-28T20:07:43.309294+0000 mgr.y (mgr.24961) 728 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:46 smithi139 bash[43301]: cluster 2022-04-28T20:07:45.310307+0000 mgr.y (mgr.24961) 729 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:46 smithi059 bash[61152]: cluster 2022-04-28T20:07:45.310307+0000 mgr.y (mgr.24961) 729 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:46.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:46 smithi059 bash[59745]: cluster 2022-04-28T20:07:45.310307+0000 mgr.y (mgr.24961) 729 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:48 smithi059 bash[61152]: cluster 2022-04-28T20:07:47.311116+0000 mgr.y (mgr.24961) 730 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:48 smithi059 bash[59745]: cluster 2022-04-28T20:07:47.311116+0000 mgr.y (mgr.24961) 730 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:48 smithi139 bash[43301]: cluster 2022-04-28T20:07:47.311116+0000 mgr.y (mgr.24961) 730 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:50 smithi059 bash[61152]: cluster 2022-04-28T20:07:49.312100+0000 mgr.y (mgr.24961) 731 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:50 smithi059 bash[59745]: cluster 2022-04-28T20:07:49.312100+0000 mgr.y (mgr.24961) 731 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:50 smithi139 bash[43301]: cluster 2022-04-28T20:07:49.312100+0000 mgr.y (mgr.24961) 731 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:51.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:52 smithi059 bash[61152]: cluster 2022-04-28T20:07:51.313192+0000 mgr.y (mgr.24961) 732 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:52.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:52 smithi059 bash[59745]: cluster 2022-04-28T20:07:51.313192+0000 mgr.y (mgr.24961) 732 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:52.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:52 smithi139 bash[43301]: cluster 2022-04-28T20:07:51.313192+0000 mgr.y (mgr.24961) 732 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:53 smithi059 bash[61152]: cluster 2022-04-28T20:07:53.313884+0000 mgr.y (mgr.24961) 733 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:07:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:07:53.579Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:07:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:53 smithi059 bash[59745]: cluster 2022-04-28T20:07:53.313884+0000 mgr.y (mgr.24961) 733 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:53 smithi139 bash[43301]: cluster 2022-04-28T20:07:53.313884+0000 mgr.y (mgr.24961) 733 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:56 smithi059 bash[61152]: cluster 2022-04-28T20:07:55.314984+0000 mgr.y (mgr.24961) 734 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:56 smithi059 bash[59745]: cluster 2022-04-28T20:07:55.314984+0000 mgr.y (mgr.24961) 734 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:56 smithi139 bash[43301]: cluster 2022-04-28T20:07:55.314984+0000 mgr.y (mgr.24961) 734 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:58 smithi059 bash[61152]: cluster 2022-04-28T20:07:57.315937+0000 mgr.y (mgr.24961) 735 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:58 smithi059 bash[59745]: cluster 2022-04-28T20:07:57.315937+0000 mgr.y (mgr.24961) 735 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:58 smithi139 bash[43301]: cluster 2022-04-28T20:07:57.315937+0000 mgr.y (mgr.24961) 735 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:07:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[59745]: audit 2022-04-28T20:07:58.838217+0000 mon.a (mon.0) 1716 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:07:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[59745]: audit 2022-04-28T20:07:58.840969+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:07:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[59745]: audit 2022-04-28T20:07:58.842992+0000 mon.a (mon.0) 1718 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:07:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[59745]: audit 2022-04-28T20:07:59.151458+0000 mon.a (mon.0) 1719 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:07:59.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[59745]: audit 2022-04-28T20:07:59.156770+0000 mon.a (mon.0) 1720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:07:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[61152]: audit 2022-04-28T20:07:58.838217+0000 mon.a (mon.0) 1716 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:07:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[61152]: audit 2022-04-28T20:07:58.840969+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:07:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[61152]: audit 2022-04-28T20:07:58.842992+0000 mon.a (mon.0) 1718 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:07:59.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[61152]: audit 2022-04-28T20:07:59.151458+0000 mon.a (mon.0) 1719 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:07:59.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:07:59 smithi059 bash[61152]: audit 2022-04-28T20:07:59.156770+0000 mon.a (mon.0) 1720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:07:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:59 smithi139 bash[43301]: audit 2022-04-28T20:07:58.838217+0000 mon.a (mon.0) 1716 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:07:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:59 smithi139 bash[43301]: audit 2022-04-28T20:07:58.840969+0000 mon.a (mon.0) 1717 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:07:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:59 smithi139 bash[43301]: audit 2022-04-28T20:07:58.842992+0000 mon.a (mon.0) 1718 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:07:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:59 smithi139 bash[43301]: audit 2022-04-28T20:07:59.151458+0000 mon.a (mon.0) 1719 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:07:59.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:07:59 smithi139 bash[43301]: audit 2022-04-28T20:07:59.156770+0000 mon.a (mon.0) 1720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:08:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:00 smithi059 bash[59745]: cephadm 2022-04-28T20:07:58.844717+0000 mgr.y (mgr.24961) 736 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:08:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:00 smithi059 bash[59745]: cluster 2022-04-28T20:07:59.316930+0000 mgr.y (mgr.24961) 737 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:00 smithi059 bash[59745]: audit 2022-04-28T20:07:59.448871+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:08:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:00 smithi059 bash[61152]: cephadm 2022-04-28T20:07:58.844717+0000 mgr.y (mgr.24961) 736 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:08:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:00 smithi059 bash[61152]: cluster 2022-04-28T20:07:59.316930+0000 mgr.y (mgr.24961) 737 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:00 smithi059 bash[61152]: audit 2022-04-28T20:07:59.448871+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:08:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:00 smithi139 bash[43301]: cephadm 2022-04-28T20:07:58.844717+0000 mgr.y (mgr.24961) 736 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:08:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:00 smithi139 bash[43301]: cluster 2022-04-28T20:07:59.316930+0000 mgr.y (mgr.24961) 737 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:00 smithi139 bash[43301]: audit 2022-04-28T20:07:59.448871+0000 mon.a (mon.0) 1721 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:08:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:01.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:02 smithi059 bash[61152]: cluster 2022-04-28T20:08:01.318027+0000 mgr.y (mgr.24961) 738 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:02.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:02 smithi059 bash[59745]: cluster 2022-04-28T20:08:01.318027+0000 mgr.y (mgr.24961) 738 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:02 smithi139 bash[43301]: cluster 2022-04-28T20:08:01.318027+0000 mgr.y (mgr.24961) 738 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:03 smithi059 bash[61152]: cluster 2022-04-28T20:08:03.318661+0000 mgr.y (mgr.24961) 739 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:03.580Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:03 smithi059 bash[59745]: cluster 2022-04-28T20:08:03.318661+0000 mgr.y (mgr.24961) 739 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:03 smithi139 bash[43301]: cluster 2022-04-28T20:08:03.318661+0000 mgr.y (mgr.24961) 739 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:06 smithi059 bash[61152]: cluster 2022-04-28T20:08:05.319608+0000 mgr.y (mgr.24961) 740 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:06 smithi059 bash[59745]: cluster 2022-04-28T20:08:05.319608+0000 mgr.y (mgr.24961) 740 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:06 smithi139 bash[43301]: cluster 2022-04-28T20:08:05.319608+0000 mgr.y (mgr.24961) 740 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:08 smithi059 bash[61152]: cluster 2022-04-28T20:08:07.320428+0000 mgr.y (mgr.24961) 741 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:08.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:08 smithi059 bash[59745]: cluster 2022-04-28T20:08:07.320428+0000 mgr.y (mgr.24961) 741 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:08.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:08 smithi139 bash[43301]: cluster 2022-04-28T20:08:07.320428+0000 mgr.y (mgr.24961) 741 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:10 smithi059 bash[61152]: cluster 2022-04-28T20:08:09.321293+0000 mgr.y (mgr.24961) 742 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:10 smithi059 bash[59745]: cluster 2022-04-28T20:08:09.321293+0000 mgr.y (mgr.24961) 742 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:10.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:10 smithi139 bash[43301]: cluster 2022-04-28T20:08:09.321293+0000 mgr.y (mgr.24961) 742 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:11.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:12 smithi059 bash[61152]: cluster 2022-04-28T20:08:11.322263+0000 mgr.y (mgr.24961) 743 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:12.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:12 smithi059 bash[59745]: cluster 2022-04-28T20:08:11.322263+0000 mgr.y (mgr.24961) 743 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:12 smithi139 bash[43301]: cluster 2022-04-28T20:08:11.322263+0000 mgr.y (mgr.24961) 743 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:13 smithi059 bash[61152]: cluster 2022-04-28T20:08:13.322929+0000 mgr.y (mgr.24961) 744 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:13.581Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:13 smithi059 bash[59745]: cluster 2022-04-28T20:08:13.322929+0000 mgr.y (mgr.24961) 744 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:13 smithi139 bash[43301]: cluster 2022-04-28T20:08:13.322929+0000 mgr.y (mgr.24961) 744 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:16 smithi059 bash[61152]: cluster 2022-04-28T20:08:15.323935+0000 mgr.y (mgr.24961) 745 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:16 smithi059 bash[59745]: cluster 2022-04-28T20:08:15.323935+0000 mgr.y (mgr.24961) 745 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:16 smithi139 bash[43301]: cluster 2022-04-28T20:08:15.323935+0000 mgr.y (mgr.24961) 745 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[59745]: cluster 2022-04-28T20:08:17.324604+0000 mgr.y (mgr.24961) 746 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[59745]: audit 2022-04-28T20:08:17.428442+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:08:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[59745]: audit 2022-04-28T20:08:17.434182+0000 mon.a (mon.0) 1723 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:08:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[59745]: audit 2022-04-28T20:08:17.442294+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:08:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[59745]: audit 2022-04-28T20:08:17.442995+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:08:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[59745]: audit 2022-04-28T20:08:17.443553+0000 mon.a (mon.0) 1726 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:08:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[61152]: cluster 2022-04-28T20:08:17.324604+0000 mgr.y (mgr.24961) 746 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[61152]: audit 2022-04-28T20:08:17.428442+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:08:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[61152]: audit 2022-04-28T20:08:17.434182+0000 mon.a (mon.0) 1723 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:08:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[61152]: audit 2022-04-28T20:08:17.442294+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:08:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[61152]: audit 2022-04-28T20:08:17.442995+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:08:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:18 smithi059 bash[61152]: audit 2022-04-28T20:08:17.443553+0000 mon.a (mon.0) 1726 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:08:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:18 smithi139 bash[43301]: cluster 2022-04-28T20:08:17.324604+0000 mgr.y (mgr.24961) 746 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:18 smithi139 bash[43301]: audit 2022-04-28T20:08:17.428442+0000 mon.a (mon.0) 1722 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:08:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:18 smithi139 bash[43301]: audit 2022-04-28T20:08:17.434182+0000 mon.a (mon.0) 1723 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:08:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:18 smithi139 bash[43301]: audit 2022-04-28T20:08:17.442294+0000 mon.a (mon.0) 1724 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:08:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:18 smithi139 bash[43301]: audit 2022-04-28T20:08:17.442995+0000 mon.a (mon.0) 1725 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:08:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:18 smithi139 bash[43301]: audit 2022-04-28T20:08:17.443553+0000 mon.a (mon.0) 1726 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:08:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:20 smithi059 bash[61152]: cluster 2022-04-28T20:08:19.325430+0000 mgr.y (mgr.24961) 747 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:20 smithi059 bash[59745]: cluster 2022-04-28T20:08:19.325430+0000 mgr.y (mgr.24961) 747 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:20 smithi139 bash[43301]: cluster 2022-04-28T20:08:19.325430+0000 mgr.y (mgr.24961) 747 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:21.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:22 smithi059 bash[61152]: cluster 2022-04-28T20:08:21.326385+0000 mgr.y (mgr.24961) 748 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:22 smithi059 bash[59745]: cluster 2022-04-28T20:08:21.326385+0000 mgr.y (mgr.24961) 748 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:22 smithi139 bash[43301]: cluster 2022-04-28T20:08:21.326385+0000 mgr.y (mgr.24961) 748 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:23 smithi059 bash[61152]: cluster 2022-04-28T20:08:23.327008+0000 mgr.y (mgr.24961) 749 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:23.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:23 smithi059 bash[59745]: cluster 2022-04-28T20:08:23.327008+0000 mgr.y (mgr.24961) 749 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:23 smithi139 bash[43301]: cluster 2022-04-28T20:08:23.327008+0000 mgr.y (mgr.24961) 749 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:26 smithi059 bash[61152]: cluster 2022-04-28T20:08:25.328127+0000 mgr.y (mgr.24961) 750 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:26 smithi059 bash[59745]: cluster 2022-04-28T20:08:25.328127+0000 mgr.y (mgr.24961) 750 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:26 smithi139 bash[43301]: cluster 2022-04-28T20:08:25.328127+0000 mgr.y (mgr.24961) 750 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:28 smithi059 bash[61152]: cluster 2022-04-28T20:08:27.328966+0000 mgr.y (mgr.24961) 751 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:28.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:28 smithi059 bash[59745]: cluster 2022-04-28T20:08:27.328966+0000 mgr.y (mgr.24961) 751 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:28.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:28 smithi139 bash[43301]: cluster 2022-04-28T20:08:27.328966+0000 mgr.y (mgr.24961) 751 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:30 smithi059 bash[61152]: cluster 2022-04-28T20:08:29.329843+0000 mgr.y (mgr.24961) 752 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:30 smithi059 bash[59745]: cluster 2022-04-28T20:08:29.329843+0000 mgr.y (mgr.24961) 752 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:30 smithi139 bash[43301]: cluster 2022-04-28T20:08:29.329843+0000 mgr.y (mgr.24961) 752 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:31.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:32 smithi059 bash[61152]: cluster 2022-04-28T20:08:31.330747+0000 mgr.y (mgr.24961) 753 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:32.853 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:32 smithi059 bash[59745]: cluster 2022-04-28T20:08:31.330747+0000 mgr.y (mgr.24961) 753 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:32 smithi139 bash[43301]: cluster 2022-04-28T20:08:31.330747+0000 mgr.y (mgr.24961) 753 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:33 smithi059 bash[61152]: cluster 2022-04-28T20:08:33.331401+0000 mgr.y (mgr.24961) 754 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:33.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:33.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:33 smithi059 bash[59745]: cluster 2022-04-28T20:08:33.331401+0000 mgr.y (mgr.24961) 754 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:33 smithi139 bash[43301]: cluster 2022-04-28T20:08:33.331401+0000 mgr.y (mgr.24961) 754 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:36 smithi059 bash[61152]: cluster 2022-04-28T20:08:35.332440+0000 mgr.y (mgr.24961) 755 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:36 smithi059 bash[59745]: cluster 2022-04-28T20:08:35.332440+0000 mgr.y (mgr.24961) 755 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:36 smithi139 bash[43301]: cluster 2022-04-28T20:08:35.332440+0000 mgr.y (mgr.24961) 755 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:38 smithi059 bash[61152]: cluster 2022-04-28T20:08:37.333270+0000 mgr.y (mgr.24961) 756 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:38.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:38 smithi059 bash[59745]: cluster 2022-04-28T20:08:37.333270+0000 mgr.y (mgr.24961) 756 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:38 smithi139 bash[43301]: cluster 2022-04-28T20:08:37.333270+0000 mgr.y (mgr.24961) 756 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:40 smithi059 bash[61152]: cluster 2022-04-28T20:08:39.334236+0000 mgr.y (mgr.24961) 757 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:40 smithi059 bash[59745]: cluster 2022-04-28T20:08:39.334236+0000 mgr.y (mgr.24961) 757 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:40 smithi139 bash[43301]: cluster 2022-04-28T20:08:39.334236+0000 mgr.y (mgr.24961) 757 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:41.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:42 smithi059 bash[61152]: cluster 2022-04-28T20:08:41.335227+0000 mgr.y (mgr.24961) 758 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:42 smithi059 bash[59745]: cluster 2022-04-28T20:08:41.335227+0000 mgr.y (mgr.24961) 758 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:42 smithi139 bash[43301]: cluster 2022-04-28T20:08:41.335227+0000 mgr.y (mgr.24961) 758 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:43 smithi059 bash[61152]: cluster 2022-04-28T20:08:43.335924+0000 mgr.y (mgr.24961) 759 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:43.583Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:43 smithi059 bash[59745]: cluster 2022-04-28T20:08:43.335924+0000 mgr.y (mgr.24961) 759 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:43 smithi139 bash[43301]: cluster 2022-04-28T20:08:43.335924+0000 mgr.y (mgr.24961) 759 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:46 smithi059 bash[61152]: cluster 2022-04-28T20:08:45.336893+0000 mgr.y (mgr.24961) 760 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:46.857 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:46 smithi059 bash[59745]: cluster 2022-04-28T20:08:45.336893+0000 mgr.y (mgr.24961) 760 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:46.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:46 smithi139 bash[43301]: cluster 2022-04-28T20:08:45.336893+0000 mgr.y (mgr.24961) 760 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:48 smithi059 bash[61152]: cluster 2022-04-28T20:08:47.337655+0000 mgr.y (mgr.24961) 761 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:48 smithi059 bash[59745]: cluster 2022-04-28T20:08:47.337655+0000 mgr.y (mgr.24961) 761 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:48 smithi139 bash[43301]: cluster 2022-04-28T20:08:47.337655+0000 mgr.y (mgr.24961) 761 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:50 smithi059 bash[59745]: cluster 2022-04-28T20:08:49.338530+0000 mgr.y (mgr.24961) 762 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:50 smithi059 bash[61152]: cluster 2022-04-28T20:08:49.338530+0000 mgr.y (mgr.24961) 762 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:50.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:50 smithi139 bash[43301]: cluster 2022-04-28T20:08:49.338530+0000 mgr.y (mgr.24961) 762 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:51.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:52 smithi059 bash[61152]: cluster 2022-04-28T20:08:51.339618+0000 mgr.y (mgr.24961) 763 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:52.888 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:52 smithi059 bash[59745]: cluster 2022-04-28T20:08:51.339618+0000 mgr.y (mgr.24961) 763 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:52.889 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:52 smithi139 bash[43301]: cluster 2022-04-28T20:08:51.339618+0000 mgr.y (mgr.24961) 763 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:53 smithi059 bash[59745]: cluster 2022-04-28T20:08:53.340306+0000 mgr.y (mgr.24961) 764 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:53 smithi059 bash[61152]: cluster 2022-04-28T20:08:53.340306+0000 mgr.y (mgr.24961) 764 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:08:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:08:53.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:08:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:53 smithi139 bash[43301]: cluster 2022-04-28T20:08:53.340306+0000 mgr.y (mgr.24961) 764 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:56 smithi059 bash[61152]: cluster 2022-04-28T20:08:55.341326+0000 mgr.y (mgr.24961) 765 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:56 smithi059 bash[59745]: cluster 2022-04-28T20:08:55.341326+0000 mgr.y (mgr.24961) 765 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:56 smithi139 bash[43301]: cluster 2022-04-28T20:08:55.341326+0000 mgr.y (mgr.24961) 765 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:08:58 smithi059 bash[61152]: cluster 2022-04-28T20:08:57.342152+0000 mgr.y (mgr.24961) 766 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:08:58 smithi059 bash[59745]: cluster 2022-04-28T20:08:57.342152+0000 mgr.y (mgr.24961) 766 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:08:58.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:08:58 smithi139 bash[43301]: cluster 2022-04-28T20:08:57.342152+0000 mgr.y (mgr.24961) 766 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[59745]: cluster 2022-04-28T20:08:59.343035+0000 mgr.y (mgr.24961) 767 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[59745]: audit 2022-04-28T20:08:59.457217+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:09:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[59745]: audit 2022-04-28T20:08:59.459964+0000 mon.a (mon.0) 1728 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:09:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[59745]: audit 2022-04-28T20:08:59.461881+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:09:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[59745]: cephadm 2022-04-28T20:08:59.463558+0000 mgr.y (mgr.24961) 768 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:09:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[59745]: audit 2022-04-28T20:08:59.465124+0000 mon.a (mon.0) 1730 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:09:00.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[61152]: cluster 2022-04-28T20:08:59.343035+0000 mgr.y (mgr.24961) 767 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:00.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[61152]: audit 2022-04-28T20:08:59.457217+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:09:00.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[61152]: audit 2022-04-28T20:08:59.459964+0000 mon.a (mon.0) 1728 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:09:00.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[61152]: audit 2022-04-28T20:08:59.461881+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:09:00.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[61152]: cephadm 2022-04-28T20:08:59.463558+0000 mgr.y (mgr.24961) 768 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:09:00.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:00 smithi059 bash[61152]: audit 2022-04-28T20:08:59.465124+0000 mon.a (mon.0) 1730 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:09:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:00 smithi139 bash[43301]: cluster 2022-04-28T20:08:59.343035+0000 mgr.y (mgr.24961) 767 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:00 smithi139 bash[43301]: audit 2022-04-28T20:08:59.457217+0000 mon.a (mon.0) 1727 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:09:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:00 smithi139 bash[43301]: audit 2022-04-28T20:08:59.459964+0000 mon.a (mon.0) 1728 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:09:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:00 smithi139 bash[43301]: audit 2022-04-28T20:08:59.461881+0000 mon.a (mon.0) 1729 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:09:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:00 smithi139 bash[43301]: cephadm 2022-04-28T20:08:59.463558+0000 mgr.y (mgr.24961) 768 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:09:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:00 smithi139 bash[43301]: audit 2022-04-28T20:08:59.465124+0000 mon.a (mon.0) 1730 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:09:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:01.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:02.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:02 smithi059 bash[59745]: cluster 2022-04-28T20:09:01.344010+0000 mgr.y (mgr.24961) 769 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:02.891 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:02 smithi059 bash[61152]: cluster 2022-04-28T20:09:01.344010+0000 mgr.y (mgr.24961) 769 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:02.892 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:02 smithi139 bash[43301]: cluster 2022-04-28T20:09:01.344010+0000 mgr.y (mgr.24961) 769 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:03.786 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:03 smithi059 bash[61152]: cluster 2022-04-28T20:09:03.344704+0000 mgr.y (mgr.24961) 770 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:03.786 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:03 smithi059 bash[59745]: cluster 2022-04-28T20:09:03.344704+0000 mgr.y (mgr.24961) 770 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:03.787 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:03.582Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:03 smithi139 bash[43301]: cluster 2022-04-28T20:09:03.344704+0000 mgr.y (mgr.24961) 770 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:06 smithi059 bash[59745]: cluster 2022-04-28T20:09:05.345538+0000 mgr.y (mgr.24961) 771 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:06 smithi059 bash[61152]: cluster 2022-04-28T20:09:05.345538+0000 mgr.y (mgr.24961) 771 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:06 smithi139 bash[43301]: cluster 2022-04-28T20:09:05.345538+0000 mgr.y (mgr.24961) 771 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:07 smithi059 bash[59745]: audit 2022-04-28T20:09:06.685788+0000 mon.a (mon.0) 1731 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:09:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:07 smithi059 bash[59745]: audit 2022-04-28T20:09:06.915443+0000 mon.a (mon.0) 1732 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:09:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:07 smithi059 bash[59745]: audit 2022-04-28T20:09:07.205161+0000 mon.a (mon.0) 1733 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:09:07.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:07 smithi059 bash[61152]: audit 2022-04-28T20:09:06.685788+0000 mon.a (mon.0) 1731 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:09:07.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:07 smithi059 bash[61152]: audit 2022-04-28T20:09:06.915443+0000 mon.a (mon.0) 1732 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:09:07.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:07 smithi059 bash[61152]: audit 2022-04-28T20:09:07.205161+0000 mon.a (mon.0) 1733 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:09:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:07 smithi139 bash[43301]: audit 2022-04-28T20:09:06.685788+0000 mon.a (mon.0) 1731 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:09:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:07 smithi139 bash[43301]: audit 2022-04-28T20:09:06.915443+0000 mon.a (mon.0) 1732 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:09:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:07 smithi139 bash[43301]: audit 2022-04-28T20:09:07.205161+0000 mon.a (mon.0) 1733 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:09:08.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:08 smithi059 bash[59745]: cluster 2022-04-28T20:09:07.346355+0000 mgr.y (mgr.24961) 772 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:08 smithi059 bash[61152]: cluster 2022-04-28T20:09:07.346355+0000 mgr.y (mgr.24961) 772 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:08 smithi139 bash[43301]: cluster 2022-04-28T20:09:07.346355+0000 mgr.y (mgr.24961) 772 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:10 smithi059 bash[61152]: cluster 2022-04-28T20:09:09.347289+0000 mgr.y (mgr.24961) 773 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:10.855 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:10 smithi059 bash[59745]: cluster 2022-04-28T20:09:09.347289+0000 mgr.y (mgr.24961) 773 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:10 smithi139 bash[43301]: cluster 2022-04-28T20:09:09.347289+0000 mgr.y (mgr.24961) 773 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:11.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:12 smithi059 bash[61152]: cluster 2022-04-28T20:09:11.348264+0000 mgr.y (mgr.24961) 774 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:12 smithi059 bash[59745]: cluster 2022-04-28T20:09:11.348264+0000 mgr.y (mgr.24961) 774 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:12 smithi139 bash[43301]: cluster 2022-04-28T20:09:11.348264+0000 mgr.y (mgr.24961) 774 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:13 smithi059 bash[61152]: cluster 2022-04-28T20:09:13.348944+0000 mgr.y (mgr.24961) 775 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:13.584Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:13 smithi059 bash[59745]: cluster 2022-04-28T20:09:13.348944+0000 mgr.y (mgr.24961) 775 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:13 smithi139 bash[43301]: cluster 2022-04-28T20:09:13.348944+0000 mgr.y (mgr.24961) 775 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:16 smithi059 bash[61152]: cluster 2022-04-28T20:09:15.349970+0000 mgr.y (mgr.24961) 776 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:16 smithi059 bash[59745]: cluster 2022-04-28T20:09:15.349970+0000 mgr.y (mgr.24961) 776 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:16 smithi139 bash[43301]: cluster 2022-04-28T20:09:15.349970+0000 mgr.y (mgr.24961) 776 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[61152]: cluster 2022-04-28T20:09:17.350799+0000 mgr.y (mgr.24961) 777 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[61152]: audit 2022-04-28T20:09:17.434732+0000 mon.a (mon.0) 1734 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:09:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[61152]: audit 2022-04-28T20:09:17.441024+0000 mon.a (mon.0) 1735 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:09:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[61152]: audit 2022-04-28T20:09:17.449778+0000 mon.a (mon.0) 1736 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:09:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[61152]: audit 2022-04-28T20:09:17.450286+0000 mon.a (mon.0) 1737 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:09:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[61152]: audit 2022-04-28T20:09:17.450796+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:09:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[59745]: cluster 2022-04-28T20:09:17.350799+0000 mgr.y (mgr.24961) 777 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[59745]: audit 2022-04-28T20:09:17.434732+0000 mon.a (mon.0) 1734 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:09:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[59745]: audit 2022-04-28T20:09:17.441024+0000 mon.a (mon.0) 1735 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:09:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[59745]: audit 2022-04-28T20:09:17.449778+0000 mon.a (mon.0) 1736 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:09:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[59745]: audit 2022-04-28T20:09:17.450286+0000 mon.a (mon.0) 1737 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:09:18.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:18 smithi059 bash[59745]: audit 2022-04-28T20:09:17.450796+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:09:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:18 smithi139 bash[43301]: cluster 2022-04-28T20:09:17.350799+0000 mgr.y (mgr.24961) 777 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:18 smithi139 bash[43301]: audit 2022-04-28T20:09:17.434732+0000 mon.a (mon.0) 1734 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:09:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:18 smithi139 bash[43301]: audit 2022-04-28T20:09:17.441024+0000 mon.a (mon.0) 1735 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:09:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:18 smithi139 bash[43301]: audit 2022-04-28T20:09:17.449778+0000 mon.a (mon.0) 1736 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:18 smithi139 bash[43301]: audit 2022-04-28T20:09:17.450286+0000 mon.a (mon.0) 1737 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:18 smithi139 bash[43301]: audit 2022-04-28T20:09:17.450796+0000 mon.a (mon.0) 1738 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:09:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:20 smithi059 bash[61152]: cluster 2022-04-28T20:09:19.351697+0000 mgr.y (mgr.24961) 778 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:20 smithi059 bash[59745]: cluster 2022-04-28T20:09:19.351697+0000 mgr.y (mgr.24961) 778 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:20 smithi139 bash[43301]: cluster 2022-04-28T20:09:19.351697+0000 mgr.y (mgr.24961) 778 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:21.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:22 smithi059 bash[61152]: cluster 2022-04-28T20:09:21.352655+0000 mgr.y (mgr.24961) 779 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:22 smithi059 bash[59745]: cluster 2022-04-28T20:09:21.352655+0000 mgr.y (mgr.24961) 779 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:22 smithi139 bash[43301]: cluster 2022-04-28T20:09:21.352655+0000 mgr.y (mgr.24961) 779 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:23 smithi059 bash[61152]: cluster 2022-04-28T20:09:23.353331+0000 mgr.y (mgr.24961) 780 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:23.585Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:23 smithi059 bash[59745]: cluster 2022-04-28T20:09:23.353331+0000 mgr.y (mgr.24961) 780 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:23 smithi139 bash[43301]: cluster 2022-04-28T20:09:23.353331+0000 mgr.y (mgr.24961) 780 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:26 smithi059 bash[61152]: cluster 2022-04-28T20:09:25.354337+0000 mgr.y (mgr.24961) 781 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:26 smithi059 bash[59745]: cluster 2022-04-28T20:09:25.354337+0000 mgr.y (mgr.24961) 781 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:26 smithi139 bash[43301]: cluster 2022-04-28T20:09:25.354337+0000 mgr.y (mgr.24961) 781 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:28 smithi059 bash[61152]: cluster 2022-04-28T20:09:27.355172+0000 mgr.y (mgr.24961) 782 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:28 smithi059 bash[59745]: cluster 2022-04-28T20:09:27.355172+0000 mgr.y (mgr.24961) 782 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:28.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:28 smithi139 bash[43301]: cluster 2022-04-28T20:09:27.355172+0000 mgr.y (mgr.24961) 782 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:30 smithi059 bash[61152]: cluster 2022-04-28T20:09:29.356093+0000 mgr.y (mgr.24961) 783 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:30 smithi059 bash[59745]: cluster 2022-04-28T20:09:29.356093+0000 mgr.y (mgr.24961) 783 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:30 smithi139 bash[43301]: cluster 2022-04-28T20:09:29.356093+0000 mgr.y (mgr.24961) 783 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:31.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:32 smithi059 bash[61152]: cluster 2022-04-28T20:09:31.357029+0000 mgr.y (mgr.24961) 784 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:32 smithi059 bash[59745]: cluster 2022-04-28T20:09:31.357029+0000 mgr.y (mgr.24961) 784 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:32 smithi139 bash[43301]: cluster 2022-04-28T20:09:31.357029+0000 mgr.y (mgr.24961) 784 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:33 smithi059 bash[61152]: cluster 2022-04-28T20:09:33.357780+0000 mgr.y (mgr.24961) 785 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:34.024 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:33.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:34.025 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:33 smithi059 bash[59745]: cluster 2022-04-28T20:09:33.357780+0000 mgr.y (mgr.24961) 785 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:33 smithi139 bash[43301]: cluster 2022-04-28T20:09:33.357780+0000 mgr.y (mgr.24961) 785 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:36 smithi059 bash[61152]: cluster 2022-04-28T20:09:35.358809+0000 mgr.y (mgr.24961) 786 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:36 smithi059 bash[59745]: cluster 2022-04-28T20:09:35.358809+0000 mgr.y (mgr.24961) 786 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:36 smithi139 bash[43301]: cluster 2022-04-28T20:09:35.358809+0000 mgr.y (mgr.24961) 786 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:38 smithi059 bash[61152]: cluster 2022-04-28T20:09:37.359680+0000 mgr.y (mgr.24961) 787 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:38 smithi059 bash[59745]: cluster 2022-04-28T20:09:37.359680+0000 mgr.y (mgr.24961) 787 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:38 smithi139 bash[43301]: cluster 2022-04-28T20:09:37.359680+0000 mgr.y (mgr.24961) 787 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:40 smithi059 bash[59745]: cluster 2022-04-28T20:09:39.360598+0000 mgr.y (mgr.24961) 788 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:40 smithi059 bash[59745]: cluster 2022-04-28T20:09:39.509370+0000 mon.a (mon.0) 1739 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2022-04-28T20:09:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:40 smithi059 bash[59745]: audit 2022-04-28T20:09:39.511349+0000 mon.a (mon.0) 1740 : audit [INF] from='client.? 172.21.15.59:0/2047634936' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75923-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:09:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:40 smithi059 bash[61152]: cluster 2022-04-28T20:09:39.360598+0000 mgr.y (mgr.24961) 788 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:40 smithi059 bash[61152]: cluster 2022-04-28T20:09:39.509370+0000 mon.a (mon.0) 1739 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2022-04-28T20:09:40.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:40 smithi059 bash[61152]: audit 2022-04-28T20:09:39.511349+0000 mon.a (mon.0) 1740 : audit [INF] from='client.? 172.21.15.59:0/2047634936' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75923-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:09:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:40 smithi139 bash[43301]: cluster 2022-04-28T20:09:39.360598+0000 mgr.y (mgr.24961) 788 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:40 smithi139 bash[43301]: cluster 2022-04-28T20:09:39.509370+0000 mon.a (mon.0) 1739 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2022-04-28T20:09:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:40 smithi139 bash[43301]: audit 2022-04-28T20:09:39.511349+0000 mon.a (mon.0) 1740 : audit [INF] from='client.? 172.21.15.59:0/2047634936' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75923-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:09:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:41 smithi059 bash[59745]: audit 2022-04-28T20:09:40.507058+0000 mon.a (mon.0) 1741 : audit [INF] from='client.? 172.21.15.59:0/2047634936' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75923-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:09:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:41 smithi059 bash[59745]: cluster 2022-04-28T20:09:40.507273+0000 mon.a (mon.0) 1742 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2022-04-28T20:09:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:41 smithi059 bash[61152]: audit 2022-04-28T20:09:40.507058+0000 mon.a (mon.0) 1741 : audit [INF] from='client.? 172.21.15.59:0/2047634936' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75923-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:09:41.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:41 smithi059 bash[61152]: cluster 2022-04-28T20:09:40.507273+0000 mon.a (mon.0) 1742 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2022-04-28T20:09:41.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:41.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:41 smithi139 bash[43301]: audit 2022-04-28T20:09:40.507058+0000 mon.a (mon.0) 1741 : audit [INF] from='client.? 172.21.15.59:0/2047634936' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-75923-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:09:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:41 smithi139 bash[43301]: cluster 2022-04-28T20:09:40.507273+0000 mon.a (mon.0) 1742 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2022-04-28T20:09:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:42 smithi059 bash[61152]: cluster 2022-04-28T20:09:41.361564+0000 mgr.y (mgr.24961) 789 : cluster [DBG] pgmap v1165: 105 pgs: 12 creating+peering, 11 unknown, 82 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:09:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:42 smithi059 bash[61152]: cluster 2022-04-28T20:09:41.521358+0000 mon.a (mon.0) 1743 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2022-04-28T20:09:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:42 smithi059 bash[59745]: cluster 2022-04-28T20:09:41.361564+0000 mgr.y (mgr.24961) 789 : cluster [DBG] pgmap v1165: 105 pgs: 12 creating+peering, 11 unknown, 82 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:09:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:42 smithi059 bash[59745]: cluster 2022-04-28T20:09:41.521358+0000 mon.a (mon.0) 1743 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2022-04-28T20:09:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:42 smithi139 bash[43301]: cluster 2022-04-28T20:09:41.361564+0000 mgr.y (mgr.24961) 789 : cluster [DBG] pgmap v1165: 105 pgs: 12 creating+peering, 11 unknown, 82 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:09:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:42 smithi139 bash[43301]: cluster 2022-04-28T20:09:41.521358+0000 mon.a (mon.0) 1743 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2022-04-28T20:09:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:43 smithi059 bash[61152]: cluster 2022-04-28T20:09:43.362348+0000 mgr.y (mgr.24961) 790 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:43.945 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:43.586Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:43.946 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:43 smithi059 bash[59745]: cluster 2022-04-28T20:09:43.362348+0000 mgr.y (mgr.24961) 790 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:43 smithi139 bash[43301]: cluster 2022-04-28T20:09:43.362348+0000 mgr.y (mgr.24961) 790 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 24 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:46 smithi059 bash[61152]: cluster 2022-04-28T20:09:45.363401+0000 mgr.y (mgr.24961) 791 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:46 smithi059 bash[59745]: cluster 2022-04-28T20:09:45.363401+0000 mgr.y (mgr.24961) 791 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:46 smithi139 bash[43301]: cluster 2022-04-28T20:09:45.363401+0000 mgr.y (mgr.24961) 791 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:48 smithi059 bash[61152]: cluster 2022-04-28T20:09:47.364256+0000 mgr.y (mgr.24961) 792 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:48 smithi059 bash[59745]: cluster 2022-04-28T20:09:47.364256+0000 mgr.y (mgr.24961) 792 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:48 smithi139 bash[43301]: cluster 2022-04-28T20:09:47.364256+0000 mgr.y (mgr.24961) 792 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:50 smithi059 bash[59745]: cluster 2022-04-28T20:09:49.365224+0000 mgr.y (mgr.24961) 793 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:50.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:50 smithi059 bash[61152]: cluster 2022-04-28T20:09:49.365224+0000 mgr.y (mgr.24961) 793 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:50 smithi139 bash[43301]: cluster 2022-04-28T20:09:49.365224+0000 mgr.y (mgr.24961) 793 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:51.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:52 smithi059 bash[61152]: cluster 2022-04-28T20:09:51.366216+0000 mgr.y (mgr.24961) 794 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:52.915 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:52 smithi059 bash[59745]: cluster 2022-04-28T20:09:51.366216+0000 mgr.y (mgr.24961) 794 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:52.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:52 smithi139 bash[43301]: cluster 2022-04-28T20:09:51.366216+0000 mgr.y (mgr.24961) 794 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:53 smithi059 bash[61152]: cluster 2022-04-28T20:09:53.366904+0000 mgr.y (mgr.24961) 795 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:09:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:09:53.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:09:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:53 smithi059 bash[59745]: cluster 2022-04-28T20:09:53.366904+0000 mgr.y (mgr.24961) 795 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:53 smithi139 bash[43301]: cluster 2022-04-28T20:09:53.366904+0000 mgr.y (mgr.24961) 795 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:56 smithi059 bash[59745]: cluster 2022-04-28T20:09:55.367938+0000 mgr.y (mgr.24961) 796 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:56.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:56 smithi059 bash[61152]: cluster 2022-04-28T20:09:55.367938+0000 mgr.y (mgr.24961) 796 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:56 smithi139 bash[43301]: cluster 2022-04-28T20:09:55.367938+0000 mgr.y (mgr.24961) 796 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:09:58 smithi059 bash[61152]: cluster 2022-04-28T20:09:57.368773+0000 mgr.y (mgr.24961) 797 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:09:58 smithi059 bash[59745]: cluster 2022-04-28T20:09:57.368773+0000 mgr.y (mgr.24961) 797 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:09:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:09:58 smithi139 bash[43301]: cluster 2022-04-28T20:09:57.368773+0000 mgr.y (mgr.24961) 797 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:00 smithi059 bash[61152]: cluster 2022-04-28T20:09:59.369599+0000 mgr.y (mgr.24961) 798 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:00.914 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:00 smithi059 bash[61152]: cluster 2022-04-28T20:10:00.000134+0000 mon.a (mon.0) 1744 : cluster [INF] overall HEALTH_OK 2022-04-28T20:10:00.915 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:00 smithi059 bash[59745]: cluster 2022-04-28T20:09:59.369599+0000 mgr.y (mgr.24961) 798 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:00.915 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:00 smithi059 bash[59745]: cluster 2022-04-28T20:10:00.000134+0000 mon.a (mon.0) 1744 : cluster [INF] overall HEALTH_OK 2022-04-28T20:10:00.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:00 smithi139 bash[43301]: cluster 2022-04-28T20:09:59.369599+0000 mgr.y (mgr.24961) 798 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:00.916 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:00 smithi139 bash[43301]: cluster 2022-04-28T20:10:00.000134+0000 mon.a (mon.0) 1744 : cluster [INF] overall HEALTH_OK 2022-04-28T20:10:02.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:01.696Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:02.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:02 smithi059 bash[59745]: cluster 2022-04-28T20:10:01.370566+0000 mgr.y (mgr.24961) 799 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:02.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:02 smithi059 bash[61152]: cluster 2022-04-28T20:10:01.370566+0000 mgr.y (mgr.24961) 799 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:02 smithi139 bash[43301]: cluster 2022-04-28T20:10:01.370566+0000 mgr.y (mgr.24961) 799 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:03 smithi059 bash[61152]: cluster 2022-04-28T20:10:03.371227+0000 mgr.y (mgr.24961) 800 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:03.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:03.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:03 smithi059 bash[59745]: cluster 2022-04-28T20:10:03.371227+0000 mgr.y (mgr.24961) 800 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:03 smithi139 bash[43301]: cluster 2022-04-28T20:10:03.371227+0000 mgr.y (mgr.24961) 800 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:06 smithi059 bash[61152]: cluster 2022-04-28T20:10:05.372346+0000 mgr.y (mgr.24961) 801 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:06 smithi059 bash[59745]: cluster 2022-04-28T20:10:05.372346+0000 mgr.y (mgr.24961) 801 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:06 smithi139 bash[43301]: cluster 2022-04-28T20:10:05.372346+0000 mgr.y (mgr.24961) 801 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[59745]: audit 2022-04-28T20:10:07.214532+0000 mon.a (mon.0) 1745 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:10:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[59745]: audit 2022-04-28T20:10:07.217582+0000 mon.a (mon.0) 1746 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:10:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[59745]: audit 2022-04-28T20:10:07.219555+0000 mon.a (mon.0) 1747 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:10:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[59745]: audit 2022-04-28T20:10:07.224015+0000 mon.a (mon.0) 1748 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:10:07.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[61152]: audit 2022-04-28T20:10:07.214532+0000 mon.a (mon.0) 1745 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:10:07.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[61152]: audit 2022-04-28T20:10:07.217582+0000 mon.a (mon.0) 1746 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:10:07.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[61152]: audit 2022-04-28T20:10:07.219555+0000 mon.a (mon.0) 1747 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:10:07.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:07 smithi059 bash[61152]: audit 2022-04-28T20:10:07.224015+0000 mon.a (mon.0) 1748 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:10:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:07 smithi139 bash[43301]: audit 2022-04-28T20:10:07.214532+0000 mon.a (mon.0) 1745 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:10:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:07 smithi139 bash[43301]: audit 2022-04-28T20:10:07.217582+0000 mon.a (mon.0) 1746 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:10:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:07 smithi139 bash[43301]: audit 2022-04-28T20:10:07.219555+0000 mon.a (mon.0) 1747 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:10:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:07 smithi139 bash[43301]: audit 2022-04-28T20:10:07.224015+0000 mon.a (mon.0) 1748 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:10:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: cephadm 2022-04-28T20:10:07.221235+0000 mgr.y (mgr.24961) 802 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:10:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: cluster 2022-04-28T20:10:07.373192+0000 mgr.y (mgr.24961) 803 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.541774+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.542424+0000 mon.a (mon.0) 1750 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.543017+0000 mon.a (mon.0) 1751 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.543586+0000 mon.a (mon.0) 1752 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.544143+0000 mon.a (mon.0) 1753 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.544689+0000 mon.a (mon.0) 1754 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.545280+0000 mon.a (mon.0) 1755 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.545905+0000 mon.a (mon.0) 1756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.546523+0000 mon.a (mon.0) 1757 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:10:08.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[61152]: audit 2022-04-28T20:10:07.547241+0000 mon.a (mon.0) 1758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: cephadm 2022-04-28T20:10:07.221235+0000 mgr.y (mgr.24961) 802 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:10:08.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: cluster 2022-04-28T20:10:07.373192+0000 mgr.y (mgr.24961) 803 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:08.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.541774+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.542424+0000 mon.a (mon.0) 1750 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.543017+0000 mon.a (mon.0) 1751 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.543586+0000 mon.a (mon.0) 1752 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.544143+0000 mon.a (mon.0) 1753 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.544689+0000 mon.a (mon.0) 1754 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.545280+0000 mon.a (mon.0) 1755 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.545905+0000 mon.a (mon.0) 1756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.546523+0000 mon.a (mon.0) 1757 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:10:08.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:08 smithi059 bash[59745]: audit 2022-04-28T20:10:07.547241+0000 mon.a (mon.0) 1758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: cephadm 2022-04-28T20:10:07.221235+0000 mgr.y (mgr.24961) 802 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:10:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: cluster 2022-04-28T20:10:07.373192+0000 mgr.y (mgr.24961) 803 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.541774+0000 mon.a (mon.0) 1749 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.542424+0000 mon.a (mon.0) 1750 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.543017+0000 mon.a (mon.0) 1751 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.543586+0000 mon.a (mon.0) 1752 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.544143+0000 mon.a (mon.0) 1753 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.544689+0000 mon.a (mon.0) 1754 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.545280+0000 mon.a (mon.0) 1755 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:08.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.545905+0000 mon.a (mon.0) 1756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:08.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.546523+0000 mon.a (mon.0) 1757 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:10:08.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:08 smithi139 bash[43301]: audit 2022-04-28T20:10:07.547241+0000 mon.a (mon.0) 1758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:10:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:10 smithi059 bash[61152]: cluster 2022-04-28T20:10:09.374143+0000 mgr.y (mgr.24961) 804 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:10.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:10 smithi059 bash[59745]: cluster 2022-04-28T20:10:09.374143+0000 mgr.y (mgr.24961) 804 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:10 smithi139 bash[43301]: cluster 2022-04-28T20:10:09.374143+0000 mgr.y (mgr.24961) 804 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:11.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:12.829 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:12 smithi139 bash[43301]: cluster 2022-04-28T20:10:11.375227+0000 mgr.y (mgr.24961) 805 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:12 smithi059 bash[61152]: cluster 2022-04-28T20:10:11.375227+0000 mgr.y (mgr.24961) 805 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:12 smithi059 bash[59745]: cluster 2022-04-28T20:10:11.375227+0000 mgr.y (mgr.24961) 805 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:13 smithi059 bash[61152]: cluster 2022-04-28T20:10:13.375929+0000 mgr.y (mgr.24961) 806 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:13.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:13 smithi059 bash[59745]: cluster 2022-04-28T20:10:13.375929+0000 mgr.y (mgr.24961) 806 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:13 smithi139 bash[43301]: cluster 2022-04-28T20:10:13.375929+0000 mgr.y (mgr.24961) 806 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:15.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:14 smithi059 bash[61152]: audit 2022-04-28T20:10:14.274961+0000 mon.a (mon.0) 1759 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:10:15.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:14 smithi059 bash[61152]: audit 2022-04-28T20:10:14.499017+0000 mon.a (mon.0) 1760 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:10:15.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:14 smithi059 bash[59745]: audit 2022-04-28T20:10:14.274961+0000 mon.a (mon.0) 1759 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:10:15.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:14 smithi059 bash[59745]: audit 2022-04-28T20:10:14.499017+0000 mon.a (mon.0) 1760 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:10:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:14 smithi139 bash[43301]: audit 2022-04-28T20:10:14.274961+0000 mon.a (mon.0) 1759 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:10:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:14 smithi139 bash[43301]: audit 2022-04-28T20:10:14.499017+0000 mon.a (mon.0) 1760 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:10:16.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.789498+0000 mon.a (mon.0) 1761 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.790703+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.791863+0000 mon.a (mon.0) 1763 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.792956+0000 mon.a (mon.0) 1764 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.794039+0000 mon.a (mon.0) 1765 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.795191+0000 mon.a (mon.0) 1766 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.796287+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.797335+0000 mon.a (mon.0) 1768 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.798271+0000 mon.a (mon.0) 1769 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:10:16.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.799195+0000 mon.a (mon.0) 1770 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: audit 2022-04-28T20:10:14.807551+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:10:16.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[59745]: cluster 2022-04-28T20:10:15.376943+0000 mgr.y (mgr.24961) 807 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:16.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.789498+0000 mon.a (mon.0) 1761 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.790703+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.791863+0000 mon.a (mon.0) 1763 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.792956+0000 mon.a (mon.0) 1764 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.794039+0000 mon.a (mon.0) 1765 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.795191+0000 mon.a (mon.0) 1766 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.796287+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.797335+0000 mon.a (mon.0) 1768 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.798271+0000 mon.a (mon.0) 1769 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:10:16.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.799195+0000 mon.a (mon.0) 1770 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: audit 2022-04-28T20:10:14.807551+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:10:16.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:15 smithi059 bash[61152]: cluster 2022-04-28T20:10:15.376943+0000 mgr.y (mgr.24961) 807 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.789498+0000 mon.a (mon.0) 1761 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.272 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.790703+0000 mon.a (mon.0) 1762 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.272 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.791863+0000 mon.a (mon.0) 1763 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.273 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.792956+0000 mon.a (mon.0) 1764 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.273 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.794039+0000 mon.a (mon.0) 1765 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.273 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.795191+0000 mon.a (mon.0) 1766 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.274 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.796287+0000 mon.a (mon.0) 1767 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:10:16.274 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.797335+0000 mon.a (mon.0) 1768 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.276 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.798271+0000 mon.a (mon.0) 1769 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:10:16.276 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.799195+0000 mon.a (mon.0) 1770 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:10:16.276 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: audit 2022-04-28T20:10:14.807551+0000 mon.a (mon.0) 1771 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:10:16.277 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:15 smithi139 bash[43301]: cluster 2022-04-28T20:10:15.376943+0000 mgr.y (mgr.24961) 807 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[61152]: cluster 2022-04-28T20:10:17.377698+0000 mgr.y (mgr.24961) 808 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[61152]: audit 2022-04-28T20:10:17.440656+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[61152]: audit 2022-04-28T20:10:17.445529+0000 mon.a (mon.0) 1773 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[61152]: audit 2022-04-28T20:10:17.457123+0000 mon.a (mon.0) 1774 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[61152]: audit 2022-04-28T20:10:17.457778+0000 mon.a (mon.0) 1775 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[61152]: audit 2022-04-28T20:10:17.458438+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[59745]: cluster 2022-04-28T20:10:17.377698+0000 mgr.y (mgr.24961) 808 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[59745]: audit 2022-04-28T20:10:17.440656+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[59745]: audit 2022-04-28T20:10:17.445529+0000 mon.a (mon.0) 1773 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:10:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[59745]: audit 2022-04-28T20:10:17.457123+0000 mon.a (mon.0) 1774 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[59745]: audit 2022-04-28T20:10:17.457778+0000 mon.a (mon.0) 1775 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:18 smithi059 bash[59745]: audit 2022-04-28T20:10:17.458438+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:18 smithi139 bash[43301]: cluster 2022-04-28T20:10:17.377698+0000 mgr.y (mgr.24961) 808 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:18 smithi139 bash[43301]: audit 2022-04-28T20:10:17.440656+0000 mon.a (mon.0) 1772 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:18 smithi139 bash[43301]: audit 2022-04-28T20:10:17.445529+0000 mon.a (mon.0) 1773 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:18 smithi139 bash[43301]: audit 2022-04-28T20:10:17.457123+0000 mon.a (mon.0) 1774 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:18 smithi139 bash[43301]: audit 2022-04-28T20:10:17.457778+0000 mon.a (mon.0) 1775 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:10:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:18 smithi139 bash[43301]: audit 2022-04-28T20:10:17.458438+0000 mon.a (mon.0) 1776 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:10:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:20 smithi059 bash[61152]: cluster 2022-04-28T20:10:19.378589+0000 mgr.y (mgr.24961) 809 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:20.916 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:20 smithi059 bash[59745]: cluster 2022-04-28T20:10:19.378589+0000 mgr.y (mgr.24961) 809 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:20.917 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:20 smithi139 bash[43301]: cluster 2022-04-28T20:10:19.378589+0000 mgr.y (mgr.24961) 809 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:22.098 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:21.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:22 smithi059 bash[59745]: cluster 2022-04-28T20:10:21.379690+0000 mgr.y (mgr.24961) 810 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:22 smithi059 bash[61152]: cluster 2022-04-28T20:10:21.379690+0000 mgr.y (mgr.24961) 810 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:22 smithi139 bash[43301]: cluster 2022-04-28T20:10:21.379690+0000 mgr.y (mgr.24961) 810 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:23 smithi059 bash[59745]: cluster 2022-04-28T20:10:23.380500+0000 mgr.y (mgr.24961) 811 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:23 smithi059 bash[61152]: cluster 2022-04-28T20:10:23.380500+0000 mgr.y (mgr.24961) 811 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:23.589Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:24.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:23 smithi139 bash[43301]: cluster 2022-04-28T20:10:23.380500+0000 mgr.y (mgr.24961) 811 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:26 smithi059 bash[61152]: cluster 2022-04-28T20:10:25.381528+0000 mgr.y (mgr.24961) 812 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:26 smithi059 bash[59745]: cluster 2022-04-28T20:10:25.381528+0000 mgr.y (mgr.24961) 812 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:26 smithi139 bash[43301]: cluster 2022-04-28T20:10:25.381528+0000 mgr.y (mgr.24961) 812 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:28 smithi059 bash[61152]: cluster 2022-04-28T20:10:27.382386+0000 mgr.y (mgr.24961) 813 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:28.955 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:28 smithi059 bash[59745]: cluster 2022-04-28T20:10:27.382386+0000 mgr.y (mgr.24961) 813 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:28.956 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:28 smithi139 bash[43301]: cluster 2022-04-28T20:10:27.382386+0000 mgr.y (mgr.24961) 813 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:30 smithi059 bash[61152]: cluster 2022-04-28T20:10:29.383253+0000 mgr.y (mgr.24961) 814 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:30 smithi059 bash[59745]: cluster 2022-04-28T20:10:29.383253+0000 mgr.y (mgr.24961) 814 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:30 smithi139 bash[43301]: cluster 2022-04-28T20:10:29.383253+0000 mgr.y (mgr.24961) 814 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:31.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:32 smithi059 bash[61152]: cluster 2022-04-28T20:10:31.384279+0000 mgr.y (mgr.24961) 815 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:32 smithi059 bash[59745]: cluster 2022-04-28T20:10:31.384279+0000 mgr.y (mgr.24961) 815 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:32 smithi139 bash[43301]: cluster 2022-04-28T20:10:31.384279+0000 mgr.y (mgr.24961) 815 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:33 smithi059 bash[61152]: cluster 2022-04-28T20:10:33.385065+0000 mgr.y (mgr.24961) 816 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:33.843 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:33.588Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:33.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:33 smithi059 bash[59745]: cluster 2022-04-28T20:10:33.385065+0000 mgr.y (mgr.24961) 816 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:33 smithi139 bash[43301]: cluster 2022-04-28T20:10:33.385065+0000 mgr.y (mgr.24961) 816 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:36.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:36 smithi059 bash[61152]: cluster 2022-04-28T20:10:35.386128+0000 mgr.y (mgr.24961) 817 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:36 smithi059 bash[59745]: cluster 2022-04-28T20:10:35.386128+0000 mgr.y (mgr.24961) 817 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:36 smithi139 bash[43301]: cluster 2022-04-28T20:10:35.386128+0000 mgr.y (mgr.24961) 817 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:38 smithi059 bash[61152]: cluster 2022-04-28T20:10:37.386978+0000 mgr.y (mgr.24961) 818 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:38.855 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:38 smithi059 bash[59745]: cluster 2022-04-28T20:10:37.386978+0000 mgr.y (mgr.24961) 818 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:38 smithi139 bash[43301]: cluster 2022-04-28T20:10:37.386978+0000 mgr.y (mgr.24961) 818 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:40 smithi059 bash[61152]: cluster 2022-04-28T20:10:39.387911+0000 mgr.y (mgr.24961) 819 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:40 smithi059 bash[59745]: cluster 2022-04-28T20:10:39.387911+0000 mgr.y (mgr.24961) 819 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:40 smithi139 bash[43301]: cluster 2022-04-28T20:10:39.387911+0000 mgr.y (mgr.24961) 819 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:41.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:42 smithi059 bash[61152]: cluster 2022-04-28T20:10:41.388852+0000 mgr.y (mgr.24961) 820 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:42.873 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:42 smithi059 bash[59745]: cluster 2022-04-28T20:10:41.388852+0000 mgr.y (mgr.24961) 820 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:42.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:42 smithi139 bash[43301]: cluster 2022-04-28T20:10:41.388852+0000 mgr.y (mgr.24961) 820 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:43 smithi059 bash[61152]: cluster 2022-04-28T20:10:43.389475+0000 mgr.y (mgr.24961) 821 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:43.590Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:43 smithi059 bash[59745]: cluster 2022-04-28T20:10:43.389475+0000 mgr.y (mgr.24961) 821 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:43 smithi139 bash[43301]: cluster 2022-04-28T20:10:43.389475+0000 mgr.y (mgr.24961) 821 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:46 smithi059 bash[61152]: cluster 2022-04-28T20:10:45.390466+0000 mgr.y (mgr.24961) 822 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:46 smithi059 bash[59745]: cluster 2022-04-28T20:10:45.390466+0000 mgr.y (mgr.24961) 822 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:46 smithi139 bash[43301]: cluster 2022-04-28T20:10:45.390466+0000 mgr.y (mgr.24961) 822 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:48.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:48 smithi059 bash[61152]: cluster 2022-04-28T20:10:47.391221+0000 mgr.y (mgr.24961) 823 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:48 smithi059 bash[59745]: cluster 2022-04-28T20:10:47.391221+0000 mgr.y (mgr.24961) 823 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:48 smithi139 bash[43301]: cluster 2022-04-28T20:10:47.391221+0000 mgr.y (mgr.24961) 823 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:50.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:50 smithi059 bash[61152]: cluster 2022-04-28T20:10:49.392118+0000 mgr.y (mgr.24961) 824 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:50.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:50 smithi059 bash[59745]: cluster 2022-04-28T20:10:49.392118+0000 mgr.y (mgr.24961) 824 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:50 smithi139 bash[43301]: cluster 2022-04-28T20:10:49.392118+0000 mgr.y (mgr.24961) 824 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:51.699Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:52 smithi059 bash[61152]: cluster 2022-04-28T20:10:51.393076+0000 mgr.y (mgr.24961) 825 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:52 smithi059 bash[59745]: cluster 2022-04-28T20:10:51.393076+0000 mgr.y (mgr.24961) 825 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:52 smithi139 bash[43301]: cluster 2022-04-28T20:10:51.393076+0000 mgr.y (mgr.24961) 825 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:53 smithi059 bash[61152]: cluster 2022-04-28T20:10:53.393747+0000 mgr.y (mgr.24961) 826 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:10:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:10:53.591Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:10:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:53 smithi059 bash[59745]: cluster 2022-04-28T20:10:53.393747+0000 mgr.y (mgr.24961) 826 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:53 smithi139 bash[43301]: cluster 2022-04-28T20:10:53.393747+0000 mgr.y (mgr.24961) 826 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:56 smithi059 bash[61152]: cluster 2022-04-28T20:10:55.394755+0000 mgr.y (mgr.24961) 827 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:56 smithi059 bash[59745]: cluster 2022-04-28T20:10:55.394755+0000 mgr.y (mgr.24961) 827 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:56 smithi139 bash[43301]: cluster 2022-04-28T20:10:55.394755+0000 mgr.y (mgr.24961) 827 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:10:58 smithi059 bash[61152]: cluster 2022-04-28T20:10:57.395672+0000 mgr.y (mgr.24961) 828 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:10:58 smithi059 bash[59745]: cluster 2022-04-28T20:10:57.395672+0000 mgr.y (mgr.24961) 828 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:10:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:10:58 smithi139 bash[43301]: cluster 2022-04-28T20:10:57.395672+0000 mgr.y (mgr.24961) 828 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:00.119 INFO:journalctl@ceph.grafana.a.smithi139.stdout:Apr 28 20:10:59 smithi139 bash[54700]: t=2022-04-28T20:10:59+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2022-04-28T20:11:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:00 smithi059 bash[61152]: cluster 2022-04-28T20:10:59.396673+0000 mgr.y (mgr.24961) 829 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:00 smithi059 bash[59745]: cluster 2022-04-28T20:10:59.396673+0000 mgr.y (mgr.24961) 829 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:00 smithi139 bash[43301]: cluster 2022-04-28T20:10:59.396673+0000 mgr.y (mgr.24961) 829 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:01.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:02 smithi059 bash[61152]: cluster 2022-04-28T20:11:01.397642+0000 mgr.y (mgr.24961) 830 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:02 smithi059 bash[59745]: cluster 2022-04-28T20:11:01.397642+0000 mgr.y (mgr.24961) 830 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:02 smithi139 bash[43301]: cluster 2022-04-28T20:11:01.397642+0000 mgr.y (mgr.24961) 830 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:03 smithi059 bash[61152]: cluster 2022-04-28T20:11:03.398282+0000 mgr.y (mgr.24961) 831 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:03.592Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:03 smithi059 bash[59745]: cluster 2022-04-28T20:11:03.398282+0000 mgr.y (mgr.24961) 831 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:03 smithi139 bash[43301]: cluster 2022-04-28T20:11:03.398282+0000 mgr.y (mgr.24961) 831 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:06 smithi059 bash[61152]: cluster 2022-04-28T20:11:05.399281+0000 mgr.y (mgr.24961) 832 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:06 smithi059 bash[59745]: cluster 2022-04-28T20:11:05.399281+0000 mgr.y (mgr.24961) 832 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:06 smithi139 bash[43301]: cluster 2022-04-28T20:11:05.399281+0000 mgr.y (mgr.24961) 832 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:08 smithi059 bash[61152]: cluster 2022-04-28T20:11:07.400118+0000 mgr.y (mgr.24961) 833 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:08 smithi059 bash[59745]: cluster 2022-04-28T20:11:07.400118+0000 mgr.y (mgr.24961) 833 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:08 smithi139 bash[43301]: cluster 2022-04-28T20:11:07.400118+0000 mgr.y (mgr.24961) 833 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:10 smithi059 bash[61152]: cluster 2022-04-28T20:11:09.401101+0000 mgr.y (mgr.24961) 834 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:10 smithi059 bash[59745]: cluster 2022-04-28T20:11:09.401101+0000 mgr.y (mgr.24961) 834 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:10 smithi139 bash[43301]: cluster 2022-04-28T20:11:09.401101+0000 mgr.y (mgr.24961) 834 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:11.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:12 smithi059 bash[61152]: cluster 2022-04-28T20:11:11.402082+0000 mgr.y (mgr.24961) 835 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:12 smithi059 bash[59745]: cluster 2022-04-28T20:11:11.402082+0000 mgr.y (mgr.24961) 835 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:12 smithi139 bash[43301]: cluster 2022-04-28T20:11:11.402082+0000 mgr.y (mgr.24961) 835 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:13 smithi059 bash[61152]: cluster 2022-04-28T20:11:13.402759+0000 mgr.y (mgr.24961) 836 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:13.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:13 smithi059 bash[59745]: cluster 2022-04-28T20:11:13.402759+0000 mgr.y (mgr.24961) 836 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:13 smithi139 bash[43301]: cluster 2022-04-28T20:11:13.402759+0000 mgr.y (mgr.24961) 836 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:14 smithi059 bash[59745]: audit 2022-04-28T20:11:14.814488+0000 mon.a (mon.0) 1777 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:11:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:14 smithi059 bash[59745]: audit 2022-04-28T20:11:14.817318+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:11:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:14 smithi059 bash[59745]: audit 2022-04-28T20:11:14.819125+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:11:15.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:14 smithi059 bash[61152]: audit 2022-04-28T20:11:14.814488+0000 mon.a (mon.0) 1777 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:11:15.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:14 smithi059 bash[61152]: audit 2022-04-28T20:11:14.817318+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:11:15.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:14 smithi059 bash[61152]: audit 2022-04-28T20:11:14.819125+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:11:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:14 smithi139 bash[43301]: audit 2022-04-28T20:11:14.814488+0000 mon.a (mon.0) 1777 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:11:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:14 smithi139 bash[43301]: audit 2022-04-28T20:11:14.817318+0000 mon.a (mon.0) 1778 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:11:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:14 smithi139 bash[43301]: audit 2022-04-28T20:11:14.819125+0000 mon.a (mon.0) 1779 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:11:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:16 smithi059 bash[61152]: cephadm 2022-04-28T20:11:14.820762+0000 mgr.y (mgr.24961) 837 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:11:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:16 smithi059 bash[61152]: audit 2022-04-28T20:11:15.117074+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:11:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:16 smithi059 bash[61152]: cluster 2022-04-28T20:11:15.403798+0000 mgr.y (mgr.24961) 838 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:16 smithi059 bash[59745]: cephadm 2022-04-28T20:11:14.820762+0000 mgr.y (mgr.24961) 837 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:11:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:16 smithi059 bash[59745]: audit 2022-04-28T20:11:15.117074+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:11:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:16 smithi059 bash[59745]: cluster 2022-04-28T20:11:15.403798+0000 mgr.y (mgr.24961) 838 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:16 smithi139 bash[43301]: cephadm 2022-04-28T20:11:14.820762+0000 mgr.y (mgr.24961) 837 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:11:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:16 smithi139 bash[43301]: audit 2022-04-28T20:11:15.117074+0000 mon.a (mon.0) 1780 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:11:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:16 smithi139 bash[43301]: cluster 2022-04-28T20:11:15.403798+0000 mgr.y (mgr.24961) 838 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[61152]: audit 2022-04-28T20:11:17.445733+0000 mon.a (mon.0) 1781 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:11:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[61152]: audit 2022-04-28T20:11:17.451545+0000 mon.a (mon.0) 1782 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:11:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[61152]: audit 2022-04-28T20:11:17.464460+0000 mon.a (mon.0) 1783 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:11:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[61152]: audit 2022-04-28T20:11:17.465002+0000 mon.a (mon.0) 1784 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:11:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[61152]: audit 2022-04-28T20:11:17.465521+0000 mon.a (mon.0) 1785 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:11:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[59745]: audit 2022-04-28T20:11:17.445733+0000 mon.a (mon.0) 1781 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:11:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[59745]: audit 2022-04-28T20:11:17.451545+0000 mon.a (mon.0) 1782 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:11:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[59745]: audit 2022-04-28T20:11:17.464460+0000 mon.a (mon.0) 1783 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:11:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[59745]: audit 2022-04-28T20:11:17.465002+0000 mon.a (mon.0) 1784 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:11:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:17 smithi059 bash[59745]: audit 2022-04-28T20:11:17.465521+0000 mon.a (mon.0) 1785 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:11:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:17 smithi139 bash[43301]: audit 2022-04-28T20:11:17.445733+0000 mon.a (mon.0) 1781 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:11:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:17 smithi139 bash[43301]: audit 2022-04-28T20:11:17.451545+0000 mon.a (mon.0) 1782 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:11:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:17 smithi139 bash[43301]: audit 2022-04-28T20:11:17.464460+0000 mon.a (mon.0) 1783 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:11:17.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:17 smithi139 bash[43301]: audit 2022-04-28T20:11:17.465002+0000 mon.a (mon.0) 1784 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:11:17.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:17 smithi139 bash[43301]: audit 2022-04-28T20:11:17.465521+0000 mon.a (mon.0) 1785 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:11:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:18 smithi059 bash[61152]: cluster 2022-04-28T20:11:17.404589+0000 mgr.y (mgr.24961) 839 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:18 smithi059 bash[59745]: cluster 2022-04-28T20:11:17.404589+0000 mgr.y (mgr.24961) 839 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:18.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:18 smithi139 bash[43301]: cluster 2022-04-28T20:11:17.404589+0000 mgr.y (mgr.24961) 839 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:20 smithi059 bash[61152]: cluster 2022-04-28T20:11:19.405429+0000 mgr.y (mgr.24961) 840 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:20 smithi059 bash[59745]: cluster 2022-04-28T20:11:19.405429+0000 mgr.y (mgr.24961) 840 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:20 smithi139 bash[43301]: cluster 2022-04-28T20:11:19.405429+0000 mgr.y (mgr.24961) 840 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:21.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:22 smithi059 bash[61152]: cluster 2022-04-28T20:11:21.406379+0000 mgr.y (mgr.24961) 841 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:22 smithi059 bash[59745]: cluster 2022-04-28T20:11:21.406379+0000 mgr.y (mgr.24961) 841 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:22 smithi139 bash[43301]: cluster 2022-04-28T20:11:21.406379+0000 mgr.y (mgr.24961) 841 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:23 smithi059 bash[61152]: cluster 2022-04-28T20:11:23.406989+0000 mgr.y (mgr.24961) 842 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:23.593Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:23 smithi059 bash[59745]: cluster 2022-04-28T20:11:23.406989+0000 mgr.y (mgr.24961) 842 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:23 smithi139 bash[43301]: cluster 2022-04-28T20:11:23.406989+0000 mgr.y (mgr.24961) 842 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:26 smithi059 bash[61152]: cluster 2022-04-28T20:11:25.408019+0000 mgr.y (mgr.24961) 843 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:26 smithi059 bash[59745]: cluster 2022-04-28T20:11:25.408019+0000 mgr.y (mgr.24961) 843 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:26 smithi139 bash[43301]: cluster 2022-04-28T20:11:25.408019+0000 mgr.y (mgr.24961) 843 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:28 smithi059 bash[61152]: cluster 2022-04-28T20:11:27.408842+0000 mgr.y (mgr.24961) 844 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:28 smithi059 bash[59745]: cluster 2022-04-28T20:11:27.408842+0000 mgr.y (mgr.24961) 844 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:28.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:28 smithi139 bash[43301]: cluster 2022-04-28T20:11:27.408842+0000 mgr.y (mgr.24961) 844 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:30 smithi059 bash[61152]: cluster 2022-04-28T20:11:29.409772+0000 mgr.y (mgr.24961) 845 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:30 smithi059 bash[59745]: cluster 2022-04-28T20:11:29.409772+0000 mgr.y (mgr.24961) 845 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:30 smithi139 bash[43301]: cluster 2022-04-28T20:11:29.409772+0000 mgr.y (mgr.24961) 845 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:31.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:32 smithi059 bash[61152]: cluster 2022-04-28T20:11:31.410747+0000 mgr.y (mgr.24961) 846 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:32.989 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:32 smithi059 bash[59745]: cluster 2022-04-28T20:11:31.410747+0000 mgr.y (mgr.24961) 846 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:32.990 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:32 smithi139 bash[43301]: cluster 2022-04-28T20:11:31.410747+0000 mgr.y (mgr.24961) 846 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:33 smithi059 bash[61152]: cluster 2022-04-28T20:11:33.411401+0000 mgr.y (mgr.24961) 847 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:34.130 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:33.594Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:34.130 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:33 smithi059 bash[59745]: cluster 2022-04-28T20:11:33.411401+0000 mgr.y (mgr.24961) 847 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:34.131 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:33 smithi139 bash[43301]: cluster 2022-04-28T20:11:33.411401+0000 mgr.y (mgr.24961) 847 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:36 smithi059 bash[61152]: cluster 2022-04-28T20:11:35.412433+0000 mgr.y (mgr.24961) 848 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:36.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:36 smithi059 bash[59745]: cluster 2022-04-28T20:11:35.412433+0000 mgr.y (mgr.24961) 848 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:36.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:36 smithi139 bash[43301]: cluster 2022-04-28T20:11:35.412433+0000 mgr.y (mgr.24961) 848 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:38 smithi059 bash[61152]: cluster 2022-04-28T20:11:37.413266+0000 mgr.y (mgr.24961) 849 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:38 smithi059 bash[59745]: cluster 2022-04-28T20:11:37.413266+0000 mgr.y (mgr.24961) 849 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:38 smithi139 bash[43301]: cluster 2022-04-28T20:11:37.413266+0000 mgr.y (mgr.24961) 849 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:40 smithi059 bash[61152]: cluster 2022-04-28T20:11:39.414145+0000 mgr.y (mgr.24961) 850 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:40 smithi059 bash[59745]: cluster 2022-04-28T20:11:39.414145+0000 mgr.y (mgr.24961) 850 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:40 smithi139 bash[43301]: cluster 2022-04-28T20:11:39.414145+0000 mgr.y (mgr.24961) 850 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:41.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:42 smithi059 bash[61152]: cluster 2022-04-28T20:11:41.415167+0000 mgr.y (mgr.24961) 851 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:42 smithi059 bash[59745]: cluster 2022-04-28T20:11:41.415167+0000 mgr.y (mgr.24961) 851 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:42 smithi139 bash[43301]: cluster 2022-04-28T20:11:41.415167+0000 mgr.y (mgr.24961) 851 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:43 smithi059 bash[61152]: cluster 2022-04-28T20:11:43.415892+0000 mgr.y (mgr.24961) 852 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:43.595Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:43 smithi059 bash[59745]: cluster 2022-04-28T20:11:43.415892+0000 mgr.y (mgr.24961) 852 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:43 smithi139 bash[43301]: cluster 2022-04-28T20:11:43.415892+0000 mgr.y (mgr.24961) 852 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:46 smithi059 bash[61152]: cluster 2022-04-28T20:11:45.416927+0000 mgr.y (mgr.24961) 853 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:46 smithi059 bash[59745]: cluster 2022-04-28T20:11:45.416927+0000 mgr.y (mgr.24961) 853 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:46 smithi139 bash[43301]: cluster 2022-04-28T20:11:45.416927+0000 mgr.y (mgr.24961) 853 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:48 smithi059 bash[61152]: cluster 2022-04-28T20:11:47.417762+0000 mgr.y (mgr.24961) 854 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:48 smithi059 bash[59745]: cluster 2022-04-28T20:11:47.417762+0000 mgr.y (mgr.24961) 854 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:48 smithi139 bash[43301]: cluster 2022-04-28T20:11:47.417762+0000 mgr.y (mgr.24961) 854 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:50 smithi059 bash[61152]: cluster 2022-04-28T20:11:49.418603+0000 mgr.y (mgr.24961) 855 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:50 smithi059 bash[59745]: cluster 2022-04-28T20:11:49.418603+0000 mgr.y (mgr.24961) 855 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:50 smithi139 bash[43301]: cluster 2022-04-28T20:11:49.418603+0000 mgr.y (mgr.24961) 855 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:51.702Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:52 smithi059 bash[61152]: cluster 2022-04-28T20:11:51.419569+0000 mgr.y (mgr.24961) 856 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:52.904 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:52 smithi059 bash[59745]: cluster 2022-04-28T20:11:51.419569+0000 mgr.y (mgr.24961) 856 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:53.039 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:52 smithi139 bash[43301]: cluster 2022-04-28T20:11:51.419569+0000 mgr.y (mgr.24961) 856 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:53 smithi059 bash[61152]: cluster 2022-04-28T20:11:53.420231+0000 mgr.y (mgr.24961) 857 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:11:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:11:53.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:11:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:53 smithi059 bash[59745]: cluster 2022-04-28T20:11:53.420231+0000 mgr.y (mgr.24961) 857 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:53 smithi139 bash[43301]: cluster 2022-04-28T20:11:53.420231+0000 mgr.y (mgr.24961) 857 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:56.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:56 smithi059 bash[61152]: cluster 2022-04-28T20:11:55.421236+0000 mgr.y (mgr.24961) 858 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:56.850 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:56 smithi059 bash[59745]: cluster 2022-04-28T20:11:55.421236+0000 mgr.y (mgr.24961) 858 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:56 smithi139 bash[43301]: cluster 2022-04-28T20:11:55.421236+0000 mgr.y (mgr.24961) 858 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:11:58 smithi059 bash[61152]: cluster 2022-04-28T20:11:57.422046+0000 mgr.y (mgr.24961) 859 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:11:58 smithi059 bash[59745]: cluster 2022-04-28T20:11:57.422046+0000 mgr.y (mgr.24961) 859 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:11:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:11:58 smithi139 bash[43301]: cluster 2022-04-28T20:11:57.422046+0000 mgr.y (mgr.24961) 859 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:00 smithi059 bash[61152]: cluster 2022-04-28T20:11:59.422864+0000 mgr.y (mgr.24961) 860 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:00 smithi059 bash[59745]: cluster 2022-04-28T20:11:59.422864+0000 mgr.y (mgr.24961) 860 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:00 smithi139 bash[43301]: cluster 2022-04-28T20:11:59.422864+0000 mgr.y (mgr.24961) 860 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:01.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:02 smithi059 bash[61152]: cluster 2022-04-28T20:12:01.423816+0000 mgr.y (mgr.24961) 861 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:02 smithi059 bash[59745]: cluster 2022-04-28T20:12:01.423816+0000 mgr.y (mgr.24961) 861 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:02 smithi139 bash[43301]: cluster 2022-04-28T20:12:01.423816+0000 mgr.y (mgr.24961) 861 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:03 smithi059 bash[61152]: cluster 2022-04-28T20:12:03.424480+0000 mgr.y (mgr.24961) 862 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:03.596Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:03 smithi059 bash[59745]: cluster 2022-04-28T20:12:03.424480+0000 mgr.y (mgr.24961) 862 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:03 smithi139 bash[43301]: cluster 2022-04-28T20:12:03.424480+0000 mgr.y (mgr.24961) 862 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:06 smithi059 bash[61152]: cluster 2022-04-28T20:12:05.425453+0000 mgr.y (mgr.24961) 863 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:06 smithi059 bash[59745]: cluster 2022-04-28T20:12:05.425453+0000 mgr.y (mgr.24961) 863 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:06 smithi139 bash[43301]: cluster 2022-04-28T20:12:05.425453+0000 mgr.y (mgr.24961) 863 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:08 smithi059 bash[61152]: cluster 2022-04-28T20:12:07.426268+0000 mgr.y (mgr.24961) 864 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:08 smithi059 bash[59745]: cluster 2022-04-28T20:12:07.426268+0000 mgr.y (mgr.24961) 864 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:08 smithi139 bash[43301]: cluster 2022-04-28T20:12:07.426268+0000 mgr.y (mgr.24961) 864 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:10 smithi059 bash[61152]: cluster 2022-04-28T20:12:09.427096+0000 mgr.y (mgr.24961) 865 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:10 smithi059 bash[59745]: cluster 2022-04-28T20:12:09.427096+0000 mgr.y (mgr.24961) 865 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:10 smithi139 bash[43301]: cluster 2022-04-28T20:12:09.427096+0000 mgr.y (mgr.24961) 865 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:12.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:11.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:12 smithi059 bash[61152]: cluster 2022-04-28T20:12:11.428121+0000 mgr.y (mgr.24961) 866 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:12 smithi059 bash[59745]: cluster 2022-04-28T20:12:11.428121+0000 mgr.y (mgr.24961) 866 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:12 smithi139 bash[43301]: cluster 2022-04-28T20:12:11.428121+0000 mgr.y (mgr.24961) 866 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:13 smithi059 bash[61152]: cluster 2022-04-28T20:12:13.428776+0000 mgr.y (mgr.24961) 867 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:13.597Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:13 smithi059 bash[59745]: cluster 2022-04-28T20:12:13.428776+0000 mgr.y (mgr.24961) 867 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:13 smithi139 bash[43301]: cluster 2022-04-28T20:12:13.428776+0000 mgr.y (mgr.24961) 867 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:15 smithi059 bash[59745]: audit 2022-04-28T20:12:15.127356+0000 mon.a (mon.0) 1786 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:12:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:15 smithi059 bash[59745]: audit 2022-04-28T20:12:15.130293+0000 mon.a (mon.0) 1787 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:12:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:15 smithi059 bash[59745]: audit 2022-04-28T20:12:15.132250+0000 mon.a (mon.0) 1788 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:12:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:15 smithi059 bash[61152]: audit 2022-04-28T20:12:15.127356+0000 mon.a (mon.0) 1786 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:12:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:15 smithi059 bash[61152]: audit 2022-04-28T20:12:15.130293+0000 mon.a (mon.0) 1787 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:12:15.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:15 smithi059 bash[61152]: audit 2022-04-28T20:12:15.132250+0000 mon.a (mon.0) 1788 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:12:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:15 smithi139 bash[43301]: audit 2022-04-28T20:12:15.127356+0000 mon.a (mon.0) 1786 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:12:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:15 smithi139 bash[43301]: audit 2022-04-28T20:12:15.130293+0000 mon.a (mon.0) 1787 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:12:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:15 smithi139 bash[43301]: audit 2022-04-28T20:12:15.132250+0000 mon.a (mon.0) 1788 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:12:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:16 smithi059 bash[61152]: cephadm 2022-04-28T20:12:15.134038+0000 mgr.y (mgr.24961) 868 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:12:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:16 smithi059 bash[61152]: cluster 2022-04-28T20:12:15.429425+0000 mgr.y (mgr.24961) 869 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:16 smithi059 bash[61152]: audit 2022-04-28T20:12:15.433446+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:12:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:16 smithi059 bash[59745]: cephadm 2022-04-28T20:12:15.134038+0000 mgr.y (mgr.24961) 868 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:12:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:16 smithi059 bash[59745]: cluster 2022-04-28T20:12:15.429425+0000 mgr.y (mgr.24961) 869 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:16 smithi059 bash[59745]: audit 2022-04-28T20:12:15.433446+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:12:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:16 smithi139 bash[43301]: cephadm 2022-04-28T20:12:15.134038+0000 mgr.y (mgr.24961) 868 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:12:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:16 smithi139 bash[43301]: cluster 2022-04-28T20:12:15.429425+0000 mgr.y (mgr.24961) 869 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:16 smithi139 bash[43301]: audit 2022-04-28T20:12:15.433446+0000 mon.a (mon.0) 1789 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:12:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[61152]: cluster 2022-04-28T20:12:17.430159+0000 mgr.y (mgr.24961) 870 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[61152]: audit 2022-04-28T20:12:17.452206+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[61152]: audit 2022-04-28T20:12:17.457507+0000 mon.a (mon.0) 1791 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[61152]: audit 2022-04-28T20:12:17.471771+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[61152]: audit 2022-04-28T20:12:17.472477+0000 mon.a (mon.0) 1793 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[61152]: audit 2022-04-28T20:12:17.473127+0000 mon.a (mon.0) 1794 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:12:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[59745]: cluster 2022-04-28T20:12:17.430159+0000 mgr.y (mgr.24961) 870 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[59745]: audit 2022-04-28T20:12:17.452206+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:12:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[59745]: audit 2022-04-28T20:12:17.457507+0000 mon.a (mon.0) 1791 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:12:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[59745]: audit 2022-04-28T20:12:17.471771+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:12:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[59745]: audit 2022-04-28T20:12:17.472477+0000 mon.a (mon.0) 1793 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:12:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:18 smithi059 bash[59745]: audit 2022-04-28T20:12:17.473127+0000 mon.a (mon.0) 1794 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:12:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:18 smithi139 bash[43301]: cluster 2022-04-28T20:12:17.430159+0000 mgr.y (mgr.24961) 870 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:18 smithi139 bash[43301]: audit 2022-04-28T20:12:17.452206+0000 mon.a (mon.0) 1790 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:12:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:18 smithi139 bash[43301]: audit 2022-04-28T20:12:17.457507+0000 mon.a (mon.0) 1791 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:12:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:18 smithi139 bash[43301]: audit 2022-04-28T20:12:17.471771+0000 mon.a (mon.0) 1792 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:12:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:18 smithi139 bash[43301]: audit 2022-04-28T20:12:17.472477+0000 mon.a (mon.0) 1793 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:12:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:18 smithi139 bash[43301]: audit 2022-04-28T20:12:17.473127+0000 mon.a (mon.0) 1794 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:12:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:20 smithi059 bash[61152]: cluster 2022-04-28T20:12:19.431005+0000 mgr.y (mgr.24961) 871 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:20 smithi059 bash[59745]: cluster 2022-04-28T20:12:19.431005+0000 mgr.y (mgr.24961) 871 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:20 smithi139 bash[43301]: cluster 2022-04-28T20:12:19.431005+0000 mgr.y (mgr.24961) 871 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:21.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:22 smithi059 bash[61152]: cluster 2022-04-28T20:12:21.431914+0000 mgr.y (mgr.24961) 872 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:22 smithi059 bash[59745]: cluster 2022-04-28T20:12:21.431914+0000 mgr.y (mgr.24961) 872 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:22 smithi139 bash[43301]: cluster 2022-04-28T20:12:21.431914+0000 mgr.y (mgr.24961) 872 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:23 smithi059 bash[61152]: cluster 2022-04-28T20:12:23.432559+0000 mgr.y (mgr.24961) 873 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:23.599Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:23 smithi059 bash[59745]: cluster 2022-04-28T20:12:23.432559+0000 mgr.y (mgr.24961) 873 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:23 smithi139 bash[43301]: cluster 2022-04-28T20:12:23.432559+0000 mgr.y (mgr.24961) 873 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:26 smithi059 bash[61152]: cluster 2022-04-28T20:12:25.433531+0000 mgr.y (mgr.24961) 874 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:26 smithi059 bash[59745]: cluster 2022-04-28T20:12:25.433531+0000 mgr.y (mgr.24961) 874 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:26.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:26 smithi139 bash[43301]: cluster 2022-04-28T20:12:25.433531+0000 mgr.y (mgr.24961) 874 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:28 smithi059 bash[61152]: cluster 2022-04-28T20:12:27.434348+0000 mgr.y (mgr.24961) 875 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:28 smithi059 bash[59745]: cluster 2022-04-28T20:12:27.434348+0000 mgr.y (mgr.24961) 875 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:28.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:28 smithi139 bash[43301]: cluster 2022-04-28T20:12:27.434348+0000 mgr.y (mgr.24961) 875 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:30 smithi059 bash[61152]: cluster 2022-04-28T20:12:29.435213+0000 mgr.y (mgr.24961) 876 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:30 smithi059 bash[59745]: cluster 2022-04-28T20:12:29.435213+0000 mgr.y (mgr.24961) 876 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:30 smithi139 bash[43301]: cluster 2022-04-28T20:12:29.435213+0000 mgr.y (mgr.24961) 876 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:32.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:31.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:32 smithi059 bash[61152]: cluster 2022-04-28T20:12:31.436253+0000 mgr.y (mgr.24961) 877 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:32 smithi059 bash[59745]: cluster 2022-04-28T20:12:31.436253+0000 mgr.y (mgr.24961) 877 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:32 smithi139 bash[43301]: cluster 2022-04-28T20:12:31.436253+0000 mgr.y (mgr.24961) 877 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:33 smithi059 bash[59745]: cluster 2022-04-28T20:12:33.436973+0000 mgr.y (mgr.24961) 878 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:34.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:33 smithi059 bash[61152]: cluster 2022-04-28T20:12:33.436973+0000 mgr.y (mgr.24961) 878 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:34.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:33.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:33 smithi139 bash[43301]: cluster 2022-04-28T20:12:33.436973+0000 mgr.y (mgr.24961) 878 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:36 smithi059 bash[59745]: cluster 2022-04-28T20:12:35.438025+0000 mgr.y (mgr.24961) 879 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:36.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:36 smithi059 bash[61152]: cluster 2022-04-28T20:12:35.438025+0000 mgr.y (mgr.24961) 879 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:36 smithi139 bash[43301]: cluster 2022-04-28T20:12:35.438025+0000 mgr.y (mgr.24961) 879 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:38 smithi059 bash[61152]: cluster 2022-04-28T20:12:37.438848+0000 mgr.y (mgr.24961) 880 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:38 smithi059 bash[59745]: cluster 2022-04-28T20:12:37.438848+0000 mgr.y (mgr.24961) 880 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:38 smithi139 bash[43301]: cluster 2022-04-28T20:12:37.438848+0000 mgr.y (mgr.24961) 880 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:40 smithi059 bash[61152]: cluster 2022-04-28T20:12:39.439715+0000 mgr.y (mgr.24961) 881 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:40.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:40 smithi059 bash[59745]: cluster 2022-04-28T20:12:39.439715+0000 mgr.y (mgr.24961) 881 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:40 smithi139 bash[43301]: cluster 2022-04-28T20:12:39.439715+0000 mgr.y (mgr.24961) 881 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:42.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:41.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:42 smithi059 bash[61152]: cluster 2022-04-28T20:12:41.440624+0000 mgr.y (mgr.24961) 882 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:42 smithi059 bash[59745]: cluster 2022-04-28T20:12:41.440624+0000 mgr.y (mgr.24961) 882 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:42 smithi139 bash[43301]: cluster 2022-04-28T20:12:41.440624+0000 mgr.y (mgr.24961) 882 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:43 smithi059 bash[61152]: cluster 2022-04-28T20:12:43.441287+0000 mgr.y (mgr.24961) 883 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:43.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:44.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:43 smithi059 bash[59745]: cluster 2022-04-28T20:12:43.441287+0000 mgr.y (mgr.24961) 883 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:43 smithi139 bash[43301]: cluster 2022-04-28T20:12:43.441287+0000 mgr.y (mgr.24961) 883 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:46.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:46 smithi059 bash[59745]: cluster 2022-04-28T20:12:45.442268+0000 mgr.y (mgr.24961) 884 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:46.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:46 smithi059 bash[61152]: cluster 2022-04-28T20:12:45.442268+0000 mgr.y (mgr.24961) 884 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:46 smithi139 bash[43301]: cluster 2022-04-28T20:12:45.442268+0000 mgr.y (mgr.24961) 884 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:48 smithi059 bash[61152]: cluster 2022-04-28T20:12:47.443070+0000 mgr.y (mgr.24961) 885 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:48 smithi059 bash[59745]: cluster 2022-04-28T20:12:47.443070+0000 mgr.y (mgr.24961) 885 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:48 smithi139 bash[43301]: cluster 2022-04-28T20:12:47.443070+0000 mgr.y (mgr.24961) 885 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:50 smithi059 bash[61152]: cluster 2022-04-28T20:12:49.443902+0000 mgr.y (mgr.24961) 886 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:50 smithi059 bash[59745]: cluster 2022-04-28T20:12:49.443902+0000 mgr.y (mgr.24961) 886 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:50 smithi139 bash[43301]: cluster 2022-04-28T20:12:49.443902+0000 mgr.y (mgr.24961) 886 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:51.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:52.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:52 smithi059 bash[59745]: cluster 2022-04-28T20:12:51.444928+0000 mgr.y (mgr.24961) 887 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:52.969 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:52 smithi059 bash[61152]: cluster 2022-04-28T20:12:51.444928+0000 mgr.y (mgr.24961) 887 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:52.971 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:52 smithi139 bash[43301]: cluster 2022-04-28T20:12:51.444928+0000 mgr.y (mgr.24961) 887 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:53 smithi059 bash[61152]: cluster 2022-04-28T20:12:53.445610+0000 mgr.y (mgr.24961) 888 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:12:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:12:53.601Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:12:54.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:53 smithi059 bash[59745]: cluster 2022-04-28T20:12:53.445610+0000 mgr.y (mgr.24961) 888 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:53 smithi139 bash[43301]: cluster 2022-04-28T20:12:53.445610+0000 mgr.y (mgr.24961) 888 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:56 smithi059 bash[61152]: cluster 2022-04-28T20:12:55.446709+0000 mgr.y (mgr.24961) 889 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:56 smithi059 bash[59745]: cluster 2022-04-28T20:12:55.446709+0000 mgr.y (mgr.24961) 889 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:56 smithi139 bash[43301]: cluster 2022-04-28T20:12:55.446709+0000 mgr.y (mgr.24961) 889 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:12:58 smithi059 bash[61152]: cluster 2022-04-28T20:12:57.447828+0000 mgr.y (mgr.24961) 890 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:12:58 smithi059 bash[59745]: cluster 2022-04-28T20:12:57.447828+0000 mgr.y (mgr.24961) 890 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:12:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:12:58 smithi139 bash[43301]: cluster 2022-04-28T20:12:57.447828+0000 mgr.y (mgr.24961) 890 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:00 smithi059 bash[59745]: cluster 2022-04-28T20:12:59.448716+0000 mgr.y (mgr.24961) 891 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:00.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:00 smithi059 bash[61152]: cluster 2022-04-28T20:12:59.448716+0000 mgr.y (mgr.24961) 891 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:00 smithi139 bash[43301]: cluster 2022-04-28T20:12:59.448716+0000 mgr.y (mgr.24961) 891 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:01.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:02 smithi059 bash[61152]: cluster 2022-04-28T20:13:01.449670+0000 mgr.y (mgr.24961) 892 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:02 smithi059 bash[59745]: cluster 2022-04-28T20:13:01.449670+0000 mgr.y (mgr.24961) 892 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:02 smithi139 bash[43301]: cluster 2022-04-28T20:13:01.449670+0000 mgr.y (mgr.24961) 892 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:03 smithi059 bash[59745]: cluster 2022-04-28T20:13:03.450295+0000 mgr.y (mgr.24961) 893 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:03 smithi059 bash[61152]: cluster 2022-04-28T20:13:03.450295+0000 mgr.y (mgr.24961) 893 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:04.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:03.600Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:03 smithi139 bash[43301]: cluster 2022-04-28T20:13:03.450295+0000 mgr.y (mgr.24961) 893 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:06 smithi059 bash[59745]: cluster 2022-04-28T20:13:05.451296+0000 mgr.y (mgr.24961) 894 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:06.935 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:06 smithi059 bash[61152]: cluster 2022-04-28T20:13:05.451296+0000 mgr.y (mgr.24961) 894 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:06.936 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:06 smithi139 bash[43301]: cluster 2022-04-28T20:13:05.451296+0000 mgr.y (mgr.24961) 894 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:08 smithi059 bash[61152]: cluster 2022-04-28T20:13:07.452114+0000 mgr.y (mgr.24961) 895 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:08 smithi059 bash[59745]: cluster 2022-04-28T20:13:07.452114+0000 mgr.y (mgr.24961) 895 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:08.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:08 smithi139 bash[43301]: cluster 2022-04-28T20:13:07.452114+0000 mgr.y (mgr.24961) 895 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:10 smithi059 bash[61152]: cluster 2022-04-28T20:13:09.452992+0000 mgr.y (mgr.24961) 896 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:10 smithi059 bash[59745]: cluster 2022-04-28T20:13:09.452992+0000 mgr.y (mgr.24961) 896 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:10 smithi139 bash[43301]: cluster 2022-04-28T20:13:09.452992+0000 mgr.y (mgr.24961) 896 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:11.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:12 smithi059 bash[61152]: cluster 2022-04-28T20:13:11.453947+0000 mgr.y (mgr.24961) 897 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:12 smithi059 bash[59745]: cluster 2022-04-28T20:13:11.453947+0000 mgr.y (mgr.24961) 897 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:12 smithi139 bash[43301]: cluster 2022-04-28T20:13:11.453947+0000 mgr.y (mgr.24961) 897 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:13 smithi059 bash[61152]: cluster 2022-04-28T20:13:13.454579+0000 mgr.y (mgr.24961) 898 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:13.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:13 smithi059 bash[59745]: cluster 2022-04-28T20:13:13.454579+0000 mgr.y (mgr.24961) 898 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:13 smithi139 bash[43301]: cluster 2022-04-28T20:13:13.454579+0000 mgr.y (mgr.24961) 898 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:15 smithi059 bash[61152]: audit 2022-04-28T20:13:15.440554+0000 mon.a (mon.0) 1795 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:13:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:15 smithi059 bash[61152]: audit 2022-04-28T20:13:15.443256+0000 mon.a (mon.0) 1796 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:13:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:15 smithi059 bash[61152]: audit 2022-04-28T20:13:15.445221+0000 mon.a (mon.0) 1797 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:13:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:15 smithi059 bash[59745]: audit 2022-04-28T20:13:15.440554+0000 mon.a (mon.0) 1795 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:13:15.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:15 smithi059 bash[59745]: audit 2022-04-28T20:13:15.443256+0000 mon.a (mon.0) 1796 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:13:15.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:15 smithi059 bash[59745]: audit 2022-04-28T20:13:15.445221+0000 mon.a (mon.0) 1797 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:13:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:15 smithi139 bash[43301]: audit 2022-04-28T20:13:15.440554+0000 mon.a (mon.0) 1795 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:13:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:15 smithi139 bash[43301]: audit 2022-04-28T20:13:15.443256+0000 mon.a (mon.0) 1796 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:13:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:15 smithi139 bash[43301]: audit 2022-04-28T20:13:15.445221+0000 mon.a (mon.0) 1797 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:13:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:16 smithi059 bash[61152]: cephadm 2022-04-28T20:13:15.446904+0000 mgr.y (mgr.24961) 899 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:13:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:16 smithi059 bash[61152]: cluster 2022-04-28T20:13:15.455931+0000 mgr.y (mgr.24961) 900 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:16 smithi059 bash[61152]: audit 2022-04-28T20:13:15.757022+0000 mon.a (mon.0) 1798 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:13:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:16 smithi059 bash[59745]: cephadm 2022-04-28T20:13:15.446904+0000 mgr.y (mgr.24961) 899 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:13:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:16 smithi059 bash[59745]: cluster 2022-04-28T20:13:15.455931+0000 mgr.y (mgr.24961) 900 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:16 smithi059 bash[59745]: audit 2022-04-28T20:13:15.757022+0000 mon.a (mon.0) 1798 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:13:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:16 smithi139 bash[43301]: cephadm 2022-04-28T20:13:15.446904+0000 mgr.y (mgr.24961) 899 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:13:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:16 smithi139 bash[43301]: cluster 2022-04-28T20:13:15.455931+0000 mgr.y (mgr.24961) 900 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:16 smithi139 bash[43301]: audit 2022-04-28T20:13:15.757022+0000 mon.a (mon.0) 1798 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:13:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[61152]: audit 2022-04-28T20:13:17.458337+0000 mon.a (mon.0) 1799 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:13:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[61152]: audit 2022-04-28T20:13:17.461909+0000 mon.a (mon.0) 1800 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:13:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[61152]: audit 2022-04-28T20:13:17.478017+0000 mon.a (mon.0) 1801 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:13:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[61152]: audit 2022-04-28T20:13:17.478651+0000 mon.a (mon.0) 1802 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:13:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[61152]: audit 2022-04-28T20:13:17.479231+0000 mon.a (mon.0) 1803 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:13:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[59745]: audit 2022-04-28T20:13:17.458337+0000 mon.a (mon.0) 1799 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:13:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[59745]: audit 2022-04-28T20:13:17.461909+0000 mon.a (mon.0) 1800 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:13:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[59745]: audit 2022-04-28T20:13:17.478017+0000 mon.a (mon.0) 1801 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:13:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[59745]: audit 2022-04-28T20:13:17.478651+0000 mon.a (mon.0) 1802 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:13:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:17 smithi059 bash[59745]: audit 2022-04-28T20:13:17.479231+0000 mon.a (mon.0) 1803 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:13:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:17 smithi139 bash[43301]: audit 2022-04-28T20:13:17.458337+0000 mon.a (mon.0) 1799 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:13:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:17 smithi139 bash[43301]: audit 2022-04-28T20:13:17.461909+0000 mon.a (mon.0) 1800 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:13:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:17 smithi139 bash[43301]: audit 2022-04-28T20:13:17.478017+0000 mon.a (mon.0) 1801 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:13:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:17 smithi139 bash[43301]: audit 2022-04-28T20:13:17.478651+0000 mon.a (mon.0) 1802 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:13:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:17 smithi139 bash[43301]: audit 2022-04-28T20:13:17.479231+0000 mon.a (mon.0) 1803 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:13:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:18 smithi059 bash[61152]: cluster 2022-04-28T20:13:17.456730+0000 mgr.y (mgr.24961) 901 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:18 smithi059 bash[59745]: cluster 2022-04-28T20:13:17.456730+0000 mgr.y (mgr.24961) 901 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:18 smithi139 bash[43301]: cluster 2022-04-28T20:13:17.456730+0000 mgr.y (mgr.24961) 901 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:20 smithi059 bash[61152]: cluster 2022-04-28T20:13:19.457608+0000 mgr.y (mgr.24961) 902 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:20 smithi059 bash[59745]: cluster 2022-04-28T20:13:19.457608+0000 mgr.y (mgr.24961) 902 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:20 smithi139 bash[43301]: cluster 2022-04-28T20:13:19.457608+0000 mgr.y (mgr.24961) 902 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:21.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:22 smithi059 bash[61152]: cluster 2022-04-28T20:13:21.458576+0000 mgr.y (mgr.24961) 903 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:22 smithi059 bash[59745]: cluster 2022-04-28T20:13:21.458576+0000 mgr.y (mgr.24961) 903 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:22 smithi139 bash[43301]: cluster 2022-04-28T20:13:21.458576+0000 mgr.y (mgr.24961) 903 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:23 smithi059 bash[61152]: cluster 2022-04-28T20:13:23.459233+0000 mgr.y (mgr.24961) 904 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:23.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:23 smithi059 bash[59745]: cluster 2022-04-28T20:13:23.459233+0000 mgr.y (mgr.24961) 904 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:23 smithi139 bash[43301]: cluster 2022-04-28T20:13:23.459233+0000 mgr.y (mgr.24961) 904 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:26 smithi059 bash[61152]: cluster 2022-04-28T20:13:25.460268+0000 mgr.y (mgr.24961) 905 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:26.961 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:26 smithi059 bash[59745]: cluster 2022-04-28T20:13:25.460268+0000 mgr.y (mgr.24961) 905 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:26.962 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:26 smithi139 bash[43301]: cluster 2022-04-28T20:13:25.460268+0000 mgr.y (mgr.24961) 905 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:28 smithi059 bash[61152]: cluster 2022-04-28T20:13:27.461059+0000 mgr.y (mgr.24961) 906 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:28 smithi059 bash[59745]: cluster 2022-04-28T20:13:27.461059+0000 mgr.y (mgr.24961) 906 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:28.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:28 smithi139 bash[43301]: cluster 2022-04-28T20:13:27.461059+0000 mgr.y (mgr.24961) 906 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:30 smithi059 bash[61152]: cluster 2022-04-28T20:13:29.461875+0000 mgr.y (mgr.24961) 907 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:30 smithi059 bash[59745]: cluster 2022-04-28T20:13:29.461875+0000 mgr.y (mgr.24961) 907 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:30.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:30 smithi139 bash[43301]: cluster 2022-04-28T20:13:29.461875+0000 mgr.y (mgr.24961) 907 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:31.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:32 smithi059 bash[61152]: cluster 2022-04-28T20:13:31.462903+0000 mgr.y (mgr.24961) 908 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:32 smithi059 bash[59745]: cluster 2022-04-28T20:13:31.462903+0000 mgr.y (mgr.24961) 908 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:32 smithi139 bash[43301]: cluster 2022-04-28T20:13:31.462903+0000 mgr.y (mgr.24961) 908 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:33 smithi059 bash[59745]: cluster 2022-04-28T20:13:33.463585+0000 mgr.y (mgr.24961) 909 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:34.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:33 smithi059 bash[61152]: cluster 2022-04-28T20:13:33.463585+0000 mgr.y (mgr.24961) 909 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:33.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:34.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:33 smithi139 bash[43301]: cluster 2022-04-28T20:13:33.463585+0000 mgr.y (mgr.24961) 909 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:36 smithi059 bash[61152]: cluster 2022-04-28T20:13:35.464572+0000 mgr.y (mgr.24961) 910 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:36 smithi059 bash[59745]: cluster 2022-04-28T20:13:35.464572+0000 mgr.y (mgr.24961) 910 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:36.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:36 smithi139 bash[43301]: cluster 2022-04-28T20:13:35.464572+0000 mgr.y (mgr.24961) 910 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:38 smithi059 bash[61152]: cluster 2022-04-28T20:13:37.465424+0000 mgr.y (mgr.24961) 911 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:38 smithi059 bash[59745]: cluster 2022-04-28T20:13:37.465424+0000 mgr.y (mgr.24961) 911 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:38 smithi139 bash[43301]: cluster 2022-04-28T20:13:37.465424+0000 mgr.y (mgr.24961) 911 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:40.533 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:40 smithi059 bash[59745]: cluster 2022-04-28T20:13:39.466310+0000 mgr.y (mgr.24961) 912 : cluster [DBG] pgmap v1285: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:40 smithi059 bash[61152]: cluster 2022-04-28T20:13:39.466310+0000 mgr.y (mgr.24961) 912 : cluster [DBG] pgmap v1285: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:40.911 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:40 smithi139 bash[43301]: cluster 2022-04-28T20:13:39.466310+0000 mgr.y (mgr.24961) 912 : cluster [DBG] pgmap v1285: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:41.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:42.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:42 smithi059 bash[61152]: cluster 2022-04-28T20:13:41.467340+0000 mgr.y (mgr.24961) 913 : cluster [DBG] pgmap v1286: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:42 smithi059 bash[59745]: cluster 2022-04-28T20:13:41.467340+0000 mgr.y (mgr.24961) 913 : cluster [DBG] pgmap v1286: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:42 smithi139 bash[43301]: cluster 2022-04-28T20:13:41.467340+0000 mgr.y (mgr.24961) 913 : cluster [DBG] pgmap v1286: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:43 smithi059 bash[61152]: cluster 2022-04-28T20:13:43.468062+0000 mgr.y (mgr.24961) 914 : cluster [DBG] pgmap v1287: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:43.603Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:43 smithi059 bash[59745]: cluster 2022-04-28T20:13:43.468062+0000 mgr.y (mgr.24961) 914 : cluster [DBG] pgmap v1287: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:43 smithi139 bash[43301]: cluster 2022-04-28T20:13:43.468062+0000 mgr.y (mgr.24961) 914 : cluster [DBG] pgmap v1287: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:46 smithi059 bash[61152]: cluster 2022-04-28T20:13:45.469115+0000 mgr.y (mgr.24961) 915 : cluster [DBG] pgmap v1288: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:46 smithi059 bash[59745]: cluster 2022-04-28T20:13:45.469115+0000 mgr.y (mgr.24961) 915 : cluster [DBG] pgmap v1288: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:46 smithi139 bash[43301]: cluster 2022-04-28T20:13:45.469115+0000 mgr.y (mgr.24961) 915 : cluster [DBG] pgmap v1288: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:48 smithi059 bash[61152]: cluster 2022-04-28T20:13:47.469927+0000 mgr.y (mgr.24961) 916 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:48 smithi059 bash[59745]: cluster 2022-04-28T20:13:47.469927+0000 mgr.y (mgr.24961) 916 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:48 smithi139 bash[43301]: cluster 2022-04-28T20:13:47.469927+0000 mgr.y (mgr.24961) 916 : cluster [DBG] pgmap v1289: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:50.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:50 smithi059 bash[61152]: cluster 2022-04-28T20:13:49.470804+0000 mgr.y (mgr.24961) 917 : cluster [DBG] pgmap v1290: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:50.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:50 smithi059 bash[59745]: cluster 2022-04-28T20:13:49.470804+0000 mgr.y (mgr.24961) 917 : cluster [DBG] pgmap v1290: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:50 smithi139 bash[43301]: cluster 2022-04-28T20:13:49.470804+0000 mgr.y (mgr.24961) 917 : cluster [DBG] pgmap v1290: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:51.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:52 smithi059 bash[61152]: cluster 2022-04-28T20:13:51.471840+0000 mgr.y (mgr.24961) 918 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:52 smithi059 bash[59745]: cluster 2022-04-28T20:13:51.471840+0000 mgr.y (mgr.24961) 918 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:52 smithi139 bash[43301]: cluster 2022-04-28T20:13:51.471840+0000 mgr.y (mgr.24961) 918 : cluster [DBG] pgmap v1291: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:53 smithi059 bash[61152]: cluster 2022-04-28T20:13:53.472525+0000 mgr.y (mgr.24961) 919 : cluster [DBG] pgmap v1292: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:13:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:13:53.602Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:13:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:53 smithi059 bash[59745]: cluster 2022-04-28T20:13:53.472525+0000 mgr.y (mgr.24961) 919 : cluster [DBG] pgmap v1292: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:53 smithi139 bash[43301]: cluster 2022-04-28T20:13:53.472525+0000 mgr.y (mgr.24961) 919 : cluster [DBG] pgmap v1292: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:56 smithi059 bash[61152]: cluster 2022-04-28T20:13:55.473512+0000 mgr.y (mgr.24961) 920 : cluster [DBG] pgmap v1293: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:56 smithi059 bash[59745]: cluster 2022-04-28T20:13:55.473512+0000 mgr.y (mgr.24961) 920 : cluster [DBG] pgmap v1293: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:56 smithi139 bash[43301]: cluster 2022-04-28T20:13:55.473512+0000 mgr.y (mgr.24961) 920 : cluster [DBG] pgmap v1293: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:13:58 smithi059 bash[61152]: cluster 2022-04-28T20:13:57.474351+0000 mgr.y (mgr.24961) 921 : cluster [DBG] pgmap v1294: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:13:58 smithi059 bash[59745]: cluster 2022-04-28T20:13:57.474351+0000 mgr.y (mgr.24961) 921 : cluster [DBG] pgmap v1294: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:13:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:13:58 smithi139 bash[43301]: cluster 2022-04-28T20:13:57.474351+0000 mgr.y (mgr.24961) 921 : cluster [DBG] pgmap v1294: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:00 smithi059 bash[61152]: cluster 2022-04-28T20:13:59.475243+0000 mgr.y (mgr.24961) 922 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:00 smithi059 bash[59745]: cluster 2022-04-28T20:13:59.475243+0000 mgr.y (mgr.24961) 922 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:00 smithi139 bash[43301]: cluster 2022-04-28T20:13:59.475243+0000 mgr.y (mgr.24961) 922 : cluster [DBG] pgmap v1295: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:01.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:02 smithi059 bash[61152]: cluster 2022-04-28T20:14:01.476211+0000 mgr.y (mgr.24961) 923 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:02 smithi059 bash[59745]: cluster 2022-04-28T20:14:01.476211+0000 mgr.y (mgr.24961) 923 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:02 smithi139 bash[43301]: cluster 2022-04-28T20:14:01.476211+0000 mgr.y (mgr.24961) 923 : cluster [DBG] pgmap v1296: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:03 smithi059 bash[59745]: cluster 2022-04-28T20:14:03.476877+0000 mgr.y (mgr.24961) 924 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:04.284 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:03 smithi059 bash[61152]: cluster 2022-04-28T20:14:03.476877+0000 mgr.y (mgr.24961) 924 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:04.284 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:03.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:04.286 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:03 smithi139 bash[43301]: cluster 2022-04-28T20:14:03.476877+0000 mgr.y (mgr.24961) 924 : cluster [DBG] pgmap v1297: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:06 smithi059 bash[61152]: cluster 2022-04-28T20:14:05.477881+0000 mgr.y (mgr.24961) 925 : cluster [DBG] pgmap v1298: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:06 smithi059 bash[59745]: cluster 2022-04-28T20:14:05.477881+0000 mgr.y (mgr.24961) 925 : cluster [DBG] pgmap v1298: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:06 smithi139 bash[43301]: cluster 2022-04-28T20:14:05.477881+0000 mgr.y (mgr.24961) 925 : cluster [DBG] pgmap v1298: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:08.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:08 smithi059 bash[59745]: cluster 2022-04-28T20:14:07.478685+0000 mgr.y (mgr.24961) 926 : cluster [DBG] pgmap v1299: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:08 smithi059 bash[61152]: cluster 2022-04-28T20:14:07.478685+0000 mgr.y (mgr.24961) 926 : cluster [DBG] pgmap v1299: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:08 smithi139 bash[43301]: cluster 2022-04-28T20:14:07.478685+0000 mgr.y (mgr.24961) 926 : cluster [DBG] pgmap v1299: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:10 smithi059 bash[61152]: cluster 2022-04-28T20:14:09.479547+0000 mgr.y (mgr.24961) 927 : cluster [DBG] pgmap v1300: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:10 smithi059 bash[59745]: cluster 2022-04-28T20:14:09.479547+0000 mgr.y (mgr.24961) 927 : cluster [DBG] pgmap v1300: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:10.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:10 smithi139 bash[43301]: cluster 2022-04-28T20:14:09.479547+0000 mgr.y (mgr.24961) 927 : cluster [DBG] pgmap v1300: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:11.710Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:12 smithi059 bash[61152]: cluster 2022-04-28T20:14:11.480501+0000 mgr.y (mgr.24961) 928 : cluster [DBG] pgmap v1301: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:12 smithi059 bash[59745]: cluster 2022-04-28T20:14:11.480501+0000 mgr.y (mgr.24961) 928 : cluster [DBG] pgmap v1301: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:12 smithi139 bash[43301]: cluster 2022-04-28T20:14:11.480501+0000 mgr.y (mgr.24961) 928 : cluster [DBG] pgmap v1301: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:13 smithi059 bash[61152]: cluster 2022-04-28T20:14:13.481190+0000 mgr.y (mgr.24961) 929 : cluster [DBG] pgmap v1302: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:13.604Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:13 smithi059 bash[59745]: cluster 2022-04-28T20:14:13.481190+0000 mgr.y (mgr.24961) 929 : cluster [DBG] pgmap v1302: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:13 smithi139 bash[43301]: cluster 2022-04-28T20:14:13.481190+0000 mgr.y (mgr.24961) 929 : cluster [DBG] pgmap v1302: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[59745]: cluster 2022-04-28T20:14:15.482187+0000 mgr.y (mgr.24961) 930 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[59745]: audit 2022-04-28T20:14:15.765606+0000 mon.a (mon.0) 1804 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:14:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[59745]: audit 2022-04-28T20:14:15.768590+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:14:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[59745]: audit 2022-04-28T20:14:15.770511+0000 mon.a (mon.0) 1806 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:14:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[59745]: audit 2022-04-28T20:14:16.091655+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:14:16.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[61152]: cluster 2022-04-28T20:14:15.482187+0000 mgr.y (mgr.24961) 930 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:16.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[61152]: audit 2022-04-28T20:14:15.765606+0000 mon.a (mon.0) 1804 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:14:16.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[61152]: audit 2022-04-28T20:14:15.768590+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:14:16.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[61152]: audit 2022-04-28T20:14:15.770511+0000 mon.a (mon.0) 1806 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:14:16.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:16 smithi059 bash[61152]: audit 2022-04-28T20:14:16.091655+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:14:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:16 smithi139 bash[43301]: cluster 2022-04-28T20:14:15.482187+0000 mgr.y (mgr.24961) 930 : cluster [DBG] pgmap v1303: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:16 smithi139 bash[43301]: audit 2022-04-28T20:14:15.765606+0000 mon.a (mon.0) 1804 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:14:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:16 smithi139 bash[43301]: audit 2022-04-28T20:14:15.768590+0000 mon.a (mon.0) 1805 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:14:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:16 smithi139 bash[43301]: audit 2022-04-28T20:14:15.770511+0000 mon.a (mon.0) 1806 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:14:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:16 smithi139 bash[43301]: audit 2022-04-28T20:14:16.091655+0000 mon.a (mon.0) 1807 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:14:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[61152]: cephadm 2022-04-28T20:14:15.772238+0000 mgr.y (mgr.24961) 931 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:14:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[61152]: audit 2022-04-28T20:14:17.464509+0000 mon.a (mon.0) 1808 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:14:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[61152]: audit 2022-04-28T20:14:17.468055+0000 mon.a (mon.0) 1809 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:14:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[61152]: audit 2022-04-28T20:14:17.485729+0000 mon.a (mon.0) 1810 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[61152]: audit 2022-04-28T20:14:17.486317+0000 mon.a (mon.0) 1811 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[61152]: audit 2022-04-28T20:14:17.486843+0000 mon.a (mon.0) 1812 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[59745]: cephadm 2022-04-28T20:14:15.772238+0000 mgr.y (mgr.24961) 931 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:14:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[59745]: audit 2022-04-28T20:14:17.464509+0000 mon.a (mon.0) 1808 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:14:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[59745]: audit 2022-04-28T20:14:17.468055+0000 mon.a (mon.0) 1809 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:14:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[59745]: audit 2022-04-28T20:14:17.485729+0000 mon.a (mon.0) 1810 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[59745]: audit 2022-04-28T20:14:17.486317+0000 mon.a (mon.0) 1811 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:17 smithi059 bash[59745]: audit 2022-04-28T20:14:17.486843+0000 mon.a (mon.0) 1812 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:17 smithi139 bash[43301]: cephadm 2022-04-28T20:14:15.772238+0000 mgr.y (mgr.24961) 931 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:14:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:17 smithi139 bash[43301]: audit 2022-04-28T20:14:17.464509+0000 mon.a (mon.0) 1808 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:14:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:17 smithi139 bash[43301]: audit 2022-04-28T20:14:17.468055+0000 mon.a (mon.0) 1809 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:14:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:17 smithi139 bash[43301]: audit 2022-04-28T20:14:17.485729+0000 mon.a (mon.0) 1810 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:17 smithi139 bash[43301]: audit 2022-04-28T20:14:17.486317+0000 mon.a (mon.0) 1811 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:14:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:17 smithi139 bash[43301]: audit 2022-04-28T20:14:17.486843+0000 mon.a (mon.0) 1812 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:14:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:18 smithi059 bash[61152]: cluster 2022-04-28T20:14:17.482942+0000 mgr.y (mgr.24961) 932 : cluster [DBG] pgmap v1304: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:18 smithi059 bash[59745]: cluster 2022-04-28T20:14:17.482942+0000 mgr.y (mgr.24961) 932 : cluster [DBG] pgmap v1304: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:18 smithi139 bash[43301]: cluster 2022-04-28T20:14:17.482942+0000 mgr.y (mgr.24961) 932 : cluster [DBG] pgmap v1304: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:19.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:19 smithi059 bash[59745]: audit 2022-04-28T20:14:19.159897+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.59:0/2936053734' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:14:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:19 smithi059 bash[61152]: audit 2022-04-28T20:14:19.159897+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.59:0/2936053734' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:14:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:19 smithi139 bash[43301]: audit 2022-04-28T20:14:19.159897+0000 mon.c (mon.1) 26 : audit [DBG] from='client.? 172.21.15.59:0/2936053734' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:14:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:20 smithi059 bash[61152]: cluster 2022-04-28T20:14:19.483936+0000 mgr.y (mgr.24961) 933 : cluster [DBG] pgmap v1305: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:20 smithi059 bash[59745]: cluster 2022-04-28T20:14:19.483936+0000 mgr.y (mgr.24961) 933 : cluster [DBG] pgmap v1305: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:20 smithi139 bash[43301]: cluster 2022-04-28T20:14:19.483936+0000 mgr.y (mgr.24961) 933 : cluster [DBG] pgmap v1305: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:21.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:22 smithi059 bash[61152]: cluster 2022-04-28T20:14:21.484931+0000 mgr.y (mgr.24961) 934 : cluster [DBG] pgmap v1306: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:22 smithi059 bash[59745]: cluster 2022-04-28T20:14:21.484931+0000 mgr.y (mgr.24961) 934 : cluster [DBG] pgmap v1306: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:22 smithi139 bash[43301]: cluster 2022-04-28T20:14:21.484931+0000 mgr.y (mgr.24961) 934 : cluster [DBG] pgmap v1306: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:23 smithi059 bash[61152]: audit 2022-04-28T20:14:22.708486+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.59:0/1177509129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T20:14:23.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:23.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:23 smithi059 bash[59745]: audit 2022-04-28T20:14:22.708486+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.59:0/1177509129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T20:14:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:23 smithi139 bash[43301]: audit 2022-04-28T20:14:22.708486+0000 mon.c (mon.1) 27 : audit [DBG] from='client.? 172.21.15.59:0/1177509129' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2022-04-28T20:14:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:24 smithi059 bash[59745]: cluster 2022-04-28T20:14:23.485613+0000 mgr.y (mgr.24961) 935 : cluster [DBG] pgmap v1307: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:24 smithi059 bash[61152]: cluster 2022-04-28T20:14:23.485613+0000 mgr.y (mgr.24961) 935 : cluster [DBG] pgmap v1307: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:24 smithi139 bash[43301]: cluster 2022-04-28T20:14:23.485613+0000 mgr.y (mgr.24961) 935 : cluster [DBG] pgmap v1307: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:26 smithi059 bash[61152]: cluster 2022-04-28T20:14:25.486597+0000 mgr.y (mgr.24961) 936 : cluster [DBG] pgmap v1308: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:26 smithi059 bash[59745]: cluster 2022-04-28T20:14:25.486597+0000 mgr.y (mgr.24961) 936 : cluster [DBG] pgmap v1308: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:26 smithi139 bash[43301]: cluster 2022-04-28T20:14:25.486597+0000 mgr.y (mgr.24961) 936 : cluster [DBG] pgmap v1308: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:28.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:28 smithi139 bash[43301]: cluster 2022-04-28T20:14:27.487429+0000 mgr.y (mgr.24961) 937 : cluster [DBG] pgmap v1309: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:28 smithi059 bash[61152]: cluster 2022-04-28T20:14:27.487429+0000 mgr.y (mgr.24961) 937 : cluster [DBG] pgmap v1309: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:28 smithi059 bash[59745]: cluster 2022-04-28T20:14:27.487429+0000 mgr.y (mgr.24961) 937 : cluster [DBG] pgmap v1309: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:30 smithi139 bash[43301]: cluster 2022-04-28T20:14:29.488361+0000 mgr.y (mgr.24961) 938 : cluster [DBG] pgmap v1310: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:30 smithi059 bash[61152]: cluster 2022-04-28T20:14:29.488361+0000 mgr.y (mgr.24961) 938 : cluster [DBG] pgmap v1310: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:30 smithi059 bash[59745]: cluster 2022-04-28T20:14:29.488361+0000 mgr.y (mgr.24961) 938 : cluster [DBG] pgmap v1310: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:31 smithi139 bash[43301]: cluster 2022-04-28T20:14:31.489332+0000 mgr.y (mgr.24961) 939 : cluster [DBG] pgmap v1311: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:31 smithi059 bash[61152]: cluster 2022-04-28T20:14:31.489332+0000 mgr.y (mgr.24961) 939 : cluster [DBG] pgmap v1311: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:32.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:31.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:31 smithi059 bash[59745]: cluster 2022-04-28T20:14:31.489332+0000 mgr.y (mgr.24961) 939 : cluster [DBG] pgmap v1311: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:34.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:33 smithi059 bash[61152]: cluster 2022-04-28T20:14:33.489996+0000 mgr.y (mgr.24961) 940 : cluster [DBG] pgmap v1312: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:34.096 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:33.606Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:34.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:33 smithi059 bash[59745]: cluster 2022-04-28T20:14:33.489996+0000 mgr.y (mgr.24961) 940 : cluster [DBG] pgmap v1312: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:33 smithi139 bash[43301]: cluster 2022-04-28T20:14:33.489996+0000 mgr.y (mgr.24961) 940 : cluster [DBG] pgmap v1312: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:36 smithi059 bash[61152]: cluster 2022-04-28T20:14:35.491008+0000 mgr.y (mgr.24961) 941 : cluster [DBG] pgmap v1313: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:36 smithi059 bash[59745]: cluster 2022-04-28T20:14:35.491008+0000 mgr.y (mgr.24961) 941 : cluster [DBG] pgmap v1313: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:36 smithi139 bash[43301]: cluster 2022-04-28T20:14:35.491008+0000 mgr.y (mgr.24961) 941 : cluster [DBG] pgmap v1313: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:38 smithi059 bash[61152]: cluster 2022-04-28T20:14:37.491854+0000 mgr.y (mgr.24961) 942 : cluster [DBG] pgmap v1314: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:38 smithi059 bash[59745]: cluster 2022-04-28T20:14:37.491854+0000 mgr.y (mgr.24961) 942 : cluster [DBG] pgmap v1314: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:38 smithi139 bash[43301]: cluster 2022-04-28T20:14:37.491854+0000 mgr.y (mgr.24961) 942 : cluster [DBG] pgmap v1314: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:40 smithi059 bash[59745]: cluster 2022-04-28T20:14:39.492703+0000 mgr.y (mgr.24961) 943 : cluster [DBG] pgmap v1315: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:40 smithi059 bash[61152]: cluster 2022-04-28T20:14:39.492703+0000 mgr.y (mgr.24961) 943 : cluster [DBG] pgmap v1315: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:40 smithi139 bash[43301]: cluster 2022-04-28T20:14:39.492703+0000 mgr.y (mgr.24961) 943 : cluster [DBG] pgmap v1315: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:41.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:42 smithi059 bash[61152]: cluster 2022-04-28T20:14:41.493686+0000 mgr.y (mgr.24961) 944 : cluster [DBG] pgmap v1316: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:42 smithi059 bash[59745]: cluster 2022-04-28T20:14:41.493686+0000 mgr.y (mgr.24961) 944 : cluster [DBG] pgmap v1316: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:42 smithi139 bash[43301]: cluster 2022-04-28T20:14:41.493686+0000 mgr.y (mgr.24961) 944 : cluster [DBG] pgmap v1316: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:43 smithi059 bash[61152]: cluster 2022-04-28T20:14:43.494304+0000 mgr.y (mgr.24961) 945 : cluster [DBG] pgmap v1317: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:43.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:43 smithi059 bash[59745]: cluster 2022-04-28T20:14:43.494304+0000 mgr.y (mgr.24961) 945 : cluster [DBG] pgmap v1317: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:43 smithi139 bash[43301]: cluster 2022-04-28T20:14:43.494304+0000 mgr.y (mgr.24961) 945 : cluster [DBG] pgmap v1317: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:46 smithi059 bash[61152]: cluster 2022-04-28T20:14:45.495313+0000 mgr.y (mgr.24961) 946 : cluster [DBG] pgmap v1318: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:46.948 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:46 smithi059 bash[59745]: cluster 2022-04-28T20:14:45.495313+0000 mgr.y (mgr.24961) 946 : cluster [DBG] pgmap v1318: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:46.949 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:46 smithi139 bash[43301]: cluster 2022-04-28T20:14:45.495313+0000 mgr.y (mgr.24961) 946 : cluster [DBG] pgmap v1318: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:48 smithi059 bash[61152]: cluster 2022-04-28T20:14:47.496274+0000 mgr.y (mgr.24961) 947 : cluster [DBG] pgmap v1319: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:48 smithi059 bash[59745]: cluster 2022-04-28T20:14:47.496274+0000 mgr.y (mgr.24961) 947 : cluster [DBG] pgmap v1319: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:48 smithi139 bash[43301]: cluster 2022-04-28T20:14:47.496274+0000 mgr.y (mgr.24961) 947 : cluster [DBG] pgmap v1319: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:50 smithi059 bash[61152]: cluster 2022-04-28T20:14:49.497173+0000 mgr.y (mgr.24961) 948 : cluster [DBG] pgmap v1320: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:50.909 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:50 smithi059 bash[59745]: cluster 2022-04-28T20:14:49.497173+0000 mgr.y (mgr.24961) 948 : cluster [DBG] pgmap v1320: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:50.910 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:50 smithi139 bash[43301]: cluster 2022-04-28T20:14:49.497173+0000 mgr.y (mgr.24961) 948 : cluster [DBG] pgmap v1320: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:51.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:52.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:52 smithi059 bash[61152]: cluster 2022-04-28T20:14:51.498138+0000 mgr.y (mgr.24961) 949 : cluster [DBG] pgmap v1321: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:52.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:52 smithi059 bash[59745]: cluster 2022-04-28T20:14:51.498138+0000 mgr.y (mgr.24961) 949 : cluster [DBG] pgmap v1321: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:52 smithi139 bash[43301]: cluster 2022-04-28T20:14:51.498138+0000 mgr.y (mgr.24961) 949 : cluster [DBG] pgmap v1321: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:53 smithi059 bash[61152]: cluster 2022-04-28T20:14:53.498812+0000 mgr.y (mgr.24961) 950 : cluster [DBG] pgmap v1322: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:14:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:14:53.607Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:14:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:53 smithi059 bash[59745]: cluster 2022-04-28T20:14:53.498812+0000 mgr.y (mgr.24961) 950 : cluster [DBG] pgmap v1322: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:53 smithi139 bash[43301]: cluster 2022-04-28T20:14:53.498812+0000 mgr.y (mgr.24961) 950 : cluster [DBG] pgmap v1322: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:56.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:56 smithi059 bash[61152]: cluster 2022-04-28T20:14:55.499848+0000 mgr.y (mgr.24961) 951 : cluster [DBG] pgmap v1323: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:56 smithi059 bash[59745]: cluster 2022-04-28T20:14:55.499848+0000 mgr.y (mgr.24961) 951 : cluster [DBG] pgmap v1323: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:56 smithi139 bash[43301]: cluster 2022-04-28T20:14:55.499848+0000 mgr.y (mgr.24961) 951 : cluster [DBG] pgmap v1323: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:14:58 smithi059 bash[61152]: cluster 2022-04-28T20:14:57.500685+0000 mgr.y (mgr.24961) 952 : cluster [DBG] pgmap v1324: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:14:58 smithi059 bash[59745]: cluster 2022-04-28T20:14:57.500685+0000 mgr.y (mgr.24961) 952 : cluster [DBG] pgmap v1324: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:14:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:14:58 smithi139 bash[43301]: cluster 2022-04-28T20:14:57.500685+0000 mgr.y (mgr.24961) 952 : cluster [DBG] pgmap v1324: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:00 smithi059 bash[61152]: cluster 2022-04-28T20:14:59.501564+0000 mgr.y (mgr.24961) 953 : cluster [DBG] pgmap v1325: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:00 smithi059 bash[59745]: cluster 2022-04-28T20:14:59.501564+0000 mgr.y (mgr.24961) 953 : cluster [DBG] pgmap v1325: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:00 smithi139 bash[43301]: cluster 2022-04-28T20:14:59.501564+0000 mgr.y (mgr.24961) 953 : cluster [DBG] pgmap v1325: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:01.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:02 smithi059 bash[61152]: cluster 2022-04-28T20:15:01.502517+0000 mgr.y (mgr.24961) 954 : cluster [DBG] pgmap v1326: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:02 smithi059 bash[59745]: cluster 2022-04-28T20:15:01.502517+0000 mgr.y (mgr.24961) 954 : cluster [DBG] pgmap v1326: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:02.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:02 smithi139 bash[43301]: cluster 2022-04-28T20:15:01.502517+0000 mgr.y (mgr.24961) 954 : cluster [DBG] pgmap v1326: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:03 smithi059 bash[61152]: cluster 2022-04-28T20:15:03.503213+0000 mgr.y (mgr.24961) 955 : cluster [DBG] pgmap v1327: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:03.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:03 smithi059 bash[59745]: cluster 2022-04-28T20:15:03.503213+0000 mgr.y (mgr.24961) 955 : cluster [DBG] pgmap v1327: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:03 smithi139 bash[43301]: cluster 2022-04-28T20:15:03.503213+0000 mgr.y (mgr.24961) 955 : cluster [DBG] pgmap v1327: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:06 smithi059 bash[59745]: cluster 2022-04-28T20:15:05.504290+0000 mgr.y (mgr.24961) 956 : cluster [DBG] pgmap v1328: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:06 smithi059 bash[61152]: cluster 2022-04-28T20:15:05.504290+0000 mgr.y (mgr.24961) 956 : cluster [DBG] pgmap v1328: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:06 smithi139 bash[43301]: cluster 2022-04-28T20:15:05.504290+0000 mgr.y (mgr.24961) 956 : cluster [DBG] pgmap v1328: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:08 smithi059 bash[61152]: cluster 2022-04-28T20:15:07.505108+0000 mgr.y (mgr.24961) 957 : cluster [DBG] pgmap v1329: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:08 smithi059 bash[59745]: cluster 2022-04-28T20:15:07.505108+0000 mgr.y (mgr.24961) 957 : cluster [DBG] pgmap v1329: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:08 smithi139 bash[43301]: cluster 2022-04-28T20:15:07.505108+0000 mgr.y (mgr.24961) 957 : cluster [DBG] pgmap v1329: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:10 smithi059 bash[61152]: cluster 2022-04-28T20:15:09.505966+0000 mgr.y (mgr.24961) 958 : cluster [DBG] pgmap v1330: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:10 smithi059 bash[59745]: cluster 2022-04-28T20:15:09.505966+0000 mgr.y (mgr.24961) 958 : cluster [DBG] pgmap v1330: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:10 smithi139 bash[43301]: cluster 2022-04-28T20:15:09.505966+0000 mgr.y (mgr.24961) 958 : cluster [DBG] pgmap v1330: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:11.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:12 smithi059 bash[61152]: cluster 2022-04-28T20:15:11.506928+0000 mgr.y (mgr.24961) 959 : cluster [DBG] pgmap v1331: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:12 smithi059 bash[59745]: cluster 2022-04-28T20:15:11.506928+0000 mgr.y (mgr.24961) 959 : cluster [DBG] pgmap v1331: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:12 smithi139 bash[43301]: cluster 2022-04-28T20:15:11.506928+0000 mgr.y (mgr.24961) 959 : cluster [DBG] pgmap v1331: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:13 smithi059 bash[61152]: cluster 2022-04-28T20:15:13.507562+0000 mgr.y (mgr.24961) 960 : cluster [DBG] pgmap v1332: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:13.608Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:13 smithi059 bash[59745]: cluster 2022-04-28T20:15:13.507562+0000 mgr.y (mgr.24961) 960 : cluster [DBG] pgmap v1332: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:13 smithi139 bash[43301]: cluster 2022-04-28T20:15:13.507562+0000 mgr.y (mgr.24961) 960 : cluster [DBG] pgmap v1332: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[59745]: cluster 2022-04-28T20:15:15.508570+0000 mgr.y (mgr.24961) 961 : cluster [DBG] pgmap v1333: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[59745]: audit 2022-04-28T20:15:16.099095+0000 mon.a (mon.0) 1813 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:15:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[59745]: audit 2022-04-28T20:15:16.101923+0000 mon.a (mon.0) 1814 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:15:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[59745]: audit 2022-04-28T20:15:16.103694+0000 mon.a (mon.0) 1815 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:15:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[59745]: audit 2022-04-28T20:15:16.432149+0000 mon.a (mon.0) 1816 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:15:16.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[61152]: cluster 2022-04-28T20:15:15.508570+0000 mgr.y (mgr.24961) 961 : cluster [DBG] pgmap v1333: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:16.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[61152]: audit 2022-04-28T20:15:16.099095+0000 mon.a (mon.0) 1813 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:15:16.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[61152]: audit 2022-04-28T20:15:16.101923+0000 mon.a (mon.0) 1814 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:15:16.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[61152]: audit 2022-04-28T20:15:16.103694+0000 mon.a (mon.0) 1815 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:15:16.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:16 smithi059 bash[61152]: audit 2022-04-28T20:15:16.432149+0000 mon.a (mon.0) 1816 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:15:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:16 smithi139 bash[43301]: cluster 2022-04-28T20:15:15.508570+0000 mgr.y (mgr.24961) 961 : cluster [DBG] pgmap v1333: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:16 smithi139 bash[43301]: audit 2022-04-28T20:15:16.099095+0000 mon.a (mon.0) 1813 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:15:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:16 smithi139 bash[43301]: audit 2022-04-28T20:15:16.101923+0000 mon.a (mon.0) 1814 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:15:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:16 smithi139 bash[43301]: audit 2022-04-28T20:15:16.103694+0000 mon.a (mon.0) 1815 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:15:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:16 smithi139 bash[43301]: audit 2022-04-28T20:15:16.432149+0000 mon.a (mon.0) 1816 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:15:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[61152]: cephadm 2022-04-28T20:15:16.105261+0000 mgr.y (mgr.24961) 962 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:15:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[61152]: audit 2022-04-28T20:15:17.471232+0000 mon.a (mon.0) 1817 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:15:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[61152]: audit 2022-04-28T20:15:17.474531+0000 mon.a (mon.0) 1818 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:15:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[61152]: audit 2022-04-28T20:15:17.493422+0000 mon.a (mon.0) 1819 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:15:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[61152]: audit 2022-04-28T20:15:17.494359+0000 mon.a (mon.0) 1820 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:15:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[61152]: audit 2022-04-28T20:15:17.495186+0000 mon.a (mon.0) 1821 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T20:15:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[59745]: cephadm 2022-04-28T20:15:16.105261+0000 mgr.y (mgr.24961) 962 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:15:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[59745]: audit 2022-04-28T20:15:17.471232+0000 mon.a (mon.0) 1817 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:15:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[59745]: audit 2022-04-28T20:15:17.474531+0000 mon.a (mon.0) 1818 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:15:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[59745]: audit 2022-04-28T20:15:17.493422+0000 mon.a (mon.0) 1819 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:15:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[59745]: audit 2022-04-28T20:15:17.494359+0000 mon.a (mon.0) 1820 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:15:17.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:17 smithi059 bash[59745]: audit 2022-04-28T20:15:17.495186+0000 mon.a (mon.0) 1821 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T20:15:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:17 smithi139 bash[43301]: cephadm 2022-04-28T20:15:16.105261+0000 mgr.y (mgr.24961) 962 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:15:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:17 smithi139 bash[43301]: audit 2022-04-28T20:15:17.471232+0000 mon.a (mon.0) 1817 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:15:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:17 smithi139 bash[43301]: audit 2022-04-28T20:15:17.474531+0000 mon.a (mon.0) 1818 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:15:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:17 smithi139 bash[43301]: audit 2022-04-28T20:15:17.493422+0000 mon.a (mon.0) 1819 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:15:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:17 smithi139 bash[43301]: audit 2022-04-28T20:15:17.494359+0000 mon.a (mon.0) 1820 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:15:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:17 smithi139 bash[43301]: audit 2022-04-28T20:15:17.495186+0000 mon.a (mon.0) 1821 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T20:15:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:18 smithi059 bash[61152]: cluster 2022-04-28T20:15:17.509362+0000 mgr.y (mgr.24961) 963 : cluster [DBG] pgmap v1334: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:18 smithi059 bash[59745]: cluster 2022-04-28T20:15:17.509362+0000 mgr.y (mgr.24961) 963 : cluster [DBG] pgmap v1334: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:18 smithi139 bash[43301]: cluster 2022-04-28T20:15:17.509362+0000 mgr.y (mgr.24961) 963 : cluster [DBG] pgmap v1334: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:20.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:20 smithi059 bash[59745]: cluster 2022-04-28T20:15:19.510225+0000 mgr.y (mgr.24961) 964 : cluster [DBG] pgmap v1335: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:20 smithi059 bash[61152]: cluster 2022-04-28T20:15:19.510225+0000 mgr.y (mgr.24961) 964 : cluster [DBG] pgmap v1335: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:20 smithi139 bash[43301]: cluster 2022-04-28T20:15:19.510225+0000 mgr.y (mgr.24961) 964 : cluster [DBG] pgmap v1335: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:21.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:22 smithi139 bash[43301]: cluster 2022-04-28T20:15:21.511171+0000 mgr.y (mgr.24961) 965 : cluster [DBG] pgmap v1336: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:22 smithi059 bash[61152]: cluster 2022-04-28T20:15:21.511171+0000 mgr.y (mgr.24961) 965 : cluster [DBG] pgmap v1336: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:22 smithi059 bash[59745]: cluster 2022-04-28T20:15:21.511171+0000 mgr.y (mgr.24961) 965 : cluster [DBG] pgmap v1336: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:23 smithi059 bash[61152]: cluster 2022-04-28T20:15:23.512025+0000 mgr.y (mgr.24961) 966 : cluster [DBG] pgmap v1337: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:23.609Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:23 smithi059 bash[59745]: cluster 2022-04-28T20:15:23.512025+0000 mgr.y (mgr.24961) 966 : cluster [DBG] pgmap v1337: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:24.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:23 smithi139 bash[43301]: cluster 2022-04-28T20:15:23.512025+0000 mgr.y (mgr.24961) 966 : cluster [DBG] pgmap v1337: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:26 smithi059 bash[61152]: cluster 2022-04-28T20:15:25.513061+0000 mgr.y (mgr.24961) 967 : cluster [DBG] pgmap v1338: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:26 smithi059 bash[59745]: cluster 2022-04-28T20:15:25.513061+0000 mgr.y (mgr.24961) 967 : cluster [DBG] pgmap v1338: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:26 smithi139 bash[43301]: cluster 2022-04-28T20:15:25.513061+0000 mgr.y (mgr.24961) 967 : cluster [DBG] pgmap v1338: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:28 smithi059 bash[61152]: cluster 2022-04-28T20:15:27.513864+0000 mgr.y (mgr.24961) 968 : cluster [DBG] pgmap v1339: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:28 smithi059 bash[59745]: cluster 2022-04-28T20:15:27.513864+0000 mgr.y (mgr.24961) 968 : cluster [DBG] pgmap v1339: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:28.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:28 smithi139 bash[43301]: cluster 2022-04-28T20:15:27.513864+0000 mgr.y (mgr.24961) 968 : cluster [DBG] pgmap v1339: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:30 smithi059 bash[61152]: cluster 2022-04-28T20:15:29.514758+0000 mgr.y (mgr.24961) 969 : cluster [DBG] pgmap v1340: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:30 smithi059 bash[59745]: cluster 2022-04-28T20:15:29.514758+0000 mgr.y (mgr.24961) 969 : cluster [DBG] pgmap v1340: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:30 smithi139 bash[43301]: cluster 2022-04-28T20:15:29.514758+0000 mgr.y (mgr.24961) 969 : cluster [DBG] pgmap v1340: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:31.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:32 smithi059 bash[61152]: cluster 2022-04-28T20:15:31.515782+0000 mgr.y (mgr.24961) 970 : cluster [DBG] pgmap v1341: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:32 smithi059 bash[59745]: cluster 2022-04-28T20:15:31.515782+0000 mgr.y (mgr.24961) 970 : cluster [DBG] pgmap v1341: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:32.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:32 smithi139 bash[43301]: cluster 2022-04-28T20:15:31.515782+0000 mgr.y (mgr.24961) 970 : cluster [DBG] pgmap v1341: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:33 smithi059 bash[61152]: cluster 2022-04-28T20:15:33.516571+0000 mgr.y (mgr.24961) 971 : cluster [DBG] pgmap v1342: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:34.108 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:33.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:34.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:33 smithi059 bash[59745]: cluster 2022-04-28T20:15:33.516571+0000 mgr.y (mgr.24961) 971 : cluster [DBG] pgmap v1342: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:34.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:33 smithi139 bash[43301]: cluster 2022-04-28T20:15:33.516571+0000 mgr.y (mgr.24961) 971 : cluster [DBG] pgmap v1342: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:36 smithi059 bash[61152]: cluster 2022-04-28T20:15:35.517599+0000 mgr.y (mgr.24961) 972 : cluster [DBG] pgmap v1343: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:36 smithi059 bash[59745]: cluster 2022-04-28T20:15:35.517599+0000 mgr.y (mgr.24961) 972 : cluster [DBG] pgmap v1343: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:36 smithi139 bash[43301]: cluster 2022-04-28T20:15:35.517599+0000 mgr.y (mgr.24961) 972 : cluster [DBG] pgmap v1343: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:38 smithi059 bash[61152]: cluster 2022-04-28T20:15:37.518453+0000 mgr.y (mgr.24961) 973 : cluster [DBG] pgmap v1344: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:38.985 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:38 smithi059 bash[59745]: cluster 2022-04-28T20:15:37.518453+0000 mgr.y (mgr.24961) 973 : cluster [DBG] pgmap v1344: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:38.986 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:38 smithi139 bash[43301]: cluster 2022-04-28T20:15:37.518453+0000 mgr.y (mgr.24961) 973 : cluster [DBG] pgmap v1344: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:40 smithi059 bash[61152]: cluster 2022-04-28T20:15:39.519414+0000 mgr.y (mgr.24961) 974 : cluster [DBG] pgmap v1345: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:40 smithi059 bash[59745]: cluster 2022-04-28T20:15:39.519414+0000 mgr.y (mgr.24961) 974 : cluster [DBG] pgmap v1345: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:40.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:40 smithi139 bash[43301]: cluster 2022-04-28T20:15:39.519414+0000 mgr.y (mgr.24961) 974 : cluster [DBG] pgmap v1345: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:41.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:42 smithi059 bash[61152]: cluster 2022-04-28T20:15:41.520450+0000 mgr.y (mgr.24961) 975 : cluster [DBG] pgmap v1346: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:42 smithi059 bash[59745]: cluster 2022-04-28T20:15:41.520450+0000 mgr.y (mgr.24961) 975 : cluster [DBG] pgmap v1346: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:42 smithi139 bash[43301]: cluster 2022-04-28T20:15:41.520450+0000 mgr.y (mgr.24961) 975 : cluster [DBG] pgmap v1346: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:43 smithi059 bash[61152]: cluster 2022-04-28T20:15:43.521092+0000 mgr.y (mgr.24961) 976 : cluster [DBG] pgmap v1347: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:43.610Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:43 smithi059 bash[59745]: cluster 2022-04-28T20:15:43.521092+0000 mgr.y (mgr.24961) 976 : cluster [DBG] pgmap v1347: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:43 smithi139 bash[43301]: cluster 2022-04-28T20:15:43.521092+0000 mgr.y (mgr.24961) 976 : cluster [DBG] pgmap v1347: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:46 smithi059 bash[61152]: cluster 2022-04-28T20:15:45.522062+0000 mgr.y (mgr.24961) 977 : cluster [DBG] pgmap v1348: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:46 smithi059 bash[59745]: cluster 2022-04-28T20:15:45.522062+0000 mgr.y (mgr.24961) 977 : cluster [DBG] pgmap v1348: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:46 smithi139 bash[43301]: cluster 2022-04-28T20:15:45.522062+0000 mgr.y (mgr.24961) 977 : cluster [DBG] pgmap v1348: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:48 smithi059 bash[61152]: cluster 2022-04-28T20:15:47.522871+0000 mgr.y (mgr.24961) 978 : cluster [DBG] pgmap v1349: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:48 smithi059 bash[59745]: cluster 2022-04-28T20:15:47.522871+0000 mgr.y (mgr.24961) 978 : cluster [DBG] pgmap v1349: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:48.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:48 smithi139 bash[43301]: cluster 2022-04-28T20:15:47.522871+0000 mgr.y (mgr.24961) 978 : cluster [DBG] pgmap v1349: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:50 smithi059 bash[61152]: cluster 2022-04-28T20:15:49.523884+0000 mgr.y (mgr.24961) 979 : cluster [DBG] pgmap v1350: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:50 smithi059 bash[59745]: cluster 2022-04-28T20:15:49.523884+0000 mgr.y (mgr.24961) 979 : cluster [DBG] pgmap v1350: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:50.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:50 smithi139 bash[43301]: cluster 2022-04-28T20:15:49.523884+0000 mgr.y (mgr.24961) 979 : cluster [DBG] pgmap v1350: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:51.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:52 smithi139 bash[43301]: cluster 2022-04-28T20:15:51.524954+0000 mgr.y (mgr.24961) 980 : cluster [DBG] pgmap v1351: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:52 smithi059 bash[61152]: cluster 2022-04-28T20:15:51.524954+0000 mgr.y (mgr.24961) 980 : cluster [DBG] pgmap v1351: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:53.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:52 smithi059 bash[59745]: cluster 2022-04-28T20:15:51.524954+0000 mgr.y (mgr.24961) 980 : cluster [DBG] pgmap v1351: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:53 smithi059 bash[61152]: cluster 2022-04-28T20:15:53.525650+0000 mgr.y (mgr.24961) 981 : cluster [DBG] pgmap v1352: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:15:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:15:53.611Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:15:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:53 smithi059 bash[59745]: cluster 2022-04-28T20:15:53.525650+0000 mgr.y (mgr.24961) 981 : cluster [DBG] pgmap v1352: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:54.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:53 smithi139 bash[43301]: cluster 2022-04-28T20:15:53.525650+0000 mgr.y (mgr.24961) 981 : cluster [DBG] pgmap v1352: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:56 smithi059 bash[61152]: cluster 2022-04-28T20:15:55.526637+0000 mgr.y (mgr.24961) 982 : cluster [DBG] pgmap v1353: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:56 smithi059 bash[59745]: cluster 2022-04-28T20:15:55.526637+0000 mgr.y (mgr.24961) 982 : cluster [DBG] pgmap v1353: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:56 smithi139 bash[43301]: cluster 2022-04-28T20:15:55.526637+0000 mgr.y (mgr.24961) 982 : cluster [DBG] pgmap v1353: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:15:58 smithi059 bash[61152]: cluster 2022-04-28T20:15:57.527456+0000 mgr.y (mgr.24961) 983 : cluster [DBG] pgmap v1354: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:58.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:15:58 smithi059 bash[59745]: cluster 2022-04-28T20:15:57.527456+0000 mgr.y (mgr.24961) 983 : cluster [DBG] pgmap v1354: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:15:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:15:58 smithi139 bash[43301]: cluster 2022-04-28T20:15:57.527456+0000 mgr.y (mgr.24961) 983 : cluster [DBG] pgmap v1354: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:00 smithi139 bash[43301]: cluster 2022-04-28T20:15:59.528407+0000 mgr.y (mgr.24961) 984 : cluster [DBG] pgmap v1355: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:00 smithi059 bash[61152]: cluster 2022-04-28T20:15:59.528407+0000 mgr.y (mgr.24961) 984 : cluster [DBG] pgmap v1355: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:00 smithi059 bash[59745]: cluster 2022-04-28T20:15:59.528407+0000 mgr.y (mgr.24961) 984 : cluster [DBG] pgmap v1355: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:01.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:02 smithi139 bash[43301]: cluster 2022-04-28T20:16:01.529346+0000 mgr.y (mgr.24961) 985 : cluster [DBG] pgmap v1356: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:02 smithi059 bash[61152]: cluster 2022-04-28T20:16:01.529346+0000 mgr.y (mgr.24961) 985 : cluster [DBG] pgmap v1356: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:02 smithi059 bash[59745]: cluster 2022-04-28T20:16:01.529346+0000 mgr.y (mgr.24961) 985 : cluster [DBG] pgmap v1356: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:03 smithi059 bash[61152]: cluster 2022-04-28T20:16:03.530001+0000 mgr.y (mgr.24961) 986 : cluster [DBG] pgmap v1357: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:03.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:03 smithi059 bash[59745]: cluster 2022-04-28T20:16:03.530001+0000 mgr.y (mgr.24961) 986 : cluster [DBG] pgmap v1357: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:03 smithi139 bash[43301]: cluster 2022-04-28T20:16:03.530001+0000 mgr.y (mgr.24961) 986 : cluster [DBG] pgmap v1357: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:06 smithi059 bash[61152]: cluster 2022-04-28T20:16:05.531086+0000 mgr.y (mgr.24961) 987 : cluster [DBG] pgmap v1358: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:06 smithi059 bash[59745]: cluster 2022-04-28T20:16:05.531086+0000 mgr.y (mgr.24961) 987 : cluster [DBG] pgmap v1358: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:06 smithi139 bash[43301]: cluster 2022-04-28T20:16:05.531086+0000 mgr.y (mgr.24961) 987 : cluster [DBG] pgmap v1358: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:08 smithi139 bash[43301]: cluster 2022-04-28T20:16:07.532092+0000 mgr.y (mgr.24961) 988 : cluster [DBG] pgmap v1359: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:08 smithi059 bash[61152]: cluster 2022-04-28T20:16:07.532092+0000 mgr.y (mgr.24961) 988 : cluster [DBG] pgmap v1359: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:08 smithi059 bash[59745]: cluster 2022-04-28T20:16:07.532092+0000 mgr.y (mgr.24961) 988 : cluster [DBG] pgmap v1359: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:10 smithi139 bash[43301]: cluster 2022-04-28T20:16:09.532969+0000 mgr.y (mgr.24961) 989 : cluster [DBG] pgmap v1360: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:10 smithi059 bash[61152]: cluster 2022-04-28T20:16:09.532969+0000 mgr.y (mgr.24961) 989 : cluster [DBG] pgmap v1360: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:10 smithi059 bash[59745]: cluster 2022-04-28T20:16:09.532969+0000 mgr.y (mgr.24961) 989 : cluster [DBG] pgmap v1360: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:11.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:12 smithi139 bash[43301]: cluster 2022-04-28T20:16:11.534015+0000 mgr.y (mgr.24961) 990 : cluster [DBG] pgmap v1361: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:12 smithi059 bash[61152]: cluster 2022-04-28T20:16:11.534015+0000 mgr.y (mgr.24961) 990 : cluster [DBG] pgmap v1361: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:13.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:12 smithi059 bash[59745]: cluster 2022-04-28T20:16:11.534015+0000 mgr.y (mgr.24961) 990 : cluster [DBG] pgmap v1361: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:13 smithi059 bash[61152]: cluster 2022-04-28T20:16:13.534673+0000 mgr.y (mgr.24961) 991 : cluster [DBG] pgmap v1362: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:13.612Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:13 smithi059 bash[59745]: cluster 2022-04-28T20:16:13.534673+0000 mgr.y (mgr.24961) 991 : cluster [DBG] pgmap v1362: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:14.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:13 smithi139 bash[43301]: cluster 2022-04-28T20:16:13.534673+0000 mgr.y (mgr.24961) 991 : cluster [DBG] pgmap v1362: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:16 smithi139 bash[43301]: cluster 2022-04-28T20:16:15.535813+0000 mgr.y (mgr.24961) 992 : cluster [DBG] pgmap v1363: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:16 smithi139 bash[43301]: audit 2022-04-28T20:16:16.439100+0000 mon.a (mon.0) 1822 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:16:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:16 smithi139 bash[43301]: audit 2022-04-28T20:16:16.442002+0000 mon.a (mon.0) 1823 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:16:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:16 smithi139 bash[43301]: audit 2022-04-28T20:16:16.443968+0000 mon.a (mon.0) 1824 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:16:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:16 smithi139 bash[43301]: audit 2022-04-28T20:16:16.447274+0000 mon.a (mon.0) 1825 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:16:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:16 smithi139 bash[43301]: audit 2022-04-28T20:16:16.448119+0000 mon.a (mon.0) 1826 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:16:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[61152]: cluster 2022-04-28T20:16:15.535813+0000 mgr.y (mgr.24961) 992 : cluster [DBG] pgmap v1363: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[61152]: audit 2022-04-28T20:16:16.439100+0000 mon.a (mon.0) 1822 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:16:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[61152]: audit 2022-04-28T20:16:16.442002+0000 mon.a (mon.0) 1823 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:16:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[61152]: audit 2022-04-28T20:16:16.443968+0000 mon.a (mon.0) 1824 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:16:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[61152]: audit 2022-04-28T20:16:16.447274+0000 mon.a (mon.0) 1825 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:16:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[61152]: audit 2022-04-28T20:16:16.448119+0000 mon.a (mon.0) 1826 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:16:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[59745]: cluster 2022-04-28T20:16:15.535813+0000 mgr.y (mgr.24961) 992 : cluster [DBG] pgmap v1363: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[59745]: audit 2022-04-28T20:16:16.439100+0000 mon.a (mon.0) 1822 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:16:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[59745]: audit 2022-04-28T20:16:16.442002+0000 mon.a (mon.0) 1823 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:16:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[59745]: audit 2022-04-28T20:16:16.443968+0000 mon.a (mon.0) 1824 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:16:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[59745]: audit 2022-04-28T20:16:16.447274+0000 mon.a (mon.0) 1825 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:16:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:16 smithi059 bash[59745]: audit 2022-04-28T20:16:16.448119+0000 mon.a (mon.0) 1826 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:16:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: cephadm 2022-04-28T20:16:16.445619+0000 mgr.y (mgr.24961) 993 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:16:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: audit 2022-04-28T20:16:17.477604+0000 mon.a (mon.0) 1827 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:16:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: audit 2022-04-28T20:16:17.481205+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:16:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: audit 2022-04-28T20:16:17.501761+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:16:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: audit 2022-04-28T20:16:17.502480+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:16:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: audit 2022-04-28T20:16:17.503030+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:16:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:17 smithi139 bash[43301]: audit 2022-04-28T20:16:17.503591+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2022-04-28T20:16:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: cephadm 2022-04-28T20:16:16.445619+0000 mgr.y (mgr.24961) 993 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:16:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: audit 2022-04-28T20:16:17.477604+0000 mon.a (mon.0) 1827 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:16:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: audit 2022-04-28T20:16:17.481205+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:16:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: audit 2022-04-28T20:16:17.501761+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:16:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: audit 2022-04-28T20:16:17.502480+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:16:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: audit 2022-04-28T20:16:17.503030+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:16:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[59745]: audit 2022-04-28T20:16:17.503591+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2022-04-28T20:16:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: cephadm 2022-04-28T20:16:16.445619+0000 mgr.y (mgr.24961) 993 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:16:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: audit 2022-04-28T20:16:17.477604+0000 mon.a (mon.0) 1827 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:16:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: audit 2022-04-28T20:16:17.481205+0000 mon.a (mon.0) 1828 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:16:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: audit 2022-04-28T20:16:17.501761+0000 mon.a (mon.0) 1829 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:16:18.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: audit 2022-04-28T20:16:17.502480+0000 mon.a (mon.0) 1830 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:16:18.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: audit 2022-04-28T20:16:17.503030+0000 mon.a (mon.0) 1831 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:16:18.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:17 smithi059 bash[61152]: audit 2022-04-28T20:16:17.503591+0000 mon.a (mon.0) 1832 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2022-04-28T20:16:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:18 smithi139 bash[43301]: cluster 2022-04-28T20:16:17.536655+0000 mgr.y (mgr.24961) 994 : cluster [DBG] pgmap v1364: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:18 smithi059 bash[59745]: cluster 2022-04-28T20:16:17.536655+0000 mgr.y (mgr.24961) 994 : cluster [DBG] pgmap v1364: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:18 smithi059 bash[61152]: cluster 2022-04-28T20:16:17.536655+0000 mgr.y (mgr.24961) 994 : cluster [DBG] pgmap v1364: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:20 smithi139 bash[43301]: cluster 2022-04-28T20:16:19.537514+0000 mgr.y (mgr.24961) 995 : cluster [DBG] pgmap v1365: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:20 smithi139 bash[43301]: audit 2022-04-28T20:16:20.509343+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:20 smithi059 bash[59745]: cluster 2022-04-28T20:16:19.537514+0000 mgr.y (mgr.24961) 995 : cluster [DBG] pgmap v1365: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:20 smithi059 bash[59745]: audit 2022-04-28T20:16:20.509343+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:20 smithi059 bash[61152]: cluster 2022-04-28T20:16:19.537514+0000 mgr.y (mgr.24961) 995 : cluster [DBG] pgmap v1365: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:20 smithi059 bash[61152]: audit 2022-04-28T20:16:20.509343+0000 mon.a (mon.0) 1833 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:21.720Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[61152]: audit 2022-04-28T20:16:20.734728+0000 mon.a (mon.0) 1834 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[61152]: audit 2022-04-28T20:16:21.026070+0000 mon.a (mon.0) 1835 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[61152]: cluster 2022-04-28T20:16:21.538582+0000 mgr.y (mgr.24961) 996 : cluster [DBG] pgmap v1366: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[59745]: audit 2022-04-28T20:16:20.734728+0000 mon.a (mon.0) 1834 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[59745]: audit 2022-04-28T20:16:21.026070+0000 mon.a (mon.0) 1835 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:21 smithi059 bash[59745]: cluster 2022-04-28T20:16:21.538582+0000 mgr.y (mgr.24961) 996 : cluster [DBG] pgmap v1366: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:21 smithi139 bash[43301]: audit 2022-04-28T20:16:20.734728+0000 mon.a (mon.0) 1834 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:21 smithi139 bash[43301]: audit 2022-04-28T20:16:21.026070+0000 mon.a (mon.0) 1835 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:16:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:21 smithi139 bash[43301]: cluster 2022-04-28T20:16:21.538582+0000 mgr.y (mgr.24961) 996 : cluster [DBG] pgmap v1366: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:23 smithi059 bash[61152]: cluster 2022-04-28T20:16:23.539271+0000 mgr.y (mgr.24961) 997 : cluster [DBG] pgmap v1367: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:23.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:23 smithi059 bash[59745]: cluster 2022-04-28T20:16:23.539271+0000 mgr.y (mgr.24961) 997 : cluster [DBG] pgmap v1367: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:24.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:23 smithi139 bash[43301]: cluster 2022-04-28T20:16:23.539271+0000 mgr.y (mgr.24961) 997 : cluster [DBG] pgmap v1367: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:26 smithi139 bash[43301]: cluster 2022-04-28T20:16:25.540268+0000 mgr.y (mgr.24961) 998 : cluster [DBG] pgmap v1368: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:26 smithi059 bash[61152]: cluster 2022-04-28T20:16:25.540268+0000 mgr.y (mgr.24961) 998 : cluster [DBG] pgmap v1368: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:27.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:26 smithi059 bash[59745]: cluster 2022-04-28T20:16:25.540268+0000 mgr.y (mgr.24961) 998 : cluster [DBG] pgmap v1368: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:28.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:28 smithi139 bash[43301]: cluster 2022-04-28T20:16:27.541117+0000 mgr.y (mgr.24961) 999 : cluster [DBG] pgmap v1369: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:28 smithi059 bash[61152]: cluster 2022-04-28T20:16:27.541117+0000 mgr.y (mgr.24961) 999 : cluster [DBG] pgmap v1369: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:28 smithi059 bash[59745]: cluster 2022-04-28T20:16:27.541117+0000 mgr.y (mgr.24961) 999 : cluster [DBG] pgmap v1369: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:30 smithi139 bash[43301]: cluster 2022-04-28T20:16:29.541958+0000 mgr.y (mgr.24961) 1000 : cluster [DBG] pgmap v1370: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:30 smithi059 bash[61152]: cluster 2022-04-28T20:16:29.541958+0000 mgr.y (mgr.24961) 1000 : cluster [DBG] pgmap v1370: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:30 smithi059 bash[59745]: cluster 2022-04-28T20:16:29.541958+0000 mgr.y (mgr.24961) 1000 : cluster [DBG] pgmap v1370: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:31.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:32 smithi139 bash[43301]: cluster 2022-04-28T20:16:31.542885+0000 mgr.y (mgr.24961) 1001 : cluster [DBG] pgmap v1371: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:32 smithi059 bash[61152]: cluster 2022-04-28T20:16:31.542885+0000 mgr.y (mgr.24961) 1001 : cluster [DBG] pgmap v1371: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:32 smithi059 bash[59745]: cluster 2022-04-28T20:16:31.542885+0000 mgr.y (mgr.24961) 1001 : cluster [DBG] pgmap v1371: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:33 smithi059 bash[61152]: cluster 2022-04-28T20:16:33.543666+0000 mgr.y (mgr.24961) 1002 : cluster [DBG] pgmap v1372: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:33.613Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:33 smithi059 bash[59745]: cluster 2022-04-28T20:16:33.543666+0000 mgr.y (mgr.24961) 1002 : cluster [DBG] pgmap v1372: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:34.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:33 smithi139 bash[43301]: cluster 2022-04-28T20:16:33.543666+0000 mgr.y (mgr.24961) 1002 : cluster [DBG] pgmap v1372: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:36 smithi139 bash[43301]: cluster 2022-04-28T20:16:35.544679+0000 mgr.y (mgr.24961) 1003 : cluster [DBG] pgmap v1373: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:36 smithi059 bash[61152]: cluster 2022-04-28T20:16:35.544679+0000 mgr.y (mgr.24961) 1003 : cluster [DBG] pgmap v1373: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:37.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:36 smithi059 bash[59745]: cluster 2022-04-28T20:16:35.544679+0000 mgr.y (mgr.24961) 1003 : cluster [DBG] pgmap v1373: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:38 smithi139 bash[43301]: cluster 2022-04-28T20:16:37.545550+0000 mgr.y (mgr.24961) 1004 : cluster [DBG] pgmap v1374: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:38 smithi059 bash[61152]: cluster 2022-04-28T20:16:37.545550+0000 mgr.y (mgr.24961) 1004 : cluster [DBG] pgmap v1374: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:38 smithi059 bash[59745]: cluster 2022-04-28T20:16:37.545550+0000 mgr.y (mgr.24961) 1004 : cluster [DBG] pgmap v1374: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:40 smithi139 bash[43301]: cluster 2022-04-28T20:16:39.546384+0000 mgr.y (mgr.24961) 1005 : cluster [DBG] pgmap v1375: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:41.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:40 smithi059 bash[61152]: cluster 2022-04-28T20:16:39.546384+0000 mgr.y (mgr.24961) 1005 : cluster [DBG] pgmap v1375: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:40 smithi059 bash[59745]: cluster 2022-04-28T20:16:39.546384+0000 mgr.y (mgr.24961) 1005 : cluster [DBG] pgmap v1375: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:41.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:43.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:42 smithi059 bash[59745]: cluster 2022-04-28T20:16:41.547326+0000 mgr.y (mgr.24961) 1006 : cluster [DBG] pgmap v1376: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:43.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:42 smithi059 bash[61152]: cluster 2022-04-28T20:16:41.547326+0000 mgr.y (mgr.24961) 1006 : cluster [DBG] pgmap v1376: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:43.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:42 smithi139 bash[43301]: cluster 2022-04-28T20:16:41.547326+0000 mgr.y (mgr.24961) 1006 : cluster [DBG] pgmap v1376: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:43.869 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:43.615Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:43.869 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:43 smithi059 bash[59745]: cluster 2022-04-28T20:16:43.547983+0000 mgr.y (mgr.24961) 1007 : cluster [DBG] pgmap v1377: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:43 smithi139 bash[43301]: cluster 2022-04-28T20:16:43.547983+0000 mgr.y (mgr.24961) 1007 : cluster [DBG] pgmap v1377: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:43 smithi059 bash[61152]: cluster 2022-04-28T20:16:43.547983+0000 mgr.y (mgr.24961) 1007 : cluster [DBG] pgmap v1377: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:46 smithi139 bash[43301]: cluster 2022-04-28T20:16:45.548979+0000 mgr.y (mgr.24961) 1008 : cluster [DBG] pgmap v1378: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:46 smithi059 bash[61152]: cluster 2022-04-28T20:16:45.548979+0000 mgr.y (mgr.24961) 1008 : cluster [DBG] pgmap v1378: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:46 smithi059 bash[59745]: cluster 2022-04-28T20:16:45.548979+0000 mgr.y (mgr.24961) 1008 : cluster [DBG] pgmap v1378: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:48 smithi139 bash[43301]: cluster 2022-04-28T20:16:47.549927+0000 mgr.y (mgr.24961) 1009 : cluster [DBG] pgmap v1379: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:48 smithi059 bash[61152]: cluster 2022-04-28T20:16:47.549927+0000 mgr.y (mgr.24961) 1009 : cluster [DBG] pgmap v1379: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:49.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:48 smithi059 bash[59745]: cluster 2022-04-28T20:16:47.549927+0000 mgr.y (mgr.24961) 1009 : cluster [DBG] pgmap v1379: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:50 smithi139 bash[43301]: cluster 2022-04-28T20:16:49.550953+0000 mgr.y (mgr.24961) 1010 : cluster [DBG] pgmap v1380: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:50 smithi059 bash[61152]: cluster 2022-04-28T20:16:49.550953+0000 mgr.y (mgr.24961) 1010 : cluster [DBG] pgmap v1380: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:50 smithi059 bash[59745]: cluster 2022-04-28T20:16:49.550953+0000 mgr.y (mgr.24961) 1010 : cluster [DBG] pgmap v1380: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:51.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:52 smithi059 bash[61152]: cluster 2022-04-28T20:16:51.551960+0000 mgr.y (mgr.24961) 1011 : cluster [DBG] pgmap v1381: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:52 smithi059 bash[59745]: cluster 2022-04-28T20:16:51.551960+0000 mgr.y (mgr.24961) 1011 : cluster [DBG] pgmap v1381: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:53.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:52 smithi139 bash[43301]: cluster 2022-04-28T20:16:51.551960+0000 mgr.y (mgr.24961) 1011 : cluster [DBG] pgmap v1381: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:53 smithi059 bash[61152]: cluster 2022-04-28T20:16:53.552736+0000 mgr.y (mgr.24961) 1012 : cluster [DBG] pgmap v1382: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:54.198 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:16:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:16:53.616Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:16:54.198 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:53 smithi059 bash[59745]: cluster 2022-04-28T20:16:53.552736+0000 mgr.y (mgr.24961) 1012 : cluster [DBG] pgmap v1382: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:54.199 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:53 smithi139 bash[43301]: cluster 2022-04-28T20:16:53.552736+0000 mgr.y (mgr.24961) 1012 : cluster [DBG] pgmap v1382: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:56 smithi139 bash[43301]: cluster 2022-04-28T20:16:55.553744+0000 mgr.y (mgr.24961) 1013 : cluster [DBG] pgmap v1383: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:56 smithi059 bash[61152]: cluster 2022-04-28T20:16:55.553744+0000 mgr.y (mgr.24961) 1013 : cluster [DBG] pgmap v1383: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:56 smithi059 bash[59745]: cluster 2022-04-28T20:16:55.553744+0000 mgr.y (mgr.24961) 1013 : cluster [DBG] pgmap v1383: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:16:58 smithi139 bash[43301]: cluster 2022-04-28T20:16:57.554553+0000 mgr.y (mgr.24961) 1014 : cluster [DBG] pgmap v1384: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:59.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:16:58 smithi059 bash[59745]: cluster 2022-04-28T20:16:57.554553+0000 mgr.y (mgr.24961) 1014 : cluster [DBG] pgmap v1384: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:16:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:16:58 smithi059 bash[61152]: cluster 2022-04-28T20:16:57.554553+0000 mgr.y (mgr.24961) 1014 : cluster [DBG] pgmap v1384: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:00 smithi059 bash[61152]: cluster 2022-04-28T20:16:59.555403+0000 mgr.y (mgr.24961) 1015 : cluster [DBG] pgmap v1385: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:00 smithi059 bash[59745]: cluster 2022-04-28T20:16:59.555403+0000 mgr.y (mgr.24961) 1015 : cluster [DBG] pgmap v1385: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:00 smithi139 bash[43301]: cluster 2022-04-28T20:16:59.555403+0000 mgr.y (mgr.24961) 1015 : cluster [DBG] pgmap v1385: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:01.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:02 smithi059 bash[61152]: cluster 2022-04-28T20:17:01.556408+0000 mgr.y (mgr.24961) 1016 : cluster [DBG] pgmap v1386: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:02 smithi059 bash[59745]: cluster 2022-04-28T20:17:01.556408+0000 mgr.y (mgr.24961) 1016 : cluster [DBG] pgmap v1386: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:03.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:02 smithi139 bash[43301]: cluster 2022-04-28T20:17:01.556408+0000 mgr.y (mgr.24961) 1016 : cluster [DBG] pgmap v1386: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:03.875 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:03.617Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:03.875 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:03 smithi059 bash[59745]: cluster 2022-04-28T20:17:03.557074+0000 mgr.y (mgr.24961) 1017 : cluster [DBG] pgmap v1387: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:03 smithi059 bash[61152]: cluster 2022-04-28T20:17:03.557074+0000 mgr.y (mgr.24961) 1017 : cluster [DBG] pgmap v1387: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:03 smithi139 bash[43301]: cluster 2022-04-28T20:17:03.557074+0000 mgr.y (mgr.24961) 1017 : cluster [DBG] pgmap v1387: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:06 smithi139 bash[43301]: cluster 2022-04-28T20:17:05.558079+0000 mgr.y (mgr.24961) 1018 : cluster [DBG] pgmap v1388: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:06 smithi059 bash[61152]: cluster 2022-04-28T20:17:05.558079+0000 mgr.y (mgr.24961) 1018 : cluster [DBG] pgmap v1388: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:06 smithi059 bash[59745]: cluster 2022-04-28T20:17:05.558079+0000 mgr.y (mgr.24961) 1018 : cluster [DBG] pgmap v1388: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:09.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:08 smithi059 bash[61152]: cluster 2022-04-28T20:17:07.558869+0000 mgr.y (mgr.24961) 1019 : cluster [DBG] pgmap v1389: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:08 smithi059 bash[59745]: cluster 2022-04-28T20:17:07.558869+0000 mgr.y (mgr.24961) 1019 : cluster [DBG] pgmap v1389: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:08 smithi139 bash[43301]: cluster 2022-04-28T20:17:07.558869+0000 mgr.y (mgr.24961) 1019 : cluster [DBG] pgmap v1389: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:10 smithi059 bash[61152]: cluster 2022-04-28T20:17:09.559757+0000 mgr.y (mgr.24961) 1020 : cluster [DBG] pgmap v1390: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:10 smithi059 bash[59745]: cluster 2022-04-28T20:17:09.559757+0000 mgr.y (mgr.24961) 1020 : cluster [DBG] pgmap v1390: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:10 smithi139 bash[43301]: cluster 2022-04-28T20:17:09.559757+0000 mgr.y (mgr.24961) 1020 : cluster [DBG] pgmap v1390: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:11 smithi059 bash[61152]: cluster 2022-04-28T20:17:11.560711+0000 mgr.y (mgr.24961) 1021 : cluster [DBG] pgmap v1391: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:11.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:11 smithi059 bash[59745]: cluster 2022-04-28T20:17:11.560711+0000 mgr.y (mgr.24961) 1021 : cluster [DBG] pgmap v1391: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:11 smithi139 bash[43301]: cluster 2022-04-28T20:17:11.560711+0000 mgr.y (mgr.24961) 1021 : cluster [DBG] pgmap v1391: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:13 smithi059 bash[61152]: cluster 2022-04-28T20:17:13.561394+0000 mgr.y (mgr.24961) 1022 : cluster [DBG] pgmap v1392: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:13.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:13 smithi059 bash[59745]: cluster 2022-04-28T20:17:13.561394+0000 mgr.y (mgr.24961) 1022 : cluster [DBG] pgmap v1392: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:13 smithi139 bash[43301]: cluster 2022-04-28T20:17:13.561394+0000 mgr.y (mgr.24961) 1022 : cluster [DBG] pgmap v1392: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:16 smithi059 bash[59745]: cluster 2022-04-28T20:17:15.562396+0000 mgr.y (mgr.24961) 1023 : cluster [DBG] pgmap v1393: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:16 smithi059 bash[61152]: cluster 2022-04-28T20:17:15.562396+0000 mgr.y (mgr.24961) 1023 : cluster [DBG] pgmap v1393: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:16 smithi139 bash[43301]: cluster 2022-04-28T20:17:15.562396+0000 mgr.y (mgr.24961) 1023 : cluster [DBG] pgmap v1393: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[59745]: audit 2022-04-28T20:17:17.484431+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:17:18.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[59745]: audit 2022-04-28T20:17:17.487874+0000 mon.a (mon.0) 1837 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:17:18.250 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[59745]: audit 2022-04-28T20:17:17.510263+0000 mon.a (mon.0) 1838 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:17:18.250 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[59745]: audit 2022-04-28T20:17:17.511004+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:17:18.260 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[59745]: audit 2022-04-28T20:17:17.511299+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:17:18.261 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[61152]: audit 2022-04-28T20:17:17.484431+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:17:18.261 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[61152]: audit 2022-04-28T20:17:17.487874+0000 mon.a (mon.0) 1837 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:17:18.261 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[61152]: audit 2022-04-28T20:17:17.510263+0000 mon.a (mon.0) 1838 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:17:18.261 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[61152]: audit 2022-04-28T20:17:17.511004+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:17:18.262 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:17 smithi059 bash[61152]: audit 2022-04-28T20:17:17.511299+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:17:18.262 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:17 smithi139 bash[43301]: audit 2022-04-28T20:17:17.484431+0000 mon.a (mon.0) 1836 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:17:18.262 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:17 smithi139 bash[43301]: audit 2022-04-28T20:17:17.487874+0000 mon.a (mon.0) 1837 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:17:18.263 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:17 smithi139 bash[43301]: audit 2022-04-28T20:17:17.510263+0000 mon.a (mon.0) 1838 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:17:18.263 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:17 smithi139 bash[43301]: audit 2022-04-28T20:17:17.511004+0000 mon.a (mon.0) 1839 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:17:18.263 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:17 smithi139 bash[43301]: audit 2022-04-28T20:17:17.511299+0000 mon.a (mon.0) 1840 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:17:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:18 smithi059 bash[59745]: cluster 2022-04-28T20:17:17.563242+0000 mgr.y (mgr.24961) 1024 : cluster [DBG] pgmap v1394: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:19.220 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:18 smithi059 bash[61152]: cluster 2022-04-28T20:17:17.563242+0000 mgr.y (mgr.24961) 1024 : cluster [DBG] pgmap v1394: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:19.221 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:18 smithi139 bash[43301]: cluster 2022-04-28T20:17:17.563242+0000 mgr.y (mgr.24961) 1024 : cluster [DBG] pgmap v1394: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:19 smithi059 bash[61152]: cluster 2022-04-28T20:17:19.564152+0000 mgr.y (mgr.24961) 1025 : cluster [DBG] pgmap v1395: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:19 smithi059 bash[59745]: cluster 2022-04-28T20:17:19.564152+0000 mgr.y (mgr.24961) 1025 : cluster [DBG] pgmap v1395: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:20.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:19 smithi139 bash[43301]: cluster 2022-04-28T20:17:19.564152+0000 mgr.y (mgr.24961) 1025 : cluster [DBG] pgmap v1395: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[59745]: audit 2022-04-28T20:17:21.034710+0000 mon.a (mon.0) 1841 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:17:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[59745]: audit 2022-04-28T20:17:21.037642+0000 mon.a (mon.0) 1842 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:17:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[59745]: audit 2022-04-28T20:17:21.039762+0000 mon.a (mon.0) 1843 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:17:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[61152]: audit 2022-04-28T20:17:21.034710+0000 mon.a (mon.0) 1841 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:17:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[61152]: audit 2022-04-28T20:17:21.037642+0000 mon.a (mon.0) 1842 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:17:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[61152]: audit 2022-04-28T20:17:21.039762+0000 mon.a (mon.0) 1843 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:17:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:21 smithi139 bash[43301]: audit 2022-04-28T20:17:21.034710+0000 mon.a (mon.0) 1841 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:17:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:21 smithi139 bash[43301]: audit 2022-04-28T20:17:21.037642+0000 mon.a (mon.0) 1842 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:17:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:21 smithi139 bash[43301]: audit 2022-04-28T20:17:21.039762+0000 mon.a (mon.0) 1843 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:17:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:21.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:22 smithi059 bash[61152]: cephadm 2022-04-28T20:17:21.041662+0000 mgr.y (mgr.24961) 1026 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:17:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:22 smithi059 bash[61152]: audit 2022-04-28T20:17:21.373678+0000 mon.a (mon.0) 1844 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:17:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:22 smithi059 bash[61152]: cluster 2022-04-28T20:17:21.565123+0000 mgr.y (mgr.24961) 1027 : cluster [DBG] pgmap v1396: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:22 smithi059 bash[59745]: cephadm 2022-04-28T20:17:21.041662+0000 mgr.y (mgr.24961) 1026 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:17:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:22 smithi059 bash[59745]: audit 2022-04-28T20:17:21.373678+0000 mon.a (mon.0) 1844 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:17:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:22 smithi059 bash[59745]: cluster 2022-04-28T20:17:21.565123+0000 mgr.y (mgr.24961) 1027 : cluster [DBG] pgmap v1396: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:22 smithi139 bash[43301]: cephadm 2022-04-28T20:17:21.041662+0000 mgr.y (mgr.24961) 1026 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:17:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:22 smithi139 bash[43301]: audit 2022-04-28T20:17:21.373678+0000 mon.a (mon.0) 1844 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:17:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:22 smithi139 bash[43301]: cluster 2022-04-28T20:17:21.565123+0000 mgr.y (mgr.24961) 1027 : cluster [DBG] pgmap v1396: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:23 smithi059 bash[61152]: cluster 2022-04-28T20:17:23.565892+0000 mgr.y (mgr.24961) 1028 : cluster [DBG] pgmap v1397: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:23.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:23 smithi059 bash[59745]: cluster 2022-04-28T20:17:23.565892+0000 mgr.y (mgr.24961) 1028 : cluster [DBG] pgmap v1397: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:23 smithi139 bash[43301]: cluster 2022-04-28T20:17:23.565892+0000 mgr.y (mgr.24961) 1028 : cluster [DBG] pgmap v1397: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:26 smithi059 bash[61152]: cluster 2022-04-28T20:17:25.566937+0000 mgr.y (mgr.24961) 1029 : cluster [DBG] pgmap v1398: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:27.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:26 smithi059 bash[59745]: cluster 2022-04-28T20:17:25.566937+0000 mgr.y (mgr.24961) 1029 : cluster [DBG] pgmap v1398: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:26 smithi139 bash[43301]: cluster 2022-04-28T20:17:25.566937+0000 mgr.y (mgr.24961) 1029 : cluster [DBG] pgmap v1398: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:28 smithi059 bash[61152]: cluster 2022-04-28T20:17:27.567898+0000 mgr.y (mgr.24961) 1030 : cluster [DBG] pgmap v1399: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:28 smithi059 bash[59745]: cluster 2022-04-28T20:17:27.567898+0000 mgr.y (mgr.24961) 1030 : cluster [DBG] pgmap v1399: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:29.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:28 smithi139 bash[43301]: cluster 2022-04-28T20:17:27.567898+0000 mgr.y (mgr.24961) 1030 : cluster [DBG] pgmap v1399: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:29 smithi059 bash[61152]: cluster 2022-04-28T20:17:29.568802+0000 mgr.y (mgr.24961) 1031 : cluster [DBG] pgmap v1400: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:29 smithi059 bash[59745]: cluster 2022-04-28T20:17:29.568802+0000 mgr.y (mgr.24961) 1031 : cluster [DBG] pgmap v1400: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:29 smithi139 bash[43301]: cluster 2022-04-28T20:17:29.568802+0000 mgr.y (mgr.24961) 1031 : cluster [DBG] pgmap v1400: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:31.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:32 smithi059 bash[61152]: cluster 2022-04-28T20:17:31.569759+0000 mgr.y (mgr.24961) 1032 : cluster [DBG] pgmap v1401: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:32 smithi059 bash[59745]: cluster 2022-04-28T20:17:31.569759+0000 mgr.y (mgr.24961) 1032 : cluster [DBG] pgmap v1401: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:32 smithi139 bash[43301]: cluster 2022-04-28T20:17:31.569759+0000 mgr.y (mgr.24961) 1032 : cluster [DBG] pgmap v1401: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:33.880 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:33.618Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:33 smithi059 bash[59745]: cluster 2022-04-28T20:17:33.570413+0000 mgr.y (mgr.24961) 1033 : cluster [DBG] pgmap v1402: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:33 smithi059 bash[61152]: cluster 2022-04-28T20:17:33.570413+0000 mgr.y (mgr.24961) 1033 : cluster [DBG] pgmap v1402: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:33 smithi139 bash[43301]: cluster 2022-04-28T20:17:33.570413+0000 mgr.y (mgr.24961) 1033 : cluster [DBG] pgmap v1402: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:36 smithi059 bash[61152]: cluster 2022-04-28T20:17:35.571425+0000 mgr.y (mgr.24961) 1034 : cluster [DBG] pgmap v1403: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:37.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:36 smithi059 bash[59745]: cluster 2022-04-28T20:17:35.571425+0000 mgr.y (mgr.24961) 1034 : cluster [DBG] pgmap v1403: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:37.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:36 smithi139 bash[43301]: cluster 2022-04-28T20:17:35.571425+0000 mgr.y (mgr.24961) 1034 : cluster [DBG] pgmap v1403: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:37 smithi059 bash[61152]: cluster 2022-04-28T20:17:37.572291+0000 mgr.y (mgr.24961) 1035 : cluster [DBG] pgmap v1404: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:38.202 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:37 smithi059 bash[59745]: cluster 2022-04-28T20:17:37.572291+0000 mgr.y (mgr.24961) 1035 : cluster [DBG] pgmap v1404: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:38.203 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:37 smithi139 bash[43301]: cluster 2022-04-28T20:17:37.572291+0000 mgr.y (mgr.24961) 1035 : cluster [DBG] pgmap v1404: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:41.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:40 smithi059 bash[61152]: cluster 2022-04-28T20:17:39.573170+0000 mgr.y (mgr.24961) 1036 : cluster [DBG] pgmap v1405: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:40 smithi059 bash[59745]: cluster 2022-04-28T20:17:39.573170+0000 mgr.y (mgr.24961) 1036 : cluster [DBG] pgmap v1405: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:40 smithi139 bash[43301]: cluster 2022-04-28T20:17:39.573170+0000 mgr.y (mgr.24961) 1036 : cluster [DBG] pgmap v1405: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:41 smithi059 bash[61152]: cluster 2022-04-28T20:17:41.574115+0000 mgr.y (mgr.24961) 1037 : cluster [DBG] pgmap v1406: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:42.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:41.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:41 smithi059 bash[59745]: cluster 2022-04-28T20:17:41.574115+0000 mgr.y (mgr.24961) 1037 : cluster [DBG] pgmap v1406: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:41 smithi139 bash[43301]: cluster 2022-04-28T20:17:41.574115+0000 mgr.y (mgr.24961) 1037 : cluster [DBG] pgmap v1406: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:43.874 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:43.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:43 smithi059 bash[61152]: cluster 2022-04-28T20:17:43.574794+0000 mgr.y (mgr.24961) 1038 : cluster [DBG] pgmap v1407: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:43 smithi059 bash[59745]: cluster 2022-04-28T20:17:43.574794+0000 mgr.y (mgr.24961) 1038 : cluster [DBG] pgmap v1407: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:43 smithi139 bash[43301]: cluster 2022-04-28T20:17:43.574794+0000 mgr.y (mgr.24961) 1038 : cluster [DBG] pgmap v1407: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:46 smithi059 bash[61152]: cluster 2022-04-28T20:17:45.575836+0000 mgr.y (mgr.24961) 1039 : cluster [DBG] pgmap v1408: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:46 smithi059 bash[59745]: cluster 2022-04-28T20:17:45.575836+0000 mgr.y (mgr.24961) 1039 : cluster [DBG] pgmap v1408: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:47.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:46 smithi139 bash[43301]: cluster 2022-04-28T20:17:45.575836+0000 mgr.y (mgr.24961) 1039 : cluster [DBG] pgmap v1408: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:47 smithi059 bash[61152]: cluster 2022-04-28T20:17:47.576672+0000 mgr.y (mgr.24961) 1040 : cluster [DBG] pgmap v1409: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:47 smithi059 bash[59745]: cluster 2022-04-28T20:17:47.576672+0000 mgr.y (mgr.24961) 1040 : cluster [DBG] pgmap v1409: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:48.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:47 smithi139 bash[43301]: cluster 2022-04-28T20:17:47.576672+0000 mgr.y (mgr.24961) 1040 : cluster [DBG] pgmap v1409: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:50 smithi059 bash[61152]: cluster 2022-04-28T20:17:49.577553+0000 mgr.y (mgr.24961) 1041 : cluster [DBG] pgmap v1410: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:50 smithi059 bash[59745]: cluster 2022-04-28T20:17:49.577553+0000 mgr.y (mgr.24961) 1041 : cluster [DBG] pgmap v1410: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:51.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:50 smithi139 bash[43301]: cluster 2022-04-28T20:17:49.577553+0000 mgr.y (mgr.24961) 1041 : cluster [DBG] pgmap v1410: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:51 smithi059 bash[61152]: cluster 2022-04-28T20:17:51.578509+0000 mgr.y (mgr.24961) 1042 : cluster [DBG] pgmap v1411: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:51.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:51 smithi059 bash[59745]: cluster 2022-04-28T20:17:51.578509+0000 mgr.y (mgr.24961) 1042 : cluster [DBG] pgmap v1411: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:51 smithi139 bash[43301]: cluster 2022-04-28T20:17:51.578509+0000 mgr.y (mgr.24961) 1042 : cluster [DBG] pgmap v1411: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:53.878 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:17:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:17:53.620Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:17:53.879 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:53 smithi059 bash[59745]: cluster 2022-04-28T20:17:53.579192+0000 mgr.y (mgr.24961) 1043 : cluster [DBG] pgmap v1412: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:53 smithi059 bash[61152]: cluster 2022-04-28T20:17:53.579192+0000 mgr.y (mgr.24961) 1043 : cluster [DBG] pgmap v1412: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:53 smithi139 bash[43301]: cluster 2022-04-28T20:17:53.579192+0000 mgr.y (mgr.24961) 1043 : cluster [DBG] pgmap v1412: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:55 smithi059 bash[61152]: cluster 2022-04-28T20:17:55.580233+0000 mgr.y (mgr.24961) 1044 : cluster [DBG] pgmap v1413: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:55 smithi059 bash[59745]: cluster 2022-04-28T20:17:55.580233+0000 mgr.y (mgr.24961) 1044 : cluster [DBG] pgmap v1413: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:56.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:55 smithi139 bash[43301]: cluster 2022-04-28T20:17:55.580233+0000 mgr.y (mgr.24961) 1044 : cluster [DBG] pgmap v1413: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:57 smithi059 bash[59745]: cluster 2022-04-28T20:17:57.581130+0000 mgr.y (mgr.24961) 1045 : cluster [DBG] pgmap v1414: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:58.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:57 smithi059 bash[61152]: cluster 2022-04-28T20:17:57.581130+0000 mgr.y (mgr.24961) 1045 : cluster [DBG] pgmap v1414: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:17:58.139 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:57 smithi139 bash[43301]: cluster 2022-04-28T20:17:57.581130+0000 mgr.y (mgr.24961) 1045 : cluster [DBG] pgmap v1414: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:17:59 smithi059 bash[61152]: cluster 2022-04-28T20:17:59.582013+0000 mgr.y (mgr.24961) 1046 : cluster [DBG] pgmap v1415: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:17:59 smithi059 bash[59745]: cluster 2022-04-28T20:17:59.582013+0000 mgr.y (mgr.24961) 1046 : cluster [DBG] pgmap v1415: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:00.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:17:59 smithi139 bash[43301]: cluster 2022-04-28T20:17:59.582013+0000 mgr.y (mgr.24961) 1046 : cluster [DBG] pgmap v1415: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:01 smithi059 bash[61152]: cluster 2022-04-28T20:18:01.582974+0000 mgr.y (mgr.24961) 1047 : cluster [DBG] pgmap v1416: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:01.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:01 smithi059 bash[59745]: cluster 2022-04-28T20:18:01.582974+0000 mgr.y (mgr.24961) 1047 : cluster [DBG] pgmap v1416: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:02.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:01 smithi139 bash[43301]: cluster 2022-04-28T20:18:01.582974+0000 mgr.y (mgr.24961) 1047 : cluster [DBG] pgmap v1416: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:03.886 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:03.619Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:03 smithi059 bash[61152]: cluster 2022-04-28T20:18:03.583682+0000 mgr.y (mgr.24961) 1048 : cluster [DBG] pgmap v1417: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:03 smithi059 bash[59745]: cluster 2022-04-28T20:18:03.583682+0000 mgr.y (mgr.24961) 1048 : cluster [DBG] pgmap v1417: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:03 smithi139 bash[43301]: cluster 2022-04-28T20:18:03.583682+0000 mgr.y (mgr.24961) 1048 : cluster [DBG] pgmap v1417: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:05 smithi059 bash[61152]: cluster 2022-04-28T20:18:05.584656+0000 mgr.y (mgr.24961) 1049 : cluster [DBG] pgmap v1418: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:05 smithi059 bash[59745]: cluster 2022-04-28T20:18:05.584656+0000 mgr.y (mgr.24961) 1049 : cluster [DBG] pgmap v1418: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:05 smithi139 bash[43301]: cluster 2022-04-28T20:18:05.584656+0000 mgr.y (mgr.24961) 1049 : cluster [DBG] pgmap v1418: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:07 smithi059 bash[61152]: cluster 2022-04-28T20:18:07.585507+0000 mgr.y (mgr.24961) 1050 : cluster [DBG] pgmap v1419: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:07 smithi059 bash[59745]: cluster 2022-04-28T20:18:07.585507+0000 mgr.y (mgr.24961) 1050 : cluster [DBG] pgmap v1419: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:08.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:07 smithi139 bash[43301]: cluster 2022-04-28T20:18:07.585507+0000 mgr.y (mgr.24961) 1050 : cluster [DBG] pgmap v1419: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:09 smithi059 bash[61152]: cluster 2022-04-28T20:18:09.586473+0000 mgr.y (mgr.24961) 1051 : cluster [DBG] pgmap v1420: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:10.155 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:09 smithi059 bash[59745]: cluster 2022-04-28T20:18:09.586473+0000 mgr.y (mgr.24961) 1051 : cluster [DBG] pgmap v1420: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:10.155 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:09 smithi139 bash[43301]: cluster 2022-04-28T20:18:09.586473+0000 mgr.y (mgr.24961) 1051 : cluster [DBG] pgmap v1420: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:11 smithi059 bash[61152]: cluster 2022-04-28T20:18:11.587584+0000 mgr.y (mgr.24961) 1052 : cluster [DBG] pgmap v1421: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:11 smithi059 bash[59745]: cluster 2022-04-28T20:18:11.587584+0000 mgr.y (mgr.24961) 1052 : cluster [DBG] pgmap v1421: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:11.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:12.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:11 smithi139 bash[43301]: cluster 2022-04-28T20:18:11.587584+0000 mgr.y (mgr.24961) 1052 : cluster [DBG] pgmap v1421: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:13.913 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:13.621Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:13 smithi059 bash[61152]: cluster 2022-04-28T20:18:13.588290+0000 mgr.y (mgr.24961) 1053 : cluster [DBG] pgmap v1422: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:13 smithi059 bash[59745]: cluster 2022-04-28T20:18:13.588290+0000 mgr.y (mgr.24961) 1053 : cluster [DBG] pgmap v1422: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:13 smithi139 bash[43301]: cluster 2022-04-28T20:18:13.588290+0000 mgr.y (mgr.24961) 1053 : cluster [DBG] pgmap v1422: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:15 smithi059 bash[61152]: cluster 2022-04-28T20:18:15.589293+0000 mgr.y (mgr.24961) 1054 : cluster [DBG] pgmap v1423: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:15 smithi059 bash[59745]: cluster 2022-04-28T20:18:15.589293+0000 mgr.y (mgr.24961) 1054 : cluster [DBG] pgmap v1423: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:15 smithi139 bash[43301]: cluster 2022-04-28T20:18:15.589293+0000 mgr.y (mgr.24961) 1054 : cluster [DBG] pgmap v1423: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[61152]: audit 2022-04-28T20:18:17.493233+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:18:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[61152]: audit 2022-04-28T20:18:17.493714+0000 mon.a (mon.0) 1846 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:18:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[61152]: audit 2022-04-28T20:18:17.516021+0000 mon.a (mon.0) 1847 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[61152]: audit 2022-04-28T20:18:17.516702+0000 mon.a (mon.0) 1848 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[61152]: audit 2022-04-28T20:18:17.517389+0000 mon.a (mon.0) 1849 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[59745]: audit 2022-04-28T20:18:17.493233+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:18:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[59745]: audit 2022-04-28T20:18:17.493714+0000 mon.a (mon.0) 1846 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:18:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[59745]: audit 2022-04-28T20:18:17.516021+0000 mon.a (mon.0) 1847 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[59745]: audit 2022-04-28T20:18:17.516702+0000 mon.a (mon.0) 1848 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:17 smithi059 bash[59745]: audit 2022-04-28T20:18:17.517389+0000 mon.a (mon.0) 1849 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:17 smithi139 bash[43301]: audit 2022-04-28T20:18:17.493233+0000 mon.a (mon.0) 1845 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:18:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:17 smithi139 bash[43301]: audit 2022-04-28T20:18:17.493714+0000 mon.a (mon.0) 1846 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:18:17.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:17 smithi139 bash[43301]: audit 2022-04-28T20:18:17.516021+0000 mon.a (mon.0) 1847 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:17 smithi139 bash[43301]: audit 2022-04-28T20:18:17.516702+0000 mon.a (mon.0) 1848 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:18:17.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:17 smithi139 bash[43301]: audit 2022-04-28T20:18:17.517389+0000 mon.a (mon.0) 1849 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:18:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:18 smithi059 bash[61152]: cluster 2022-04-28T20:18:17.590147+0000 mgr.y (mgr.24961) 1055 : cluster [DBG] pgmap v1424: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:18 smithi059 bash[59745]: cluster 2022-04-28T20:18:17.590147+0000 mgr.y (mgr.24961) 1055 : cluster [DBG] pgmap v1424: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:18.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:18 smithi139 bash[43301]: cluster 2022-04-28T20:18:17.590147+0000 mgr.y (mgr.24961) 1055 : cluster [DBG] pgmap v1424: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:19 smithi059 bash[61152]: cluster 2022-04-28T20:18:19.591041+0000 mgr.y (mgr.24961) 1056 : cluster [DBG] pgmap v1425: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:19 smithi059 bash[59745]: cluster 2022-04-28T20:18:19.591041+0000 mgr.y (mgr.24961) 1056 : cluster [DBG] pgmap v1425: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:20.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:19 smithi139 bash[43301]: cluster 2022-04-28T20:18:19.591041+0000 mgr.y (mgr.24961) 1056 : cluster [DBG] pgmap v1425: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:21.725 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[61152]: audit 2022-04-28T20:18:21.384916+0000 mon.a (mon.0) 1850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:18:21.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[61152]: audit 2022-04-28T20:18:21.387798+0000 mon.a (mon.0) 1851 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:18:21.726 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[61152]: audit 2022-04-28T20:18:21.389620+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:18:21.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[59745]: audit 2022-04-28T20:18:21.384916+0000 mon.a (mon.0) 1850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:18:21.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[59745]: audit 2022-04-28T20:18:21.387798+0000 mon.a (mon.0) 1851 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:18:21.726 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[59745]: audit 2022-04-28T20:18:21.389620+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:18:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:21 smithi139 bash[43301]: audit 2022-04-28T20:18:21.384916+0000 mon.a (mon.0) 1850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:18:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:21 smithi139 bash[43301]: audit 2022-04-28T20:18:21.387798+0000 mon.a (mon.0) 1851 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:18:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:21 smithi139 bash[43301]: audit 2022-04-28T20:18:21.389620+0000 mon.a (mon.0) 1852 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:18:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:21.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[59745]: cephadm 2022-04-28T20:18:21.391284+0000 mgr.y (mgr.24961) 1057 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:18:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[59745]: cluster 2022-04-28T20:18:21.592346+0000 mgr.y (mgr.24961) 1058 : cluster [DBG] pgmap v1426: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[59745]: audit 2022-04-28T20:18:21.701635+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[59745]: audit 2022-04-28T20:18:21.706846+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[59745]: audit 2022-04-28T20:18:22.026524+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[61152]: cephadm 2022-04-28T20:18:21.391284+0000 mgr.y (mgr.24961) 1057 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:18:22.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[61152]: cluster 2022-04-28T20:18:21.592346+0000 mgr.y (mgr.24961) 1058 : cluster [DBG] pgmap v1426: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:22.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[61152]: audit 2022-04-28T20:18:21.701635+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[61152]: audit 2022-04-28T20:18:21.706846+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:22 smithi059 bash[61152]: audit 2022-04-28T20:18:22.026524+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:22 smithi139 bash[43301]: cephadm 2022-04-28T20:18:21.391284+0000 mgr.y (mgr.24961) 1057 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:18:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:22 smithi139 bash[43301]: cluster 2022-04-28T20:18:21.592346+0000 mgr.y (mgr.24961) 1058 : cluster [DBG] pgmap v1426: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:22 smithi139 bash[43301]: audit 2022-04-28T20:18:21.701635+0000 mon.a (mon.0) 1853 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:22 smithi139 bash[43301]: audit 2022-04-28T20:18:21.706846+0000 mon.a (mon.0) 1854 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:22 smithi139 bash[43301]: audit 2022-04-28T20:18:22.026524+0000 mon.a (mon.0) 1855 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:18:23.921 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:23.622Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:23 smithi059 bash[59745]: cluster 2022-04-28T20:18:23.593123+0000 mgr.y (mgr.24961) 1059 : cluster [DBG] pgmap v1427: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:23 smithi059 bash[61152]: cluster 2022-04-28T20:18:23.593123+0000 mgr.y (mgr.24961) 1059 : cluster [DBG] pgmap v1427: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:23 smithi139 bash[43301]: cluster 2022-04-28T20:18:23.593123+0000 mgr.y (mgr.24961) 1059 : cluster [DBG] pgmap v1427: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:25 smithi059 bash[61152]: cluster 2022-04-28T20:18:25.594097+0000 mgr.y (mgr.24961) 1060 : cluster [DBG] pgmap v1428: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:25 smithi059 bash[59745]: cluster 2022-04-28T20:18:25.594097+0000 mgr.y (mgr.24961) 1060 : cluster [DBG] pgmap v1428: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:25 smithi139 bash[43301]: cluster 2022-04-28T20:18:25.594097+0000 mgr.y (mgr.24961) 1060 : cluster [DBG] pgmap v1428: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:27 smithi059 bash[61152]: cluster 2022-04-28T20:18:27.594926+0000 mgr.y (mgr.24961) 1061 : cluster [DBG] pgmap v1429: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:27 smithi059 bash[59745]: cluster 2022-04-28T20:18:27.594926+0000 mgr.y (mgr.24961) 1061 : cluster [DBG] pgmap v1429: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:27 smithi139 bash[43301]: cluster 2022-04-28T20:18:27.594926+0000 mgr.y (mgr.24961) 1061 : cluster [DBG] pgmap v1429: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:29 smithi059 bash[61152]: cluster 2022-04-28T20:18:29.595809+0000 mgr.y (mgr.24961) 1062 : cluster [DBG] pgmap v1430: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:29 smithi059 bash[59745]: cluster 2022-04-28T20:18:29.595809+0000 mgr.y (mgr.24961) 1062 : cluster [DBG] pgmap v1430: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:29 smithi139 bash[43301]: cluster 2022-04-28T20:18:29.595809+0000 mgr.y (mgr.24961) 1062 : cluster [DBG] pgmap v1430: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:31 smithi059 bash[61152]: cluster 2022-04-28T20:18:31.596768+0000 mgr.y (mgr.24961) 1063 : cluster [DBG] pgmap v1431: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:32.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:31.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:31 smithi059 bash[59745]: cluster 2022-04-28T20:18:31.596768+0000 mgr.y (mgr.24961) 1063 : cluster [DBG] pgmap v1431: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:32.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:31 smithi139 bash[43301]: cluster 2022-04-28T20:18:31.596768+0000 mgr.y (mgr.24961) 1063 : cluster [DBG] pgmap v1431: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:33.924 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:33.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:33.925 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:33 smithi059 bash[59745]: cluster 2022-04-28T20:18:33.597444+0000 mgr.y (mgr.24961) 1064 : cluster [DBG] pgmap v1432: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:33 smithi059 bash[61152]: cluster 2022-04-28T20:18:33.597444+0000 mgr.y (mgr.24961) 1064 : cluster [DBG] pgmap v1432: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:33 smithi139 bash[43301]: cluster 2022-04-28T20:18:33.597444+0000 mgr.y (mgr.24961) 1064 : cluster [DBG] pgmap v1432: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:35 smithi059 bash[61152]: cluster 2022-04-28T20:18:35.598466+0000 mgr.y (mgr.24961) 1065 : cluster [DBG] pgmap v1433: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:35 smithi059 bash[59745]: cluster 2022-04-28T20:18:35.598466+0000 mgr.y (mgr.24961) 1065 : cluster [DBG] pgmap v1433: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:35 smithi139 bash[43301]: cluster 2022-04-28T20:18:35.598466+0000 mgr.y (mgr.24961) 1065 : cluster [DBG] pgmap v1433: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:37 smithi059 bash[61152]: cluster 2022-04-28T20:18:37.599274+0000 mgr.y (mgr.24961) 1066 : cluster [DBG] pgmap v1434: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:37 smithi059 bash[59745]: cluster 2022-04-28T20:18:37.599274+0000 mgr.y (mgr.24961) 1066 : cluster [DBG] pgmap v1434: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:38.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:37 smithi139 bash[43301]: cluster 2022-04-28T20:18:37.599274+0000 mgr.y (mgr.24961) 1066 : cluster [DBG] pgmap v1434: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:39 smithi059 bash[61152]: cluster 2022-04-28T20:18:39.600145+0000 mgr.y (mgr.24961) 1067 : cluster [DBG] pgmap v1435: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:39 smithi059 bash[59745]: cluster 2022-04-28T20:18:39.600145+0000 mgr.y (mgr.24961) 1067 : cluster [DBG] pgmap v1435: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:39 smithi139 bash[43301]: cluster 2022-04-28T20:18:39.600145+0000 mgr.y (mgr.24961) 1067 : cluster [DBG] pgmap v1435: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:41 smithi059 bash[61152]: cluster 2022-04-28T20:18:41.601107+0000 mgr.y (mgr.24961) 1068 : cluster [DBG] pgmap v1436: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:42.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:41.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:41 smithi059 bash[59745]: cluster 2022-04-28T20:18:41.601107+0000 mgr.y (mgr.24961) 1068 : cluster [DBG] pgmap v1436: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:41 smithi139 bash[43301]: cluster 2022-04-28T20:18:41.601107+0000 mgr.y (mgr.24961) 1068 : cluster [DBG] pgmap v1436: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:43.922 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:43.623Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:43.922 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:43 smithi059 bash[59745]: cluster 2022-04-28T20:18:43.601787+0000 mgr.y (mgr.24961) 1069 : cluster [DBG] pgmap v1437: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:43 smithi059 bash[61152]: cluster 2022-04-28T20:18:43.601787+0000 mgr.y (mgr.24961) 1069 : cluster [DBG] pgmap v1437: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:43 smithi139 bash[43301]: cluster 2022-04-28T20:18:43.601787+0000 mgr.y (mgr.24961) 1069 : cluster [DBG] pgmap v1437: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:45 smithi059 bash[61152]: cluster 2022-04-28T20:18:45.602738+0000 mgr.y (mgr.24961) 1070 : cluster [DBG] pgmap v1438: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:46.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:45 smithi059 bash[59745]: cluster 2022-04-28T20:18:45.602738+0000 mgr.y (mgr.24961) 1070 : cluster [DBG] pgmap v1438: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:46.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:45 smithi139 bash[43301]: cluster 2022-04-28T20:18:45.602738+0000 mgr.y (mgr.24961) 1070 : cluster [DBG] pgmap v1438: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:47 smithi059 bash[61152]: cluster 2022-04-28T20:18:47.603585+0000 mgr.y (mgr.24961) 1071 : cluster [DBG] pgmap v1439: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:47 smithi059 bash[59745]: cluster 2022-04-28T20:18:47.603585+0000 mgr.y (mgr.24961) 1071 : cluster [DBG] pgmap v1439: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:47 smithi139 bash[43301]: cluster 2022-04-28T20:18:47.603585+0000 mgr.y (mgr.24961) 1071 : cluster [DBG] pgmap v1439: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:49 smithi059 bash[61152]: cluster 2022-04-28T20:18:49.604465+0000 mgr.y (mgr.24961) 1072 : cluster [DBG] pgmap v1440: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:49 smithi059 bash[59745]: cluster 2022-04-28T20:18:49.604465+0000 mgr.y (mgr.24961) 1072 : cluster [DBG] pgmap v1440: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:50.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:49 smithi139 bash[43301]: cluster 2022-04-28T20:18:49.604465+0000 mgr.y (mgr.24961) 1072 : cluster [DBG] pgmap v1440: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:51 smithi059 bash[61152]: cluster 2022-04-28T20:18:51.605422+0000 mgr.y (mgr.24961) 1073 : cluster [DBG] pgmap v1441: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:52.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:51.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:51 smithi059 bash[59745]: cluster 2022-04-28T20:18:51.605422+0000 mgr.y (mgr.24961) 1073 : cluster [DBG] pgmap v1441: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:51 smithi139 bash[43301]: cluster 2022-04-28T20:18:51.605422+0000 mgr.y (mgr.24961) 1073 : cluster [DBG] pgmap v1441: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:53.923 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:18:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:18:53.625Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:18:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:53 smithi059 bash[61152]: cluster 2022-04-28T20:18:53.606127+0000 mgr.y (mgr.24961) 1074 : cluster [DBG] pgmap v1442: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:53 smithi059 bash[59745]: cluster 2022-04-28T20:18:53.606127+0000 mgr.y (mgr.24961) 1074 : cluster [DBG] pgmap v1442: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:53 smithi139 bash[43301]: cluster 2022-04-28T20:18:53.606127+0000 mgr.y (mgr.24961) 1074 : cluster [DBG] pgmap v1442: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:55 smithi059 bash[59745]: cluster 2022-04-28T20:18:55.607248+0000 mgr.y (mgr.24961) 1075 : cluster [DBG] pgmap v1443: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:55 smithi059 bash[61152]: cluster 2022-04-28T20:18:55.607248+0000 mgr.y (mgr.24961) 1075 : cluster [DBG] pgmap v1443: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:55 smithi139 bash[43301]: cluster 2022-04-28T20:18:55.607248+0000 mgr.y (mgr.24961) 1075 : cluster [DBG] pgmap v1443: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:57 smithi059 bash[61152]: cluster 2022-04-28T20:18:57.608114+0000 mgr.y (mgr.24961) 1076 : cluster [DBG] pgmap v1444: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:57 smithi059 bash[59745]: cluster 2022-04-28T20:18:57.608114+0000 mgr.y (mgr.24961) 1076 : cluster [DBG] pgmap v1444: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:18:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:57 smithi139 bash[43301]: cluster 2022-04-28T20:18:57.608114+0000 mgr.y (mgr.24961) 1076 : cluster [DBG] pgmap v1444: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:18:59 smithi059 bash[61152]: cluster 2022-04-28T20:18:59.608975+0000 mgr.y (mgr.24961) 1077 : cluster [DBG] pgmap v1445: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:18:59 smithi059 bash[59745]: cluster 2022-04-28T20:18:59.608975+0000 mgr.y (mgr.24961) 1077 : cluster [DBG] pgmap v1445: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:18:59 smithi139 bash[43301]: cluster 2022-04-28T20:18:59.608975+0000 mgr.y (mgr.24961) 1077 : cluster [DBG] pgmap v1445: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:01 smithi059 bash[61152]: cluster 2022-04-28T20:19:01.609917+0000 mgr.y (mgr.24961) 1078 : cluster [DBG] pgmap v1446: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:02.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:01.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:01 smithi059 bash[59745]: cluster 2022-04-28T20:19:01.609917+0000 mgr.y (mgr.24961) 1078 : cluster [DBG] pgmap v1446: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:01 smithi139 bash[43301]: cluster 2022-04-28T20:19:01.609917+0000 mgr.y (mgr.24961) 1078 : cluster [DBG] pgmap v1446: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:03.928 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:03.624Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:03.929 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:03 smithi059 bash[59745]: cluster 2022-04-28T20:19:03.610596+0000 mgr.y (mgr.24961) 1079 : cluster [DBG] pgmap v1447: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:03 smithi059 bash[61152]: cluster 2022-04-28T20:19:03.610596+0000 mgr.y (mgr.24961) 1079 : cluster [DBG] pgmap v1447: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:03 smithi139 bash[43301]: cluster 2022-04-28T20:19:03.610596+0000 mgr.y (mgr.24961) 1079 : cluster [DBG] pgmap v1447: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:05 smithi059 bash[61152]: cluster 2022-04-28T20:19:05.611659+0000 mgr.y (mgr.24961) 1080 : cluster [DBG] pgmap v1448: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:05 smithi059 bash[59745]: cluster 2022-04-28T20:19:05.611659+0000 mgr.y (mgr.24961) 1080 : cluster [DBG] pgmap v1448: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:05 smithi139 bash[43301]: cluster 2022-04-28T20:19:05.611659+0000 mgr.y (mgr.24961) 1080 : cluster [DBG] pgmap v1448: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:07 smithi059 bash[61152]: cluster 2022-04-28T20:19:07.612504+0000 mgr.y (mgr.24961) 1081 : cluster [DBG] pgmap v1449: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:07 smithi059 bash[59745]: cluster 2022-04-28T20:19:07.612504+0000 mgr.y (mgr.24961) 1081 : cluster [DBG] pgmap v1449: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:07 smithi139 bash[43301]: cluster 2022-04-28T20:19:07.612504+0000 mgr.y (mgr.24961) 1081 : cluster [DBG] pgmap v1449: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:09 smithi059 bash[61152]: cluster 2022-04-28T20:19:09.613419+0000 mgr.y (mgr.24961) 1082 : cluster [DBG] pgmap v1450: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:09 smithi059 bash[59745]: cluster 2022-04-28T20:19:09.613419+0000 mgr.y (mgr.24961) 1082 : cluster [DBG] pgmap v1450: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:09 smithi139 bash[43301]: cluster 2022-04-28T20:19:09.613419+0000 mgr.y (mgr.24961) 1082 : cluster [DBG] pgmap v1450: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:11 smithi059 bash[61152]: cluster 2022-04-28T20:19:11.614373+0000 mgr.y (mgr.24961) 1083 : cluster [DBG] pgmap v1451: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:11 smithi059 bash[59745]: cluster 2022-04-28T20:19:11.614373+0000 mgr.y (mgr.24961) 1083 : cluster [DBG] pgmap v1451: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:11.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:11 smithi139 bash[43301]: cluster 2022-04-28T20:19:11.614373+0000 mgr.y (mgr.24961) 1083 : cluster [DBG] pgmap v1451: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:13.929 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:13.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:13 smithi059 bash[59745]: cluster 2022-04-28T20:19:13.615166+0000 mgr.y (mgr.24961) 1084 : cluster [DBG] pgmap v1452: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:14.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:13 smithi059 bash[61152]: cluster 2022-04-28T20:19:13.615166+0000 mgr.y (mgr.24961) 1084 : cluster [DBG] pgmap v1452: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:13 smithi139 bash[43301]: cluster 2022-04-28T20:19:13.615166+0000 mgr.y (mgr.24961) 1084 : cluster [DBG] pgmap v1452: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:15 smithi059 bash[59745]: cluster 2022-04-28T20:19:15.616231+0000 mgr.y (mgr.24961) 1085 : cluster [DBG] pgmap v1453: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:15 smithi059 bash[61152]: cluster 2022-04-28T20:19:15.616231+0000 mgr.y (mgr.24961) 1085 : cluster [DBG] pgmap v1453: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:15 smithi139 bash[43301]: cluster 2022-04-28T20:19:15.616231+0000 mgr.y (mgr.24961) 1085 : cluster [DBG] pgmap v1453: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[61152]: audit 2022-04-28T20:19:17.501497+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[61152]: audit 2022-04-28T20:19:17.501983+0000 mon.a (mon.0) 1857 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:19:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[61152]: audit 2022-04-28T20:19:17.524627+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:19:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[61152]: audit 2022-04-28T20:19:17.525239+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:19:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[61152]: audit 2022-04-28T20:19:17.525795+0000 mon.a (mon.0) 1860 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:19:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[59745]: audit 2022-04-28T20:19:17.501497+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:19:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[59745]: audit 2022-04-28T20:19:17.501983+0000 mon.a (mon.0) 1857 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:19:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[59745]: audit 2022-04-28T20:19:17.524627+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:19:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[59745]: audit 2022-04-28T20:19:17.525239+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:19:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:17 smithi059 bash[59745]: audit 2022-04-28T20:19:17.525795+0000 mon.a (mon.0) 1860 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:19:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:17 smithi139 bash[43301]: audit 2022-04-28T20:19:17.501497+0000 mon.a (mon.0) 1856 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:19:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:17 smithi139 bash[43301]: audit 2022-04-28T20:19:17.501983+0000 mon.a (mon.0) 1857 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:19:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:17 smithi139 bash[43301]: audit 2022-04-28T20:19:17.524627+0000 mon.a (mon.0) 1858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:19:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:17 smithi139 bash[43301]: audit 2022-04-28T20:19:17.525239+0000 mon.a (mon.0) 1859 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:19:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:17 smithi139 bash[43301]: audit 2022-04-28T20:19:17.525795+0000 mon.a (mon.0) 1860 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:19:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:18 smithi059 bash[61152]: cluster 2022-04-28T20:19:17.617049+0000 mgr.y (mgr.24961) 1086 : cluster [DBG] pgmap v1454: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:18 smithi059 bash[59745]: cluster 2022-04-28T20:19:17.617049+0000 mgr.y (mgr.24961) 1086 : cluster [DBG] pgmap v1454: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:18 smithi139 bash[43301]: cluster 2022-04-28T20:19:17.617049+0000 mgr.y (mgr.24961) 1086 : cluster [DBG] pgmap v1454: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:19 smithi059 bash[61152]: cluster 2022-04-28T20:19:19.617918+0000 mgr.y (mgr.24961) 1087 : cluster [DBG] pgmap v1455: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:19 smithi059 bash[59745]: cluster 2022-04-28T20:19:19.617918+0000 mgr.y (mgr.24961) 1087 : cluster [DBG] pgmap v1455: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:19 smithi139 bash[43301]: cluster 2022-04-28T20:19:19.617918+0000 mgr.y (mgr.24961) 1087 : cluster [DBG] pgmap v1455: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:21 smithi059 bash[61152]: cluster 2022-04-28T20:19:21.618869+0000 mgr.y (mgr.24961) 1088 : cluster [DBG] pgmap v1456: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:21.730Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:21 smithi059 bash[59745]: cluster 2022-04-28T20:19:21.618869+0000 mgr.y (mgr.24961) 1088 : cluster [DBG] pgmap v1456: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:21 smithi139 bash[43301]: cluster 2022-04-28T20:19:21.618869+0000 mgr.y (mgr.24961) 1088 : cluster [DBG] pgmap v1456: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[61152]: audit 2022-04-28T20:19:22.037072+0000 mon.a (mon.0) 1861 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:19:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[61152]: audit 2022-04-28T20:19:22.039901+0000 mon.a (mon.0) 1862 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:19:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[61152]: audit 2022-04-28T20:19:22.041766+0000 mon.a (mon.0) 1863 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:19:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[61152]: cephadm 2022-04-28T20:19:22.043453+0000 mgr.y (mgr.24961) 1089 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:19:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[61152]: audit 2022-04-28T20:19:22.379081+0000 mon.a (mon.0) 1864 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:19:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[59745]: audit 2022-04-28T20:19:22.037072+0000 mon.a (mon.0) 1861 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:19:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[59745]: audit 2022-04-28T20:19:22.039901+0000 mon.a (mon.0) 1862 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:19:23.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[59745]: audit 2022-04-28T20:19:22.041766+0000 mon.a (mon.0) 1863 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:19:23.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[59745]: cephadm 2022-04-28T20:19:22.043453+0000 mgr.y (mgr.24961) 1089 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:19:23.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:22 smithi059 bash[59745]: audit 2022-04-28T20:19:22.379081+0000 mon.a (mon.0) 1864 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:19:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:22 smithi139 bash[43301]: audit 2022-04-28T20:19:22.037072+0000 mon.a (mon.0) 1861 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:19:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:22 smithi139 bash[43301]: audit 2022-04-28T20:19:22.039901+0000 mon.a (mon.0) 1862 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:19:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:22 smithi139 bash[43301]: audit 2022-04-28T20:19:22.041766+0000 mon.a (mon.0) 1863 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:19:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:22 smithi139 bash[43301]: cephadm 2022-04-28T20:19:22.043453+0000 mgr.y (mgr.24961) 1089 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:19:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:22 smithi139 bash[43301]: audit 2022-04-28T20:19:22.379081+0000 mon.a (mon.0) 1864 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:19:23.936 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:23.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:23 smithi059 bash[61152]: cluster 2022-04-28T20:19:23.619585+0000 mgr.y (mgr.24961) 1090 : cluster [DBG] pgmap v1457: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:23 smithi059 bash[59745]: cluster 2022-04-28T20:19:23.619585+0000 mgr.y (mgr.24961) 1090 : cluster [DBG] pgmap v1457: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:23 smithi139 bash[43301]: cluster 2022-04-28T20:19:23.619585+0000 mgr.y (mgr.24961) 1090 : cluster [DBG] pgmap v1457: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:25 smithi059 bash[61152]: cluster 2022-04-28T20:19:25.620617+0000 mgr.y (mgr.24961) 1091 : cluster [DBG] pgmap v1458: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:25 smithi059 bash[59745]: cluster 2022-04-28T20:19:25.620617+0000 mgr.y (mgr.24961) 1091 : cluster [DBG] pgmap v1458: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:25 smithi139 bash[43301]: cluster 2022-04-28T20:19:25.620617+0000 mgr.y (mgr.24961) 1091 : cluster [DBG] pgmap v1458: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:27 smithi059 bash[61152]: cluster 2022-04-28T20:19:27.621512+0000 mgr.y (mgr.24961) 1092 : cluster [DBG] pgmap v1459: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:27 smithi059 bash[59745]: cluster 2022-04-28T20:19:27.621512+0000 mgr.y (mgr.24961) 1092 : cluster [DBG] pgmap v1459: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:27 smithi139 bash[43301]: cluster 2022-04-28T20:19:27.621512+0000 mgr.y (mgr.24961) 1092 : cluster [DBG] pgmap v1459: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:29 smithi059 bash[61152]: cluster 2022-04-28T20:19:29.622397+0000 mgr.y (mgr.24961) 1093 : cluster [DBG] pgmap v1460: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:29 smithi059 bash[59745]: cluster 2022-04-28T20:19:29.622397+0000 mgr.y (mgr.24961) 1093 : cluster [DBG] pgmap v1460: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:29 smithi139 bash[43301]: cluster 2022-04-28T20:19:29.622397+0000 mgr.y (mgr.24961) 1093 : cluster [DBG] pgmap v1460: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:31 smithi059 bash[61152]: cluster 2022-04-28T20:19:31.623374+0000 mgr.y (mgr.24961) 1094 : cluster [DBG] pgmap v1461: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:32.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:31.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:31 smithi059 bash[59745]: cluster 2022-04-28T20:19:31.623374+0000 mgr.y (mgr.24961) 1094 : cluster [DBG] pgmap v1461: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:32.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:31 smithi139 bash[43301]: cluster 2022-04-28T20:19:31.623374+0000 mgr.y (mgr.24961) 1094 : cluster [DBG] pgmap v1461: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:33.939 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:33 smithi059 bash[59745]: cluster 2022-04-28T20:19:33.624235+0000 mgr.y (mgr.24961) 1095 : cluster [DBG] pgmap v1462: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:33.939 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:33.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:33 smithi059 bash[61152]: cluster 2022-04-28T20:19:33.624235+0000 mgr.y (mgr.24961) 1095 : cluster [DBG] pgmap v1462: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:33 smithi139 bash[43301]: cluster 2022-04-28T20:19:33.624235+0000 mgr.y (mgr.24961) 1095 : cluster [DBG] pgmap v1462: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:35 smithi059 bash[61152]: cluster 2022-04-28T20:19:35.625292+0000 mgr.y (mgr.24961) 1096 : cluster [DBG] pgmap v1463: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:35 smithi059 bash[59745]: cluster 2022-04-28T20:19:35.625292+0000 mgr.y (mgr.24961) 1096 : cluster [DBG] pgmap v1463: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:35 smithi139 bash[43301]: cluster 2022-04-28T20:19:35.625292+0000 mgr.y (mgr.24961) 1096 : cluster [DBG] pgmap v1463: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:37 smithi059 bash[61152]: cluster 2022-04-28T20:19:37.626121+0000 mgr.y (mgr.24961) 1097 : cluster [DBG] pgmap v1464: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:37 smithi059 bash[59745]: cluster 2022-04-28T20:19:37.626121+0000 mgr.y (mgr.24961) 1097 : cluster [DBG] pgmap v1464: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:37 smithi139 bash[43301]: cluster 2022-04-28T20:19:37.626121+0000 mgr.y (mgr.24961) 1097 : cluster [DBG] pgmap v1464: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:39 smithi059 bash[61152]: cluster 2022-04-28T20:19:39.627016+0000 mgr.y (mgr.24961) 1098 : cluster [DBG] pgmap v1465: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:39 smithi059 bash[59745]: cluster 2022-04-28T20:19:39.627016+0000 mgr.y (mgr.24961) 1098 : cluster [DBG] pgmap v1465: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:39 smithi139 bash[43301]: cluster 2022-04-28T20:19:39.627016+0000 mgr.y (mgr.24961) 1098 : cluster [DBG] pgmap v1465: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:41 smithi059 bash[61152]: cluster 2022-04-28T20:19:41.628026+0000 mgr.y (mgr.24961) 1099 : cluster [DBG] pgmap v1466: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:41.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:41 smithi059 bash[59745]: cluster 2022-04-28T20:19:41.628026+0000 mgr.y (mgr.24961) 1099 : cluster [DBG] pgmap v1466: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:42.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:41 smithi139 bash[43301]: cluster 2022-04-28T20:19:41.628026+0000 mgr.y (mgr.24961) 1099 : cluster [DBG] pgmap v1466: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:43.938 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:43.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:43 smithi059 bash[61152]: cluster 2022-04-28T20:19:43.628830+0000 mgr.y (mgr.24961) 1100 : cluster [DBG] pgmap v1467: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:43 smithi059 bash[59745]: cluster 2022-04-28T20:19:43.628830+0000 mgr.y (mgr.24961) 1100 : cluster [DBG] pgmap v1467: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:43 smithi139 bash[43301]: cluster 2022-04-28T20:19:43.628830+0000 mgr.y (mgr.24961) 1100 : cluster [DBG] pgmap v1467: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:45 smithi059 bash[61152]: cluster 2022-04-28T20:19:45.629909+0000 mgr.y (mgr.24961) 1101 : cluster [DBG] pgmap v1468: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:45 smithi059 bash[59745]: cluster 2022-04-28T20:19:45.629909+0000 mgr.y (mgr.24961) 1101 : cluster [DBG] pgmap v1468: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:45 smithi139 bash[43301]: cluster 2022-04-28T20:19:45.629909+0000 mgr.y (mgr.24961) 1101 : cluster [DBG] pgmap v1468: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:47 smithi059 bash[61152]: cluster 2022-04-28T20:19:47.630753+0000 mgr.y (mgr.24961) 1102 : cluster [DBG] pgmap v1469: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:47 smithi059 bash[59745]: cluster 2022-04-28T20:19:47.630753+0000 mgr.y (mgr.24961) 1102 : cluster [DBG] pgmap v1469: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:47 smithi139 bash[43301]: cluster 2022-04-28T20:19:47.630753+0000 mgr.y (mgr.24961) 1102 : cluster [DBG] pgmap v1469: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:49 smithi059 bash[61152]: cluster 2022-04-28T20:19:49.631662+0000 mgr.y (mgr.24961) 1103 : cluster [DBG] pgmap v1470: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:49 smithi059 bash[59745]: cluster 2022-04-28T20:19:49.631662+0000 mgr.y (mgr.24961) 1103 : cluster [DBG] pgmap v1470: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:49 smithi139 bash[43301]: cluster 2022-04-28T20:19:49.631662+0000 mgr.y (mgr.24961) 1103 : cluster [DBG] pgmap v1470: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:51 smithi059 bash[61152]: cluster 2022-04-28T20:19:51.632626+0000 mgr.y (mgr.24961) 1104 : cluster [DBG] pgmap v1471: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:51.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:51 smithi059 bash[59745]: cluster 2022-04-28T20:19:51.632626+0000 mgr.y (mgr.24961) 1104 : cluster [DBG] pgmap v1471: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:51 smithi139 bash[43301]: cluster 2022-04-28T20:19:51.632626+0000 mgr.y (mgr.24961) 1104 : cluster [DBG] pgmap v1471: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:53.941 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:19:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:19:53.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:19:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:53 smithi059 bash[61152]: cluster 2022-04-28T20:19:53.633211+0000 mgr.y (mgr.24961) 1105 : cluster [DBG] pgmap v1472: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:53 smithi059 bash[59745]: cluster 2022-04-28T20:19:53.633211+0000 mgr.y (mgr.24961) 1105 : cluster [DBG] pgmap v1472: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:53 smithi139 bash[43301]: cluster 2022-04-28T20:19:53.633211+0000 mgr.y (mgr.24961) 1105 : cluster [DBG] pgmap v1472: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:55 smithi059 bash[61152]: cluster 2022-04-28T20:19:55.634176+0000 mgr.y (mgr.24961) 1106 : cluster [DBG] pgmap v1473: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:55 smithi059 bash[59745]: cluster 2022-04-28T20:19:55.634176+0000 mgr.y (mgr.24961) 1106 : cluster [DBG] pgmap v1473: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:55 smithi139 bash[43301]: cluster 2022-04-28T20:19:55.634176+0000 mgr.y (mgr.24961) 1106 : cluster [DBG] pgmap v1473: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:57 smithi059 bash[61152]: cluster 2022-04-28T20:19:57.635015+0000 mgr.y (mgr.24961) 1107 : cluster [DBG] pgmap v1474: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:57 smithi059 bash[59745]: cluster 2022-04-28T20:19:57.635015+0000 mgr.y (mgr.24961) 1107 : cluster [DBG] pgmap v1474: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:19:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:57 smithi139 bash[43301]: cluster 2022-04-28T20:19:57.635015+0000 mgr.y (mgr.24961) 1107 : cluster [DBG] pgmap v1474: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:19:59 smithi059 bash[61152]: cluster 2022-04-28T20:19:59.635930+0000 mgr.y (mgr.24961) 1108 : cluster [DBG] pgmap v1475: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:19:59 smithi059 bash[59745]: cluster 2022-04-28T20:19:59.635930+0000 mgr.y (mgr.24961) 1108 : cluster [DBG] pgmap v1475: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:19:59 smithi139 bash[43301]: cluster 2022-04-28T20:19:59.635930+0000 mgr.y (mgr.24961) 1108 : cluster [DBG] pgmap v1475: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:00 smithi059 bash[61152]: cluster 2022-04-28T20:20:00.000164+0000 mon.a (mon.0) 1865 : cluster [INF] overall HEALTH_OK 2022-04-28T20:20:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:00 smithi059 bash[59745]: cluster 2022-04-28T20:20:00.000164+0000 mon.a (mon.0) 1865 : cluster [INF] overall HEALTH_OK 2022-04-28T20:20:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:00 smithi139 bash[43301]: cluster 2022-04-28T20:20:00.000164+0000 mon.a (mon.0) 1865 : cluster [INF] overall HEALTH_OK 2022-04-28T20:20:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:01 smithi059 bash[61152]: cluster 2022-04-28T20:20:01.636835+0000 mgr.y (mgr.24961) 1109 : cluster [DBG] pgmap v1476: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:01.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:01 smithi059 bash[59745]: cluster 2022-04-28T20:20:01.636835+0000 mgr.y (mgr.24961) 1109 : cluster [DBG] pgmap v1476: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:01 smithi139 bash[43301]: cluster 2022-04-28T20:20:01.636835+0000 mgr.y (mgr.24961) 1109 : cluster [DBG] pgmap v1476: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:03.947 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:03.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:03.947 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:03 smithi059 bash[59745]: cluster 2022-04-28T20:20:03.637551+0000 mgr.y (mgr.24961) 1110 : cluster [DBG] pgmap v1477: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:03 smithi059 bash[61152]: cluster 2022-04-28T20:20:03.637551+0000 mgr.y (mgr.24961) 1110 : cluster [DBG] pgmap v1477: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:03 smithi139 bash[43301]: cluster 2022-04-28T20:20:03.637551+0000 mgr.y (mgr.24961) 1110 : cluster [DBG] pgmap v1477: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:05 smithi059 bash[61152]: cluster 2022-04-28T20:20:05.638539+0000 mgr.y (mgr.24961) 1111 : cluster [DBG] pgmap v1478: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:06.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:05 smithi059 bash[59745]: cluster 2022-04-28T20:20:05.638539+0000 mgr.y (mgr.24961) 1111 : cluster [DBG] pgmap v1478: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:05 smithi139 bash[43301]: cluster 2022-04-28T20:20:05.638539+0000 mgr.y (mgr.24961) 1111 : cluster [DBG] pgmap v1478: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:07 smithi059 bash[61152]: cluster 2022-04-28T20:20:07.639362+0000 mgr.y (mgr.24961) 1112 : cluster [DBG] pgmap v1479: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:07 smithi059 bash[59745]: cluster 2022-04-28T20:20:07.639362+0000 mgr.y (mgr.24961) 1112 : cluster [DBG] pgmap v1479: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:07 smithi139 bash[43301]: cluster 2022-04-28T20:20:07.639362+0000 mgr.y (mgr.24961) 1112 : cluster [DBG] pgmap v1479: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:09 smithi059 bash[59745]: cluster 2022-04-28T20:20:09.640296+0000 mgr.y (mgr.24961) 1113 : cluster [DBG] pgmap v1480: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:09 smithi059 bash[61152]: cluster 2022-04-28T20:20:09.640296+0000 mgr.y (mgr.24961) 1113 : cluster [DBG] pgmap v1480: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:09 smithi139 bash[43301]: cluster 2022-04-28T20:20:09.640296+0000 mgr.y (mgr.24961) 1113 : cluster [DBG] pgmap v1480: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:11 smithi059 bash[59745]: cluster 2022-04-28T20:20:11.641266+0000 mgr.y (mgr.24961) 1114 : cluster [DBG] pgmap v1481: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:11 smithi059 bash[61152]: cluster 2022-04-28T20:20:11.641266+0000 mgr.y (mgr.24961) 1114 : cluster [DBG] pgmap v1481: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:11.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:11 smithi139 bash[43301]: cluster 2022-04-28T20:20:11.641266+0000 mgr.y (mgr.24961) 1114 : cluster [DBG] pgmap v1481: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:13.948 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:13.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:13 smithi059 bash[61152]: cluster 2022-04-28T20:20:13.641949+0000 mgr.y (mgr.24961) 1115 : cluster [DBG] pgmap v1482: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:13 smithi059 bash[59745]: cluster 2022-04-28T20:20:13.641949+0000 mgr.y (mgr.24961) 1115 : cluster [DBG] pgmap v1482: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:13 smithi139 bash[43301]: cluster 2022-04-28T20:20:13.641949+0000 mgr.y (mgr.24961) 1115 : cluster [DBG] pgmap v1482: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:15 smithi059 bash[61152]: cluster 2022-04-28T20:20:15.643069+0000 mgr.y (mgr.24961) 1116 : cluster [DBG] pgmap v1483: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:15 smithi059 bash[59745]: cluster 2022-04-28T20:20:15.643069+0000 mgr.y (mgr.24961) 1116 : cluster [DBG] pgmap v1483: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:15 smithi139 bash[43301]: cluster 2022-04-28T20:20:15.643069+0000 mgr.y (mgr.24961) 1116 : cluster [DBG] pgmap v1483: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[61152]: audit 2022-04-28T20:20:17.510021+0000 mon.a (mon.0) 1866 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:20:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[61152]: audit 2022-04-28T20:20:17.510583+0000 mon.a (mon.0) 1867 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:20:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[61152]: audit 2022-04-28T20:20:17.532679+0000 mon.a (mon.0) 1868 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:20:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[61152]: audit 2022-04-28T20:20:17.533211+0000 mon.a (mon.0) 1869 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T20:20:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[61152]: audit 2022-04-28T20:20:17.533721+0000 mon.a (mon.0) 1870 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:20:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[59745]: audit 2022-04-28T20:20:17.510021+0000 mon.a (mon.0) 1866 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:20:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[59745]: audit 2022-04-28T20:20:17.510583+0000 mon.a (mon.0) 1867 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:20:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[59745]: audit 2022-04-28T20:20:17.532679+0000 mon.a (mon.0) 1868 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:20:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[59745]: audit 2022-04-28T20:20:17.533211+0000 mon.a (mon.0) 1869 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T20:20:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:17 smithi059 bash[59745]: audit 2022-04-28T20:20:17.533721+0000 mon.a (mon.0) 1870 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:20:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:17 smithi139 bash[43301]: audit 2022-04-28T20:20:17.510021+0000 mon.a (mon.0) 1866 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:20:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:17 smithi139 bash[43301]: audit 2022-04-28T20:20:17.510583+0000 mon.a (mon.0) 1867 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:20:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:17 smithi139 bash[43301]: audit 2022-04-28T20:20:17.532679+0000 mon.a (mon.0) 1868 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:20:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:17 smithi139 bash[43301]: audit 2022-04-28T20:20:17.533211+0000 mon.a (mon.0) 1869 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T20:20:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:17 smithi139 bash[43301]: audit 2022-04-28T20:20:17.533721+0000 mon.a (mon.0) 1870 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:20:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:18 smithi059 bash[61152]: cluster 2022-04-28T20:20:17.644083+0000 mgr.y (mgr.24961) 1117 : cluster [DBG] pgmap v1484: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:18 smithi059 bash[59745]: cluster 2022-04-28T20:20:17.644083+0000 mgr.y (mgr.24961) 1117 : cluster [DBG] pgmap v1484: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:18 smithi139 bash[43301]: cluster 2022-04-28T20:20:17.644083+0000 mgr.y (mgr.24961) 1117 : cluster [DBG] pgmap v1484: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:19 smithi059 bash[61152]: cluster 2022-04-28T20:20:19.645029+0000 mgr.y (mgr.24961) 1118 : cluster [DBG] pgmap v1485: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:19 smithi059 bash[59745]: cluster 2022-04-28T20:20:19.645029+0000 mgr.y (mgr.24961) 1118 : cluster [DBG] pgmap v1485: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:19 smithi139 bash[43301]: cluster 2022-04-28T20:20:19.645029+0000 mgr.y (mgr.24961) 1118 : cluster [DBG] pgmap v1485: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:21 smithi059 bash[61152]: cluster 2022-04-28T20:20:21.645978+0000 mgr.y (mgr.24961) 1119 : cluster [DBG] pgmap v1486: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:21.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:21 smithi059 bash[59745]: cluster 2022-04-28T20:20:21.645978+0000 mgr.y (mgr.24961) 1119 : cluster [DBG] pgmap v1486: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:21 smithi139 bash[43301]: cluster 2022-04-28T20:20:21.645978+0000 mgr.y (mgr.24961) 1119 : cluster [DBG] pgmap v1486: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.387955+0000 mon.a (mon.0) 1871 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:20:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.390672+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:20:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.392542+0000 mon.a (mon.0) 1873 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:20:23.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: cephadm 2022-04-28T20:20:22.394266+0000 mgr.y (mgr.24961) 1120 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:20:23.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.704753+0000 mon.a (mon.0) 1874 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.707665+0000 mon.a (mon.0) 1875 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.709493+0000 mon.a (mon.0) 1876 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.710284+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.710856+0000 mon.a (mon.0) 1878 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.711429+0000 mon.a (mon.0) 1879 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.712012+0000 mon.a (mon.0) 1880 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.712565+0000 mon.a (mon.0) 1881 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.713116+0000 mon.a (mon.0) 1882 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.713693+0000 mon.a (mon.0) 1883 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.714242+0000 mon.a (mon.0) 1884 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.714795+0000 mon.a (mon.0) 1885 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.387955+0000 mon.a (mon.0) 1871 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:20:23.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.390672+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:20:23.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.392542+0000 mon.a (mon.0) 1873 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:20:23.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: cephadm 2022-04-28T20:20:22.394266+0000 mgr.y (mgr.24961) 1120 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:20:23.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.704753+0000 mon.a (mon.0) 1874 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.707665+0000 mon.a (mon.0) 1875 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.709493+0000 mon.a (mon.0) 1876 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.710284+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.710856+0000 mon.a (mon.0) 1878 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.711429+0000 mon.a (mon.0) 1879 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.712012+0000 mon.a (mon.0) 1880 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.101 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.712565+0000 mon.a (mon.0) 1881 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.101 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.713116+0000 mon.a (mon.0) 1882 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.101 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.713693+0000 mon.a (mon.0) 1883 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.101 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.714242+0000 mon.a (mon.0) 1884 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.102 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.714795+0000 mon.a (mon.0) 1885 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.715329+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.715755+0000 mon.a (mon.0) 1887 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.716177+0000 mon.a (mon.0) 1888 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:20:23.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.716556+0000 mon.a (mon.0) 1889 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.716984+0000 mon.a (mon.0) 1890 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.717388+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[59745]: audit 2022-04-28T20:20:22.718021+0000 mon.a (mon.0) 1892 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:20:23.104 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.715329+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.104 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.715755+0000 mon.a (mon.0) 1887 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.105 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.716177+0000 mon.a (mon.0) 1888 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:20:23.105 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.716556+0000 mon.a (mon.0) 1889 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.105 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.716984+0000 mon.a (mon.0) 1890 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.105 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.717388+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.105 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:22 smithi059 bash[61152]: audit 2022-04-28T20:20:22.718021+0000 mon.a (mon.0) 1892 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:20:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.387955+0000 mon.a (mon.0) 1871 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:20:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.390672+0000 mon.a (mon.0) 1872 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:20:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.392542+0000 mon.a (mon.0) 1873 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:20:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: cephadm 2022-04-28T20:20:22.394266+0000 mgr.y (mgr.24961) 1120 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:20:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.704753+0000 mon.a (mon.0) 1874 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.707665+0000 mon.a (mon.0) 1875 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.709493+0000 mon.a (mon.0) 1876 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.710284+0000 mon.a (mon.0) 1877 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.710856+0000 mon.a (mon.0) 1878 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.711429+0000 mon.a (mon.0) 1879 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.712012+0000 mon.a (mon.0) 1880 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.712565+0000 mon.a (mon.0) 1881 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.713116+0000 mon.a (mon.0) 1882 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.713693+0000 mon.a (mon.0) 1883 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.714242+0000 mon.a (mon.0) 1884 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.714795+0000 mon.a (mon.0) 1885 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.715329+0000 mon.a (mon.0) 1886 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.715755+0000 mon.a (mon.0) 1887 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.716177+0000 mon.a (mon.0) 1888 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:20:23.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.716556+0000 mon.a (mon.0) 1889 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:20:23.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.716984+0000 mon.a (mon.0) 1890 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.717388+0000 mon.a (mon.0) 1891 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:20:23.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:22 smithi139 bash[43301]: audit 2022-04-28T20:20:22.718021+0000 mon.a (mon.0) 1892 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:20:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:23.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[61152]: audit 2022-04-28T20:20:22.718649+0000 mon.a (mon.0) 1893 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:20:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[61152]: audit 2022-04-28T20:20:22.732407+0000 mon.a (mon.0) 1894 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:20:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[61152]: cluster 2022-04-28T20:20:23.646734+0000 mgr.y (mgr.24961) 1121 : cluster [DBG] pgmap v1487: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[59745]: audit 2022-04-28T20:20:22.718649+0000 mon.a (mon.0) 1893 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:20:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[59745]: audit 2022-04-28T20:20:22.732407+0000 mon.a (mon.0) 1894 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:20:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:23 smithi059 bash[59745]: cluster 2022-04-28T20:20:23.646734+0000 mgr.y (mgr.24961) 1121 : cluster [DBG] pgmap v1487: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:23 smithi139 bash[43301]: audit 2022-04-28T20:20:22.718649+0000 mon.a (mon.0) 1893 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:20:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:23 smithi139 bash[43301]: audit 2022-04-28T20:20:22.732407+0000 mon.a (mon.0) 1894 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:20:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:23 smithi139 bash[43301]: cluster 2022-04-28T20:20:23.646734+0000 mgr.y (mgr.24961) 1121 : cluster [DBG] pgmap v1487: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:25 smithi059 bash[61152]: cluster 2022-04-28T20:20:25.647842+0000 mgr.y (mgr.24961) 1122 : cluster [DBG] pgmap v1488: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:25 smithi059 bash[59745]: cluster 2022-04-28T20:20:25.647842+0000 mgr.y (mgr.24961) 1122 : cluster [DBG] pgmap v1488: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:25 smithi139 bash[43301]: cluster 2022-04-28T20:20:25.647842+0000 mgr.y (mgr.24961) 1122 : cluster [DBG] pgmap v1488: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:27 smithi059 bash[61152]: cluster 2022-04-28T20:20:27.648653+0000 mgr.y (mgr.24961) 1123 : cluster [DBG] pgmap v1489: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:27 smithi059 bash[59745]: cluster 2022-04-28T20:20:27.648653+0000 mgr.y (mgr.24961) 1123 : cluster [DBG] pgmap v1489: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:27 smithi139 bash[43301]: cluster 2022-04-28T20:20:27.648653+0000 mgr.y (mgr.24961) 1123 : cluster [DBG] pgmap v1489: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:29 smithi059 bash[61152]: cluster 2022-04-28T20:20:29.649537+0000 mgr.y (mgr.24961) 1124 : cluster [DBG] pgmap v1490: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:29 smithi059 bash[59745]: cluster 2022-04-28T20:20:29.649537+0000 mgr.y (mgr.24961) 1124 : cluster [DBG] pgmap v1490: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:30.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:29 smithi139 bash[43301]: cluster 2022-04-28T20:20:29.649537+0000 mgr.y (mgr.24961) 1124 : cluster [DBG] pgmap v1490: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:31 smithi059 bash[61152]: cluster 2022-04-28T20:20:31.650550+0000 mgr.y (mgr.24961) 1125 : cluster [DBG] pgmap v1491: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:31.735Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:31 smithi059 bash[59745]: cluster 2022-04-28T20:20:31.650550+0000 mgr.y (mgr.24961) 1125 : cluster [DBG] pgmap v1491: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:31 smithi139 bash[43301]: cluster 2022-04-28T20:20:31.650550+0000 mgr.y (mgr.24961) 1125 : cluster [DBG] pgmap v1491: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:33.954 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:33.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:33 smithi059 bash[61152]: cluster 2022-04-28T20:20:33.651270+0000 mgr.y (mgr.24961) 1126 : cluster [DBG] pgmap v1492: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:33 smithi059 bash[59745]: cluster 2022-04-28T20:20:33.651270+0000 mgr.y (mgr.24961) 1126 : cluster [DBG] pgmap v1492: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:33 smithi139 bash[43301]: cluster 2022-04-28T20:20:33.651270+0000 mgr.y (mgr.24961) 1126 : cluster [DBG] pgmap v1492: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:35 smithi059 bash[61152]: cluster 2022-04-28T20:20:35.652265+0000 mgr.y (mgr.24961) 1127 : cluster [DBG] pgmap v1493: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:35 smithi059 bash[59745]: cluster 2022-04-28T20:20:35.652265+0000 mgr.y (mgr.24961) 1127 : cluster [DBG] pgmap v1493: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:35 smithi139 bash[43301]: cluster 2022-04-28T20:20:35.652265+0000 mgr.y (mgr.24961) 1127 : cluster [DBG] pgmap v1493: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:37 smithi059 bash[61152]: cluster 2022-04-28T20:20:37.653081+0000 mgr.y (mgr.24961) 1128 : cluster [DBG] pgmap v1494: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:37 smithi059 bash[59745]: cluster 2022-04-28T20:20:37.653081+0000 mgr.y (mgr.24961) 1128 : cluster [DBG] pgmap v1494: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:37 smithi139 bash[43301]: cluster 2022-04-28T20:20:37.653081+0000 mgr.y (mgr.24961) 1128 : cluster [DBG] pgmap v1494: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:39 smithi059 bash[61152]: cluster 2022-04-28T20:20:39.653938+0000 mgr.y (mgr.24961) 1129 : cluster [DBG] pgmap v1495: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:39 smithi059 bash[59745]: cluster 2022-04-28T20:20:39.653938+0000 mgr.y (mgr.24961) 1129 : cluster [DBG] pgmap v1495: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:39 smithi139 bash[43301]: cluster 2022-04-28T20:20:39.653938+0000 mgr.y (mgr.24961) 1129 : cluster [DBG] pgmap v1495: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:41 smithi059 bash[61152]: cluster 2022-04-28T20:20:41.654865+0000 mgr.y (mgr.24961) 1130 : cluster [DBG] pgmap v1496: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:41.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:41 smithi059 bash[59745]: cluster 2022-04-28T20:20:41.654865+0000 mgr.y (mgr.24961) 1130 : cluster [DBG] pgmap v1496: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:41 smithi139 bash[43301]: cluster 2022-04-28T20:20:41.654865+0000 mgr.y (mgr.24961) 1130 : cluster [DBG] pgmap v1496: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:43.960 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:43.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:43 smithi059 bash[59745]: cluster 2022-04-28T20:20:43.655587+0000 mgr.y (mgr.24961) 1131 : cluster [DBG] pgmap v1497: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:43 smithi059 bash[61152]: cluster 2022-04-28T20:20:43.655587+0000 mgr.y (mgr.24961) 1131 : cluster [DBG] pgmap v1497: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:43 smithi139 bash[43301]: cluster 2022-04-28T20:20:43.655587+0000 mgr.y (mgr.24961) 1131 : cluster [DBG] pgmap v1497: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:45 smithi059 bash[61152]: cluster 2022-04-28T20:20:45.656569+0000 mgr.y (mgr.24961) 1132 : cluster [DBG] pgmap v1498: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:45 smithi059 bash[59745]: cluster 2022-04-28T20:20:45.656569+0000 mgr.y (mgr.24961) 1132 : cluster [DBG] pgmap v1498: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:45 smithi139 bash[43301]: cluster 2022-04-28T20:20:45.656569+0000 mgr.y (mgr.24961) 1132 : cluster [DBG] pgmap v1498: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:47 smithi059 bash[61152]: cluster 2022-04-28T20:20:47.657402+0000 mgr.y (mgr.24961) 1133 : cluster [DBG] pgmap v1499: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:47 smithi059 bash[59745]: cluster 2022-04-28T20:20:47.657402+0000 mgr.y (mgr.24961) 1133 : cluster [DBG] pgmap v1499: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:47 smithi139 bash[43301]: cluster 2022-04-28T20:20:47.657402+0000 mgr.y (mgr.24961) 1133 : cluster [DBG] pgmap v1499: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:49 smithi059 bash[61152]: cluster 2022-04-28T20:20:49.658250+0000 mgr.y (mgr.24961) 1134 : cluster [DBG] pgmap v1500: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:49 smithi059 bash[59745]: cluster 2022-04-28T20:20:49.658250+0000 mgr.y (mgr.24961) 1134 : cluster [DBG] pgmap v1500: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:49 smithi139 bash[43301]: cluster 2022-04-28T20:20:49.658250+0000 mgr.y (mgr.24961) 1134 : cluster [DBG] pgmap v1500: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:51 smithi059 bash[61152]: cluster 2022-04-28T20:20:51.659316+0000 mgr.y (mgr.24961) 1135 : cluster [DBG] pgmap v1501: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:51.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:51 smithi059 bash[59745]: cluster 2022-04-28T20:20:51.659316+0000 mgr.y (mgr.24961) 1135 : cluster [DBG] pgmap v1501: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:51 smithi139 bash[43301]: cluster 2022-04-28T20:20:51.659316+0000 mgr.y (mgr.24961) 1135 : cluster [DBG] pgmap v1501: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:53.961 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:20:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:20:53.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:20:53.962 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:53 smithi059 bash[59745]: cluster 2022-04-28T20:20:53.660025+0000 mgr.y (mgr.24961) 1136 : cluster [DBG] pgmap v1502: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:53 smithi059 bash[61152]: cluster 2022-04-28T20:20:53.660025+0000 mgr.y (mgr.24961) 1136 : cluster [DBG] pgmap v1502: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:53 smithi139 bash[43301]: cluster 2022-04-28T20:20:53.660025+0000 mgr.y (mgr.24961) 1136 : cluster [DBG] pgmap v1502: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:55 smithi059 bash[61152]: cluster 2022-04-28T20:20:55.661043+0000 mgr.y (mgr.24961) 1137 : cluster [DBG] pgmap v1503: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:55 smithi059 bash[59745]: cluster 2022-04-28T20:20:55.661043+0000 mgr.y (mgr.24961) 1137 : cluster [DBG] pgmap v1503: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:55 smithi139 bash[43301]: cluster 2022-04-28T20:20:55.661043+0000 mgr.y (mgr.24961) 1137 : cluster [DBG] pgmap v1503: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:57 smithi059 bash[61152]: cluster 2022-04-28T20:20:57.661864+0000 mgr.y (mgr.24961) 1138 : cluster [DBG] pgmap v1504: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:57 smithi059 bash[59745]: cluster 2022-04-28T20:20:57.661864+0000 mgr.y (mgr.24961) 1138 : cluster [DBG] pgmap v1504: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:20:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:57 smithi139 bash[43301]: cluster 2022-04-28T20:20:57.661864+0000 mgr.y (mgr.24961) 1138 : cluster [DBG] pgmap v1504: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:20:59 smithi059 bash[61152]: cluster 2022-04-28T20:20:59.662730+0000 mgr.y (mgr.24961) 1139 : cluster [DBG] pgmap v1505: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:20:59 smithi059 bash[59745]: cluster 2022-04-28T20:20:59.662730+0000 mgr.y (mgr.24961) 1139 : cluster [DBG] pgmap v1505: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:20:59 smithi139 bash[43301]: cluster 2022-04-28T20:20:59.662730+0000 mgr.y (mgr.24961) 1139 : cluster [DBG] pgmap v1505: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:01 smithi059 bash[61152]: cluster 2022-04-28T20:21:01.663738+0000 mgr.y (mgr.24961) 1140 : cluster [DBG] pgmap v1506: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:01.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:01 smithi059 bash[59745]: cluster 2022-04-28T20:21:01.663738+0000 mgr.y (mgr.24961) 1140 : cluster [DBG] pgmap v1506: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:01 smithi139 bash[43301]: cluster 2022-04-28T20:21:01.663738+0000 mgr.y (mgr.24961) 1140 : cluster [DBG] pgmap v1506: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:03.964 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:03.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:03 smithi059 bash[61152]: cluster 2022-04-28T20:21:03.664382+0000 mgr.y (mgr.24961) 1141 : cluster [DBG] pgmap v1507: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:03 smithi059 bash[59745]: cluster 2022-04-28T20:21:03.664382+0000 mgr.y (mgr.24961) 1141 : cluster [DBG] pgmap v1507: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:03 smithi139 bash[43301]: cluster 2022-04-28T20:21:03.664382+0000 mgr.y (mgr.24961) 1141 : cluster [DBG] pgmap v1507: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:05 smithi059 bash[61152]: cluster 2022-04-28T20:21:05.665316+0000 mgr.y (mgr.24961) 1142 : cluster [DBG] pgmap v1508: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:05 smithi059 bash[59745]: cluster 2022-04-28T20:21:05.665316+0000 mgr.y (mgr.24961) 1142 : cluster [DBG] pgmap v1508: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:06.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:05 smithi139 bash[43301]: cluster 2022-04-28T20:21:05.665316+0000 mgr.y (mgr.24961) 1142 : cluster [DBG] pgmap v1508: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:07 smithi059 bash[61152]: cluster 2022-04-28T20:21:07.666198+0000 mgr.y (mgr.24961) 1143 : cluster [DBG] pgmap v1509: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:07 smithi059 bash[59745]: cluster 2022-04-28T20:21:07.666198+0000 mgr.y (mgr.24961) 1143 : cluster [DBG] pgmap v1509: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:07 smithi139 bash[43301]: cluster 2022-04-28T20:21:07.666198+0000 mgr.y (mgr.24961) 1143 : cluster [DBG] pgmap v1509: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:09 smithi059 bash[61152]: cluster 2022-04-28T20:21:09.667078+0000 mgr.y (mgr.24961) 1144 : cluster [DBG] pgmap v1510: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:09 smithi059 bash[59745]: cluster 2022-04-28T20:21:09.667078+0000 mgr.y (mgr.24961) 1144 : cluster [DBG] pgmap v1510: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:09 smithi139 bash[43301]: cluster 2022-04-28T20:21:09.667078+0000 mgr.y (mgr.24961) 1144 : cluster [DBG] pgmap v1510: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:11 smithi059 bash[61152]: cluster 2022-04-28T20:21:11.668041+0000 mgr.y (mgr.24961) 1145 : cluster [DBG] pgmap v1511: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:11 smithi059 bash[59745]: cluster 2022-04-28T20:21:11.668041+0000 mgr.y (mgr.24961) 1145 : cluster [DBG] pgmap v1511: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:11.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:11 smithi139 bash[43301]: cluster 2022-04-28T20:21:11.668041+0000 mgr.y (mgr.24961) 1145 : cluster [DBG] pgmap v1511: 73 pgs: 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[59745]: cluster 2022-04-28T20:21:12.741330+0000 mon.a (mon.0) 1895 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2022-04-28T20:21:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[59745]: audit 2022-04-28T20:21:12.742599+0000 mon.a (mon.0) 1896 : audit [INF] from='client.? 172.21.15.59:0/1743827415' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-76512-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:21:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[59745]: cluster 2022-04-28T20:21:13.668724+0000 mgr.y (mgr.24961) 1146 : cluster [DBG] pgmap v1513: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[61152]: cluster 2022-04-28T20:21:12.741330+0000 mon.a (mon.0) 1895 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2022-04-28T20:21:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[61152]: audit 2022-04-28T20:21:12.742599+0000 mon.a (mon.0) 1896 : audit [INF] from='client.? 172.21.15.59:0/1743827415' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-76512-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:21:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[61152]: cluster 2022-04-28T20:21:13.668724+0000 mgr.y (mgr.24961) 1146 : cluster [DBG] pgmap v1513: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:14.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:13.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:13 smithi139 bash[43301]: cluster 2022-04-28T20:21:12.741330+0000 mon.a (mon.0) 1895 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2022-04-28T20:21:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:13 smithi139 bash[43301]: audit 2022-04-28T20:21:12.742599+0000 mon.a (mon.0) 1896 : audit [INF] from='client.? 172.21.15.59:0/1743827415' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-76512-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:21:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:13 smithi139 bash[43301]: cluster 2022-04-28T20:21:13.668724+0000 mgr.y (mgr.24961) 1146 : cluster [DBG] pgmap v1513: 105 pgs: 32 unknown, 73 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:14 smithi059 bash[61152]: audit 2022-04-28T20:21:13.744979+0000 mon.a (mon.0) 1897 : audit [INF] from='client.? 172.21.15.59:0/1743827415' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-76512-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:21:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:14 smithi059 bash[61152]: cluster 2022-04-28T20:21:13.746720+0000 mon.a (mon.0) 1898 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2022-04-28T20:21:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:14 smithi059 bash[59745]: audit 2022-04-28T20:21:13.744979+0000 mon.a (mon.0) 1897 : audit [INF] from='client.? 172.21.15.59:0/1743827415' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-76512-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:21:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:14 smithi059 bash[59745]: cluster 2022-04-28T20:21:13.746720+0000 mon.a (mon.0) 1898 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2022-04-28T20:21:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:14 smithi139 bash[43301]: audit 2022-04-28T20:21:13.744979+0000 mon.a (mon.0) 1897 : audit [INF] from='client.? 172.21.15.59:0/1743827415' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-76512-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:21:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:14 smithi139 bash[43301]: cluster 2022-04-28T20:21:13.746720+0000 mon.a (mon.0) 1898 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2022-04-28T20:21:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:15 smithi059 bash[61152]: cluster 2022-04-28T20:21:14.767746+0000 mon.a (mon.0) 1899 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2022-04-28T20:21:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:15 smithi059 bash[61152]: cluster 2022-04-28T20:21:15.669650+0000 mgr.y (mgr.24961) 1147 : cluster [DBG] pgmap v1516: 105 pgs: 8 unknown, 97 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:15 smithi059 bash[59745]: cluster 2022-04-28T20:21:14.767746+0000 mon.a (mon.0) 1899 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2022-04-28T20:21:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:15 smithi059 bash[59745]: cluster 2022-04-28T20:21:15.669650+0000 mgr.y (mgr.24961) 1147 : cluster [DBG] pgmap v1516: 105 pgs: 8 unknown, 97 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:15 smithi139 bash[43301]: cluster 2022-04-28T20:21:14.767746+0000 mon.a (mon.0) 1899 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2022-04-28T20:21:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:15 smithi139 bash[43301]: cluster 2022-04-28T20:21:15.669650+0000 mgr.y (mgr.24961) 1147 : cluster [DBG] pgmap v1516: 105 pgs: 8 unknown, 97 active+clean; 24 KiB data, 697 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:17 smithi059 bash[61152]: audit 2022-04-28T20:21:17.519042+0000 mon.a (mon.0) 1900 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:21:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:17 smithi059 bash[61152]: audit 2022-04-28T20:21:17.519602+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:21:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:17 smithi059 bash[59745]: audit 2022-04-28T20:21:17.519042+0000 mon.a (mon.0) 1900 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:21:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:17 smithi059 bash[59745]: audit 2022-04-28T20:21:17.519602+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:21:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:17 smithi139 bash[43301]: audit 2022-04-28T20:21:17.519042+0000 mon.a (mon.0) 1900 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:21:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:17 smithi139 bash[43301]: audit 2022-04-28T20:21:17.519602+0000 mon.a (mon.0) 1901 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:21:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:18 smithi059 bash[61152]: cluster 2022-04-28T20:21:17.670652+0000 mgr.y (mgr.24961) 1148 : cluster [DBG] pgmap v1517: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 1.3 KiB/s wr, 2 op/s 2022-04-28T20:21:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:18 smithi059 bash[59745]: cluster 2022-04-28T20:21:17.670652+0000 mgr.y (mgr.24961) 1148 : cluster [DBG] pgmap v1517: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 1.3 KiB/s wr, 2 op/s 2022-04-28T20:21:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:18 smithi139 bash[43301]: cluster 2022-04-28T20:21:17.670652+0000 mgr.y (mgr.24961) 1148 : cluster [DBG] pgmap v1517: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 1.3 KiB/s wr, 2 op/s 2022-04-28T20:21:20.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:19 smithi059 bash[61152]: cluster 2022-04-28T20:21:19.671765+0000 mgr.y (mgr.24961) 1149 : cluster [DBG] pgmap v1518: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 1.2 KiB/s wr, 1 op/s 2022-04-28T20:21:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:19 smithi059 bash[59745]: cluster 2022-04-28T20:21:19.671765+0000 mgr.y (mgr.24961) 1149 : cluster [DBG] pgmap v1518: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 1.2 KiB/s wr, 1 op/s 2022-04-28T20:21:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:19 smithi139 bash[43301]: cluster 2022-04-28T20:21:19.671765+0000 mgr.y (mgr.24961) 1149 : cluster [DBG] pgmap v1518: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 738 B/s rd, 1.2 KiB/s wr, 1 op/s 2022-04-28T20:21:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:21 smithi059 bash[61152]: cluster 2022-04-28T20:21:21.672863+0000 mgr.y (mgr.24961) 1150 : cluster [DBG] pgmap v1519: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:21:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:21 smithi059 bash[59745]: cluster 2022-04-28T20:21:21.672863+0000 mgr.y (mgr.24961) 1150 : cluster [DBG] pgmap v1519: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:21:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:21.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:21 smithi139 bash[43301]: cluster 2022-04-28T20:21:21.672863+0000 mgr.y (mgr.24961) 1150 : cluster [DBG] pgmap v1519: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T20:21:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:22 smithi059 bash[59745]: audit 2022-04-28T20:21:22.740385+0000 mon.a (mon.0) 1902 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:21:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:22 smithi059 bash[59745]: audit 2022-04-28T20:21:22.743133+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:21:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:22 smithi059 bash[59745]: audit 2022-04-28T20:21:22.744875+0000 mon.a (mon.0) 1904 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:21:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:22 smithi059 bash[61152]: audit 2022-04-28T20:21:22.740385+0000 mon.a (mon.0) 1902 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:21:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:22 smithi059 bash[61152]: audit 2022-04-28T20:21:22.743133+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:21:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:22 smithi059 bash[61152]: audit 2022-04-28T20:21:22.744875+0000 mon.a (mon.0) 1904 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:21:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:22 smithi139 bash[43301]: audit 2022-04-28T20:21:22.740385+0000 mon.a (mon.0) 1902 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:21:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:22 smithi139 bash[43301]: audit 2022-04-28T20:21:22.743133+0000 mon.a (mon.0) 1903 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:21:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:22 smithi139 bash[43301]: audit 2022-04-28T20:21:22.744875+0000 mon.a (mon.0) 1904 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:21:24.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:23.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:24.083 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:24 smithi059 bash[59745]: cephadm 2022-04-28T20:21:22.746461+0000 mgr.y (mgr.24961) 1151 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:21:24.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:24 smithi059 bash[59745]: audit 2022-04-28T20:21:23.072563+0000 mon.a (mon.0) 1905 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:21:24.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:24 smithi059 bash[59745]: cluster 2022-04-28T20:21:23.673555+0000 mgr.y (mgr.24961) 1152 : cluster [DBG] pgmap v1520: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 824 B/s wr, 1 op/s 2022-04-28T20:21:24.084 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:24 smithi059 bash[61152]: cephadm 2022-04-28T20:21:22.746461+0000 mgr.y (mgr.24961) 1151 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:21:24.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:24 smithi059 bash[61152]: audit 2022-04-28T20:21:23.072563+0000 mon.a (mon.0) 1905 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:21:24.085 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:24 smithi059 bash[61152]: cluster 2022-04-28T20:21:23.673555+0000 mgr.y (mgr.24961) 1152 : cluster [DBG] pgmap v1520: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 824 B/s wr, 1 op/s 2022-04-28T20:21:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:24 smithi139 bash[43301]: cephadm 2022-04-28T20:21:22.746461+0000 mgr.y (mgr.24961) 1151 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:21:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:24 smithi139 bash[43301]: audit 2022-04-28T20:21:23.072563+0000 mon.a (mon.0) 1905 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:21:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:24 smithi139 bash[43301]: cluster 2022-04-28T20:21:23.673555+0000 mgr.y (mgr.24961) 1152 : cluster [DBG] pgmap v1520: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 515 B/s rd, 824 B/s wr, 1 op/s 2022-04-28T20:21:27.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:26 smithi059 bash[61152]: cluster 2022-04-28T20:21:25.674884+0000 mgr.y (mgr.24961) 1153 : cluster [DBG] pgmap v1521: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 469 B/s rd, 750 B/s wr, 1 op/s 2022-04-28T20:21:27.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:26 smithi059 bash[59745]: cluster 2022-04-28T20:21:25.674884+0000 mgr.y (mgr.24961) 1153 : cluster [DBG] pgmap v1521: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 469 B/s rd, 750 B/s wr, 1 op/s 2022-04-28T20:21:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:26 smithi139 bash[43301]: cluster 2022-04-28T20:21:25.674884+0000 mgr.y (mgr.24961) 1153 : cluster [DBG] pgmap v1521: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 469 B/s rd, 750 B/s wr, 1 op/s 2022-04-28T20:21:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:28 smithi059 bash[61152]: cluster 2022-04-28T20:21:27.675943+0000 mgr.y (mgr.24961) 1154 : cluster [DBG] pgmap v1522: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 682 B/s wr, 1 op/s 2022-04-28T20:21:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:28 smithi059 bash[59745]: cluster 2022-04-28T20:21:27.675943+0000 mgr.y (mgr.24961) 1154 : cluster [DBG] pgmap v1522: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 682 B/s wr, 1 op/s 2022-04-28T20:21:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:28 smithi139 bash[43301]: cluster 2022-04-28T20:21:27.675943+0000 mgr.y (mgr.24961) 1154 : cluster [DBG] pgmap v1522: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 682 B/s wr, 1 op/s 2022-04-28T20:21:31.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:30 smithi059 bash[61152]: cluster 2022-04-28T20:21:29.676877+0000 mgr.y (mgr.24961) 1155 : cluster [DBG] pgmap v1523: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:30 smithi059 bash[59745]: cluster 2022-04-28T20:21:29.676877+0000 mgr.y (mgr.24961) 1155 : cluster [DBG] pgmap v1523: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:30 smithi139 bash[43301]: cluster 2022-04-28T20:21:29.676877+0000 mgr.y (mgr.24961) 1155 : cluster [DBG] pgmap v1523: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:31.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:33.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:32 smithi059 bash[61152]: cluster 2022-04-28T20:21:31.678026+0000 mgr.y (mgr.24961) 1156 : cluster [DBG] pgmap v1524: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:32 smithi059 bash[59745]: cluster 2022-04-28T20:21:31.678026+0000 mgr.y (mgr.24961) 1156 : cluster [DBG] pgmap v1524: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:32 smithi139 bash[43301]: cluster 2022-04-28T20:21:31.678026+0000 mgr.y (mgr.24961) 1156 : cluster [DBG] pgmap v1524: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:33.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:33.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:34 smithi059 bash[61152]: cluster 2022-04-28T20:21:33.678832+0000 mgr.y (mgr.24961) 1157 : cluster [DBG] pgmap v1525: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:34 smithi059 bash[59745]: cluster 2022-04-28T20:21:33.678832+0000 mgr.y (mgr.24961) 1157 : cluster [DBG] pgmap v1525: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:34 smithi139 bash[43301]: cluster 2022-04-28T20:21:33.678832+0000 mgr.y (mgr.24961) 1157 : cluster [DBG] pgmap v1525: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:36 smithi059 bash[61152]: cluster 2022-04-28T20:21:35.680169+0000 mgr.y (mgr.24961) 1158 : cluster [DBG] pgmap v1526: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:36 smithi059 bash[59745]: cluster 2022-04-28T20:21:35.680169+0000 mgr.y (mgr.24961) 1158 : cluster [DBG] pgmap v1526: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:36 smithi139 bash[43301]: cluster 2022-04-28T20:21:35.680169+0000 mgr.y (mgr.24961) 1158 : cluster [DBG] pgmap v1526: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:39.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:38 smithi059 bash[59745]: cluster 2022-04-28T20:21:37.681259+0000 mgr.y (mgr.24961) 1159 : cluster [DBG] pgmap v1527: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:38 smithi059 bash[61152]: cluster 2022-04-28T20:21:37.681259+0000 mgr.y (mgr.24961) 1159 : cluster [DBG] pgmap v1527: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:38 smithi139 bash[43301]: cluster 2022-04-28T20:21:37.681259+0000 mgr.y (mgr.24961) 1159 : cluster [DBG] pgmap v1527: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:40 smithi059 bash[61152]: audit 2022-04-28T20:21:39.595405+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.59:0/3269346696' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:21:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:40 smithi059 bash[59745]: audit 2022-04-28T20:21:39.595405+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.59:0/3269346696' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:21:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:40 smithi139 bash[43301]: audit 2022-04-28T20:21:39.595405+0000 mon.c (mon.1) 28 : audit [DBG] from='client.? 172.21.15.59:0/3269346696' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:21:41.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:41 smithi059 bash[61152]: cluster 2022-04-28T20:21:39.682294+0000 mgr.y (mgr.24961) 1160 : cluster [DBG] pgmap v1528: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:41 smithi059 bash[59745]: cluster 2022-04-28T20:21:39.682294+0000 mgr.y (mgr.24961) 1160 : cluster [DBG] pgmap v1528: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:41 smithi139 bash[43301]: cluster 2022-04-28T20:21:39.682294+0000 mgr.y (mgr.24961) 1160 : cluster [DBG] pgmap v1528: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:41.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:43 smithi059 bash[61152]: cluster 2022-04-28T20:21:41.683461+0000 mgr.y (mgr.24961) 1161 : cluster [DBG] pgmap v1529: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:43 smithi059 bash[59745]: cluster 2022-04-28T20:21:41.683461+0000 mgr.y (mgr.24961) 1161 : cluster [DBG] pgmap v1529: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:43 smithi139 bash[43301]: cluster 2022-04-28T20:21:41.683461+0000 mgr.y (mgr.24961) 1161 : cluster [DBG] pgmap v1529: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:44.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:43.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:45.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:45 smithi059 bash[61152]: cluster 2022-04-28T20:21:43.684188+0000 mgr.y (mgr.24961) 1162 : cluster [DBG] pgmap v1530: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:45 smithi059 bash[59745]: cluster 2022-04-28T20:21:43.684188+0000 mgr.y (mgr.24961) 1162 : cluster [DBG] pgmap v1530: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:45 smithi139 bash[43301]: cluster 2022-04-28T20:21:43.684188+0000 mgr.y (mgr.24961) 1162 : cluster [DBG] pgmap v1530: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:47 smithi059 bash[61152]: cluster 2022-04-28T20:21:45.685370+0000 mgr.y (mgr.24961) 1163 : cluster [DBG] pgmap v1531: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:47 smithi059 bash[59745]: cluster 2022-04-28T20:21:45.685370+0000 mgr.y (mgr.24961) 1163 : cluster [DBG] pgmap v1531: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:47 smithi139 bash[43301]: cluster 2022-04-28T20:21:45.685370+0000 mgr.y (mgr.24961) 1163 : cluster [DBG] pgmap v1531: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:49 smithi059 bash[61152]: cluster 2022-04-28T20:21:47.686268+0000 mgr.y (mgr.24961) 1164 : cluster [DBG] pgmap v1532: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:49 smithi059 bash[59745]: cluster 2022-04-28T20:21:47.686268+0000 mgr.y (mgr.24961) 1164 : cluster [DBG] pgmap v1532: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:49 smithi139 bash[43301]: cluster 2022-04-28T20:21:47.686268+0000 mgr.y (mgr.24961) 1164 : cluster [DBG] pgmap v1532: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:51 smithi059 bash[61152]: cluster 2022-04-28T20:21:49.687284+0000 mgr.y (mgr.24961) 1165 : cluster [DBG] pgmap v1533: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:51 smithi059 bash[59745]: cluster 2022-04-28T20:21:49.687284+0000 mgr.y (mgr.24961) 1165 : cluster [DBG] pgmap v1533: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:51 smithi139 bash[43301]: cluster 2022-04-28T20:21:49.687284+0000 mgr.y (mgr.24961) 1165 : cluster [DBG] pgmap v1533: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:51.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:53.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:53 smithi059 bash[59745]: cluster 2022-04-28T20:21:51.688417+0000 mgr.y (mgr.24961) 1166 : cluster [DBG] pgmap v1534: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:53 smithi059 bash[61152]: cluster 2022-04-28T20:21:51.688417+0000 mgr.y (mgr.24961) 1166 : cluster [DBG] pgmap v1534: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:53 smithi139 bash[43301]: cluster 2022-04-28T20:21:51.688417+0000 mgr.y (mgr.24961) 1166 : cluster [DBG] pgmap v1534: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:54.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:21:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:21:53.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:21:55.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:55 smithi059 bash[59745]: cluster 2022-04-28T20:21:53.689111+0000 mgr.y (mgr.24961) 1167 : cluster [DBG] pgmap v1535: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:55 smithi059 bash[61152]: cluster 2022-04-28T20:21:53.689111+0000 mgr.y (mgr.24961) 1167 : cluster [DBG] pgmap v1535: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:55 smithi139 bash[43301]: cluster 2022-04-28T20:21:53.689111+0000 mgr.y (mgr.24961) 1167 : cluster [DBG] pgmap v1535: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:57.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:57 smithi059 bash[59745]: cluster 2022-04-28T20:21:55.690299+0000 mgr.y (mgr.24961) 1168 : cluster [DBG] pgmap v1536: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:57 smithi059 bash[61152]: cluster 2022-04-28T20:21:55.690299+0000 mgr.y (mgr.24961) 1168 : cluster [DBG] pgmap v1536: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:57 smithi139 bash[43301]: cluster 2022-04-28T20:21:55.690299+0000 mgr.y (mgr.24961) 1168 : cluster [DBG] pgmap v1536: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:21:59 smithi059 bash[61152]: cluster 2022-04-28T20:21:57.691210+0000 mgr.y (mgr.24961) 1169 : cluster [DBG] pgmap v1537: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:21:59 smithi059 bash[59745]: cluster 2022-04-28T20:21:57.691210+0000 mgr.y (mgr.24961) 1169 : cluster [DBG] pgmap v1537: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:21:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:21:59 smithi139 bash[43301]: cluster 2022-04-28T20:21:57.691210+0000 mgr.y (mgr.24961) 1169 : cluster [DBG] pgmap v1537: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:01 smithi059 bash[61152]: cluster 2022-04-28T20:21:59.692237+0000 mgr.y (mgr.24961) 1170 : cluster [DBG] pgmap v1538: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:01 smithi059 bash[59745]: cluster 2022-04-28T20:21:59.692237+0000 mgr.y (mgr.24961) 1170 : cluster [DBG] pgmap v1538: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:01 smithi139 bash[43301]: cluster 2022-04-28T20:21:59.692237+0000 mgr.y (mgr.24961) 1170 : cluster [DBG] pgmap v1538: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:01.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:03.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:03 smithi059 bash[59745]: cluster 2022-04-28T20:22:01.693466+0000 mgr.y (mgr.24961) 1171 : cluster [DBG] pgmap v1539: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:03 smithi059 bash[61152]: cluster 2022-04-28T20:22:01.693466+0000 mgr.y (mgr.24961) 1171 : cluster [DBG] pgmap v1539: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:03 smithi139 bash[43301]: cluster 2022-04-28T20:22:01.693466+0000 mgr.y (mgr.24961) 1171 : cluster [DBG] pgmap v1539: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:04.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:03.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:05.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:05 smithi059 bash[61152]: cluster 2022-04-28T20:22:03.694160+0000 mgr.y (mgr.24961) 1172 : cluster [DBG] pgmap v1540: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:05 smithi059 bash[59745]: cluster 2022-04-28T20:22:03.694160+0000 mgr.y (mgr.24961) 1172 : cluster [DBG] pgmap v1540: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:05 smithi139 bash[43301]: cluster 2022-04-28T20:22:03.694160+0000 mgr.y (mgr.24961) 1172 : cluster [DBG] pgmap v1540: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:07 smithi059 bash[61152]: cluster 2022-04-28T20:22:05.695351+0000 mgr.y (mgr.24961) 1173 : cluster [DBG] pgmap v1541: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:07 smithi059 bash[59745]: cluster 2022-04-28T20:22:05.695351+0000 mgr.y (mgr.24961) 1173 : cluster [DBG] pgmap v1541: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:07 smithi139 bash[43301]: cluster 2022-04-28T20:22:05.695351+0000 mgr.y (mgr.24961) 1173 : cluster [DBG] pgmap v1541: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:09 smithi059 bash[61152]: cluster 2022-04-28T20:22:07.696375+0000 mgr.y (mgr.24961) 1174 : cluster [DBG] pgmap v1542: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:09 smithi059 bash[59745]: cluster 2022-04-28T20:22:07.696375+0000 mgr.y (mgr.24961) 1174 : cluster [DBG] pgmap v1542: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:09 smithi139 bash[43301]: cluster 2022-04-28T20:22:07.696375+0000 mgr.y (mgr.24961) 1174 : cluster [DBG] pgmap v1542: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:11.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:11 smithi059 bash[61152]: cluster 2022-04-28T20:22:09.697493+0000 mgr.y (mgr.24961) 1175 : cluster [DBG] pgmap v1543: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:11 smithi059 bash[59745]: cluster 2022-04-28T20:22:09.697493+0000 mgr.y (mgr.24961) 1175 : cluster [DBG] pgmap v1543: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:11 smithi139 bash[43301]: cluster 2022-04-28T20:22:09.697493+0000 mgr.y (mgr.24961) 1175 : cluster [DBG] pgmap v1543: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:11.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:13 smithi139 bash[43301]: cluster 2022-04-28T20:22:11.698659+0000 mgr.y (mgr.24961) 1176 : cluster [DBG] pgmap v1544: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:13.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:13 smithi059 bash[59745]: cluster 2022-04-28T20:22:11.698659+0000 mgr.y (mgr.24961) 1176 : cluster [DBG] pgmap v1544: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:13 smithi059 bash[61152]: cluster 2022-04-28T20:22:11.698659+0000 mgr.y (mgr.24961) 1176 : cluster [DBG] pgmap v1544: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:14.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:13.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:15 smithi139 bash[43301]: cluster 2022-04-28T20:22:13.699396+0000 mgr.y (mgr.24961) 1177 : cluster [DBG] pgmap v1545: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:15 smithi059 bash[59745]: cluster 2022-04-28T20:22:13.699396+0000 mgr.y (mgr.24961) 1177 : cluster [DBG] pgmap v1545: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:15 smithi059 bash[61152]: cluster 2022-04-28T20:22:13.699396+0000 mgr.y (mgr.24961) 1177 : cluster [DBG] pgmap v1545: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:17 smithi139 bash[43301]: cluster 2022-04-28T20:22:15.700633+0000 mgr.y (mgr.24961) 1178 : cluster [DBG] pgmap v1546: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:17.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:17 smithi059 bash[61152]: cluster 2022-04-28T20:22:15.700633+0000 mgr.y (mgr.24961) 1178 : cluster [DBG] pgmap v1546: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:17 smithi059 bash[59745]: cluster 2022-04-28T20:22:15.700633+0000 mgr.y (mgr.24961) 1178 : cluster [DBG] pgmap v1546: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[61152]: audit 2022-04-28T20:22:17.527317+0000 mon.a (mon.0) 1906 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:22:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[61152]: audit 2022-04-28T20:22:17.527697+0000 mon.a (mon.0) 1907 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:22:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[61152]: audit 2022-04-28T20:22:17.541782+0000 mon.a (mon.0) 1908 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:22:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[61152]: audit 2022-04-28T20:22:17.542270+0000 mon.a (mon.0) 1909 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:22:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[61152]: audit 2022-04-28T20:22:17.542768+0000 mon.a (mon.0) 1910 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:22:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[59745]: audit 2022-04-28T20:22:17.527317+0000 mon.a (mon.0) 1906 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:22:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[59745]: audit 2022-04-28T20:22:17.527697+0000 mon.a (mon.0) 1907 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:22:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[59745]: audit 2022-04-28T20:22:17.541782+0000 mon.a (mon.0) 1908 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:22:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[59745]: audit 2022-04-28T20:22:17.542270+0000 mon.a (mon.0) 1909 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:22:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:18 smithi059 bash[59745]: audit 2022-04-28T20:22:17.542768+0000 mon.a (mon.0) 1910 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:22:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:18 smithi139 bash[43301]: audit 2022-04-28T20:22:17.527317+0000 mon.a (mon.0) 1906 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:22:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:18 smithi139 bash[43301]: audit 2022-04-28T20:22:17.527697+0000 mon.a (mon.0) 1907 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:22:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:18 smithi139 bash[43301]: audit 2022-04-28T20:22:17.541782+0000 mon.a (mon.0) 1908 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:22:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:18 smithi139 bash[43301]: audit 2022-04-28T20:22:17.542270+0000 mon.a (mon.0) 1909 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:22:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:18 smithi139 bash[43301]: audit 2022-04-28T20:22:17.542768+0000 mon.a (mon.0) 1910 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:22:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:19 smithi059 bash[61152]: cluster 2022-04-28T20:22:17.701517+0000 mgr.y (mgr.24961) 1179 : cluster [DBG] pgmap v1547: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:19 smithi059 bash[59745]: cluster 2022-04-28T20:22:17.701517+0000 mgr.y (mgr.24961) 1179 : cluster [DBG] pgmap v1547: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:19 smithi139 bash[43301]: cluster 2022-04-28T20:22:17.701517+0000 mgr.y (mgr.24961) 1179 : cluster [DBG] pgmap v1547: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:21 smithi059 bash[61152]: cluster 2022-04-28T20:22:19.702553+0000 mgr.y (mgr.24961) 1180 : cluster [DBG] pgmap v1548: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:21 smithi059 bash[59745]: cluster 2022-04-28T20:22:19.702553+0000 mgr.y (mgr.24961) 1180 : cluster [DBG] pgmap v1548: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:21 smithi139 bash[43301]: cluster 2022-04-28T20:22:19.702553+0000 mgr.y (mgr.24961) 1180 : cluster [DBG] pgmap v1548: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:21.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[59745]: cluster 2022-04-28T20:22:21.703746+0000 mgr.y (mgr.24961) 1181 : cluster [DBG] pgmap v1549: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[59745]: audit 2022-04-28T20:22:23.080423+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:22:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[59745]: audit 2022-04-28T20:22:23.083275+0000 mon.a (mon.0) 1912 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:22:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[59745]: audit 2022-04-28T20:22:23.085216+0000 mon.a (mon.0) 1913 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:22:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[61152]: cluster 2022-04-28T20:22:21.703746+0000 mgr.y (mgr.24961) 1181 : cluster [DBG] pgmap v1549: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:23.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[61152]: audit 2022-04-28T20:22:23.080423+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:22:23.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[61152]: audit 2022-04-28T20:22:23.083275+0000 mon.a (mon.0) 1912 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:22:23.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[61152]: audit 2022-04-28T20:22:23.085216+0000 mon.a (mon.0) 1913 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:22:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:23 smithi139 bash[43301]: cluster 2022-04-28T20:22:21.703746+0000 mgr.y (mgr.24961) 1181 : cluster [DBG] pgmap v1549: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:23 smithi139 bash[43301]: audit 2022-04-28T20:22:23.080423+0000 mon.a (mon.0) 1911 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:22:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:23 smithi139 bash[43301]: audit 2022-04-28T20:22:23.083275+0000 mon.a (mon.0) 1912 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:22:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:23 smithi139 bash[43301]: audit 2022-04-28T20:22:23.085216+0000 mon.a (mon.0) 1913 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:22:24.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:23.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:24 smithi059 bash[59745]: cephadm 2022-04-28T20:22:23.086855+0000 mgr.y (mgr.24961) 1182 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:22:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:24 smithi059 bash[59745]: audit 2022-04-28T20:22:23.408759+0000 mon.a (mon.0) 1914 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:22:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:24 smithi059 bash[61152]: cephadm 2022-04-28T20:22:23.086855+0000 mgr.y (mgr.24961) 1182 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:22:24.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:24 smithi059 bash[61152]: audit 2022-04-28T20:22:23.408759+0000 mon.a (mon.0) 1914 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:22:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:24 smithi139 bash[43301]: cephadm 2022-04-28T20:22:23.086855+0000 mgr.y (mgr.24961) 1182 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:22:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:24 smithi139 bash[43301]: audit 2022-04-28T20:22:23.408759+0000 mon.a (mon.0) 1914 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:22:25.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:25 smithi059 bash[61152]: cluster 2022-04-28T20:22:23.704446+0000 mgr.y (mgr.24961) 1183 : cluster [DBG] pgmap v1550: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:25 smithi059 bash[59745]: cluster 2022-04-28T20:22:23.704446+0000 mgr.y (mgr.24961) 1183 : cluster [DBG] pgmap v1550: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:25 smithi139 bash[43301]: cluster 2022-04-28T20:22:23.704446+0000 mgr.y (mgr.24961) 1183 : cluster [DBG] pgmap v1550: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:27.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:27 smithi059 bash[61152]: cluster 2022-04-28T20:22:25.705659+0000 mgr.y (mgr.24961) 1184 : cluster [DBG] pgmap v1551: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:27 smithi059 bash[59745]: cluster 2022-04-28T20:22:25.705659+0000 mgr.y (mgr.24961) 1184 : cluster [DBG] pgmap v1551: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:27.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:27 smithi139 bash[43301]: cluster 2022-04-28T20:22:25.705659+0000 mgr.y (mgr.24961) 1184 : cluster [DBG] pgmap v1551: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:29.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:29 smithi059 bash[61152]: cluster 2022-04-28T20:22:27.706617+0000 mgr.y (mgr.24961) 1185 : cluster [DBG] pgmap v1552: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:29 smithi059 bash[59745]: cluster 2022-04-28T20:22:27.706617+0000 mgr.y (mgr.24961) 1185 : cluster [DBG] pgmap v1552: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:29 smithi139 bash[43301]: cluster 2022-04-28T20:22:27.706617+0000 mgr.y (mgr.24961) 1185 : cluster [DBG] pgmap v1552: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:31.745 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:31 smithi059 bash[61152]: cluster 2022-04-28T20:22:29.707704+0000 mgr.y (mgr.24961) 1186 : cluster [DBG] pgmap v1553: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:31.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:31 smithi059 bash[59745]: cluster 2022-04-28T20:22:29.707704+0000 mgr.y (mgr.24961) 1186 : cluster [DBG] pgmap v1553: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:31 smithi139 bash[43301]: cluster 2022-04-28T20:22:29.707704+0000 mgr.y (mgr.24961) 1186 : cluster [DBG] pgmap v1553: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:31.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:33 smithi059 bash[61152]: cluster 2022-04-28T20:22:31.708859+0000 mgr.y (mgr.24961) 1187 : cluster [DBG] pgmap v1554: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:33.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:33 smithi059 bash[59745]: cluster 2022-04-28T20:22:31.708859+0000 mgr.y (mgr.24961) 1187 : cluster [DBG] pgmap v1554: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:33 smithi139 bash[43301]: cluster 2022-04-28T20:22:31.708859+0000 mgr.y (mgr.24961) 1187 : cluster [DBG] pgmap v1554: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:35.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:35 smithi059 bash[61152]: cluster 2022-04-28T20:22:33.709515+0000 mgr.y (mgr.24961) 1188 : cluster [DBG] pgmap v1555: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:35 smithi059 bash[59745]: cluster 2022-04-28T20:22:33.709515+0000 mgr.y (mgr.24961) 1188 : cluster [DBG] pgmap v1555: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:35 smithi139 bash[43301]: cluster 2022-04-28T20:22:33.709515+0000 mgr.y (mgr.24961) 1188 : cluster [DBG] pgmap v1555: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:37 smithi059 bash[59745]: cluster 2022-04-28T20:22:35.710730+0000 mgr.y (mgr.24961) 1189 : cluster [DBG] pgmap v1556: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:37 smithi059 bash[61152]: cluster 2022-04-28T20:22:35.710730+0000 mgr.y (mgr.24961) 1189 : cluster [DBG] pgmap v1556: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:37 smithi139 bash[43301]: cluster 2022-04-28T20:22:35.710730+0000 mgr.y (mgr.24961) 1189 : cluster [DBG] pgmap v1556: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:39.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:39 smithi059 bash[61152]: cluster 2022-04-28T20:22:37.711638+0000 mgr.y (mgr.24961) 1190 : cluster [DBG] pgmap v1557: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:39 smithi059 bash[59745]: cluster 2022-04-28T20:22:37.711638+0000 mgr.y (mgr.24961) 1190 : cluster [DBG] pgmap v1557: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:39.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:39 smithi139 bash[43301]: cluster 2022-04-28T20:22:37.711638+0000 mgr.y (mgr.24961) 1190 : cluster [DBG] pgmap v1557: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:41.745 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:41 smithi059 bash[61152]: cluster 2022-04-28T20:22:39.712643+0000 mgr.y (mgr.24961) 1191 : cluster [DBG] pgmap v1558: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:41.746 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:41.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:41.746 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:41 smithi059 bash[59745]: cluster 2022-04-28T20:22:39.712643+0000 mgr.y (mgr.24961) 1191 : cluster [DBG] pgmap v1558: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:41 smithi139 bash[43301]: cluster 2022-04-28T20:22:39.712643+0000 mgr.y (mgr.24961) 1191 : cluster [DBG] pgmap v1558: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:43 smithi059 bash[61152]: cluster 2022-04-28T20:22:41.713789+0000 mgr.y (mgr.24961) 1192 : cluster [DBG] pgmap v1559: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:43.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:43 smithi059 bash[59745]: cluster 2022-04-28T20:22:41.713789+0000 mgr.y (mgr.24961) 1192 : cluster [DBG] pgmap v1559: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:43 smithi139 bash[43301]: cluster 2022-04-28T20:22:41.713789+0000 mgr.y (mgr.24961) 1192 : cluster [DBG] pgmap v1559: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:45.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:45 smithi059 bash[61152]: cluster 2022-04-28T20:22:43.714482+0000 mgr.y (mgr.24961) 1193 : cluster [DBG] pgmap v1560: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:45 smithi059 bash[59745]: cluster 2022-04-28T20:22:43.714482+0000 mgr.y (mgr.24961) 1193 : cluster [DBG] pgmap v1560: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:45 smithi139 bash[43301]: cluster 2022-04-28T20:22:43.714482+0000 mgr.y (mgr.24961) 1193 : cluster [DBG] pgmap v1560: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:47.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:47 smithi059 bash[61152]: cluster 2022-04-28T20:22:45.715730+0000 mgr.y (mgr.24961) 1194 : cluster [DBG] pgmap v1561: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:47 smithi059 bash[59745]: cluster 2022-04-28T20:22:45.715730+0000 mgr.y (mgr.24961) 1194 : cluster [DBG] pgmap v1561: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:47 smithi139 bash[43301]: cluster 2022-04-28T20:22:45.715730+0000 mgr.y (mgr.24961) 1194 : cluster [DBG] pgmap v1561: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:49.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:49 smithi059 bash[61152]: cluster 2022-04-28T20:22:47.716668+0000 mgr.y (mgr.24961) 1195 : cluster [DBG] pgmap v1562: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:49 smithi059 bash[59745]: cluster 2022-04-28T20:22:47.716668+0000 mgr.y (mgr.24961) 1195 : cluster [DBG] pgmap v1562: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:49 smithi139 bash[43301]: cluster 2022-04-28T20:22:47.716668+0000 mgr.y (mgr.24961) 1195 : cluster [DBG] pgmap v1562: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:51.747 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:51 smithi059 bash[61152]: cluster 2022-04-28T20:22:49.717776+0000 mgr.y (mgr.24961) 1196 : cluster [DBG] pgmap v1563: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:51.747 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:51 smithi059 bash[59745]: cluster 2022-04-28T20:22:49.717776+0000 mgr.y (mgr.24961) 1196 : cluster [DBG] pgmap v1563: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:51 smithi139 bash[43301]: cluster 2022-04-28T20:22:49.717776+0000 mgr.y (mgr.24961) 1196 : cluster [DBG] pgmap v1563: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:51.746Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:53 smithi059 bash[61152]: cluster 2022-04-28T20:22:51.718974+0000 mgr.y (mgr.24961) 1197 : cluster [DBG] pgmap v1564: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:22:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:22:53.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:22:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:53 smithi059 bash[59745]: cluster 2022-04-28T20:22:51.718974+0000 mgr.y (mgr.24961) 1197 : cluster [DBG] pgmap v1564: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:53 smithi139 bash[43301]: cluster 2022-04-28T20:22:51.718974+0000 mgr.y (mgr.24961) 1197 : cluster [DBG] pgmap v1564: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:55 smithi059 bash[61152]: cluster 2022-04-28T20:22:53.719658+0000 mgr.y (mgr.24961) 1198 : cluster [DBG] pgmap v1565: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:55 smithi059 bash[59745]: cluster 2022-04-28T20:22:53.719658+0000 mgr.y (mgr.24961) 1198 : cluster [DBG] pgmap v1565: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:55 smithi139 bash[43301]: cluster 2022-04-28T20:22:53.719658+0000 mgr.y (mgr.24961) 1198 : cluster [DBG] pgmap v1565: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:57.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:57 smithi059 bash[61152]: cluster 2022-04-28T20:22:55.720866+0000 mgr.y (mgr.24961) 1199 : cluster [DBG] pgmap v1566: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:57 smithi059 bash[59745]: cluster 2022-04-28T20:22:55.720866+0000 mgr.y (mgr.24961) 1199 : cluster [DBG] pgmap v1566: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:57 smithi139 bash[43301]: cluster 2022-04-28T20:22:55.720866+0000 mgr.y (mgr.24961) 1199 : cluster [DBG] pgmap v1566: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:22:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:22:59 smithi059 bash[61152]: cluster 2022-04-28T20:22:57.721828+0000 mgr.y (mgr.24961) 1200 : cluster [DBG] pgmap v1567: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:22:59 smithi059 bash[59745]: cluster 2022-04-28T20:22:57.721828+0000 mgr.y (mgr.24961) 1200 : cluster [DBG] pgmap v1567: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:22:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:22:59 smithi139 bash[43301]: cluster 2022-04-28T20:22:57.721828+0000 mgr.y (mgr.24961) 1200 : cluster [DBG] pgmap v1567: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:01 smithi059 bash[61152]: cluster 2022-04-28T20:22:59.722804+0000 mgr.y (mgr.24961) 1201 : cluster [DBG] pgmap v1568: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:01.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:01.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:01 smithi059 bash[59745]: cluster 2022-04-28T20:22:59.722804+0000 mgr.y (mgr.24961) 1201 : cluster [DBG] pgmap v1568: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:01 smithi139 bash[43301]: cluster 2022-04-28T20:22:59.722804+0000 mgr.y (mgr.24961) 1201 : cluster [DBG] pgmap v1568: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:03 smithi059 bash[61152]: cluster 2022-04-28T20:23:01.723967+0000 mgr.y (mgr.24961) 1202 : cluster [DBG] pgmap v1569: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:03.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:03 smithi059 bash[59745]: cluster 2022-04-28T20:23:01.723967+0000 mgr.y (mgr.24961) 1202 : cluster [DBG] pgmap v1569: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:03 smithi139 bash[43301]: cluster 2022-04-28T20:23:01.723967+0000 mgr.y (mgr.24961) 1202 : cluster [DBG] pgmap v1569: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:05.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:05 smithi059 bash[61152]: cluster 2022-04-28T20:23:03.724706+0000 mgr.y (mgr.24961) 1203 : cluster [DBG] pgmap v1570: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:05 smithi059 bash[59745]: cluster 2022-04-28T20:23:03.724706+0000 mgr.y (mgr.24961) 1203 : cluster [DBG] pgmap v1570: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:05 smithi139 bash[43301]: cluster 2022-04-28T20:23:03.724706+0000 mgr.y (mgr.24961) 1203 : cluster [DBG] pgmap v1570: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:07.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:07 smithi059 bash[59745]: cluster 2022-04-28T20:23:05.725956+0000 mgr.y (mgr.24961) 1204 : cluster [DBG] pgmap v1571: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:07 smithi059 bash[61152]: cluster 2022-04-28T20:23:05.725956+0000 mgr.y (mgr.24961) 1204 : cluster [DBG] pgmap v1571: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:07 smithi139 bash[43301]: cluster 2022-04-28T20:23:05.725956+0000 mgr.y (mgr.24961) 1204 : cluster [DBG] pgmap v1571: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:09.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:09 smithi059 bash[61152]: cluster 2022-04-28T20:23:07.726902+0000 mgr.y (mgr.24961) 1205 : cluster [DBG] pgmap v1572: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:09 smithi059 bash[59745]: cluster 2022-04-28T20:23:07.726902+0000 mgr.y (mgr.24961) 1205 : cluster [DBG] pgmap v1572: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:09.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:09 smithi139 bash[43301]: cluster 2022-04-28T20:23:07.726902+0000 mgr.y (mgr.24961) 1205 : cluster [DBG] pgmap v1572: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:11.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:11 smithi059 bash[61152]: cluster 2022-04-28T20:23:09.727966+0000 mgr.y (mgr.24961) 1206 : cluster [DBG] pgmap v1573: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:11.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:11 smithi059 bash[59745]: cluster 2022-04-28T20:23:09.727966+0000 mgr.y (mgr.24961) 1206 : cluster [DBG] pgmap v1573: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:11.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:11.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:11.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:11 smithi139 bash[43301]: cluster 2022-04-28T20:23:09.727966+0000 mgr.y (mgr.24961) 1206 : cluster [DBG] pgmap v1573: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:13 smithi059 bash[61152]: cluster 2022-04-28T20:23:11.729135+0000 mgr.y (mgr.24961) 1207 : cluster [DBG] pgmap v1574: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:13.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:13 smithi059 bash[59745]: cluster 2022-04-28T20:23:11.729135+0000 mgr.y (mgr.24961) 1207 : cluster [DBG] pgmap v1574: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:13 smithi139 bash[43301]: cluster 2022-04-28T20:23:11.729135+0000 mgr.y (mgr.24961) 1207 : cluster [DBG] pgmap v1574: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:15.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:15 smithi059 bash[61152]: cluster 2022-04-28T20:23:13.729837+0000 mgr.y (mgr.24961) 1208 : cluster [DBG] pgmap v1575: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:15.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:15 smithi059 bash[59745]: cluster 2022-04-28T20:23:13.729837+0000 mgr.y (mgr.24961) 1208 : cluster [DBG] pgmap v1575: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:15 smithi139 bash[43301]: cluster 2022-04-28T20:23:13.729837+0000 mgr.y (mgr.24961) 1208 : cluster [DBG] pgmap v1575: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:17.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[59745]: cluster 2022-04-28T20:23:15.731052+0000 mgr.y (mgr.24961) 1209 : cluster [DBG] pgmap v1576: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[59745]: audit 2022-04-28T20:23:17.535620+0000 mon.a (mon.0) 1915 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:23:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[59745]: audit 2022-04-28T20:23:17.536259+0000 mon.a (mon.0) 1916 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:23:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[59745]: audit 2022-04-28T20:23:17.548411+0000 mon.a (mon.0) 1917 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:23:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[59745]: audit 2022-04-28T20:23:17.549093+0000 mon.a (mon.0) 1918 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:23:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[59745]: audit 2022-04-28T20:23:17.549762+0000 mon.a (mon.0) 1919 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:23:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[61152]: cluster 2022-04-28T20:23:15.731052+0000 mgr.y (mgr.24961) 1209 : cluster [DBG] pgmap v1576: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[61152]: audit 2022-04-28T20:23:17.535620+0000 mon.a (mon.0) 1915 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:23:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[61152]: audit 2022-04-28T20:23:17.536259+0000 mon.a (mon.0) 1916 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:23:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[61152]: audit 2022-04-28T20:23:17.548411+0000 mon.a (mon.0) 1917 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:23:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[61152]: audit 2022-04-28T20:23:17.549093+0000 mon.a (mon.0) 1918 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:23:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:17 smithi059 bash[61152]: audit 2022-04-28T20:23:17.549762+0000 mon.a (mon.0) 1919 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:23:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:17 smithi139 bash[43301]: cluster 2022-04-28T20:23:15.731052+0000 mgr.y (mgr.24961) 1209 : cluster [DBG] pgmap v1576: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:17 smithi139 bash[43301]: audit 2022-04-28T20:23:17.535620+0000 mon.a (mon.0) 1915 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:23:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:17 smithi139 bash[43301]: audit 2022-04-28T20:23:17.536259+0000 mon.a (mon.0) 1916 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:23:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:17 smithi139 bash[43301]: audit 2022-04-28T20:23:17.548411+0000 mon.a (mon.0) 1917 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:23:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:17 smithi139 bash[43301]: audit 2022-04-28T20:23:17.549093+0000 mon.a (mon.0) 1918 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:23:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:17 smithi139 bash[43301]: audit 2022-04-28T20:23:17.549762+0000 mon.a (mon.0) 1919 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:23:19.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:19 smithi059 bash[61152]: cluster 2022-04-28T20:23:17.732019+0000 mgr.y (mgr.24961) 1210 : cluster [DBG] pgmap v1577: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:19.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:19 smithi059 bash[59745]: cluster 2022-04-28T20:23:17.732019+0000 mgr.y (mgr.24961) 1210 : cluster [DBG] pgmap v1577: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:19 smithi139 bash[43301]: cluster 2022-04-28T20:23:17.732019+0000 mgr.y (mgr.24961) 1210 : cluster [DBG] pgmap v1577: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:21.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:21 smithi059 bash[61152]: cluster 2022-04-28T20:23:19.733049+0000 mgr.y (mgr.24961) 1211 : cluster [DBG] pgmap v1578: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:21.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:21.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:21 smithi059 bash[59745]: cluster 2022-04-28T20:23:19.733049+0000 mgr.y (mgr.24961) 1211 : cluster [DBG] pgmap v1578: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:21 smithi139 bash[43301]: cluster 2022-04-28T20:23:19.733049+0000 mgr.y (mgr.24961) 1211 : cluster [DBG] pgmap v1578: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:23.840 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:23.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[61152]: cluster 2022-04-28T20:23:21.734182+0000 mgr.y (mgr.24961) 1212 : cluster [DBG] pgmap v1579: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[61152]: audit 2022-04-28T20:23:23.416546+0000 mon.a (mon.0) 1920 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:23:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[61152]: audit 2022-04-28T20:23:23.419460+0000 mon.a (mon.0) 1921 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:23:23.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[61152]: audit 2022-04-28T20:23:23.421436+0000 mon.a (mon.0) 1922 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:23:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[59745]: cluster 2022-04-28T20:23:21.734182+0000 mgr.y (mgr.24961) 1212 : cluster [DBG] pgmap v1579: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:23.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[59745]: audit 2022-04-28T20:23:23.416546+0000 mon.a (mon.0) 1920 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:23:23.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[59745]: audit 2022-04-28T20:23:23.419460+0000 mon.a (mon.0) 1921 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:23:23.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:23 smithi059 bash[59745]: audit 2022-04-28T20:23:23.421436+0000 mon.a (mon.0) 1922 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:23:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:23 smithi139 bash[43301]: cluster 2022-04-28T20:23:21.734182+0000 mgr.y (mgr.24961) 1212 : cluster [DBG] pgmap v1579: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:23 smithi139 bash[43301]: audit 2022-04-28T20:23:23.416546+0000 mon.a (mon.0) 1920 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:23:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:23 smithi139 bash[43301]: audit 2022-04-28T20:23:23.419460+0000 mon.a (mon.0) 1921 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:23:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:23 smithi139 bash[43301]: audit 2022-04-28T20:23:23.421436+0000 mon.a (mon.0) 1922 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:23:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:24 smithi139 bash[43301]: cephadm 2022-04-28T20:23:23.423183+0000 mgr.y (mgr.24961) 1213 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:23:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:24 smithi139 bash[43301]: audit 2022-04-28T20:23:23.744486+0000 mon.a (mon.0) 1923 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:23:25.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:24 smithi059 bash[61152]: cephadm 2022-04-28T20:23:23.423183+0000 mgr.y (mgr.24961) 1213 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:23:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:24 smithi059 bash[61152]: audit 2022-04-28T20:23:23.744486+0000 mon.a (mon.0) 1923 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:23:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:24 smithi059 bash[59745]: cephadm 2022-04-28T20:23:23.423183+0000 mgr.y (mgr.24961) 1213 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:23:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:24 smithi059 bash[59745]: audit 2022-04-28T20:23:23.744486+0000 mon.a (mon.0) 1923 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:23:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:25 smithi139 bash[43301]: cluster 2022-04-28T20:23:23.734683+0000 mgr.y (mgr.24961) 1214 : cluster [DBG] pgmap v1580: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:26.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:25 smithi059 bash[61152]: cluster 2022-04-28T20:23:23.734683+0000 mgr.y (mgr.24961) 1214 : cluster [DBG] pgmap v1580: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:25 smithi059 bash[59745]: cluster 2022-04-28T20:23:23.734683+0000 mgr.y (mgr.24961) 1214 : cluster [DBG] pgmap v1580: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:27.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:27 smithi139 bash[43301]: cluster 2022-04-28T20:23:25.735856+0000 mgr.y (mgr.24961) 1215 : cluster [DBG] pgmap v1581: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:28.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:27 smithi059 bash[61152]: cluster 2022-04-28T20:23:25.735856+0000 mgr.y (mgr.24961) 1215 : cluster [DBG] pgmap v1581: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:27 smithi059 bash[59745]: cluster 2022-04-28T20:23:25.735856+0000 mgr.y (mgr.24961) 1215 : cluster [DBG] pgmap v1581: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:29 smithi139 bash[43301]: cluster 2022-04-28T20:23:27.736787+0000 mgr.y (mgr.24961) 1216 : cluster [DBG] pgmap v1582: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:30.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:29 smithi059 bash[61152]: cluster 2022-04-28T20:23:27.736787+0000 mgr.y (mgr.24961) 1216 : cluster [DBG] pgmap v1582: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:29 smithi059 bash[59745]: cluster 2022-04-28T20:23:27.736787+0000 mgr.y (mgr.24961) 1216 : cluster [DBG] pgmap v1582: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:31 smithi139 bash[43301]: cluster 2022-04-28T20:23:29.737805+0000 mgr.y (mgr.24961) 1217 : cluster [DBG] pgmap v1583: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:32.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:31 smithi059 bash[61152]: cluster 2022-04-28T20:23:29.737805+0000 mgr.y (mgr.24961) 1217 : cluster [DBG] pgmap v1583: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:31.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:31 smithi059 bash[59745]: cluster 2022-04-28T20:23:29.737805+0000 mgr.y (mgr.24961) 1217 : cluster [DBG] pgmap v1583: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:34.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:33 smithi059 bash[61152]: cluster 2022-04-28T20:23:31.738961+0000 mgr.y (mgr.24961) 1218 : cluster [DBG] pgmap v1584: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:33.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:33 smithi059 bash[59745]: cluster 2022-04-28T20:23:31.738961+0000 mgr.y (mgr.24961) 1218 : cluster [DBG] pgmap v1584: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:33 smithi139 bash[43301]: cluster 2022-04-28T20:23:31.738961+0000 mgr.y (mgr.24961) 1218 : cluster [DBG] pgmap v1584: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:36.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:35 smithi059 bash[61152]: cluster 2022-04-28T20:23:33.739709+0000 mgr.y (mgr.24961) 1219 : cluster [DBG] pgmap v1585: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:35 smithi059 bash[59745]: cluster 2022-04-28T20:23:33.739709+0000 mgr.y (mgr.24961) 1219 : cluster [DBG] pgmap v1585: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:35 smithi139 bash[43301]: cluster 2022-04-28T20:23:33.739709+0000 mgr.y (mgr.24961) 1219 : cluster [DBG] pgmap v1585: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:38.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:37 smithi059 bash[59745]: cluster 2022-04-28T20:23:35.741057+0000 mgr.y (mgr.24961) 1220 : cluster [DBG] pgmap v1586: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:37 smithi059 bash[61152]: cluster 2022-04-28T20:23:35.741057+0000 mgr.y (mgr.24961) 1220 : cluster [DBG] pgmap v1586: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:37 smithi139 bash[43301]: cluster 2022-04-28T20:23:35.741057+0000 mgr.y (mgr.24961) 1220 : cluster [DBG] pgmap v1586: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:40.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:39 smithi059 bash[59745]: cluster 2022-04-28T20:23:37.742022+0000 mgr.y (mgr.24961) 1221 : cluster [DBG] pgmap v1587: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:39 smithi059 bash[61152]: cluster 2022-04-28T20:23:37.742022+0000 mgr.y (mgr.24961) 1221 : cluster [DBG] pgmap v1587: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:39 smithi139 bash[43301]: cluster 2022-04-28T20:23:37.742022+0000 mgr.y (mgr.24961) 1221 : cluster [DBG] pgmap v1587: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:41 smithi059 bash[61152]: cluster 2022-04-28T20:23:39.743095+0000 mgr.y (mgr.24961) 1222 : cluster [DBG] pgmap v1588: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:41.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:41 smithi059 bash[59745]: cluster 2022-04-28T20:23:39.743095+0000 mgr.y (mgr.24961) 1222 : cluster [DBG] pgmap v1588: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:41 smithi139 bash[43301]: cluster 2022-04-28T20:23:39.743095+0000 mgr.y (mgr.24961) 1222 : cluster [DBG] pgmap v1588: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:44.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:43 smithi059 bash[61152]: cluster 2022-04-28T20:23:41.744269+0000 mgr.y (mgr.24961) 1223 : cluster [DBG] pgmap v1589: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:43.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:43 smithi059 bash[59745]: cluster 2022-04-28T20:23:41.744269+0000 mgr.y (mgr.24961) 1223 : cluster [DBG] pgmap v1589: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:43 smithi139 bash[43301]: cluster 2022-04-28T20:23:41.744269+0000 mgr.y (mgr.24961) 1223 : cluster [DBG] pgmap v1589: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:46.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:45 smithi059 bash[59745]: cluster 2022-04-28T20:23:43.745026+0000 mgr.y (mgr.24961) 1224 : cluster [DBG] pgmap v1590: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:45 smithi059 bash[61152]: cluster 2022-04-28T20:23:43.745026+0000 mgr.y (mgr.24961) 1224 : cluster [DBG] pgmap v1590: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:45 smithi139 bash[43301]: cluster 2022-04-28T20:23:43.745026+0000 mgr.y (mgr.24961) 1224 : cluster [DBG] pgmap v1590: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:48.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:47 smithi059 bash[61152]: cluster 2022-04-28T20:23:45.746236+0000 mgr.y (mgr.24961) 1225 : cluster [DBG] pgmap v1591: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:47 smithi059 bash[59745]: cluster 2022-04-28T20:23:45.746236+0000 mgr.y (mgr.24961) 1225 : cluster [DBG] pgmap v1591: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:47 smithi139 bash[43301]: cluster 2022-04-28T20:23:45.746236+0000 mgr.y (mgr.24961) 1225 : cluster [DBG] pgmap v1591: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:50.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:49 smithi059 bash[61152]: cluster 2022-04-28T20:23:47.747275+0000 mgr.y (mgr.24961) 1226 : cluster [DBG] pgmap v1592: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:49 smithi059 bash[59745]: cluster 2022-04-28T20:23:47.747275+0000 mgr.y (mgr.24961) 1226 : cluster [DBG] pgmap v1592: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:49 smithi139 bash[43301]: cluster 2022-04-28T20:23:47.747275+0000 mgr.y (mgr.24961) 1226 : cluster [DBG] pgmap v1592: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:52.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:51 smithi059 bash[61152]: cluster 2022-04-28T20:23:49.748344+0000 mgr.y (mgr.24961) 1227 : cluster [DBG] pgmap v1593: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:51.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:51 smithi059 bash[59745]: cluster 2022-04-28T20:23:49.748344+0000 mgr.y (mgr.24961) 1227 : cluster [DBG] pgmap v1593: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:51 smithi139 bash[43301]: cluster 2022-04-28T20:23:49.748344+0000 mgr.y (mgr.24961) 1227 : cluster [DBG] pgmap v1593: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:54.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:53 smithi059 bash[61152]: cluster 2022-04-28T20:23:51.749474+0000 mgr.y (mgr.24961) 1228 : cluster [DBG] pgmap v1594: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:23:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:23:53.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:23:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:53 smithi059 bash[59745]: cluster 2022-04-28T20:23:51.749474+0000 mgr.y (mgr.24961) 1228 : cluster [DBG] pgmap v1594: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:53 smithi139 bash[43301]: cluster 2022-04-28T20:23:51.749474+0000 mgr.y (mgr.24961) 1228 : cluster [DBG] pgmap v1594: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:23:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:55 smithi059 bash[61152]: cluster 2022-04-28T20:23:53.750212+0000 mgr.y (mgr.24961) 1229 : cluster [DBG] pgmap v1595: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:55 smithi059 bash[59745]: cluster 2022-04-28T20:23:53.750212+0000 mgr.y (mgr.24961) 1229 : cluster [DBG] pgmap v1595: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:55 smithi139 bash[43301]: cluster 2022-04-28T20:23:53.750212+0000 mgr.y (mgr.24961) 1229 : cluster [DBG] pgmap v1595: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:58.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:57 smithi059 bash[61152]: cluster 2022-04-28T20:23:55.751401+0000 mgr.y (mgr.24961) 1230 : cluster [DBG] pgmap v1596: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:57 smithi059 bash[59745]: cluster 2022-04-28T20:23:55.751401+0000 mgr.y (mgr.24961) 1230 : cluster [DBG] pgmap v1596: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:23:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:57 smithi139 bash[43301]: cluster 2022-04-28T20:23:55.751401+0000 mgr.y (mgr.24961) 1230 : cluster [DBG] pgmap v1596: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:23:59 smithi059 bash[61152]: cluster 2022-04-28T20:23:57.752383+0000 mgr.y (mgr.24961) 1231 : cluster [DBG] pgmap v1597: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:23:59 smithi059 bash[59745]: cluster 2022-04-28T20:23:57.752383+0000 mgr.y (mgr.24961) 1231 : cluster [DBG] pgmap v1597: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:23:59 smithi139 bash[43301]: cluster 2022-04-28T20:23:57.752383+0000 mgr.y (mgr.24961) 1231 : cluster [DBG] pgmap v1597: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:01 smithi059 bash[61152]: cluster 2022-04-28T20:23:59.753418+0000 mgr.y (mgr.24961) 1232 : cluster [DBG] pgmap v1598: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:01.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:02.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:01 smithi059 bash[59745]: cluster 2022-04-28T20:23:59.753418+0000 mgr.y (mgr.24961) 1232 : cluster [DBG] pgmap v1598: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:01 smithi139 bash[43301]: cluster 2022-04-28T20:23:59.753418+0000 mgr.y (mgr.24961) 1232 : cluster [DBG] pgmap v1598: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:04.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:03 smithi059 bash[61152]: cluster 2022-04-28T20:24:01.754504+0000 mgr.y (mgr.24961) 1233 : cluster [DBG] pgmap v1599: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:03.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:03 smithi059 bash[59745]: cluster 2022-04-28T20:24:01.754504+0000 mgr.y (mgr.24961) 1233 : cluster [DBG] pgmap v1599: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:03 smithi139 bash[43301]: cluster 2022-04-28T20:24:01.754504+0000 mgr.y (mgr.24961) 1233 : cluster [DBG] pgmap v1599: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:06.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:05 smithi059 bash[61152]: cluster 2022-04-28T20:24:03.755213+0000 mgr.y (mgr.24961) 1234 : cluster [DBG] pgmap v1600: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:05 smithi059 bash[59745]: cluster 2022-04-28T20:24:03.755213+0000 mgr.y (mgr.24961) 1234 : cluster [DBG] pgmap v1600: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:05 smithi139 bash[43301]: cluster 2022-04-28T20:24:03.755213+0000 mgr.y (mgr.24961) 1234 : cluster [DBG] pgmap v1600: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:07.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:06 smithi059 bash[61152]: cluster 2022-04-28T20:24:05.756437+0000 mgr.y (mgr.24961) 1235 : cluster [DBG] pgmap v1601: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:06 smithi059 bash[59745]: cluster 2022-04-28T20:24:05.756437+0000 mgr.y (mgr.24961) 1235 : cluster [DBG] pgmap v1601: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:06 smithi139 bash[43301]: cluster 2022-04-28T20:24:05.756437+0000 mgr.y (mgr.24961) 1235 : cluster [DBG] pgmap v1601: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:09 smithi059 bash[61152]: cluster 2022-04-28T20:24:07.757374+0000 mgr.y (mgr.24961) 1236 : cluster [DBG] pgmap v1602: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:09 smithi059 bash[59745]: cluster 2022-04-28T20:24:07.757374+0000 mgr.y (mgr.24961) 1236 : cluster [DBG] pgmap v1602: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:09 smithi139 bash[43301]: cluster 2022-04-28T20:24:07.757374+0000 mgr.y (mgr.24961) 1236 : cluster [DBG] pgmap v1602: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:11.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:11 smithi059 bash[61152]: cluster 2022-04-28T20:24:09.758426+0000 mgr.y (mgr.24961) 1237 : cluster [DBG] pgmap v1603: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:11 smithi059 bash[59745]: cluster 2022-04-28T20:24:09.758426+0000 mgr.y (mgr.24961) 1237 : cluster [DBG] pgmap v1603: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:11 smithi139 bash[43301]: cluster 2022-04-28T20:24:09.758426+0000 mgr.y (mgr.24961) 1237 : cluster [DBG] pgmap v1603: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:11.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:13.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:13 smithi059 bash[61152]: cluster 2022-04-28T20:24:11.759284+0000 mgr.y (mgr.24961) 1238 : cluster [DBG] pgmap v1604: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:13 smithi059 bash[59745]: cluster 2022-04-28T20:24:11.759284+0000 mgr.y (mgr.24961) 1238 : cluster [DBG] pgmap v1604: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:13 smithi139 bash[43301]: cluster 2022-04-28T20:24:11.759284+0000 mgr.y (mgr.24961) 1238 : cluster [DBG] pgmap v1604: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:14.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:13.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:15.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:15 smithi059 bash[61152]: cluster 2022-04-28T20:24:13.759947+0000 mgr.y (mgr.24961) 1239 : cluster [DBG] pgmap v1605: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:15 smithi059 bash[59745]: cluster 2022-04-28T20:24:13.759947+0000 mgr.y (mgr.24961) 1239 : cluster [DBG] pgmap v1605: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:15 smithi139 bash[43301]: cluster 2022-04-28T20:24:13.759947+0000 mgr.y (mgr.24961) 1239 : cluster [DBG] pgmap v1605: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:17.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:17 smithi059 bash[61152]: cluster 2022-04-28T20:24:15.761148+0000 mgr.y (mgr.24961) 1240 : cluster [DBG] pgmap v1606: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:17 smithi059 bash[59745]: cluster 2022-04-28T20:24:15.761148+0000 mgr.y (mgr.24961) 1240 : cluster [DBG] pgmap v1606: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:17 smithi139 bash[43301]: cluster 2022-04-28T20:24:15.761148+0000 mgr.y (mgr.24961) 1240 : cluster [DBG] pgmap v1606: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:18.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[59745]: audit 2022-04-28T20:24:17.544112+0000 mon.a (mon.0) 1924 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:24:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[59745]: audit 2022-04-28T20:24:17.544657+0000 mon.a (mon.0) 1925 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:24:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[59745]: audit 2022-04-28T20:24:17.556800+0000 mon.a (mon.0) 1926 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:24:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[59745]: audit 2022-04-28T20:24:17.557529+0000 mon.a (mon.0) 1927 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:24:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[59745]: audit 2022-04-28T20:24:17.558248+0000 mon.a (mon.0) 1928 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:24:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[61152]: audit 2022-04-28T20:24:17.544112+0000 mon.a (mon.0) 1924 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:24:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[61152]: audit 2022-04-28T20:24:17.544657+0000 mon.a (mon.0) 1925 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:24:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[61152]: audit 2022-04-28T20:24:17.556800+0000 mon.a (mon.0) 1926 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:24:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[61152]: audit 2022-04-28T20:24:17.557529+0000 mon.a (mon.0) 1927 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:24:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:18 smithi059 bash[61152]: audit 2022-04-28T20:24:17.558248+0000 mon.a (mon.0) 1928 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:24:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:18 smithi139 bash[43301]: audit 2022-04-28T20:24:17.544112+0000 mon.a (mon.0) 1924 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:24:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:18 smithi139 bash[43301]: audit 2022-04-28T20:24:17.544657+0000 mon.a (mon.0) 1925 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:24:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:18 smithi139 bash[43301]: audit 2022-04-28T20:24:17.556800+0000 mon.a (mon.0) 1926 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:24:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:18 smithi139 bash[43301]: audit 2022-04-28T20:24:17.557529+0000 mon.a (mon.0) 1927 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:24:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:18 smithi139 bash[43301]: audit 2022-04-28T20:24:17.558248+0000 mon.a (mon.0) 1928 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:24:19.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:19 smithi059 bash[61152]: cluster 2022-04-28T20:24:17.762185+0000 mgr.y (mgr.24961) 1241 : cluster [DBG] pgmap v1607: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:19 smithi059 bash[59745]: cluster 2022-04-28T20:24:17.762185+0000 mgr.y (mgr.24961) 1241 : cluster [DBG] pgmap v1607: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:19 smithi139 bash[43301]: cluster 2022-04-28T20:24:17.762185+0000 mgr.y (mgr.24961) 1241 : cluster [DBG] pgmap v1607: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:21.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:21 smithi059 bash[59745]: cluster 2022-04-28T20:24:19.763393+0000 mgr.y (mgr.24961) 1242 : cluster [DBG] pgmap v1608: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:21 smithi059 bash[61152]: cluster 2022-04-28T20:24:19.763393+0000 mgr.y (mgr.24961) 1242 : cluster [DBG] pgmap v1608: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:21 smithi139 bash[43301]: cluster 2022-04-28T20:24:19.763393+0000 mgr.y (mgr.24961) 1242 : cluster [DBG] pgmap v1608: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:21.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:23.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:23 smithi059 bash[61152]: cluster 2022-04-28T20:24:21.764518+0000 mgr.y (mgr.24961) 1243 : cluster [DBG] pgmap v1609: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:23 smithi059 bash[59745]: cluster 2022-04-28T20:24:21.764518+0000 mgr.y (mgr.24961) 1243 : cluster [DBG] pgmap v1609: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:23 smithi139 bash[43301]: cluster 2022-04-28T20:24:21.764518+0000 mgr.y (mgr.24961) 1243 : cluster [DBG] pgmap v1609: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:24.071 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:23.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:24.071 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[59745]: audit 2022-04-28T20:24:23.752699+0000 mon.a (mon.0) 1929 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:24:24.072 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[59745]: audit 2022-04-28T20:24:23.755603+0000 mon.a (mon.0) 1930 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:24:24.072 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[59745]: audit 2022-04-28T20:24:23.757736+0000 mon.a (mon.0) 1931 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:24:24.072 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[59745]: audit 2022-04-28T20:24:24.061183+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:24:24.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[61152]: audit 2022-04-28T20:24:23.752699+0000 mon.a (mon.0) 1929 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:24:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[61152]: audit 2022-04-28T20:24:23.755603+0000 mon.a (mon.0) 1930 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:24:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[61152]: audit 2022-04-28T20:24:23.757736+0000 mon.a (mon.0) 1931 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:24:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:24 smithi059 bash[61152]: audit 2022-04-28T20:24:24.061183+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:24:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:24 smithi139 bash[43301]: audit 2022-04-28T20:24:23.752699+0000 mon.a (mon.0) 1929 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:24:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:24 smithi139 bash[43301]: audit 2022-04-28T20:24:23.755603+0000 mon.a (mon.0) 1930 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:24:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:24 smithi139 bash[43301]: audit 2022-04-28T20:24:23.757736+0000 mon.a (mon.0) 1931 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:24:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:24 smithi139 bash[43301]: audit 2022-04-28T20:24:24.061183+0000 mon.a (mon.0) 1932 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:24:25.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:25 smithi059 bash[61152]: cephadm 2022-04-28T20:24:23.759813+0000 mgr.y (mgr.24961) 1244 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:24:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:25 smithi059 bash[61152]: cluster 2022-04-28T20:24:23.765290+0000 mgr.y (mgr.24961) 1245 : cluster [DBG] pgmap v1610: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:25 smithi059 bash[59745]: cephadm 2022-04-28T20:24:23.759813+0000 mgr.y (mgr.24961) 1244 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:24:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:25 smithi059 bash[59745]: cluster 2022-04-28T20:24:23.765290+0000 mgr.y (mgr.24961) 1245 : cluster [DBG] pgmap v1610: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:25 smithi139 bash[43301]: cephadm 2022-04-28T20:24:23.759813+0000 mgr.y (mgr.24961) 1244 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:24:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:25 smithi139 bash[43301]: cluster 2022-04-28T20:24:23.765290+0000 mgr.y (mgr.24961) 1245 : cluster [DBG] pgmap v1610: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:27.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:27 smithi059 bash[61152]: cluster 2022-04-28T20:24:25.766509+0000 mgr.y (mgr.24961) 1246 : cluster [DBG] pgmap v1611: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:27 smithi059 bash[59745]: cluster 2022-04-28T20:24:25.766509+0000 mgr.y (mgr.24961) 1246 : cluster [DBG] pgmap v1611: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:27 smithi139 bash[43301]: cluster 2022-04-28T20:24:25.766509+0000 mgr.y (mgr.24961) 1246 : cluster [DBG] pgmap v1611: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:29.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:29 smithi059 bash[61152]: cluster 2022-04-28T20:24:27.767432+0000 mgr.y (mgr.24961) 1247 : cluster [DBG] pgmap v1612: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:29 smithi059 bash[59745]: cluster 2022-04-28T20:24:27.767432+0000 mgr.y (mgr.24961) 1247 : cluster [DBG] pgmap v1612: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:29 smithi139 bash[43301]: cluster 2022-04-28T20:24:27.767432+0000 mgr.y (mgr.24961) 1247 : cluster [DBG] pgmap v1612: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:31 smithi139 bash[43301]: cluster 2022-04-28T20:24:29.768963+0000 mgr.y (mgr.24961) 1248 : cluster [DBG] pgmap v1613: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:31 smithi059 bash[61152]: cluster 2022-04-28T20:24:29.768963+0000 mgr.y (mgr.24961) 1248 : cluster [DBG] pgmap v1613: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:31 smithi059 bash[59745]: cluster 2022-04-28T20:24:29.768963+0000 mgr.y (mgr.24961) 1248 : cluster [DBG] pgmap v1613: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:32.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:31.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:33 smithi139 bash[43301]: cluster 2022-04-28T20:24:31.770056+0000 mgr.y (mgr.24961) 1249 : cluster [DBG] pgmap v1614: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:33.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:33 smithi059 bash[61152]: cluster 2022-04-28T20:24:31.770056+0000 mgr.y (mgr.24961) 1249 : cluster [DBG] pgmap v1614: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:33 smithi059 bash[59745]: cluster 2022-04-28T20:24:31.770056+0000 mgr.y (mgr.24961) 1249 : cluster [DBG] pgmap v1614: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:34.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:33.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:35 smithi139 bash[43301]: cluster 2022-04-28T20:24:33.770871+0000 mgr.y (mgr.24961) 1250 : cluster [DBG] pgmap v1615: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:35.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:35 smithi059 bash[59745]: cluster 2022-04-28T20:24:33.770871+0000 mgr.y (mgr.24961) 1250 : cluster [DBG] pgmap v1615: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:35 smithi059 bash[61152]: cluster 2022-04-28T20:24:33.770871+0000 mgr.y (mgr.24961) 1250 : cluster [DBG] pgmap v1615: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:37 smithi139 bash[43301]: cluster 2022-04-28T20:24:35.772105+0000 mgr.y (mgr.24961) 1251 : cluster [DBG] pgmap v1616: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:37 smithi059 bash[61152]: cluster 2022-04-28T20:24:35.772105+0000 mgr.y (mgr.24961) 1251 : cluster [DBG] pgmap v1616: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:37 smithi059 bash[59745]: cluster 2022-04-28T20:24:35.772105+0000 mgr.y (mgr.24961) 1251 : cluster [DBG] pgmap v1616: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:39 smithi139 bash[43301]: cluster 2022-04-28T20:24:37.773047+0000 mgr.y (mgr.24961) 1252 : cluster [DBG] pgmap v1617: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:39.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:39 smithi059 bash[61152]: cluster 2022-04-28T20:24:37.773047+0000 mgr.y (mgr.24961) 1252 : cluster [DBG] pgmap v1617: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:39.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:39 smithi059 bash[59745]: cluster 2022-04-28T20:24:37.773047+0000 mgr.y (mgr.24961) 1252 : cluster [DBG] pgmap v1617: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:41.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:41 smithi059 bash[61152]: cluster 2022-04-28T20:24:39.773683+0000 mgr.y (mgr.24961) 1253 : cluster [DBG] pgmap v1618: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:41 smithi059 bash[59745]: cluster 2022-04-28T20:24:39.773683+0000 mgr.y (mgr.24961) 1253 : cluster [DBG] pgmap v1618: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:41 smithi139 bash[43301]: cluster 2022-04-28T20:24:39.773683+0000 mgr.y (mgr.24961) 1253 : cluster [DBG] pgmap v1618: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:41.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:43.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:43 smithi059 bash[61152]: cluster 2022-04-28T20:24:41.774839+0000 mgr.y (mgr.24961) 1254 : cluster [DBG] pgmap v1619: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:43 smithi059 bash[59745]: cluster 2022-04-28T20:24:41.774839+0000 mgr.y (mgr.24961) 1254 : cluster [DBG] pgmap v1619: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:43 smithi139 bash[43301]: cluster 2022-04-28T20:24:41.774839+0000 mgr.y (mgr.24961) 1254 : cluster [DBG] pgmap v1619: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:44.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:43.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:45.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:45 smithi059 bash[61152]: cluster 2022-04-28T20:24:43.775640+0000 mgr.y (mgr.24961) 1255 : cluster [DBG] pgmap v1620: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:45 smithi059 bash[59745]: cluster 2022-04-28T20:24:43.775640+0000 mgr.y (mgr.24961) 1255 : cluster [DBG] pgmap v1620: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:45 smithi139 bash[43301]: cluster 2022-04-28T20:24:43.775640+0000 mgr.y (mgr.24961) 1255 : cluster [DBG] pgmap v1620: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:47.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:47 smithi059 bash[59745]: cluster 2022-04-28T20:24:45.776848+0000 mgr.y (mgr.24961) 1256 : cluster [DBG] pgmap v1621: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:47 smithi059 bash[61152]: cluster 2022-04-28T20:24:45.776848+0000 mgr.y (mgr.24961) 1256 : cluster [DBG] pgmap v1621: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:47 smithi139 bash[43301]: cluster 2022-04-28T20:24:45.776848+0000 mgr.y (mgr.24961) 1256 : cluster [DBG] pgmap v1621: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:49 smithi059 bash[61152]: cluster 2022-04-28T20:24:47.777860+0000 mgr.y (mgr.24961) 1257 : cluster [DBG] pgmap v1622: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:49.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:49 smithi059 bash[59745]: cluster 2022-04-28T20:24:47.777860+0000 mgr.y (mgr.24961) 1257 : cluster [DBG] pgmap v1622: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:49 smithi139 bash[43301]: cluster 2022-04-28T20:24:47.777860+0000 mgr.y (mgr.24961) 1257 : cluster [DBG] pgmap v1622: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:51.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:51 smithi059 bash[61152]: cluster 2022-04-28T20:24:49.778516+0000 mgr.y (mgr.24961) 1258 : cluster [DBG] pgmap v1623: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:51 smithi059 bash[59745]: cluster 2022-04-28T20:24:49.778516+0000 mgr.y (mgr.24961) 1258 : cluster [DBG] pgmap v1623: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:51 smithi139 bash[43301]: cluster 2022-04-28T20:24:49.778516+0000 mgr.y (mgr.24961) 1258 : cluster [DBG] pgmap v1623: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:24:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:51.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:53 smithi059 bash[61152]: cluster 2022-04-28T20:24:51.779722+0000 mgr.y (mgr.24961) 1259 : cluster [DBG] pgmap v1624: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:53 smithi059 bash[59745]: cluster 2022-04-28T20:24:51.779722+0000 mgr.y (mgr.24961) 1259 : cluster [DBG] pgmap v1624: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:53 smithi139 bash[43301]: cluster 2022-04-28T20:24:51.779722+0000 mgr.y (mgr.24961) 1259 : cluster [DBG] pgmap v1624: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:54.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:24:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:24:53.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:24:55.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:55 smithi059 bash[59745]: cluster 2022-04-28T20:24:53.780498+0000 mgr.y (mgr.24961) 1260 : cluster [DBG] pgmap v1625: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:55 smithi059 bash[61152]: cluster 2022-04-28T20:24:53.780498+0000 mgr.y (mgr.24961) 1260 : cluster [DBG] pgmap v1625: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:55 smithi139 bash[43301]: cluster 2022-04-28T20:24:53.780498+0000 mgr.y (mgr.24961) 1260 : cluster [DBG] pgmap v1625: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:57.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:57 smithi059 bash[61152]: cluster 2022-04-28T20:24:55.781708+0000 mgr.y (mgr.24961) 1261 : cluster [DBG] pgmap v1626: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:57 smithi059 bash[59745]: cluster 2022-04-28T20:24:55.781708+0000 mgr.y (mgr.24961) 1261 : cluster [DBG] pgmap v1626: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:57 smithi139 bash[43301]: cluster 2022-04-28T20:24:55.781708+0000 mgr.y (mgr.24961) 1261 : cluster [DBG] pgmap v1626: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:59.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:24:59 smithi059 bash[61152]: cluster 2022-04-28T20:24:57.782475+0000 mgr.y (mgr.24961) 1262 : cluster [DBG] pgmap v1627: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:24:59 smithi059 bash[59745]: cluster 2022-04-28T20:24:57.782475+0000 mgr.y (mgr.24961) 1262 : cluster [DBG] pgmap v1627: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:24:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:24:59 smithi139 bash[43301]: cluster 2022-04-28T20:24:57.782475+0000 mgr.y (mgr.24961) 1262 : cluster [DBG] pgmap v1627: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:01.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:01 smithi059 bash[61152]: cluster 2022-04-28T20:24:59.783297+0000 mgr.y (mgr.24961) 1263 : cluster [DBG] pgmap v1628: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:01 smithi059 bash[59745]: cluster 2022-04-28T20:24:59.783297+0000 mgr.y (mgr.24961) 1263 : cluster [DBG] pgmap v1628: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:01 smithi139 bash[43301]: cluster 2022-04-28T20:24:59.783297+0000 mgr.y (mgr.24961) 1263 : cluster [DBG] pgmap v1628: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:01.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:03 smithi059 bash[61152]: cluster 2022-04-28T20:25:01.784498+0000 mgr.y (mgr.24961) 1264 : cluster [DBG] pgmap v1629: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:03 smithi059 bash[59745]: cluster 2022-04-28T20:25:01.784498+0000 mgr.y (mgr.24961) 1264 : cluster [DBG] pgmap v1629: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:03 smithi139 bash[43301]: cluster 2022-04-28T20:25:01.784498+0000 mgr.y (mgr.24961) 1264 : cluster [DBG] pgmap v1629: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:04.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:03.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:05 smithi059 bash[59745]: cluster 2022-04-28T20:25:03.785221+0000 mgr.y (mgr.24961) 1265 : cluster [DBG] pgmap v1630: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:05 smithi059 bash[61152]: cluster 2022-04-28T20:25:03.785221+0000 mgr.y (mgr.24961) 1265 : cluster [DBG] pgmap v1630: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:05 smithi139 bash[43301]: cluster 2022-04-28T20:25:03.785221+0000 mgr.y (mgr.24961) 1265 : cluster [DBG] pgmap v1630: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:07 smithi059 bash[59745]: cluster 2022-04-28T20:25:05.786212+0000 mgr.y (mgr.24961) 1266 : cluster [DBG] pgmap v1631: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:07 smithi059 bash[61152]: cluster 2022-04-28T20:25:05.786212+0000 mgr.y (mgr.24961) 1266 : cluster [DBG] pgmap v1631: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:07 smithi139 bash[43301]: cluster 2022-04-28T20:25:05.786212+0000 mgr.y (mgr.24961) 1266 : cluster [DBG] pgmap v1631: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T20:25:09.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:09 smithi059 bash[61152]: cluster 2022-04-28T20:25:07.787081+0000 mgr.y (mgr.24961) 1267 : cluster [DBG] pgmap v1632: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:09 smithi059 bash[59745]: cluster 2022-04-28T20:25:07.787081+0000 mgr.y (mgr.24961) 1267 : cluster [DBG] pgmap v1632: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:09 smithi139 bash[43301]: cluster 2022-04-28T20:25:07.787081+0000 mgr.y (mgr.24961) 1267 : cluster [DBG] pgmap v1632: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:11.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:11 smithi059 bash[59745]: cluster 2022-04-28T20:25:09.787844+0000 mgr.y (mgr.24961) 1268 : cluster [DBG] pgmap v1633: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:11 smithi059 bash[61152]: cluster 2022-04-28T20:25:09.787844+0000 mgr.y (mgr.24961) 1268 : cluster [DBG] pgmap v1633: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:11 smithi139 bash[43301]: cluster 2022-04-28T20:25:09.787844+0000 mgr.y (mgr.24961) 1268 : cluster [DBG] pgmap v1633: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:12.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:11.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:13.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:13 smithi059 bash[61152]: cluster 2022-04-28T20:25:11.788880+0000 mgr.y (mgr.24961) 1269 : cluster [DBG] pgmap v1634: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:13 smithi059 bash[59745]: cluster 2022-04-28T20:25:11.788880+0000 mgr.y (mgr.24961) 1269 : cluster [DBG] pgmap v1634: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:13 smithi139 bash[43301]: cluster 2022-04-28T20:25:11.788880+0000 mgr.y (mgr.24961) 1269 : cluster [DBG] pgmap v1634: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:14.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:13.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:15.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:15 smithi059 bash[61152]: cluster 2022-04-28T20:25:13.789417+0000 mgr.y (mgr.24961) 1270 : cluster [DBG] pgmap v1635: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:15 smithi059 bash[59745]: cluster 2022-04-28T20:25:13.789417+0000 mgr.y (mgr.24961) 1270 : cluster [DBG] pgmap v1635: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:15 smithi139 bash[43301]: cluster 2022-04-28T20:25:13.789417+0000 mgr.y (mgr.24961) 1270 : cluster [DBG] pgmap v1635: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:17.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:17 smithi059 bash[61152]: cluster 2022-04-28T20:25:15.790561+0000 mgr.y (mgr.24961) 1271 : cluster [DBG] pgmap v1636: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:17 smithi059 bash[59745]: cluster 2022-04-28T20:25:15.790561+0000 mgr.y (mgr.24961) 1271 : cluster [DBG] pgmap v1636: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:17 smithi139 bash[43301]: cluster 2022-04-28T20:25:15.790561+0000 mgr.y (mgr.24961) 1271 : cluster [DBG] pgmap v1636: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[61152]: audit 2022-04-28T20:25:17.552536+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:25:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[61152]: audit 2022-04-28T20:25:17.553095+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:25:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[61152]: audit 2022-04-28T20:25:17.566001+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:25:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[61152]: audit 2022-04-28T20:25:17.566390+0000 mon.a (mon.0) 1936 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:25:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[61152]: audit 2022-04-28T20:25:17.566885+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:25:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[59745]: audit 2022-04-28T20:25:17.552536+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:25:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[59745]: audit 2022-04-28T20:25:17.553095+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:25:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[59745]: audit 2022-04-28T20:25:17.566001+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:25:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[59745]: audit 2022-04-28T20:25:17.566390+0000 mon.a (mon.0) 1936 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:25:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:18 smithi059 bash[59745]: audit 2022-04-28T20:25:17.566885+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:25:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:18 smithi139 bash[43301]: audit 2022-04-28T20:25:17.552536+0000 mon.a (mon.0) 1933 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:25:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:18 smithi139 bash[43301]: audit 2022-04-28T20:25:17.553095+0000 mon.a (mon.0) 1934 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:25:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:18 smithi139 bash[43301]: audit 2022-04-28T20:25:17.566001+0000 mon.a (mon.0) 1935 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:25:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:18 smithi139 bash[43301]: audit 2022-04-28T20:25:17.566390+0000 mon.a (mon.0) 1936 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:25:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:18 smithi139 bash[43301]: audit 2022-04-28T20:25:17.566885+0000 mon.a (mon.0) 1937 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:25:19.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:19 smithi059 bash[61152]: cluster 2022-04-28T20:25:17.791460+0000 mgr.y (mgr.24961) 1272 : cluster [DBG] pgmap v1637: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:19 smithi059 bash[59745]: cluster 2022-04-28T20:25:17.791460+0000 mgr.y (mgr.24961) 1272 : cluster [DBG] pgmap v1637: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:19 smithi139 bash[43301]: cluster 2022-04-28T20:25:17.791460+0000 mgr.y (mgr.24961) 1272 : cluster [DBG] pgmap v1637: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:25:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:21 smithi059 bash[61152]: cluster 2022-04-28T20:25:19.792477+0000 mgr.y (mgr.24961) 1273 : cluster [DBG] pgmap v1638: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:21 smithi059 bash[59745]: cluster 2022-04-28T20:25:19.792477+0000 mgr.y (mgr.24961) 1273 : cluster [DBG] pgmap v1638: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:21 smithi139 bash[43301]: cluster 2022-04-28T20:25:19.792477+0000 mgr.y (mgr.24961) 1273 : cluster [DBG] pgmap v1638: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:21.755Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:23 smithi059 bash[61152]: cluster 2022-04-28T20:25:21.793424+0000 mgr.y (mgr.24961) 1274 : cluster [DBG] pgmap v1639: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:23 smithi059 bash[59745]: cluster 2022-04-28T20:25:21.793424+0000 mgr.y (mgr.24961) 1274 : cluster [DBG] pgmap v1639: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:23 smithi139 bash[43301]: cluster 2022-04-28T20:25:21.793424+0000 mgr.y (mgr.24961) 1274 : cluster [DBG] pgmap v1639: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:24.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:23.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:24 smithi059 bash[59745]: audit 2022-04-28T20:25:24.068423+0000 mon.a (mon.0) 1938 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:25:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:24 smithi059 bash[59745]: audit 2022-04-28T20:25:24.071366+0000 mon.a (mon.0) 1939 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:25:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:24 smithi059 bash[59745]: audit 2022-04-28T20:25:24.073507+0000 mon.a (mon.0) 1940 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:25:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:24 smithi059 bash[61152]: audit 2022-04-28T20:25:24.068423+0000 mon.a (mon.0) 1938 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:25:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:24 smithi059 bash[61152]: audit 2022-04-28T20:25:24.071366+0000 mon.a (mon.0) 1939 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:25:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:24 smithi059 bash[61152]: audit 2022-04-28T20:25:24.073507+0000 mon.a (mon.0) 1940 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:25:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:24 smithi139 bash[43301]: audit 2022-04-28T20:25:24.068423+0000 mon.a (mon.0) 1938 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:25:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:24 smithi139 bash[43301]: audit 2022-04-28T20:25:24.071366+0000 mon.a (mon.0) 1939 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:25:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:24 smithi139 bash[43301]: audit 2022-04-28T20:25:24.073507+0000 mon.a (mon.0) 1940 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:25:25.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:25 smithi059 bash[59745]: cluster 2022-04-28T20:25:23.794080+0000 mgr.y (mgr.24961) 1275 : cluster [DBG] pgmap v1640: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:25 smithi059 bash[59745]: cephadm 2022-04-28T20:25:24.075413+0000 mgr.y (mgr.24961) 1276 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:25:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:25 smithi059 bash[59745]: audit 2022-04-28T20:25:24.376611+0000 mon.a (mon.0) 1941 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:25:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:25 smithi059 bash[61152]: cluster 2022-04-28T20:25:23.794080+0000 mgr.y (mgr.24961) 1275 : cluster [DBG] pgmap v1640: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:25 smithi059 bash[61152]: cephadm 2022-04-28T20:25:24.075413+0000 mgr.y (mgr.24961) 1276 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:25:25.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:25 smithi059 bash[61152]: audit 2022-04-28T20:25:24.376611+0000 mon.a (mon.0) 1941 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:25:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:25 smithi139 bash[43301]: cluster 2022-04-28T20:25:23.794080+0000 mgr.y (mgr.24961) 1275 : cluster [DBG] pgmap v1640: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:25 smithi139 bash[43301]: cephadm 2022-04-28T20:25:24.075413+0000 mgr.y (mgr.24961) 1276 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:25:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:25 smithi139 bash[43301]: audit 2022-04-28T20:25:24.376611+0000 mon.a (mon.0) 1941 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:25:27.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:27 smithi059 bash[61152]: cluster 2022-04-28T20:25:25.795312+0000 mgr.y (mgr.24961) 1277 : cluster [DBG] pgmap v1641: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:27 smithi059 bash[59745]: cluster 2022-04-28T20:25:25.795312+0000 mgr.y (mgr.24961) 1277 : cluster [DBG] pgmap v1641: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:27.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:27 smithi139 bash[43301]: cluster 2022-04-28T20:25:25.795312+0000 mgr.y (mgr.24961) 1277 : cluster [DBG] pgmap v1641: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:29.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:29 smithi059 bash[61152]: cluster 2022-04-28T20:25:27.796336+0000 mgr.y (mgr.24961) 1278 : cluster [DBG] pgmap v1642: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:29 smithi059 bash[59745]: cluster 2022-04-28T20:25:27.796336+0000 mgr.y (mgr.24961) 1278 : cluster [DBG] pgmap v1642: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:29 smithi139 bash[43301]: cluster 2022-04-28T20:25:27.796336+0000 mgr.y (mgr.24961) 1278 : cluster [DBG] pgmap v1642: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:31.757 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:31 smithi059 bash[61152]: cluster 2022-04-28T20:25:29.797226+0000 mgr.y (mgr.24961) 1279 : cluster [DBG] pgmap v1643: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:31.758 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:31 smithi059 bash[59745]: cluster 2022-04-28T20:25:29.797226+0000 mgr.y (mgr.24961) 1279 : cluster [DBG] pgmap v1643: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:31.759 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:31.757Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:31 smithi139 bash[43301]: cluster 2022-04-28T20:25:29.797226+0000 mgr.y (mgr.24961) 1279 : cluster [DBG] pgmap v1643: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:33 smithi059 bash[61152]: cluster 2022-04-28T20:25:31.798403+0000 mgr.y (mgr.24961) 1280 : cluster [DBG] pgmap v1644: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:33.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:33 smithi059 bash[59745]: cluster 2022-04-28T20:25:31.798403+0000 mgr.y (mgr.24961) 1280 : cluster [DBG] pgmap v1644: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:33 smithi139 bash[43301]: cluster 2022-04-28T20:25:31.798403+0000 mgr.y (mgr.24961) 1280 : cluster [DBG] pgmap v1644: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:35 smithi059 bash[61152]: cluster 2022-04-28T20:25:33.799187+0000 mgr.y (mgr.24961) 1281 : cluster [DBG] pgmap v1645: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:35 smithi059 bash[59745]: cluster 2022-04-28T20:25:33.799187+0000 mgr.y (mgr.24961) 1281 : cluster [DBG] pgmap v1645: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:35 smithi139 bash[43301]: cluster 2022-04-28T20:25:33.799187+0000 mgr.y (mgr.24961) 1281 : cluster [DBG] pgmap v1645: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:37 smithi059 bash[61152]: cluster 2022-04-28T20:25:35.800452+0000 mgr.y (mgr.24961) 1282 : cluster [DBG] pgmap v1646: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:37 smithi059 bash[59745]: cluster 2022-04-28T20:25:35.800452+0000 mgr.y (mgr.24961) 1282 : cluster [DBG] pgmap v1646: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:37 smithi139 bash[43301]: cluster 2022-04-28T20:25:35.800452+0000 mgr.y (mgr.24961) 1282 : cluster [DBG] pgmap v1646: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:39 smithi059 bash[61152]: cluster 2022-04-28T20:25:37.801362+0000 mgr.y (mgr.24961) 1283 : cluster [DBG] pgmap v1647: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:39 smithi059 bash[59745]: cluster 2022-04-28T20:25:37.801362+0000 mgr.y (mgr.24961) 1283 : cluster [DBG] pgmap v1647: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:39.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:39 smithi139 bash[43301]: cluster 2022-04-28T20:25:37.801362+0000 mgr.y (mgr.24961) 1283 : cluster [DBG] pgmap v1647: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:41.759 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:41 smithi059 bash[61152]: cluster 2022-04-28T20:25:39.802348+0000 mgr.y (mgr.24961) 1284 : cluster [DBG] pgmap v1648: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:41.759 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:41 smithi059 bash[59745]: cluster 2022-04-28T20:25:39.802348+0000 mgr.y (mgr.24961) 1284 : cluster [DBG] pgmap v1648: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:41 smithi139 bash[43301]: cluster 2022-04-28T20:25:39.802348+0000 mgr.y (mgr.24961) 1284 : cluster [DBG] pgmap v1648: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:41.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:43.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:43 smithi059 bash[59745]: cluster 2022-04-28T20:25:41.803468+0000 mgr.y (mgr.24961) 1285 : cluster [DBG] pgmap v1649: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:43 smithi059 bash[61152]: cluster 2022-04-28T20:25:41.803468+0000 mgr.y (mgr.24961) 1285 : cluster [DBG] pgmap v1649: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:43.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:43 smithi139 bash[43301]: cluster 2022-04-28T20:25:41.803468+0000 mgr.y (mgr.24961) 1285 : cluster [DBG] pgmap v1649: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:45.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:45 smithi059 bash[61152]: cluster 2022-04-28T20:25:43.804193+0000 mgr.y (mgr.24961) 1286 : cluster [DBG] pgmap v1650: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:45 smithi059 bash[59745]: cluster 2022-04-28T20:25:43.804193+0000 mgr.y (mgr.24961) 1286 : cluster [DBG] pgmap v1650: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:45 smithi139 bash[43301]: cluster 2022-04-28T20:25:43.804193+0000 mgr.y (mgr.24961) 1286 : cluster [DBG] pgmap v1650: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:47.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:47 smithi059 bash[61152]: cluster 2022-04-28T20:25:45.805344+0000 mgr.y (mgr.24961) 1287 : cluster [DBG] pgmap v1651: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:47 smithi059 bash[59745]: cluster 2022-04-28T20:25:45.805344+0000 mgr.y (mgr.24961) 1287 : cluster [DBG] pgmap v1651: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:47 smithi139 bash[43301]: cluster 2022-04-28T20:25:45.805344+0000 mgr.y (mgr.24961) 1287 : cluster [DBG] pgmap v1651: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:49 smithi059 bash[61152]: cluster 2022-04-28T20:25:47.806283+0000 mgr.y (mgr.24961) 1288 : cluster [DBG] pgmap v1652: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:49 smithi059 bash[59745]: cluster 2022-04-28T20:25:47.806283+0000 mgr.y (mgr.24961) 1288 : cluster [DBG] pgmap v1652: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:49 smithi139 bash[43301]: cluster 2022-04-28T20:25:47.806283+0000 mgr.y (mgr.24961) 1288 : cluster [DBG] pgmap v1652: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:51.759 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:51 smithi059 bash[61152]: cluster 2022-04-28T20:25:49.807181+0000 mgr.y (mgr.24961) 1289 : cluster [DBG] pgmap v1653: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:51.759 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:51 smithi059 bash[59745]: cluster 2022-04-28T20:25:49.807181+0000 mgr.y (mgr.24961) 1289 : cluster [DBG] pgmap v1653: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:51 smithi139 bash[43301]: cluster 2022-04-28T20:25:49.807181+0000 mgr.y (mgr.24961) 1289 : cluster [DBG] pgmap v1653: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:51.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:53 smithi059 bash[61152]: cluster 2022-04-28T20:25:51.808331+0000 mgr.y (mgr.24961) 1290 : cluster [DBG] pgmap v1654: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:25:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:25:53.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:25:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:53 smithi059 bash[59745]: cluster 2022-04-28T20:25:51.808331+0000 mgr.y (mgr.24961) 1290 : cluster [DBG] pgmap v1654: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:53 smithi139 bash[43301]: cluster 2022-04-28T20:25:51.808331+0000 mgr.y (mgr.24961) 1290 : cluster [DBG] pgmap v1654: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:55 smithi059 bash[61152]: cluster 2022-04-28T20:25:53.808968+0000 mgr.y (mgr.24961) 1291 : cluster [DBG] pgmap v1655: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:55 smithi059 bash[59745]: cluster 2022-04-28T20:25:53.808968+0000 mgr.y (mgr.24961) 1291 : cluster [DBG] pgmap v1655: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:55 smithi139 bash[43301]: cluster 2022-04-28T20:25:53.808968+0000 mgr.y (mgr.24961) 1291 : cluster [DBG] pgmap v1655: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:57.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:57 smithi059 bash[61152]: cluster 2022-04-28T20:25:55.810176+0000 mgr.y (mgr.24961) 1292 : cluster [DBG] pgmap v1656: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:57 smithi059 bash[59745]: cluster 2022-04-28T20:25:55.810176+0000 mgr.y (mgr.24961) 1292 : cluster [DBG] pgmap v1656: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:57 smithi139 bash[43301]: cluster 2022-04-28T20:25:55.810176+0000 mgr.y (mgr.24961) 1292 : cluster [DBG] pgmap v1656: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:25:59 smithi059 bash[61152]: cluster 2022-04-28T20:25:57.811102+0000 mgr.y (mgr.24961) 1293 : cluster [DBG] pgmap v1657: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:25:59 smithi059 bash[59745]: cluster 2022-04-28T20:25:57.811102+0000 mgr.y (mgr.24961) 1293 : cluster [DBG] pgmap v1657: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:25:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:25:59 smithi139 bash[43301]: cluster 2022-04-28T20:25:57.811102+0000 mgr.y (mgr.24961) 1293 : cluster [DBG] pgmap v1657: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:01.760 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:01 smithi059 bash[61152]: cluster 2022-04-28T20:25:59.812194+0000 mgr.y (mgr.24961) 1294 : cluster [DBG] pgmap v1658: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:01.760 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:01.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:01.761 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:01 smithi059 bash[59745]: cluster 2022-04-28T20:25:59.812194+0000 mgr.y (mgr.24961) 1294 : cluster [DBG] pgmap v1658: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:01 smithi139 bash[43301]: cluster 2022-04-28T20:25:59.812194+0000 mgr.y (mgr.24961) 1294 : cluster [DBG] pgmap v1658: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:03 smithi059 bash[61152]: cluster 2022-04-28T20:26:01.813431+0000 mgr.y (mgr.24961) 1295 : cluster [DBG] pgmap v1659: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:03.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:03 smithi059 bash[59745]: cluster 2022-04-28T20:26:01.813431+0000 mgr.y (mgr.24961) 1295 : cluster [DBG] pgmap v1659: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:03 smithi139 bash[43301]: cluster 2022-04-28T20:26:01.813431+0000 mgr.y (mgr.24961) 1295 : cluster [DBG] pgmap v1659: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:05.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:05 smithi059 bash[59745]: cluster 2022-04-28T20:26:03.814155+0000 mgr.y (mgr.24961) 1296 : cluster [DBG] pgmap v1660: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:05 smithi059 bash[61152]: cluster 2022-04-28T20:26:03.814155+0000 mgr.y (mgr.24961) 1296 : cluster [DBG] pgmap v1660: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:05 smithi139 bash[43301]: cluster 2022-04-28T20:26:03.814155+0000 mgr.y (mgr.24961) 1296 : cluster [DBG] pgmap v1660: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:07 smithi059 bash[61152]: cluster 2022-04-28T20:26:05.815386+0000 mgr.y (mgr.24961) 1297 : cluster [DBG] pgmap v1661: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:07 smithi059 bash[59745]: cluster 2022-04-28T20:26:05.815386+0000 mgr.y (mgr.24961) 1297 : cluster [DBG] pgmap v1661: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:07 smithi139 bash[43301]: cluster 2022-04-28T20:26:05.815386+0000 mgr.y (mgr.24961) 1297 : cluster [DBG] pgmap v1661: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:09.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:09 smithi059 bash[61152]: cluster 2022-04-28T20:26:07.816461+0000 mgr.y (mgr.24961) 1298 : cluster [DBG] pgmap v1662: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:09 smithi059 bash[59745]: cluster 2022-04-28T20:26:07.816461+0000 mgr.y (mgr.24961) 1298 : cluster [DBG] pgmap v1662: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:09.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:09 smithi139 bash[43301]: cluster 2022-04-28T20:26:07.816461+0000 mgr.y (mgr.24961) 1298 : cluster [DBG] pgmap v1662: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:11.760 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:11 smithi059 bash[61152]: cluster 2022-04-28T20:26:09.817519+0000 mgr.y (mgr.24961) 1299 : cluster [DBG] pgmap v1663: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:11.761 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:11.760Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:11.762 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:11 smithi059 bash[59745]: cluster 2022-04-28T20:26:09.817519+0000 mgr.y (mgr.24961) 1299 : cluster [DBG] pgmap v1663: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:11.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:11 smithi139 bash[43301]: cluster 2022-04-28T20:26:09.817519+0000 mgr.y (mgr.24961) 1299 : cluster [DBG] pgmap v1663: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:13 smithi059 bash[61152]: cluster 2022-04-28T20:26:11.818696+0000 mgr.y (mgr.24961) 1300 : cluster [DBG] pgmap v1664: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:13.867 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:13.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:13.867 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:13 smithi059 bash[59745]: cluster 2022-04-28T20:26:11.818696+0000 mgr.y (mgr.24961) 1300 : cluster [DBG] pgmap v1664: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:13 smithi139 bash[43301]: cluster 2022-04-28T20:26:11.818696+0000 mgr.y (mgr.24961) 1300 : cluster [DBG] pgmap v1664: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:15.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:15 smithi059 bash[59745]: cluster 2022-04-28T20:26:13.819392+0000 mgr.y (mgr.24961) 1301 : cluster [DBG] pgmap v1665: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:15 smithi059 bash[61152]: cluster 2022-04-28T20:26:13.819392+0000 mgr.y (mgr.24961) 1301 : cluster [DBG] pgmap v1665: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:15 smithi139 bash[43301]: cluster 2022-04-28T20:26:13.819392+0000 mgr.y (mgr.24961) 1301 : cluster [DBG] pgmap v1665: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:17 smithi059 bash[61152]: cluster 2022-04-28T20:26:15.820666+0000 mgr.y (mgr.24961) 1302 : cluster [DBG] pgmap v1666: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:17 smithi059 bash[59745]: cluster 2022-04-28T20:26:15.820666+0000 mgr.y (mgr.24961) 1302 : cluster [DBG] pgmap v1666: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:17 smithi139 bash[43301]: cluster 2022-04-28T20:26:15.820666+0000 mgr.y (mgr.24961) 1302 : cluster [DBG] pgmap v1666: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[61152]: audit 2022-04-28T20:26:17.560846+0000 mon.a (mon.0) 1942 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:26:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[61152]: audit 2022-04-28T20:26:17.561513+0000 mon.a (mon.0) 1943 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:26:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[61152]: audit 2022-04-28T20:26:17.574230+0000 mon.a (mon.0) 1944 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:26:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[61152]: audit 2022-04-28T20:26:17.574652+0000 mon.a (mon.0) 1945 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:26:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[61152]: audit 2022-04-28T20:26:17.575038+0000 mon.a (mon.0) 1946 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:26:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[59745]: audit 2022-04-28T20:26:17.560846+0000 mon.a (mon.0) 1942 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:26:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[59745]: audit 2022-04-28T20:26:17.561513+0000 mon.a (mon.0) 1943 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:26:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[59745]: audit 2022-04-28T20:26:17.574230+0000 mon.a (mon.0) 1944 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:26:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[59745]: audit 2022-04-28T20:26:17.574652+0000 mon.a (mon.0) 1945 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:26:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:18 smithi059 bash[59745]: audit 2022-04-28T20:26:17.575038+0000 mon.a (mon.0) 1946 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:26:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:18 smithi139 bash[43301]: audit 2022-04-28T20:26:17.560846+0000 mon.a (mon.0) 1942 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:26:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:18 smithi139 bash[43301]: audit 2022-04-28T20:26:17.561513+0000 mon.a (mon.0) 1943 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:26:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:18 smithi139 bash[43301]: audit 2022-04-28T20:26:17.574230+0000 mon.a (mon.0) 1944 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:26:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:18 smithi139 bash[43301]: audit 2022-04-28T20:26:17.574652+0000 mon.a (mon.0) 1945 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:26:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:18 smithi139 bash[43301]: audit 2022-04-28T20:26:17.575038+0000 mon.a (mon.0) 1946 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:26:19.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:19 smithi059 bash[61152]: cluster 2022-04-28T20:26:17.821620+0000 mgr.y (mgr.24961) 1303 : cluster [DBG] pgmap v1667: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:19.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:19 smithi059 bash[59745]: cluster 2022-04-28T20:26:17.821620+0000 mgr.y (mgr.24961) 1303 : cluster [DBG] pgmap v1667: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:19 smithi139 bash[43301]: cluster 2022-04-28T20:26:17.821620+0000 mgr.y (mgr.24961) 1303 : cluster [DBG] pgmap v1667: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:21.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:21 smithi059 bash[61152]: cluster 2022-04-28T20:26:19.822671+0000 mgr.y (mgr.24961) 1304 : cluster [DBG] pgmap v1668: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:21.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:21.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:21.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:21 smithi059 bash[59745]: cluster 2022-04-28T20:26:19.822671+0000 mgr.y (mgr.24961) 1304 : cluster [DBG] pgmap v1668: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:21 smithi139 bash[43301]: cluster 2022-04-28T20:26:19.822671+0000 mgr.y (mgr.24961) 1304 : cluster [DBG] pgmap v1668: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:23 smithi059 bash[61152]: cluster 2022-04-28T20:26:21.823805+0000 mgr.y (mgr.24961) 1305 : cluster [DBG] pgmap v1669: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:23.892 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:23.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:23.893 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:23 smithi059 bash[59745]: cluster 2022-04-28T20:26:21.823805+0000 mgr.y (mgr.24961) 1305 : cluster [DBG] pgmap v1669: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:23.893 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:23 smithi139 bash[43301]: cluster 2022-04-28T20:26:21.823805+0000 mgr.y (mgr.24961) 1305 : cluster [DBG] pgmap v1669: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:24.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[59745]: audit 2022-04-28T20:26:24.382884+0000 mon.a (mon.0) 1947 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:26:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[59745]: audit 2022-04-28T20:26:24.385635+0000 mon.a (mon.0) 1948 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:26:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[59745]: audit 2022-04-28T20:26:24.387425+0000 mon.a (mon.0) 1949 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:26:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[59745]: audit 2022-04-28T20:26:24.390705+0000 mon.a (mon.0) 1950 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:26:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[59745]: audit 2022-04-28T20:26:24.391263+0000 mon.a (mon.0) 1951 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:26:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[61152]: audit 2022-04-28T20:26:24.382884+0000 mon.a (mon.0) 1947 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:26:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[61152]: audit 2022-04-28T20:26:24.385635+0000 mon.a (mon.0) 1948 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:26:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[61152]: audit 2022-04-28T20:26:24.387425+0000 mon.a (mon.0) 1949 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:26:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[61152]: audit 2022-04-28T20:26:24.390705+0000 mon.a (mon.0) 1950 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:26:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:24 smithi059 bash[61152]: audit 2022-04-28T20:26:24.391263+0000 mon.a (mon.0) 1951 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:26:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:24 smithi139 bash[43301]: audit 2022-04-28T20:26:24.382884+0000 mon.a (mon.0) 1947 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:26:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:24 smithi139 bash[43301]: audit 2022-04-28T20:26:24.385635+0000 mon.a (mon.0) 1948 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:26:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:24 smithi139 bash[43301]: audit 2022-04-28T20:26:24.387425+0000 mon.a (mon.0) 1949 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:26:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:24 smithi139 bash[43301]: audit 2022-04-28T20:26:24.390705+0000 mon.a (mon.0) 1950 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:26:24.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:24 smithi139 bash[43301]: audit 2022-04-28T20:26:24.391263+0000 mon.a (mon.0) 1951 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:26:25.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:25 smithi059 bash[61152]: cluster 2022-04-28T20:26:23.824471+0000 mgr.y (mgr.24961) 1306 : cluster [DBG] pgmap v1670: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:25 smithi059 bash[61152]: cephadm 2022-04-28T20:26:24.389149+0000 mgr.y (mgr.24961) 1307 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:26:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:25 smithi059 bash[59745]: cluster 2022-04-28T20:26:23.824471+0000 mgr.y (mgr.24961) 1306 : cluster [DBG] pgmap v1670: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:25 smithi059 bash[59745]: cephadm 2022-04-28T20:26:24.389149+0000 mgr.y (mgr.24961) 1307 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:26:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:25 smithi139 bash[43301]: cluster 2022-04-28T20:26:23.824471+0000 mgr.y (mgr.24961) 1306 : cluster [DBG] pgmap v1670: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:25 smithi139 bash[43301]: cephadm 2022-04-28T20:26:24.389149+0000 mgr.y (mgr.24961) 1307 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:26:27.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:27 smithi059 bash[61152]: cluster 2022-04-28T20:26:25.825661+0000 mgr.y (mgr.24961) 1308 : cluster [DBG] pgmap v1671: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:27 smithi059 bash[59745]: cluster 2022-04-28T20:26:25.825661+0000 mgr.y (mgr.24961) 1308 : cluster [DBG] pgmap v1671: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:27.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:27 smithi139 bash[43301]: cluster 2022-04-28T20:26:25.825661+0000 mgr.y (mgr.24961) 1308 : cluster [DBG] pgmap v1671: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[59745]: cluster 2022-04-28T20:26:27.826413+0000 mgr.y (mgr.24961) 1309 : cluster [DBG] pgmap v1672: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[59745]: audit 2022-04-28T20:26:28.524095+0000 mon.a (mon.0) 1952 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[59745]: audit 2022-04-28T20:26:28.754423+0000 mon.a (mon.0) 1953 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[59745]: audit 2022-04-28T20:26:29.054196+0000 mon.a (mon.0) 1954 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[61152]: cluster 2022-04-28T20:26:27.826413+0000 mgr.y (mgr.24961) 1309 : cluster [DBG] pgmap v1672: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[61152]: audit 2022-04-28T20:26:28.524095+0000 mon.a (mon.0) 1952 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[61152]: audit 2022-04-28T20:26:28.754423+0000 mon.a (mon.0) 1953 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:29 smithi059 bash[61152]: audit 2022-04-28T20:26:29.054196+0000 mon.a (mon.0) 1954 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:29 smithi139 bash[43301]: cluster 2022-04-28T20:26:27.826413+0000 mgr.y (mgr.24961) 1309 : cluster [DBG] pgmap v1672: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:29 smithi139 bash[43301]: audit 2022-04-28T20:26:28.524095+0000 mon.a (mon.0) 1952 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:29 smithi139 bash[43301]: audit 2022-04-28T20:26:28.754423+0000 mon.a (mon.0) 1953 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:29 smithi139 bash[43301]: audit 2022-04-28T20:26:29.054196+0000 mon.a (mon.0) 1954 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:26:31.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:31 smithi059 bash[61152]: cluster 2022-04-28T20:26:29.827386+0000 mgr.y (mgr.24961) 1310 : cluster [DBG] pgmap v1673: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:31.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:31.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:31.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:31 smithi059 bash[59745]: cluster 2022-04-28T20:26:29.827386+0000 mgr.y (mgr.24961) 1310 : cluster [DBG] pgmap v1673: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:31 smithi139 bash[43301]: cluster 2022-04-28T20:26:29.827386+0000 mgr.y (mgr.24961) 1310 : cluster [DBG] pgmap v1673: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:33 smithi059 bash[61152]: cluster 2022-04-28T20:26:31.828485+0000 mgr.y (mgr.24961) 1311 : cluster [DBG] pgmap v1674: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:33.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:33.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:33 smithi059 bash[59745]: cluster 2022-04-28T20:26:31.828485+0000 mgr.y (mgr.24961) 1311 : cluster [DBG] pgmap v1674: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:33 smithi139 bash[43301]: cluster 2022-04-28T20:26:31.828485+0000 mgr.y (mgr.24961) 1311 : cluster [DBG] pgmap v1674: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:35.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:35 smithi059 bash[61152]: cluster 2022-04-28T20:26:33.829207+0000 mgr.y (mgr.24961) 1312 : cluster [DBG] pgmap v1675: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:35 smithi059 bash[59745]: cluster 2022-04-28T20:26:33.829207+0000 mgr.y (mgr.24961) 1312 : cluster [DBG] pgmap v1675: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:35 smithi139 bash[43301]: cluster 2022-04-28T20:26:33.829207+0000 mgr.y (mgr.24961) 1312 : cluster [DBG] pgmap v1675: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:37.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:37 smithi059 bash[61152]: cluster 2022-04-28T20:26:35.830438+0000 mgr.y (mgr.24961) 1313 : cluster [DBG] pgmap v1676: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:37.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:37 smithi059 bash[59745]: cluster 2022-04-28T20:26:35.830438+0000 mgr.y (mgr.24961) 1313 : cluster [DBG] pgmap v1676: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:37 smithi139 bash[43301]: cluster 2022-04-28T20:26:35.830438+0000 mgr.y (mgr.24961) 1313 : cluster [DBG] pgmap v1676: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:39.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:39 smithi059 bash[61152]: cluster 2022-04-28T20:26:37.831130+0000 mgr.y (mgr.24961) 1314 : cluster [DBG] pgmap v1677: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:39 smithi059 bash[59745]: cluster 2022-04-28T20:26:37.831130+0000 mgr.y (mgr.24961) 1314 : cluster [DBG] pgmap v1677: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:39.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:39 smithi139 bash[43301]: cluster 2022-04-28T20:26:37.831130+0000 mgr.y (mgr.24961) 1314 : cluster [DBG] pgmap v1677: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:41.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:41 smithi059 bash[61152]: cluster 2022-04-28T20:26:39.832156+0000 mgr.y (mgr.24961) 1315 : cluster [DBG] pgmap v1678: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:41.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:41.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:41 smithi059 bash[59745]: cluster 2022-04-28T20:26:39.832156+0000 mgr.y (mgr.24961) 1315 : cluster [DBG] pgmap v1678: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:41.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:41 smithi139 bash[43301]: cluster 2022-04-28T20:26:39.832156+0000 mgr.y (mgr.24961) 1315 : cluster [DBG] pgmap v1678: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:43 smithi059 bash[61152]: cluster 2022-04-28T20:26:41.833398+0000 mgr.y (mgr.24961) 1316 : cluster [DBG] pgmap v1679: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:43.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:43 smithi059 bash[59745]: cluster 2022-04-28T20:26:41.833398+0000 mgr.y (mgr.24961) 1316 : cluster [DBG] pgmap v1679: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:43 smithi139 bash[43301]: cluster 2022-04-28T20:26:41.833398+0000 mgr.y (mgr.24961) 1316 : cluster [DBG] pgmap v1679: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:45.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:45 smithi059 bash[61152]: cluster 2022-04-28T20:26:43.834232+0000 mgr.y (mgr.24961) 1317 : cluster [DBG] pgmap v1680: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:45 smithi059 bash[59745]: cluster 2022-04-28T20:26:43.834232+0000 mgr.y (mgr.24961) 1317 : cluster [DBG] pgmap v1680: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:45 smithi139 bash[43301]: cluster 2022-04-28T20:26:43.834232+0000 mgr.y (mgr.24961) 1317 : cluster [DBG] pgmap v1680: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:47.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:47 smithi059 bash[61152]: cluster 2022-04-28T20:26:45.835535+0000 mgr.y (mgr.24961) 1318 : cluster [DBG] pgmap v1681: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:47 smithi059 bash[59745]: cluster 2022-04-28T20:26:45.835535+0000 mgr.y (mgr.24961) 1318 : cluster [DBG] pgmap v1681: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:47 smithi139 bash[43301]: cluster 2022-04-28T20:26:45.835535+0000 mgr.y (mgr.24961) 1318 : cluster [DBG] pgmap v1681: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:49 smithi059 bash[61152]: cluster 2022-04-28T20:26:47.836345+0000 mgr.y (mgr.24961) 1319 : cluster [DBG] pgmap v1682: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:49 smithi059 bash[59745]: cluster 2022-04-28T20:26:47.836345+0000 mgr.y (mgr.24961) 1319 : cluster [DBG] pgmap v1682: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:49 smithi139 bash[43301]: cluster 2022-04-28T20:26:47.836345+0000 mgr.y (mgr.24961) 1319 : cluster [DBG] pgmap v1682: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:51.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:51 smithi059 bash[61152]: cluster 2022-04-28T20:26:49.837291+0000 mgr.y (mgr.24961) 1320 : cluster [DBG] pgmap v1683: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:51.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:51.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:51 smithi059 bash[59745]: cluster 2022-04-28T20:26:49.837291+0000 mgr.y (mgr.24961) 1320 : cluster [DBG] pgmap v1683: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:51 smithi139 bash[43301]: cluster 2022-04-28T20:26:49.837291+0000 mgr.y (mgr.24961) 1320 : cluster [DBG] pgmap v1683: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:53 smithi059 bash[61152]: cluster 2022-04-28T20:26:51.838495+0000 mgr.y (mgr.24961) 1321 : cluster [DBG] pgmap v1684: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:53.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:26:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:26:53.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:26:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:53 smithi059 bash[59745]: cluster 2022-04-28T20:26:51.838495+0000 mgr.y (mgr.24961) 1321 : cluster [DBG] pgmap v1684: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:53 smithi139 bash[43301]: cluster 2022-04-28T20:26:51.838495+0000 mgr.y (mgr.24961) 1321 : cluster [DBG] pgmap v1684: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:55 smithi139 bash[43301]: cluster 2022-04-28T20:26:53.839206+0000 mgr.y (mgr.24961) 1322 : cluster [DBG] pgmap v1685: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:55 smithi059 bash[61152]: cluster 2022-04-28T20:26:53.839206+0000 mgr.y (mgr.24961) 1322 : cluster [DBG] pgmap v1685: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:55 smithi059 bash[59745]: cluster 2022-04-28T20:26:53.839206+0000 mgr.y (mgr.24961) 1322 : cluster [DBG] pgmap v1685: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:57 smithi139 bash[43301]: cluster 2022-04-28T20:26:55.840440+0000 mgr.y (mgr.24961) 1323 : cluster [DBG] pgmap v1686: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:58.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:57 smithi059 bash[61152]: cluster 2022-04-28T20:26:55.840440+0000 mgr.y (mgr.24961) 1323 : cluster [DBG] pgmap v1686: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:57 smithi059 bash[59745]: cluster 2022-04-28T20:26:55.840440+0000 mgr.y (mgr.24961) 1323 : cluster [DBG] pgmap v1686: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:26:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:26:59 smithi139 bash[43301]: cluster 2022-04-28T20:26:57.841318+0000 mgr.y (mgr.24961) 1324 : cluster [DBG] pgmap v1687: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:00.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:26:59 smithi059 bash[61152]: cluster 2022-04-28T20:26:57.841318+0000 mgr.y (mgr.24961) 1324 : cluster [DBG] pgmap v1687: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:26:59 smithi059 bash[59745]: cluster 2022-04-28T20:26:57.841318+0000 mgr.y (mgr.24961) 1324 : cluster [DBG] pgmap v1687: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:01 smithi139 bash[43301]: cluster 2022-04-28T20:26:59.842380+0000 mgr.y (mgr.24961) 1325 : cluster [DBG] pgmap v1688: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:01 smithi059 bash[61152]: cluster 2022-04-28T20:26:59.842380+0000 mgr.y (mgr.24961) 1325 : cluster [DBG] pgmap v1688: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:01.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:01 smithi059 bash[59745]: cluster 2022-04-28T20:26:59.842380+0000 mgr.y (mgr.24961) 1325 : cluster [DBG] pgmap v1688: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:04.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:03 smithi059 bash[61152]: cluster 2022-04-28T20:27:01.843572+0000 mgr.y (mgr.24961) 1326 : cluster [DBG] pgmap v1689: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:03.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:03 smithi059 bash[59745]: cluster 2022-04-28T20:27:01.843572+0000 mgr.y (mgr.24961) 1326 : cluster [DBG] pgmap v1689: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:03 smithi139 bash[43301]: cluster 2022-04-28T20:27:01.843572+0000 mgr.y (mgr.24961) 1326 : cluster [DBG] pgmap v1689: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:06.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:05 smithi059 bash[61152]: cluster 2022-04-28T20:27:03.844285+0000 mgr.y (mgr.24961) 1327 : cluster [DBG] pgmap v1690: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:05 smithi059 bash[59745]: cluster 2022-04-28T20:27:03.844285+0000 mgr.y (mgr.24961) 1327 : cluster [DBG] pgmap v1690: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:05 smithi139 bash[43301]: cluster 2022-04-28T20:27:03.844285+0000 mgr.y (mgr.24961) 1327 : cluster [DBG] pgmap v1690: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:07 smithi059 bash[61152]: cluster 2022-04-28T20:27:05.845492+0000 mgr.y (mgr.24961) 1328 : cluster [DBG] pgmap v1691: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:07 smithi059 bash[59745]: cluster 2022-04-28T20:27:05.845492+0000 mgr.y (mgr.24961) 1328 : cluster [DBG] pgmap v1691: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:07 smithi139 bash[43301]: cluster 2022-04-28T20:27:05.845492+0000 mgr.y (mgr.24961) 1328 : cluster [DBG] pgmap v1691: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:10.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:09 smithi059 bash[61152]: cluster 2022-04-28T20:27:07.846410+0000 mgr.y (mgr.24961) 1329 : cluster [DBG] pgmap v1692: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:10.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:09 smithi059 bash[59745]: cluster 2022-04-28T20:27:07.846410+0000 mgr.y (mgr.24961) 1329 : cluster [DBG] pgmap v1692: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:09 smithi139 bash[43301]: cluster 2022-04-28T20:27:07.846410+0000 mgr.y (mgr.24961) 1329 : cluster [DBG] pgmap v1692: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:11 smithi059 bash[61152]: cluster 2022-04-28T20:27:09.847401+0000 mgr.y (mgr.24961) 1330 : cluster [DBG] pgmap v1693: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:11 smithi059 bash[59745]: cluster 2022-04-28T20:27:09.847401+0000 mgr.y (mgr.24961) 1330 : cluster [DBG] pgmap v1693: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:11.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:11 smithi139 bash[43301]: cluster 2022-04-28T20:27:09.847401+0000 mgr.y (mgr.24961) 1330 : cluster [DBG] pgmap v1693: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:13 smithi059 bash[61152]: cluster 2022-04-28T20:27:11.848581+0000 mgr.y (mgr.24961) 1331 : cluster [DBG] pgmap v1694: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:13.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:13 smithi059 bash[59745]: cluster 2022-04-28T20:27:11.848581+0000 mgr.y (mgr.24961) 1331 : cluster [DBG] pgmap v1694: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:13 smithi139 bash[43301]: cluster 2022-04-28T20:27:11.848581+0000 mgr.y (mgr.24961) 1331 : cluster [DBG] pgmap v1694: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:15 smithi059 bash[61152]: cluster 2022-04-28T20:27:13.849257+0000 mgr.y (mgr.24961) 1332 : cluster [DBG] pgmap v1695: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:15 smithi059 bash[59745]: cluster 2022-04-28T20:27:13.849257+0000 mgr.y (mgr.24961) 1332 : cluster [DBG] pgmap v1695: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:15 smithi139 bash[43301]: cluster 2022-04-28T20:27:13.849257+0000 mgr.y (mgr.24961) 1332 : cluster [DBG] pgmap v1695: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[59745]: cluster 2022-04-28T20:27:15.850427+0000 mgr.y (mgr.24961) 1333 : cluster [DBG] pgmap v1696: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[59745]: audit 2022-04-28T20:27:17.569461+0000 mon.a (mon.0) 1955 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:27:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[59745]: audit 2022-04-28T20:27:17.570008+0000 mon.a (mon.0) 1956 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:27:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[59745]: audit 2022-04-28T20:27:17.581312+0000 mon.a (mon.0) 1957 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:27:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[59745]: audit 2022-04-28T20:27:17.581787+0000 mon.a (mon.0) 1958 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:27:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[59745]: audit 2022-04-28T20:27:17.582272+0000 mon.a (mon.0) 1959 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:27:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[61152]: cluster 2022-04-28T20:27:15.850427+0000 mgr.y (mgr.24961) 1333 : cluster [DBG] pgmap v1696: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[61152]: audit 2022-04-28T20:27:17.569461+0000 mon.a (mon.0) 1955 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:27:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[61152]: audit 2022-04-28T20:27:17.570008+0000 mon.a (mon.0) 1956 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:27:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[61152]: audit 2022-04-28T20:27:17.581312+0000 mon.a (mon.0) 1957 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:27:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[61152]: audit 2022-04-28T20:27:17.581787+0000 mon.a (mon.0) 1958 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:27:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:17 smithi059 bash[61152]: audit 2022-04-28T20:27:17.582272+0000 mon.a (mon.0) 1959 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:27:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:17 smithi139 bash[43301]: cluster 2022-04-28T20:27:15.850427+0000 mgr.y (mgr.24961) 1333 : cluster [DBG] pgmap v1696: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:17 smithi139 bash[43301]: audit 2022-04-28T20:27:17.569461+0000 mon.a (mon.0) 1955 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:27:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:17 smithi139 bash[43301]: audit 2022-04-28T20:27:17.570008+0000 mon.a (mon.0) 1956 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:27:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:17 smithi139 bash[43301]: audit 2022-04-28T20:27:17.581312+0000 mon.a (mon.0) 1957 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:27:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:17 smithi139 bash[43301]: audit 2022-04-28T20:27:17.581787+0000 mon.a (mon.0) 1958 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:27:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:17 smithi139 bash[43301]: audit 2022-04-28T20:27:17.582272+0000 mon.a (mon.0) 1959 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:27:20.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:19 smithi059 bash[61152]: cluster 2022-04-28T20:27:17.851450+0000 mgr.y (mgr.24961) 1334 : cluster [DBG] pgmap v1697: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:19 smithi059 bash[59745]: cluster 2022-04-28T20:27:17.851450+0000 mgr.y (mgr.24961) 1334 : cluster [DBG] pgmap v1697: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:19 smithi139 bash[43301]: cluster 2022-04-28T20:27:17.851450+0000 mgr.y (mgr.24961) 1334 : cluster [DBG] pgmap v1697: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:21 smithi059 bash[61152]: cluster 2022-04-28T20:27:19.852552+0000 mgr.y (mgr.24961) 1335 : cluster [DBG] pgmap v1698: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:21 smithi059 bash[59745]: cluster 2022-04-28T20:27:19.852552+0000 mgr.y (mgr.24961) 1335 : cluster [DBG] pgmap v1698: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:21.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:21 smithi139 bash[43301]: cluster 2022-04-28T20:27:19.852552+0000 mgr.y (mgr.24961) 1335 : cluster [DBG] pgmap v1698: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:23 smithi059 bash[61152]: cluster 2022-04-28T20:27:21.853721+0000 mgr.y (mgr.24961) 1336 : cluster [DBG] pgmap v1699: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:23.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:23 smithi059 bash[59745]: cluster 2022-04-28T20:27:21.853721+0000 mgr.y (mgr.24961) 1336 : cluster [DBG] pgmap v1699: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:23 smithi139 bash[43301]: cluster 2022-04-28T20:27:21.853721+0000 mgr.y (mgr.24961) 1336 : cluster [DBG] pgmap v1699: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:26.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:25 smithi059 bash[61152]: cluster 2022-04-28T20:27:23.854435+0000 mgr.y (mgr.24961) 1337 : cluster [DBG] pgmap v1700: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:25 smithi059 bash[59745]: cluster 2022-04-28T20:27:23.854435+0000 mgr.y (mgr.24961) 1337 : cluster [DBG] pgmap v1700: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:25 smithi139 bash[43301]: cluster 2022-04-28T20:27:23.854435+0000 mgr.y (mgr.24961) 1337 : cluster [DBG] pgmap v1700: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:28.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:27 smithi059 bash[61152]: cluster 2022-04-28T20:27:25.855711+0000 mgr.y (mgr.24961) 1338 : cluster [DBG] pgmap v1701: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:27 smithi059 bash[59745]: cluster 2022-04-28T20:27:25.855711+0000 mgr.y (mgr.24961) 1338 : cluster [DBG] pgmap v1701: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:27 smithi139 bash[43301]: cluster 2022-04-28T20:27:25.855711+0000 mgr.y (mgr.24961) 1338 : cluster [DBG] pgmap v1701: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[61152]: cluster 2022-04-28T20:27:27.856655+0000 mgr.y (mgr.24961) 1339 : cluster [DBG] pgmap v1702: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[61152]: audit 2022-04-28T20:27:29.062022+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:27:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[61152]: audit 2022-04-28T20:27:29.064542+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:27:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[61152]: audit 2022-04-28T20:27:29.066156+0000 mon.a (mon.0) 1962 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:27:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[61152]: audit 2022-04-28T20:27:29.370181+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:27:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[59745]: cluster 2022-04-28T20:27:27.856655+0000 mgr.y (mgr.24961) 1339 : cluster [DBG] pgmap v1702: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:30.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[59745]: audit 2022-04-28T20:27:29.062022+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:27:30.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[59745]: audit 2022-04-28T20:27:29.064542+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:27:30.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[59745]: audit 2022-04-28T20:27:29.066156+0000 mon.a (mon.0) 1962 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:27:30.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:29 smithi059 bash[59745]: audit 2022-04-28T20:27:29.370181+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:27:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:29 smithi139 bash[43301]: cluster 2022-04-28T20:27:27.856655+0000 mgr.y (mgr.24961) 1339 : cluster [DBG] pgmap v1702: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:29 smithi139 bash[43301]: audit 2022-04-28T20:27:29.062022+0000 mon.a (mon.0) 1960 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:27:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:29 smithi139 bash[43301]: audit 2022-04-28T20:27:29.064542+0000 mon.a (mon.0) 1961 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:27:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:29 smithi139 bash[43301]: audit 2022-04-28T20:27:29.066156+0000 mon.a (mon.0) 1962 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:27:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:29 smithi139 bash[43301]: audit 2022-04-28T20:27:29.370181+0000 mon.a (mon.0) 1963 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:27:31.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:30 smithi059 bash[61152]: cephadm 2022-04-28T20:27:29.067667+0000 mgr.y (mgr.24961) 1340 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:27:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:30 smithi059 bash[59745]: cephadm 2022-04-28T20:27:29.067667+0000 mgr.y (mgr.24961) 1340 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:27:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:30 smithi139 bash[43301]: cephadm 2022-04-28T20:27:29.067667+0000 mgr.y (mgr.24961) 1340 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:27:32.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:31 smithi059 bash[61152]: cluster 2022-04-28T20:27:29.857650+0000 mgr.y (mgr.24961) 1341 : cluster [DBG] pgmap v1703: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:31.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:31 smithi059 bash[59745]: cluster 2022-04-28T20:27:29.857650+0000 mgr.y (mgr.24961) 1341 : cluster [DBG] pgmap v1703: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:31 smithi139 bash[43301]: cluster 2022-04-28T20:27:29.857650+0000 mgr.y (mgr.24961) 1341 : cluster [DBG] pgmap v1703: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:34.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:33 smithi059 bash[61152]: cluster 2022-04-28T20:27:31.858819+0000 mgr.y (mgr.24961) 1342 : cluster [DBG] pgmap v1704: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:33.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:33 smithi059 bash[59745]: cluster 2022-04-28T20:27:31.858819+0000 mgr.y (mgr.24961) 1342 : cluster [DBG] pgmap v1704: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:33 smithi139 bash[43301]: cluster 2022-04-28T20:27:31.858819+0000 mgr.y (mgr.24961) 1342 : cluster [DBG] pgmap v1704: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:35 smithi059 bash[61152]: cluster 2022-04-28T20:27:33.859570+0000 mgr.y (mgr.24961) 1343 : cluster [DBG] pgmap v1705: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:35 smithi059 bash[59745]: cluster 2022-04-28T20:27:33.859570+0000 mgr.y (mgr.24961) 1343 : cluster [DBG] pgmap v1705: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:35 smithi139 bash[43301]: cluster 2022-04-28T20:27:33.859570+0000 mgr.y (mgr.24961) 1343 : cluster [DBG] pgmap v1705: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:37 smithi059 bash[61152]: cluster 2022-04-28T20:27:35.860795+0000 mgr.y (mgr.24961) 1344 : cluster [DBG] pgmap v1706: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:37 smithi059 bash[59745]: cluster 2022-04-28T20:27:35.860795+0000 mgr.y (mgr.24961) 1344 : cluster [DBG] pgmap v1706: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:37 smithi139 bash[43301]: cluster 2022-04-28T20:27:35.860795+0000 mgr.y (mgr.24961) 1344 : cluster [DBG] pgmap v1706: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:39 smithi059 bash[61152]: cluster 2022-04-28T20:27:37.861745+0000 mgr.y (mgr.24961) 1345 : cluster [DBG] pgmap v1707: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:39 smithi059 bash[59745]: cluster 2022-04-28T20:27:37.861745+0000 mgr.y (mgr.24961) 1345 : cluster [DBG] pgmap v1707: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:39 smithi139 bash[43301]: cluster 2022-04-28T20:27:37.861745+0000 mgr.y (mgr.24961) 1345 : cluster [DBG] pgmap v1707: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:41 smithi059 bash[61152]: cluster 2022-04-28T20:27:39.862736+0000 mgr.y (mgr.24961) 1346 : cluster [DBG] pgmap v1708: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:41.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:41 smithi059 bash[59745]: cluster 2022-04-28T20:27:39.862736+0000 mgr.y (mgr.24961) 1346 : cluster [DBG] pgmap v1708: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:41 smithi139 bash[43301]: cluster 2022-04-28T20:27:39.862736+0000 mgr.y (mgr.24961) 1346 : cluster [DBG] pgmap v1708: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:44.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:43 smithi059 bash[61152]: cluster 2022-04-28T20:27:41.863960+0000 mgr.y (mgr.24961) 1347 : cluster [DBG] pgmap v1709: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:43.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:43 smithi059 bash[59745]: cluster 2022-04-28T20:27:41.863960+0000 mgr.y (mgr.24961) 1347 : cluster [DBG] pgmap v1709: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:43 smithi139 bash[43301]: cluster 2022-04-28T20:27:41.863960+0000 mgr.y (mgr.24961) 1347 : cluster [DBG] pgmap v1709: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:45 smithi059 bash[61152]: cluster 2022-04-28T20:27:43.864690+0000 mgr.y (mgr.24961) 1348 : cluster [DBG] pgmap v1710: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:45 smithi059 bash[59745]: cluster 2022-04-28T20:27:43.864690+0000 mgr.y (mgr.24961) 1348 : cluster [DBG] pgmap v1710: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:45 smithi139 bash[43301]: cluster 2022-04-28T20:27:43.864690+0000 mgr.y (mgr.24961) 1348 : cluster [DBG] pgmap v1710: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:47 smithi059 bash[61152]: cluster 2022-04-28T20:27:45.866016+0000 mgr.y (mgr.24961) 1349 : cluster [DBG] pgmap v1711: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:47 smithi059 bash[59745]: cluster 2022-04-28T20:27:45.866016+0000 mgr.y (mgr.24961) 1349 : cluster [DBG] pgmap v1711: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:47 smithi139 bash[43301]: cluster 2022-04-28T20:27:45.866016+0000 mgr.y (mgr.24961) 1349 : cluster [DBG] pgmap v1711: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:50.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:49 smithi059 bash[61152]: cluster 2022-04-28T20:27:47.867068+0000 mgr.y (mgr.24961) 1350 : cluster [DBG] pgmap v1712: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:49 smithi059 bash[59745]: cluster 2022-04-28T20:27:47.867068+0000 mgr.y (mgr.24961) 1350 : cluster [DBG] pgmap v1712: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:49 smithi139 bash[43301]: cluster 2022-04-28T20:27:47.867068+0000 mgr.y (mgr.24961) 1350 : cluster [DBG] pgmap v1712: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:52.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:51 smithi059 bash[61152]: cluster 2022-04-28T20:27:49.868157+0000 mgr.y (mgr.24961) 1351 : cluster [DBG] pgmap v1713: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:51.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:51 smithi059 bash[59745]: cluster 2022-04-28T20:27:49.868157+0000 mgr.y (mgr.24961) 1351 : cluster [DBG] pgmap v1713: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:51 smithi139 bash[43301]: cluster 2022-04-28T20:27:49.868157+0000 mgr.y (mgr.24961) 1351 : cluster [DBG] pgmap v1713: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:52 smithi059 bash[61152]: cluster 2022-04-28T20:27:51.869313+0000 mgr.y (mgr.24961) 1352 : cluster [DBG] pgmap v1714: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:52 smithi059 bash[59745]: cluster 2022-04-28T20:27:51.869313+0000 mgr.y (mgr.24961) 1352 : cluster [DBG] pgmap v1714: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:52 smithi139 bash[43301]: cluster 2022-04-28T20:27:51.869313+0000 mgr.y (mgr.24961) 1352 : cluster [DBG] pgmap v1714: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:27:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:27:53.659Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:27:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:55 smithi139 bash[43301]: cluster 2022-04-28T20:27:53.869994+0000 mgr.y (mgr.24961) 1353 : cluster [DBG] pgmap v1715: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:55 smithi059 bash[61152]: cluster 2022-04-28T20:27:53.869994+0000 mgr.y (mgr.24961) 1353 : cluster [DBG] pgmap v1715: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:55 smithi059 bash[59745]: cluster 2022-04-28T20:27:53.869994+0000 mgr.y (mgr.24961) 1353 : cluster [DBG] pgmap v1715: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:57 smithi139 bash[43301]: cluster 2022-04-28T20:27:55.871169+0000 mgr.y (mgr.24961) 1354 : cluster [DBG] pgmap v1716: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:57 smithi059 bash[61152]: cluster 2022-04-28T20:27:55.871169+0000 mgr.y (mgr.24961) 1354 : cluster [DBG] pgmap v1716: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:57 smithi059 bash[59745]: cluster 2022-04-28T20:27:55.871169+0000 mgr.y (mgr.24961) 1354 : cluster [DBG] pgmap v1716: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:59.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:27:59 smithi059 bash[61152]: cluster 2022-04-28T20:27:57.872107+0000 mgr.y (mgr.24961) 1355 : cluster [DBG] pgmap v1717: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:27:59 smithi059 bash[59745]: cluster 2022-04-28T20:27:57.872107+0000 mgr.y (mgr.24961) 1355 : cluster [DBG] pgmap v1717: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:27:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:27:59 smithi139 bash[43301]: cluster 2022-04-28T20:27:57.872107+0000 mgr.y (mgr.24961) 1355 : cluster [DBG] pgmap v1717: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:01.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:01 smithi059 bash[61152]: cluster 2022-04-28T20:27:59.873125+0000 mgr.y (mgr.24961) 1356 : cluster [DBG] pgmap v1718: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:01 smithi059 bash[59745]: cluster 2022-04-28T20:27:59.873125+0000 mgr.y (mgr.24961) 1356 : cluster [DBG] pgmap v1718: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:01 smithi139 bash[43301]: cluster 2022-04-28T20:27:59.873125+0000 mgr.y (mgr.24961) 1356 : cluster [DBG] pgmap v1718: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:01.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:03 smithi059 bash[61152]: cluster 2022-04-28T20:28:01.874272+0000 mgr.y (mgr.24961) 1357 : cluster [DBG] pgmap v1719: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:03 smithi059 bash[59745]: cluster 2022-04-28T20:28:01.874272+0000 mgr.y (mgr.24961) 1357 : cluster [DBG] pgmap v1719: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:03 smithi139 bash[43301]: cluster 2022-04-28T20:28:01.874272+0000 mgr.y (mgr.24961) 1357 : cluster [DBG] pgmap v1719: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:04.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:03.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:05 smithi059 bash[61152]: cluster 2022-04-28T20:28:03.874993+0000 mgr.y (mgr.24961) 1358 : cluster [DBG] pgmap v1720: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:05 smithi059 bash[59745]: cluster 2022-04-28T20:28:03.874993+0000 mgr.y (mgr.24961) 1358 : cluster [DBG] pgmap v1720: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:05 smithi139 bash[43301]: cluster 2022-04-28T20:28:03.874993+0000 mgr.y (mgr.24961) 1358 : cluster [DBG] pgmap v1720: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:07 smithi059 bash[61152]: cluster 2022-04-28T20:28:05.876231+0000 mgr.y (mgr.24961) 1359 : cluster [DBG] pgmap v1721: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:07 smithi059 bash[59745]: cluster 2022-04-28T20:28:05.876231+0000 mgr.y (mgr.24961) 1359 : cluster [DBG] pgmap v1721: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:07 smithi139 bash[43301]: cluster 2022-04-28T20:28:05.876231+0000 mgr.y (mgr.24961) 1359 : cluster [DBG] pgmap v1721: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:09 smithi059 bash[61152]: cluster 2022-04-28T20:28:07.877157+0000 mgr.y (mgr.24961) 1360 : cluster [DBG] pgmap v1722: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:09 smithi059 bash[59745]: cluster 2022-04-28T20:28:07.877157+0000 mgr.y (mgr.24961) 1360 : cluster [DBG] pgmap v1722: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:09 smithi139 bash[43301]: cluster 2022-04-28T20:28:07.877157+0000 mgr.y (mgr.24961) 1360 : cluster [DBG] pgmap v1722: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:11 smithi059 bash[61152]: cluster 2022-04-28T20:28:09.878141+0000 mgr.y (mgr.24961) 1361 : cluster [DBG] pgmap v1723: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:11 smithi059 bash[59745]: cluster 2022-04-28T20:28:09.878141+0000 mgr.y (mgr.24961) 1361 : cluster [DBG] pgmap v1723: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:11 smithi139 bash[43301]: cluster 2022-04-28T20:28:09.878141+0000 mgr.y (mgr.24961) 1361 : cluster [DBG] pgmap v1723: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:11.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:13.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:13 smithi059 bash[61152]: cluster 2022-04-28T20:28:11.879291+0000 mgr.y (mgr.24961) 1362 : cluster [DBG] pgmap v1724: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:13 smithi059 bash[59745]: cluster 2022-04-28T20:28:11.879291+0000 mgr.y (mgr.24961) 1362 : cluster [DBG] pgmap v1724: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:13 smithi139 bash[43301]: cluster 2022-04-28T20:28:11.879291+0000 mgr.y (mgr.24961) 1362 : cluster [DBG] pgmap v1724: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:14.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:13.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:15 smithi059 bash[61152]: cluster 2022-04-28T20:28:13.880011+0000 mgr.y (mgr.24961) 1363 : cluster [DBG] pgmap v1725: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:15 smithi059 bash[59745]: cluster 2022-04-28T20:28:13.880011+0000 mgr.y (mgr.24961) 1363 : cluster [DBG] pgmap v1725: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:15 smithi139 bash[43301]: cluster 2022-04-28T20:28:13.880011+0000 mgr.y (mgr.24961) 1363 : cluster [DBG] pgmap v1725: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:17 smithi059 bash[61152]: cluster 2022-04-28T20:28:15.881203+0000 mgr.y (mgr.24961) 1364 : cluster [DBG] pgmap v1726: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:17 smithi059 bash[59745]: cluster 2022-04-28T20:28:15.881203+0000 mgr.y (mgr.24961) 1364 : cluster [DBG] pgmap v1726: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:17 smithi139 bash[43301]: cluster 2022-04-28T20:28:15.881203+0000 mgr.y (mgr.24961) 1364 : cluster [DBG] pgmap v1726: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[61152]: audit 2022-04-28T20:28:17.577741+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:28:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[61152]: audit 2022-04-28T20:28:17.578263+0000 mon.a (mon.0) 1965 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:28:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[61152]: audit 2022-04-28T20:28:17.590384+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:28:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[61152]: audit 2022-04-28T20:28:17.590953+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:28:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[61152]: audit 2022-04-28T20:28:17.591653+0000 mon.a (mon.0) 1968 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T20:28:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[59745]: audit 2022-04-28T20:28:17.577741+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:28:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[59745]: audit 2022-04-28T20:28:17.578263+0000 mon.a (mon.0) 1965 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:28:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[59745]: audit 2022-04-28T20:28:17.590384+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:28:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[59745]: audit 2022-04-28T20:28:17.590953+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:28:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:18 smithi059 bash[59745]: audit 2022-04-28T20:28:17.591653+0000 mon.a (mon.0) 1968 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T20:28:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:18 smithi139 bash[43301]: audit 2022-04-28T20:28:17.577741+0000 mon.a (mon.0) 1964 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:28:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:18 smithi139 bash[43301]: audit 2022-04-28T20:28:17.578263+0000 mon.a (mon.0) 1965 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:28:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:18 smithi139 bash[43301]: audit 2022-04-28T20:28:17.590384+0000 mon.a (mon.0) 1966 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:28:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:18 smithi139 bash[43301]: audit 2022-04-28T20:28:17.590953+0000 mon.a (mon.0) 1967 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:28:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:18 smithi139 bash[43301]: audit 2022-04-28T20:28:17.591653+0000 mon.a (mon.0) 1968 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T20:28:19.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:19 smithi059 bash[61152]: cluster 2022-04-28T20:28:17.882117+0000 mgr.y (mgr.24961) 1365 : cluster [DBG] pgmap v1727: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:19 smithi059 bash[59745]: cluster 2022-04-28T20:28:17.882117+0000 mgr.y (mgr.24961) 1365 : cluster [DBG] pgmap v1727: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:19 smithi139 bash[43301]: cluster 2022-04-28T20:28:17.882117+0000 mgr.y (mgr.24961) 1365 : cluster [DBG] pgmap v1727: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:21 smithi059 bash[61152]: cluster 2022-04-28T20:28:19.883075+0000 mgr.y (mgr.24961) 1366 : cluster [DBG] pgmap v1728: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:21 smithi059 bash[59745]: cluster 2022-04-28T20:28:19.883075+0000 mgr.y (mgr.24961) 1366 : cluster [DBG] pgmap v1728: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:21 smithi139 bash[43301]: cluster 2022-04-28T20:28:19.883075+0000 mgr.y (mgr.24961) 1366 : cluster [DBG] pgmap v1728: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:22.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:21.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:23 smithi059 bash[61152]: cluster 2022-04-28T20:28:21.884251+0000 mgr.y (mgr.24961) 1367 : cluster [DBG] pgmap v1729: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:23 smithi059 bash[59745]: cluster 2022-04-28T20:28:21.884251+0000 mgr.y (mgr.24961) 1367 : cluster [DBG] pgmap v1729: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:23 smithi139 bash[43301]: cluster 2022-04-28T20:28:21.884251+0000 mgr.y (mgr.24961) 1367 : cluster [DBG] pgmap v1729: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:24.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:23.660Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:25 smithi059 bash[61152]: cluster 2022-04-28T20:28:23.884997+0000 mgr.y (mgr.24961) 1368 : cluster [DBG] pgmap v1730: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:25 smithi059 bash[59745]: cluster 2022-04-28T20:28:23.884997+0000 mgr.y (mgr.24961) 1368 : cluster [DBG] pgmap v1730: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:25 smithi139 bash[43301]: cluster 2022-04-28T20:28:23.884997+0000 mgr.y (mgr.24961) 1368 : cluster [DBG] pgmap v1730: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:27 smithi059 bash[61152]: cluster 2022-04-28T20:28:25.886252+0000 mgr.y (mgr.24961) 1369 : cluster [DBG] pgmap v1731: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:27 smithi059 bash[59745]: cluster 2022-04-28T20:28:25.886252+0000 mgr.y (mgr.24961) 1369 : cluster [DBG] pgmap v1731: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:27 smithi139 bash[43301]: cluster 2022-04-28T20:28:25.886252+0000 mgr.y (mgr.24961) 1369 : cluster [DBG] pgmap v1731: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:29 smithi059 bash[59745]: cluster 2022-04-28T20:28:27.887212+0000 mgr.y (mgr.24961) 1370 : cluster [DBG] pgmap v1732: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:29 smithi059 bash[61152]: cluster 2022-04-28T20:28:27.887212+0000 mgr.y (mgr.24961) 1370 : cluster [DBG] pgmap v1732: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:29 smithi139 bash[43301]: cluster 2022-04-28T20:28:27.887212+0000 mgr.y (mgr.24961) 1370 : cluster [DBG] pgmap v1732: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: audit 2022-04-28T20:28:29.379691+0000 mon.a (mon.0) 1969 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:28:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: audit 2022-04-28T20:28:29.382643+0000 mon.a (mon.0) 1970 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:28:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: audit 2022-04-28T20:28:29.384527+0000 mon.a (mon.0) 1971 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:28:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: cephadm 2022-04-28T20:28:29.386227+0000 mgr.y (mgr.24961) 1371 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:28:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: audit 2022-04-28T20:28:29.696661+0000 mon.a (mon.0) 1972 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: audit 2022-04-28T20:28:29.703939+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[59745]: audit 2022-04-28T20:28:30.000157+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: audit 2022-04-28T20:28:29.379691+0000 mon.a (mon.0) 1969 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:28:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: audit 2022-04-28T20:28:29.382643+0000 mon.a (mon.0) 1970 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:28:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: audit 2022-04-28T20:28:29.384527+0000 mon.a (mon.0) 1971 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:28:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: cephadm 2022-04-28T20:28:29.386227+0000 mgr.y (mgr.24961) 1371 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:28:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: audit 2022-04-28T20:28:29.696661+0000 mon.a (mon.0) 1972 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: audit 2022-04-28T20:28:29.703939+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:30 smithi059 bash[61152]: audit 2022-04-28T20:28:30.000157+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: audit 2022-04-28T20:28:29.379691+0000 mon.a (mon.0) 1969 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:28:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: audit 2022-04-28T20:28:29.382643+0000 mon.a (mon.0) 1970 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:28:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: audit 2022-04-28T20:28:29.384527+0000 mon.a (mon.0) 1971 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:28:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: cephadm 2022-04-28T20:28:29.386227+0000 mgr.y (mgr.24961) 1371 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:28:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: audit 2022-04-28T20:28:29.696661+0000 mon.a (mon.0) 1972 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: audit 2022-04-28T20:28:29.703939+0000 mon.a (mon.0) 1973 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:30 smithi139 bash[43301]: audit 2022-04-28T20:28:30.000157+0000 mon.a (mon.0) 1974 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:28:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:31 smithi059 bash[61152]: cluster 2022-04-28T20:28:29.888233+0000 mgr.y (mgr.24961) 1372 : cluster [DBG] pgmap v1733: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:31.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:31 smithi059 bash[59745]: cluster 2022-04-28T20:28:29.888233+0000 mgr.y (mgr.24961) 1372 : cluster [DBG] pgmap v1733: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:31 smithi139 bash[43301]: cluster 2022-04-28T20:28:29.888233+0000 mgr.y (mgr.24961) 1372 : cluster [DBG] pgmap v1733: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:33 smithi059 bash[61152]: cluster 2022-04-28T20:28:31.889375+0000 mgr.y (mgr.24961) 1373 : cluster [DBG] pgmap v1734: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:33.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:34.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:33 smithi059 bash[59745]: cluster 2022-04-28T20:28:31.889375+0000 mgr.y (mgr.24961) 1373 : cluster [DBG] pgmap v1734: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:33 smithi139 bash[43301]: cluster 2022-04-28T20:28:31.889375+0000 mgr.y (mgr.24961) 1373 : cluster [DBG] pgmap v1734: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:35 smithi059 bash[61152]: cluster 2022-04-28T20:28:33.890103+0000 mgr.y (mgr.24961) 1374 : cluster [DBG] pgmap v1735: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:36.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:35 smithi059 bash[59745]: cluster 2022-04-28T20:28:33.890103+0000 mgr.y (mgr.24961) 1374 : cluster [DBG] pgmap v1735: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:35 smithi139 bash[43301]: cluster 2022-04-28T20:28:33.890103+0000 mgr.y (mgr.24961) 1374 : cluster [DBG] pgmap v1735: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:37 smithi059 bash[61152]: cluster 2022-04-28T20:28:35.891308+0000 mgr.y (mgr.24961) 1375 : cluster [DBG] pgmap v1736: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:37 smithi059 bash[59745]: cluster 2022-04-28T20:28:35.891308+0000 mgr.y (mgr.24961) 1375 : cluster [DBG] pgmap v1736: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:37 smithi139 bash[43301]: cluster 2022-04-28T20:28:35.891308+0000 mgr.y (mgr.24961) 1375 : cluster [DBG] pgmap v1736: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:39 smithi059 bash[61152]: cluster 2022-04-28T20:28:37.892250+0000 mgr.y (mgr.24961) 1376 : cluster [DBG] pgmap v1737: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:39 smithi059 bash[59745]: cluster 2022-04-28T20:28:37.892250+0000 mgr.y (mgr.24961) 1376 : cluster [DBG] pgmap v1737: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:39 smithi139 bash[43301]: cluster 2022-04-28T20:28:37.892250+0000 mgr.y (mgr.24961) 1376 : cluster [DBG] pgmap v1737: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:42.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:41 smithi059 bash[61152]: cluster 2022-04-28T20:28:39.893392+0000 mgr.y (mgr.24961) 1377 : cluster [DBG] pgmap v1738: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:42.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:41.769Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:42.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:41 smithi059 bash[59745]: cluster 2022-04-28T20:28:39.893392+0000 mgr.y (mgr.24961) 1377 : cluster [DBG] pgmap v1738: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:41 smithi139 bash[43301]: cluster 2022-04-28T20:28:39.893392+0000 mgr.y (mgr.24961) 1377 : cluster [DBG] pgmap v1738: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:43 smithi059 bash[61152]: cluster 2022-04-28T20:28:41.894521+0000 mgr.y (mgr.24961) 1378 : cluster [DBG] pgmap v1739: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:43.663Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:43 smithi059 bash[59745]: cluster 2022-04-28T20:28:41.894521+0000 mgr.y (mgr.24961) 1378 : cluster [DBG] pgmap v1739: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:43 smithi139 bash[43301]: cluster 2022-04-28T20:28:41.894521+0000 mgr.y (mgr.24961) 1378 : cluster [DBG] pgmap v1739: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:45 smithi059 bash[61152]: cluster 2022-04-28T20:28:43.895247+0000 mgr.y (mgr.24961) 1379 : cluster [DBG] pgmap v1740: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:45 smithi059 bash[59745]: cluster 2022-04-28T20:28:43.895247+0000 mgr.y (mgr.24961) 1379 : cluster [DBG] pgmap v1740: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:45 smithi139 bash[43301]: cluster 2022-04-28T20:28:43.895247+0000 mgr.y (mgr.24961) 1379 : cluster [DBG] pgmap v1740: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:47 smithi059 bash[59745]: cluster 2022-04-28T20:28:45.896481+0000 mgr.y (mgr.24961) 1380 : cluster [DBG] pgmap v1741: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:48.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:47 smithi059 bash[61152]: cluster 2022-04-28T20:28:45.896481+0000 mgr.y (mgr.24961) 1380 : cluster [DBG] pgmap v1741: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:48.141 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:47 smithi139 bash[43301]: cluster 2022-04-28T20:28:45.896481+0000 mgr.y (mgr.24961) 1380 : cluster [DBG] pgmap v1741: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:49 smithi059 bash[61152]: cluster 2022-04-28T20:28:47.897382+0000 mgr.y (mgr.24961) 1381 : cluster [DBG] pgmap v1742: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:49 smithi059 bash[59745]: cluster 2022-04-28T20:28:47.897382+0000 mgr.y (mgr.24961) 1381 : cluster [DBG] pgmap v1742: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:49 smithi139 bash[43301]: cluster 2022-04-28T20:28:47.897382+0000 mgr.y (mgr.24961) 1381 : cluster [DBG] pgmap v1742: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:51 smithi059 bash[61152]: cluster 2022-04-28T20:28:49.898048+0000 mgr.y (mgr.24961) 1382 : cluster [DBG] pgmap v1743: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:51.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:51 smithi059 bash[59745]: cluster 2022-04-28T20:28:49.898048+0000 mgr.y (mgr.24961) 1382 : cluster [DBG] pgmap v1743: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:51 smithi139 bash[43301]: cluster 2022-04-28T20:28:49.898048+0000 mgr.y (mgr.24961) 1382 : cluster [DBG] pgmap v1743: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:53 smithi059 bash[61152]: cluster 2022-04-28T20:28:51.899060+0000 mgr.y (mgr.24961) 1383 : cluster [DBG] pgmap v1744: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:28:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:28:53.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:28:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:53 smithi059 bash[59745]: cluster 2022-04-28T20:28:51.899060+0000 mgr.y (mgr.24961) 1383 : cluster [DBG] pgmap v1744: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:53 smithi139 bash[43301]: cluster 2022-04-28T20:28:51.899060+0000 mgr.y (mgr.24961) 1383 : cluster [DBG] pgmap v1744: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:55 smithi059 bash[61152]: cluster 2022-04-28T20:28:53.899778+0000 mgr.y (mgr.24961) 1384 : cluster [DBG] pgmap v1745: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:56.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:55 smithi059 bash[59745]: cluster 2022-04-28T20:28:53.899778+0000 mgr.y (mgr.24961) 1384 : cluster [DBG] pgmap v1745: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:55 smithi139 bash[43301]: cluster 2022-04-28T20:28:53.899778+0000 mgr.y (mgr.24961) 1384 : cluster [DBG] pgmap v1745: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:56 smithi059 bash[61152]: cluster 2022-04-28T20:28:55.901039+0000 mgr.y (mgr.24961) 1385 : cluster [DBG] pgmap v1746: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:56 smithi059 bash[59745]: cluster 2022-04-28T20:28:55.901039+0000 mgr.y (mgr.24961) 1385 : cluster [DBG] pgmap v1746: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:56 smithi139 bash[43301]: cluster 2022-04-28T20:28:55.901039+0000 mgr.y (mgr.24961) 1385 : cluster [DBG] pgmap v1746: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:28:59 smithi059 bash[61152]: cluster 2022-04-28T20:28:57.901910+0000 mgr.y (mgr.24961) 1386 : cluster [DBG] pgmap v1747: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:28:59 smithi059 bash[59745]: cluster 2022-04-28T20:28:57.901910+0000 mgr.y (mgr.24961) 1386 : cluster [DBG] pgmap v1747: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:28:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:28:59 smithi139 bash[43301]: cluster 2022-04-28T20:28:57.901910+0000 mgr.y (mgr.24961) 1386 : cluster [DBG] pgmap v1747: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:01 smithi059 bash[61152]: cluster 2022-04-28T20:28:59.902906+0000 mgr.y (mgr.24961) 1387 : cluster [DBG] pgmap v1748: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:01 smithi059 bash[59745]: cluster 2022-04-28T20:28:59.902906+0000 mgr.y (mgr.24961) 1387 : cluster [DBG] pgmap v1748: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:01 smithi139 bash[43301]: cluster 2022-04-28T20:28:59.902906+0000 mgr.y (mgr.24961) 1387 : cluster [DBG] pgmap v1748: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:01.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:03 smithi059 bash[61152]: cluster 2022-04-28T20:29:01.904093+0000 mgr.y (mgr.24961) 1388 : cluster [DBG] pgmap v1749: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:03 smithi059 bash[59745]: cluster 2022-04-28T20:29:01.904093+0000 mgr.y (mgr.24961) 1388 : cluster [DBG] pgmap v1749: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:03 smithi139 bash[43301]: cluster 2022-04-28T20:29:01.904093+0000 mgr.y (mgr.24961) 1388 : cluster [DBG] pgmap v1749: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:03.664Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:05 smithi059 bash[61152]: cluster 2022-04-28T20:29:03.904790+0000 mgr.y (mgr.24961) 1389 : cluster [DBG] pgmap v1750: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:05 smithi059 bash[59745]: cluster 2022-04-28T20:29:03.904790+0000 mgr.y (mgr.24961) 1389 : cluster [DBG] pgmap v1750: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:05 smithi139 bash[43301]: cluster 2022-04-28T20:29:03.904790+0000 mgr.y (mgr.24961) 1389 : cluster [DBG] pgmap v1750: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:07 smithi059 bash[59745]: cluster 2022-04-28T20:29:05.905975+0000 mgr.y (mgr.24961) 1390 : cluster [DBG] pgmap v1751: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:07 smithi059 bash[61152]: cluster 2022-04-28T20:29:05.905975+0000 mgr.y (mgr.24961) 1390 : cluster [DBG] pgmap v1751: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:07 smithi139 bash[43301]: cluster 2022-04-28T20:29:05.905975+0000 mgr.y (mgr.24961) 1390 : cluster [DBG] pgmap v1751: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:09 smithi059 bash[61152]: cluster 2022-04-28T20:29:07.906885+0000 mgr.y (mgr.24961) 1391 : cluster [DBG] pgmap v1752: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:09 smithi059 bash[59745]: cluster 2022-04-28T20:29:07.906885+0000 mgr.y (mgr.24961) 1391 : cluster [DBG] pgmap v1752: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:09 smithi139 bash[43301]: cluster 2022-04-28T20:29:07.906885+0000 mgr.y (mgr.24961) 1391 : cluster [DBG] pgmap v1752: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:11 smithi059 bash[61152]: cluster 2022-04-28T20:29:09.907904+0000 mgr.y (mgr.24961) 1392 : cluster [DBG] pgmap v1753: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:11 smithi059 bash[59745]: cluster 2022-04-28T20:29:09.907904+0000 mgr.y (mgr.24961) 1392 : cluster [DBG] pgmap v1753: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:11 smithi139 bash[43301]: cluster 2022-04-28T20:29:09.907904+0000 mgr.y (mgr.24961) 1392 : cluster [DBG] pgmap v1753: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:11.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:13 smithi059 bash[59745]: cluster 2022-04-28T20:29:11.909071+0000 mgr.y (mgr.24961) 1393 : cluster [DBG] pgmap v1754: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:13 smithi059 bash[61152]: cluster 2022-04-28T20:29:11.909071+0000 mgr.y (mgr.24961) 1393 : cluster [DBG] pgmap v1754: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:13 smithi139 bash[43301]: cluster 2022-04-28T20:29:11.909071+0000 mgr.y (mgr.24961) 1393 : cluster [DBG] pgmap v1754: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:14.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:13.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:15 smithi059 bash[61152]: cluster 2022-04-28T20:29:13.909802+0000 mgr.y (mgr.24961) 1394 : cluster [DBG] pgmap v1755: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:15 smithi059 bash[59745]: cluster 2022-04-28T20:29:13.909802+0000 mgr.y (mgr.24961) 1394 : cluster [DBG] pgmap v1755: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:15 smithi139 bash[43301]: cluster 2022-04-28T20:29:13.909802+0000 mgr.y (mgr.24961) 1394 : cluster [DBG] pgmap v1755: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:17 smithi059 bash[61152]: cluster 2022-04-28T20:29:15.911051+0000 mgr.y (mgr.24961) 1395 : cluster [DBG] pgmap v1756: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:17 smithi059 bash[59745]: cluster 2022-04-28T20:29:15.911051+0000 mgr.y (mgr.24961) 1395 : cluster [DBG] pgmap v1756: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:17 smithi139 bash[43301]: cluster 2022-04-28T20:29:15.911051+0000 mgr.y (mgr.24961) 1395 : cluster [DBG] pgmap v1756: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[61152]: audit 2022-04-28T20:29:17.586059+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:29:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[61152]: audit 2022-04-28T20:29:17.586566+0000 mon.a (mon.0) 1976 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:29:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[61152]: audit 2022-04-28T20:29:17.598500+0000 mon.a (mon.0) 1977 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T20:29:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[61152]: audit 2022-04-28T20:29:17.599304+0000 mon.a (mon.0) 1978 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:29:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[61152]: audit 2022-04-28T20:29:17.599611+0000 mon.a (mon.0) 1979 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:29:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[59745]: audit 2022-04-28T20:29:17.586059+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:29:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[59745]: audit 2022-04-28T20:29:17.586566+0000 mon.a (mon.0) 1976 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:29:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[59745]: audit 2022-04-28T20:29:17.598500+0000 mon.a (mon.0) 1977 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T20:29:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[59745]: audit 2022-04-28T20:29:17.599304+0000 mon.a (mon.0) 1978 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:29:18.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:18 smithi059 bash[59745]: audit 2022-04-28T20:29:17.599611+0000 mon.a (mon.0) 1979 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:29:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:18 smithi139 bash[43301]: audit 2022-04-28T20:29:17.586059+0000 mon.a (mon.0) 1975 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:29:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:18 smithi139 bash[43301]: audit 2022-04-28T20:29:17.586566+0000 mon.a (mon.0) 1976 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:29:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:18 smithi139 bash[43301]: audit 2022-04-28T20:29:17.598500+0000 mon.a (mon.0) 1977 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T20:29:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:18 smithi139 bash[43301]: audit 2022-04-28T20:29:17.599304+0000 mon.a (mon.0) 1978 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:29:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:18 smithi139 bash[43301]: audit 2022-04-28T20:29:17.599611+0000 mon.a (mon.0) 1979 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:29:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:19 smithi059 bash[61152]: cluster 2022-04-28T20:29:17.911976+0000 mgr.y (mgr.24961) 1396 : cluster [DBG] pgmap v1757: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:19 smithi059 bash[59745]: cluster 2022-04-28T20:29:17.911976+0000 mgr.y (mgr.24961) 1396 : cluster [DBG] pgmap v1757: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:19 smithi139 bash[43301]: cluster 2022-04-28T20:29:17.911976+0000 mgr.y (mgr.24961) 1396 : cluster [DBG] pgmap v1757: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:21 smithi059 bash[61152]: cluster 2022-04-28T20:29:19.912995+0000 mgr.y (mgr.24961) 1397 : cluster [DBG] pgmap v1758: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:21 smithi059 bash[59745]: cluster 2022-04-28T20:29:19.912995+0000 mgr.y (mgr.24961) 1397 : cluster [DBG] pgmap v1758: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:21 smithi139 bash[43301]: cluster 2022-04-28T20:29:19.912995+0000 mgr.y (mgr.24961) 1397 : cluster [DBG] pgmap v1758: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:21.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:23 smithi059 bash[59745]: cluster 2022-04-28T20:29:21.914044+0000 mgr.y (mgr.24961) 1398 : cluster [DBG] pgmap v1759: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:23 smithi059 bash[61152]: cluster 2022-04-28T20:29:21.914044+0000 mgr.y (mgr.24961) 1398 : cluster [DBG] pgmap v1759: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:23 smithi139 bash[43301]: cluster 2022-04-28T20:29:21.914044+0000 mgr.y (mgr.24961) 1398 : cluster [DBG] pgmap v1759: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:23.666Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:25 smithi059 bash[61152]: cluster 2022-04-28T20:29:23.914700+0000 mgr.y (mgr.24961) 1399 : cluster [DBG] pgmap v1760: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:25 smithi059 bash[59745]: cluster 2022-04-28T20:29:23.914700+0000 mgr.y (mgr.24961) 1399 : cluster [DBG] pgmap v1760: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:25 smithi139 bash[43301]: cluster 2022-04-28T20:29:23.914700+0000 mgr.y (mgr.24961) 1399 : cluster [DBG] pgmap v1760: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:27 smithi059 bash[61152]: cluster 2022-04-28T20:29:25.915971+0000 mgr.y (mgr.24961) 1400 : cluster [DBG] pgmap v1761: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:27 smithi059 bash[59745]: cluster 2022-04-28T20:29:25.915971+0000 mgr.y (mgr.24961) 1400 : cluster [DBG] pgmap v1761: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:27 smithi139 bash[43301]: cluster 2022-04-28T20:29:25.915971+0000 mgr.y (mgr.24961) 1400 : cluster [DBG] pgmap v1761: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:29 smithi059 bash[61152]: cluster 2022-04-28T20:29:27.916793+0000 mgr.y (mgr.24961) 1401 : cluster [DBG] pgmap v1762: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:29 smithi059 bash[59745]: cluster 2022-04-28T20:29:27.916793+0000 mgr.y (mgr.24961) 1401 : cluster [DBG] pgmap v1762: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:29 smithi139 bash[43301]: cluster 2022-04-28T20:29:27.916793+0000 mgr.y (mgr.24961) 1401 : cluster [DBG] pgmap v1762: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:30 smithi059 bash[59745]: audit 2022-04-28T20:29:30.007005+0000 mon.a (mon.0) 1980 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:29:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:30 smithi059 bash[59745]: audit 2022-04-28T20:29:30.009941+0000 mon.a (mon.0) 1981 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:29:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:30 smithi059 bash[59745]: audit 2022-04-28T20:29:30.011784+0000 mon.a (mon.0) 1982 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:29:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:30 smithi059 bash[61152]: audit 2022-04-28T20:29:30.007005+0000 mon.a (mon.0) 1980 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:29:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:30 smithi059 bash[61152]: audit 2022-04-28T20:29:30.009941+0000 mon.a (mon.0) 1981 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:29:30.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:30 smithi059 bash[61152]: audit 2022-04-28T20:29:30.011784+0000 mon.a (mon.0) 1982 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:29:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:30 smithi139 bash[43301]: audit 2022-04-28T20:29:30.007005+0000 mon.a (mon.0) 1980 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:29:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:30 smithi139 bash[43301]: audit 2022-04-28T20:29:30.009941+0000 mon.a (mon.0) 1981 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:29:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:30 smithi139 bash[43301]: audit 2022-04-28T20:29:30.011784+0000 mon.a (mon.0) 1982 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:29:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[61152]: cluster 2022-04-28T20:29:29.917822+0000 mgr.y (mgr.24961) 1402 : cluster [DBG] pgmap v1763: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[61152]: cephadm 2022-04-28T20:29:30.013393+0000 mgr.y (mgr.24961) 1403 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:29:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[61152]: audit 2022-04-28T20:29:30.319317+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:29:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[59745]: cluster 2022-04-28T20:29:29.917822+0000 mgr.y (mgr.24961) 1402 : cluster [DBG] pgmap v1763: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[59745]: cephadm 2022-04-28T20:29:30.013393+0000 mgr.y (mgr.24961) 1403 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:29:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[59745]: audit 2022-04-28T20:29:30.319317+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:29:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:31 smithi139 bash[43301]: cluster 2022-04-28T20:29:29.917822+0000 mgr.y (mgr.24961) 1402 : cluster [DBG] pgmap v1763: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:31 smithi139 bash[43301]: cephadm 2022-04-28T20:29:30.013393+0000 mgr.y (mgr.24961) 1403 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:29:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:31 smithi139 bash[43301]: audit 2022-04-28T20:29:30.319317+0000 mon.a (mon.0) 1983 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:29:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:31.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:33.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:33 smithi059 bash[61152]: cluster 2022-04-28T20:29:31.919087+0000 mgr.y (mgr.24961) 1404 : cluster [DBG] pgmap v1764: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:33 smithi059 bash[59745]: cluster 2022-04-28T20:29:31.919087+0000 mgr.y (mgr.24961) 1404 : cluster [DBG] pgmap v1764: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:33 smithi139 bash[43301]: cluster 2022-04-28T20:29:31.919087+0000 mgr.y (mgr.24961) 1404 : cluster [DBG] pgmap v1764: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:33.665Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:35 smithi059 bash[61152]: cluster 2022-04-28T20:29:33.919851+0000 mgr.y (mgr.24961) 1405 : cluster [DBG] pgmap v1765: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:35 smithi059 bash[59745]: cluster 2022-04-28T20:29:33.919851+0000 mgr.y (mgr.24961) 1405 : cluster [DBG] pgmap v1765: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:35 smithi139 bash[43301]: cluster 2022-04-28T20:29:33.919851+0000 mgr.y (mgr.24961) 1405 : cluster [DBG] pgmap v1765: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:37 smithi059 bash[61152]: cluster 2022-04-28T20:29:35.921109+0000 mgr.y (mgr.24961) 1406 : cluster [DBG] pgmap v1766: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:37 smithi059 bash[59745]: cluster 2022-04-28T20:29:35.921109+0000 mgr.y (mgr.24961) 1406 : cluster [DBG] pgmap v1766: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:37 smithi139 bash[43301]: cluster 2022-04-28T20:29:35.921109+0000 mgr.y (mgr.24961) 1406 : cluster [DBG] pgmap v1766: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:39 smithi139 bash[43301]: cluster 2022-04-28T20:29:37.922117+0000 mgr.y (mgr.24961) 1407 : cluster [DBG] pgmap v1767: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:39 smithi059 bash[61152]: cluster 2022-04-28T20:29:37.922117+0000 mgr.y (mgr.24961) 1407 : cluster [DBG] pgmap v1767: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:39 smithi059 bash[59745]: cluster 2022-04-28T20:29:37.922117+0000 mgr.y (mgr.24961) 1407 : cluster [DBG] pgmap v1767: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:41 smithi139 bash[43301]: cluster 2022-04-28T20:29:39.923235+0000 mgr.y (mgr.24961) 1408 : cluster [DBG] pgmap v1768: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:41.775 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:41 smithi059 bash[61152]: cluster 2022-04-28T20:29:39.923235+0000 mgr.y (mgr.24961) 1408 : cluster [DBG] pgmap v1768: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:41.775 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:41 smithi059 bash[59745]: cluster 2022-04-28T20:29:39.923235+0000 mgr.y (mgr.24961) 1408 : cluster [DBG] pgmap v1768: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:41.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:43 smithi139 bash[43301]: cluster 2022-04-28T20:29:41.924420+0000 mgr.y (mgr.24961) 1409 : cluster [DBG] pgmap v1769: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:43.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:43 smithi059 bash[61152]: cluster 2022-04-28T20:29:41.924420+0000 mgr.y (mgr.24961) 1409 : cluster [DBG] pgmap v1769: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:43.668 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:43.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:43.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:43 smithi059 bash[59745]: cluster 2022-04-28T20:29:41.924420+0000 mgr.y (mgr.24961) 1409 : cluster [DBG] pgmap v1769: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:45 smithi139 bash[43301]: cluster 2022-04-28T20:29:43.925249+0000 mgr.y (mgr.24961) 1410 : cluster [DBG] pgmap v1770: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:45 smithi059 bash[61152]: cluster 2022-04-28T20:29:43.925249+0000 mgr.y (mgr.24961) 1410 : cluster [DBG] pgmap v1770: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:45 smithi059 bash[59745]: cluster 2022-04-28T20:29:43.925249+0000 mgr.y (mgr.24961) 1410 : cluster [DBG] pgmap v1770: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:47 smithi139 bash[43301]: cluster 2022-04-28T20:29:45.926524+0000 mgr.y (mgr.24961) 1411 : cluster [DBG] pgmap v1771: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:47 smithi059 bash[61152]: cluster 2022-04-28T20:29:45.926524+0000 mgr.y (mgr.24961) 1411 : cluster [DBG] pgmap v1771: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:47.908 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:47 smithi059 bash[59745]: cluster 2022-04-28T20:29:45.926524+0000 mgr.y (mgr.24961) 1411 : cluster [DBG] pgmap v1771: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:49 smithi059 bash[61152]: cluster 2022-04-28T20:29:47.927578+0000 mgr.y (mgr.24961) 1412 : cluster [DBG] pgmap v1772: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:49 smithi059 bash[59745]: cluster 2022-04-28T20:29:47.927578+0000 mgr.y (mgr.24961) 1412 : cluster [DBG] pgmap v1772: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:49 smithi139 bash[43301]: cluster 2022-04-28T20:29:47.927578+0000 mgr.y (mgr.24961) 1412 : cluster [DBG] pgmap v1772: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:51.776 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:51 smithi059 bash[61152]: cluster 2022-04-28T20:29:49.928621+0000 mgr.y (mgr.24961) 1413 : cluster [DBG] pgmap v1773: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:51.776 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:51 smithi059 bash[59745]: cluster 2022-04-28T20:29:49.928621+0000 mgr.y (mgr.24961) 1413 : cluster [DBG] pgmap v1773: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:51.777 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:51.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:51 smithi139 bash[43301]: cluster 2022-04-28T20:29:49.928621+0000 mgr.y (mgr.24961) 1413 : cluster [DBG] pgmap v1773: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:53.668 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:53 smithi059 bash[61152]: cluster 2022-04-28T20:29:51.929730+0000 mgr.y (mgr.24961) 1414 : cluster [DBG] pgmap v1774: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:53.669 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:53 smithi059 bash[59745]: cluster 2022-04-28T20:29:51.929730+0000 mgr.y (mgr.24961) 1414 : cluster [DBG] pgmap v1774: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:53 smithi139 bash[43301]: cluster 2022-04-28T20:29:51.929730+0000 mgr.y (mgr.24961) 1414 : cluster [DBG] pgmap v1774: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:29:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:29:53.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:29:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:55 smithi059 bash[61152]: cluster 2022-04-28T20:29:53.930433+0000 mgr.y (mgr.24961) 1415 : cluster [DBG] pgmap v1775: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:55 smithi059 bash[59745]: cluster 2022-04-28T20:29:53.930433+0000 mgr.y (mgr.24961) 1415 : cluster [DBG] pgmap v1775: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:55 smithi139 bash[43301]: cluster 2022-04-28T20:29:53.930433+0000 mgr.y (mgr.24961) 1415 : cluster [DBG] pgmap v1775: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:57 smithi059 bash[61152]: cluster 2022-04-28T20:29:55.931676+0000 mgr.y (mgr.24961) 1416 : cluster [DBG] pgmap v1776: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:57 smithi059 bash[59745]: cluster 2022-04-28T20:29:55.931676+0000 mgr.y (mgr.24961) 1416 : cluster [DBG] pgmap v1776: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:57 smithi139 bash[43301]: cluster 2022-04-28T20:29:55.931676+0000 mgr.y (mgr.24961) 1416 : cluster [DBG] pgmap v1776: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:29:59 smithi059 bash[61152]: cluster 2022-04-28T20:29:57.932562+0000 mgr.y (mgr.24961) 1417 : cluster [DBG] pgmap v1777: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:29:59 smithi059 bash[59745]: cluster 2022-04-28T20:29:57.932562+0000 mgr.y (mgr.24961) 1417 : cluster [DBG] pgmap v1777: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:29:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:29:59 smithi139 bash[43301]: cluster 2022-04-28T20:29:57.932562+0000 mgr.y (mgr.24961) 1417 : cluster [DBG] pgmap v1777: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:00 smithi059 bash[61152]: cluster 2022-04-28T20:30:00.000144+0000 mon.a (mon.0) 1984 : cluster [INF] overall HEALTH_OK 2022-04-28T20:30:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:00 smithi059 bash[59745]: cluster 2022-04-28T20:30:00.000144+0000 mon.a (mon.0) 1984 : cluster [INF] overall HEALTH_OK 2022-04-28T20:30:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:00 smithi139 bash[43301]: cluster 2022-04-28T20:30:00.000144+0000 mon.a (mon.0) 1984 : cluster [INF] overall HEALTH_OK 2022-04-28T20:30:01.776 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:01 smithi059 bash[61152]: cluster 2022-04-28T20:29:59.933502+0000 mgr.y (mgr.24961) 1418 : cluster [DBG] pgmap v1778: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:01.776 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:01 smithi059 bash[59745]: cluster 2022-04-28T20:29:59.933502+0000 mgr.y (mgr.24961) 1418 : cluster [DBG] pgmap v1778: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:01 smithi139 bash[43301]: cluster 2022-04-28T20:29:59.933502+0000 mgr.y (mgr.24961) 1418 : cluster [DBG] pgmap v1778: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:01.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:03.667 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:03 smithi059 bash[61152]: cluster 2022-04-28T20:30:01.934644+0000 mgr.y (mgr.24961) 1419 : cluster [DBG] pgmap v1779: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:03.668 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:03 smithi059 bash[59745]: cluster 2022-04-28T20:30:01.934644+0000 mgr.y (mgr.24961) 1419 : cluster [DBG] pgmap v1779: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:03 smithi139 bash[43301]: cluster 2022-04-28T20:30:01.934644+0000 mgr.y (mgr.24961) 1419 : cluster [DBG] pgmap v1779: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:03.667Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:05 smithi059 bash[61152]: cluster 2022-04-28T20:30:03.935300+0000 mgr.y (mgr.24961) 1420 : cluster [DBG] pgmap v1780: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:05.851 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:05 smithi059 bash[59745]: cluster 2022-04-28T20:30:03.935300+0000 mgr.y (mgr.24961) 1420 : cluster [DBG] pgmap v1780: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:05 smithi139 bash[43301]: cluster 2022-04-28T20:30:03.935300+0000 mgr.y (mgr.24961) 1420 : cluster [DBG] pgmap v1780: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:07 smithi059 bash[61152]: cluster 2022-04-28T20:30:05.936552+0000 mgr.y (mgr.24961) 1421 : cluster [DBG] pgmap v1781: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:07.879 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:07 smithi059 bash[59745]: cluster 2022-04-28T20:30:05.936552+0000 mgr.y (mgr.24961) 1421 : cluster [DBG] pgmap v1781: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:07.880 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:07 smithi139 bash[43301]: cluster 2022-04-28T20:30:05.936552+0000 mgr.y (mgr.24961) 1421 : cluster [DBG] pgmap v1781: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:09 smithi059 bash[59745]: cluster 2022-04-28T20:30:07.937568+0000 mgr.y (mgr.24961) 1422 : cluster [DBG] pgmap v1782: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:09 smithi059 bash[61152]: cluster 2022-04-28T20:30:07.937568+0000 mgr.y (mgr.24961) 1422 : cluster [DBG] pgmap v1782: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:09.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:09 smithi139 bash[43301]: cluster 2022-04-28T20:30:07.937568+0000 mgr.y (mgr.24961) 1422 : cluster [DBG] pgmap v1782: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:11.777 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:11 smithi059 bash[61152]: cluster 2022-04-28T20:30:09.938689+0000 mgr.y (mgr.24961) 1423 : cluster [DBG] pgmap v1783: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:11.777 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:11 smithi059 bash[59745]: cluster 2022-04-28T20:30:09.938689+0000 mgr.y (mgr.24961) 1423 : cluster [DBG] pgmap v1783: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:11.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:11 smithi139 bash[43301]: cluster 2022-04-28T20:30:09.938689+0000 mgr.y (mgr.24961) 1423 : cluster [DBG] pgmap v1783: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:11.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:13 smithi059 bash[61152]: cluster 2022-04-28T20:30:11.939896+0000 mgr.y (mgr.24961) 1424 : cluster [DBG] pgmap v1784: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:13.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:13.668Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:13 smithi059 bash[59745]: cluster 2022-04-28T20:30:11.939896+0000 mgr.y (mgr.24961) 1424 : cluster [DBG] pgmap v1784: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:13 smithi139 bash[43301]: cluster 2022-04-28T20:30:11.939896+0000 mgr.y (mgr.24961) 1424 : cluster [DBG] pgmap v1784: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:15 smithi059 bash[61152]: cluster 2022-04-28T20:30:13.940611+0000 mgr.y (mgr.24961) 1425 : cluster [DBG] pgmap v1785: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:15 smithi059 bash[59745]: cluster 2022-04-28T20:30:13.940611+0000 mgr.y (mgr.24961) 1425 : cluster [DBG] pgmap v1785: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:15 smithi139 bash[43301]: cluster 2022-04-28T20:30:13.940611+0000 mgr.y (mgr.24961) 1425 : cluster [DBG] pgmap v1785: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:17 smithi059 bash[59745]: cluster 2022-04-28T20:30:15.941796+0000 mgr.y (mgr.24961) 1426 : cluster [DBG] pgmap v1786: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:17 smithi059 bash[61152]: cluster 2022-04-28T20:30:15.941796+0000 mgr.y (mgr.24961) 1426 : cluster [DBG] pgmap v1786: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:17 smithi139 bash[43301]: cluster 2022-04-28T20:30:15.941796+0000 mgr.y (mgr.24961) 1426 : cluster [DBG] pgmap v1786: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[61152]: audit 2022-04-28T20:30:17.593863+0000 mon.a (mon.0) 1985 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:30:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[61152]: audit 2022-04-28T20:30:17.594346+0000 mon.a (mon.0) 1986 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:30:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[61152]: audit 2022-04-28T20:30:17.606072+0000 mon.a (mon.0) 1987 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:30:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[61152]: audit 2022-04-28T20:30:17.606438+0000 mon.a (mon.0) 1988 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:30:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[61152]: audit 2022-04-28T20:30:17.606898+0000 mon.a (mon.0) 1989 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:30:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[59745]: audit 2022-04-28T20:30:17.593863+0000 mon.a (mon.0) 1985 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:30:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[59745]: audit 2022-04-28T20:30:17.594346+0000 mon.a (mon.0) 1986 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:30:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[59745]: audit 2022-04-28T20:30:17.606072+0000 mon.a (mon.0) 1987 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:30:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[59745]: audit 2022-04-28T20:30:17.606438+0000 mon.a (mon.0) 1988 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:30:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:18 smithi059 bash[59745]: audit 2022-04-28T20:30:17.606898+0000 mon.a (mon.0) 1989 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:30:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:18 smithi139 bash[43301]: audit 2022-04-28T20:30:17.593863+0000 mon.a (mon.0) 1985 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:30:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:18 smithi139 bash[43301]: audit 2022-04-28T20:30:17.594346+0000 mon.a (mon.0) 1986 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:30:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:18 smithi139 bash[43301]: audit 2022-04-28T20:30:17.606072+0000 mon.a (mon.0) 1987 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:30:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:18 smithi139 bash[43301]: audit 2022-04-28T20:30:17.606438+0000 mon.a (mon.0) 1988 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T20:30:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:18 smithi139 bash[43301]: audit 2022-04-28T20:30:17.606898+0000 mon.a (mon.0) 1989 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:30:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:19 smithi059 bash[61152]: cluster 2022-04-28T20:30:17.942648+0000 mgr.y (mgr.24961) 1427 : cluster [DBG] pgmap v1787: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:19.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:19 smithi059 bash[59745]: cluster 2022-04-28T20:30:17.942648+0000 mgr.y (mgr.24961) 1427 : cluster [DBG] pgmap v1787: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:19 smithi139 bash[43301]: cluster 2022-04-28T20:30:17.942648+0000 mgr.y (mgr.24961) 1427 : cluster [DBG] pgmap v1787: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:21.778 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:21 smithi059 bash[61152]: cluster 2022-04-28T20:30:19.943697+0000 mgr.y (mgr.24961) 1428 : cluster [DBG] pgmap v1788: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:21.779 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:21 smithi059 bash[59745]: cluster 2022-04-28T20:30:19.943697+0000 mgr.y (mgr.24961) 1428 : cluster [DBG] pgmap v1788: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:21 smithi139 bash[43301]: cluster 2022-04-28T20:30:19.943697+0000 mgr.y (mgr.24961) 1428 : cluster [DBG] pgmap v1788: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:21.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:23 smithi059 bash[61152]: cluster 2022-04-28T20:30:21.944834+0000 mgr.y (mgr.24961) 1429 : cluster [DBG] pgmap v1789: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:23.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:23.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:23 smithi059 bash[59745]: cluster 2022-04-28T20:30:21.944834+0000 mgr.y (mgr.24961) 1429 : cluster [DBG] pgmap v1789: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:23 smithi139 bash[43301]: cluster 2022-04-28T20:30:21.944834+0000 mgr.y (mgr.24961) 1429 : cluster [DBG] pgmap v1789: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:25 smithi059 bash[61152]: cluster 2022-04-28T20:30:23.945537+0000 mgr.y (mgr.24961) 1430 : cluster [DBG] pgmap v1790: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:25 smithi059 bash[59745]: cluster 2022-04-28T20:30:23.945537+0000 mgr.y (mgr.24961) 1430 : cluster [DBG] pgmap v1790: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:25 smithi139 bash[43301]: cluster 2022-04-28T20:30:23.945537+0000 mgr.y (mgr.24961) 1430 : cluster [DBG] pgmap v1790: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:27 smithi059 bash[61152]: cluster 2022-04-28T20:30:25.946746+0000 mgr.y (mgr.24961) 1431 : cluster [DBG] pgmap v1791: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:27 smithi059 bash[59745]: cluster 2022-04-28T20:30:25.946746+0000 mgr.y (mgr.24961) 1431 : cluster [DBG] pgmap v1791: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:27.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:27 smithi139 bash[43301]: cluster 2022-04-28T20:30:25.946746+0000 mgr.y (mgr.24961) 1431 : cluster [DBG] pgmap v1791: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:29 smithi059 bash[61152]: cluster 2022-04-28T20:30:27.947698+0000 mgr.y (mgr.24961) 1432 : cluster [DBG] pgmap v1792: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:29 smithi059 bash[59745]: cluster 2022-04-28T20:30:27.947698+0000 mgr.y (mgr.24961) 1432 : cluster [DBG] pgmap v1792: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:29 smithi139 bash[43301]: cluster 2022-04-28T20:30:27.947698+0000 mgr.y (mgr.24961) 1432 : cluster [DBG] pgmap v1792: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:30 smithi059 bash[59745]: audit 2022-04-28T20:30:30.327915+0000 mon.a (mon.0) 1990 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:30:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:30 smithi059 bash[59745]: audit 2022-04-28T20:30:30.330727+0000 mon.a (mon.0) 1991 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:30:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:30 smithi059 bash[59745]: audit 2022-04-28T20:30:30.332605+0000 mon.a (mon.0) 1992 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:30:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:30 smithi059 bash[61152]: audit 2022-04-28T20:30:30.327915+0000 mon.a (mon.0) 1990 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:30:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:30 smithi059 bash[61152]: audit 2022-04-28T20:30:30.330727+0000 mon.a (mon.0) 1991 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:30:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:30 smithi059 bash[61152]: audit 2022-04-28T20:30:30.332605+0000 mon.a (mon.0) 1992 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:30:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:30 smithi139 bash[43301]: audit 2022-04-28T20:30:30.327915+0000 mon.a (mon.0) 1990 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:30:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:30 smithi139 bash[43301]: audit 2022-04-28T20:30:30.330727+0000 mon.a (mon.0) 1991 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:30:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:30 smithi139 bash[43301]: audit 2022-04-28T20:30:30.332605+0000 mon.a (mon.0) 1992 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:30:31.779 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: cluster 2022-04-28T20:30:29.948722+0000 mgr.y (mgr.24961) 1433 : cluster [DBG] pgmap v1793: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:31.796 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: cephadm 2022-04-28T20:30:30.334291+0000 mgr.y (mgr.24961) 1434 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:30:31.796 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.622853+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.796 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.623771+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.796 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.624567+0000 mon.a (mon.0) 1995 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.797 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.625437+0000 mon.a (mon.0) 1996 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.797 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.626212+0000 mon.a (mon.0) 1997 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.797 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.627082+0000 mon.a (mon.0) 1998 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.797 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.628054+0000 mon.a (mon.0) 1999 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.797 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.632705+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.799 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.633056+0000 mon.a (mon.0) 2001 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.799 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.633341+0000 mon.a (mon.0) 2002 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.799 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.633625+0000 mon.a (mon.0) 2003 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:30:31.799 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.633897+0000 mon.a (mon.0) 2004 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.800 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.634195+0000 mon.a (mon.0) 2005 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.800 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: cluster 2022-04-28T20:30:29.948722+0000 mgr.y (mgr.24961) 1433 : cluster [DBG] pgmap v1793: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:31.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: cephadm 2022-04-28T20:30:30.334291+0000 mgr.y (mgr.24961) 1434 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:30:31.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.622853+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.623771+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.624567+0000 mon.a (mon.0) 1995 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.801 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.625437+0000 mon.a (mon.0) 1996 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.802 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.626212+0000 mon.a (mon.0) 1997 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.802 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.627082+0000 mon.a (mon.0) 1998 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.802 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.628054+0000 mon.a (mon.0) 1999 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.802 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.632705+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.802 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.633056+0000 mon.a (mon.0) 2001 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.803 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.633341+0000 mon.a (mon.0) 2002 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.803 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.633625+0000 mon.a (mon.0) 2003 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:30:31.803 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.633897+0000 mon.a (mon.0) 2004 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.803 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.634195+0000 mon.a (mon.0) 2005 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.803 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.634421+0000 mon.a (mon.0) 2006 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.634943+0000 mon.a (mon.0) 2007 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.635541+0000 mon.a (mon.0) 2008 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.636095+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.636621+0000 mon.a (mon.0) 2010 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.804 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.637156+0000 mon.a (mon.0) 2011 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:30:31.805 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.637745+0000 mon.a (mon.0) 2012 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.805 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[59745]: audit 2022-04-28T20:30:30.643925+0000 mon.a (mon.0) 2013 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:30:31.805 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.634421+0000 mon.a (mon.0) 2006 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.805 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.634943+0000 mon.a (mon.0) 2007 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.806 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.635541+0000 mon.a (mon.0) 2008 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.806 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.636095+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.806 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.636621+0000 mon.a (mon.0) 2010 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.807 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.637156+0000 mon.a (mon.0) 2011 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:30:31.807 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.637745+0000 mon.a (mon.0) 2012 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.807 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[61152]: audit 2022-04-28T20:30:30.643925+0000 mon.a (mon.0) 2013 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:30:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: cluster 2022-04-28T20:30:29.948722+0000 mgr.y (mgr.24961) 1433 : cluster [DBG] pgmap v1793: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: cephadm 2022-04-28T20:30:30.334291+0000 mgr.y (mgr.24961) 1434 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:30:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.622853+0000 mon.a (mon.0) 1993 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.623771+0000 mon.a (mon.0) 1994 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.624567+0000 mon.a (mon.0) 1995 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.625437+0000 mon.a (mon.0) 1996 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.626212+0000 mon.a (mon.0) 1997 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.627082+0000 mon.a (mon.0) 1998 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.628054+0000 mon.a (mon.0) 1999 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.632705+0000 mon.a (mon.0) 2000 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.633056+0000 mon.a (mon.0) 2001 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.633341+0000 mon.a (mon.0) 2002 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.633625+0000 mon.a (mon.0) 2003 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:30:31.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.633897+0000 mon.a (mon.0) 2004 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.634195+0000 mon.a (mon.0) 2005 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.634421+0000 mon.a (mon.0) 2006 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.634943+0000 mon.a (mon.0) 2007 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.635541+0000 mon.a (mon.0) 2008 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.636095+0000 mon.a (mon.0) 2009 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:30:31.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.636621+0000 mon.a (mon.0) 2010 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.637156+0000 mon.a (mon.0) 2011 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:30:31.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.637745+0000 mon.a (mon.0) 2012 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:30:31.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:31 smithi139 bash[43301]: audit 2022-04-28T20:30:30.643925+0000 mon.a (mon.0) 2013 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:30:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:31.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:33 smithi059 bash[61152]: cluster 2022-04-28T20:30:31.949867+0000 mgr.y (mgr.24961) 1435 : cluster [DBG] pgmap v1794: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:33.849 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:33.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:33.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:33 smithi059 bash[59745]: cluster 2022-04-28T20:30:31.949867+0000 mgr.y (mgr.24961) 1435 : cluster [DBG] pgmap v1794: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:33 smithi139 bash[43301]: cluster 2022-04-28T20:30:31.949867+0000 mgr.y (mgr.24961) 1435 : cluster [DBG] pgmap v1794: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:35 smithi059 bash[59745]: cluster 2022-04-28T20:30:33.950609+0000 mgr.y (mgr.24961) 1436 : cluster [DBG] pgmap v1795: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:35 smithi059 bash[61152]: cluster 2022-04-28T20:30:33.950609+0000 mgr.y (mgr.24961) 1436 : cluster [DBG] pgmap v1795: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:35 smithi139 bash[43301]: cluster 2022-04-28T20:30:33.950609+0000 mgr.y (mgr.24961) 1436 : cluster [DBG] pgmap v1795: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:37 smithi059 bash[61152]: cluster 2022-04-28T20:30:35.951909+0000 mgr.y (mgr.24961) 1437 : cluster [DBG] pgmap v1796: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:37 smithi059 bash[59745]: cluster 2022-04-28T20:30:35.951909+0000 mgr.y (mgr.24961) 1437 : cluster [DBG] pgmap v1796: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:37 smithi139 bash[43301]: cluster 2022-04-28T20:30:35.951909+0000 mgr.y (mgr.24961) 1437 : cluster [DBG] pgmap v1796: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:39 smithi059 bash[61152]: cluster 2022-04-28T20:30:37.952833+0000 mgr.y (mgr.24961) 1438 : cluster [DBG] pgmap v1797: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:39 smithi059 bash[59745]: cluster 2022-04-28T20:30:37.952833+0000 mgr.y (mgr.24961) 1438 : cluster [DBG] pgmap v1797: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:39.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:39 smithi139 bash[43301]: cluster 2022-04-28T20:30:37.952833+0000 mgr.y (mgr.24961) 1438 : cluster [DBG] pgmap v1797: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:41.780 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:41 smithi059 bash[59745]: cluster 2022-04-28T20:30:39.953850+0000 mgr.y (mgr.24961) 1439 : cluster [DBG] pgmap v1798: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:41.780 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:41 smithi059 bash[61152]: cluster 2022-04-28T20:30:39.953850+0000 mgr.y (mgr.24961) 1439 : cluster [DBG] pgmap v1798: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:41 smithi139 bash[43301]: cluster 2022-04-28T20:30:39.953850+0000 mgr.y (mgr.24961) 1439 : cluster [DBG] pgmap v1798: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:41.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:43 smithi059 bash[59745]: cluster 2022-04-28T20:30:41.954965+0000 mgr.y (mgr.24961) 1440 : cluster [DBG] pgmap v1799: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:43 smithi059 bash[61152]: cluster 2022-04-28T20:30:41.954965+0000 mgr.y (mgr.24961) 1440 : cluster [DBG] pgmap v1799: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:43.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:43.670Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:43 smithi139 bash[43301]: cluster 2022-04-28T20:30:41.954965+0000 mgr.y (mgr.24961) 1440 : cluster [DBG] pgmap v1799: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:44.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[59745]: cluster 2022-04-28T20:30:43.536751+0000 mon.a (mon.0) 2014 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2022-04-28T20:30:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[59745]: audit 2022-04-28T20:30:43.542608+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.59:0/1412334371' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77329-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[59745]: audit 2022-04-28T20:30:44.539194+0000 mon.a (mon.0) 2016 : audit [INF] from='client.? 172.21.15.59:0/1412334371' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77329-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:30:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[59745]: cluster 2022-04-28T20:30:44.539535+0000 mon.a (mon.0) 2017 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2022-04-28T20:30:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[61152]: cluster 2022-04-28T20:30:43.536751+0000 mon.a (mon.0) 2014 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2022-04-28T20:30:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[61152]: audit 2022-04-28T20:30:43.542608+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.59:0/1412334371' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77329-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:44.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[61152]: audit 2022-04-28T20:30:44.539194+0000 mon.a (mon.0) 2016 : audit [INF] from='client.? 172.21.15.59:0/1412334371' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77329-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:30:44.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:44 smithi059 bash[61152]: cluster 2022-04-28T20:30:44.539535+0000 mon.a (mon.0) 2017 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2022-04-28T20:30:44.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:44 smithi139 bash[43301]: cluster 2022-04-28T20:30:43.536751+0000 mon.a (mon.0) 2014 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2022-04-28T20:30:44.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:44 smithi139 bash[43301]: audit 2022-04-28T20:30:43.542608+0000 mon.a (mon.0) 2015 : audit [INF] from='client.? 172.21.15.59:0/1412334371' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77329-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:44 smithi139 bash[43301]: audit 2022-04-28T20:30:44.539194+0000 mon.a (mon.0) 2016 : audit [INF] from='client.? 172.21.15.59:0/1412334371' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77329-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:30:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:44 smithi139 bash[43301]: cluster 2022-04-28T20:30:44.539535+0000 mon.a (mon.0) 2017 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2022-04-28T20:30:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:45 smithi059 bash[61152]: cluster 2022-04-28T20:30:43.955756+0000 mgr.y (mgr.24961) 1441 : cluster [DBG] pgmap v1801: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:45 smithi059 bash[59745]: cluster 2022-04-28T20:30:43.955756+0000 mgr.y (mgr.24961) 1441 : cluster [DBG] pgmap v1801: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:45 smithi139 bash[43301]: cluster 2022-04-28T20:30:43.955756+0000 mgr.y (mgr.24961) 1441 : cluster [DBG] pgmap v1801: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:46 smithi059 bash[61152]: cluster 2022-04-28T20:30:45.558450+0000 mon.a (mon.0) 2018 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2022-04-28T20:30:46.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:46 smithi059 bash[59745]: cluster 2022-04-28T20:30:45.558450+0000 mon.a (mon.0) 2018 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2022-04-28T20:30:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:46 smithi139 bash[43301]: cluster 2022-04-28T20:30:45.558450+0000 mon.a (mon.0) 2018 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2022-04-28T20:30:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:47 smithi059 bash[59745]: cluster 2022-04-28T20:30:45.956480+0000 mgr.y (mgr.24961) 1442 : cluster [DBG] pgmap v1804: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:47 smithi059 bash[61152]: cluster 2022-04-28T20:30:45.956480+0000 mgr.y (mgr.24961) 1442 : cluster [DBG] pgmap v1804: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:47 smithi139 bash[43301]: cluster 2022-04-28T20:30:45.956480+0000 mgr.y (mgr.24961) 1442 : cluster [DBG] pgmap v1804: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:49 smithi059 bash[61152]: cluster 2022-04-28T20:30:47.957405+0000 mgr.y (mgr.24961) 1443 : cluster [DBG] pgmap v1805: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:49 smithi059 bash[59745]: cluster 2022-04-28T20:30:47.957405+0000 mgr.y (mgr.24961) 1443 : cluster [DBG] pgmap v1805: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:49 smithi139 bash[43301]: cluster 2022-04-28T20:30:47.957405+0000 mgr.y (mgr.24961) 1443 : cluster [DBG] pgmap v1805: 105 pgs: 105 active+clean; 24 KiB data, 702 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:50 smithi139 bash[43301]: cluster 2022-04-28T20:30:49.595242+0000 mon.a (mon.0) 2019 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2022-04-28T20:30:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:50 smithi139 bash[43301]: audit 2022-04-28T20:30:49.600987+0000 mon.c (mon.1) 29 : audit [INF] from='client.? 172.21.15.59:0/2591027546' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:50 smithi139 bash[43301]: audit 2022-04-28T20:30:49.601788+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:50 smithi059 bash[61152]: cluster 2022-04-28T20:30:49.595242+0000 mon.a (mon.0) 2019 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2022-04-28T20:30:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:50 smithi059 bash[61152]: audit 2022-04-28T20:30:49.600987+0000 mon.c (mon.1) 29 : audit [INF] from='client.? 172.21.15.59:0/2591027546' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:50 smithi059 bash[61152]: audit 2022-04-28T20:30:49.601788+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:50 smithi059 bash[59745]: cluster 2022-04-28T20:30:49.595242+0000 mon.a (mon.0) 2019 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2022-04-28T20:30:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:50 smithi059 bash[59745]: audit 2022-04-28T20:30:49.600987+0000 mon.c (mon.1) 29 : audit [INF] from='client.? 172.21.15.59:0/2591027546' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:50 smithi059 bash[59745]: audit 2022-04-28T20:30:49.601788+0000 mon.a (mon.0) 2020 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:30:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:51 smithi139 bash[43301]: cluster 2022-04-28T20:30:49.958136+0000 mgr.y (mgr.24961) 1444 : cluster [DBG] pgmap v1807: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:51 smithi139 bash[43301]: audit 2022-04-28T20:30:50.606941+0000 mon.a (mon.0) 2021 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:30:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:51 smithi139 bash[43301]: cluster 2022-04-28T20:30:50.607222+0000 mon.a (mon.0) 2022 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2022-04-28T20:30:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[59745]: cluster 2022-04-28T20:30:49.958136+0000 mgr.y (mgr.24961) 1444 : cluster [DBG] pgmap v1807: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[59745]: audit 2022-04-28T20:30:50.606941+0000 mon.a (mon.0) 2021 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:30:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[59745]: cluster 2022-04-28T20:30:50.607222+0000 mon.a (mon.0) 2022 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2022-04-28T20:30:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[61152]: cluster 2022-04-28T20:30:49.958136+0000 mgr.y (mgr.24961) 1444 : cluster [DBG] pgmap v1807: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[61152]: audit 2022-04-28T20:30:50.606941+0000 mon.a (mon.0) 2021 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77352-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:30:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[61152]: cluster 2022-04-28T20:30:50.607222+0000 mon.a (mon.0) 2022 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2022-04-28T20:30:52.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:51.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:52 smithi059 bash[59745]: cluster 2022-04-28T20:30:51.613285+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2022-04-28T20:30:53.145 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:52 smithi059 bash[61152]: cluster 2022-04-28T20:30:51.613285+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2022-04-28T20:30:53.146 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:52 smithi139 bash[43301]: cluster 2022-04-28T20:30:51.613285+0000 mon.a (mon.0) 2023 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2022-04-28T20:30:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:53 smithi059 bash[59745]: cluster 2022-04-28T20:30:51.958853+0000 mgr.y (mgr.24961) 1445 : cluster [DBG] pgmap v1810: 105 pgs: 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:53 smithi059 bash[61152]: cluster 2022-04-28T20:30:51.958853+0000 mgr.y (mgr.24961) 1445 : cluster [DBG] pgmap v1810: 105 pgs: 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:30:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:30:53.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:30:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:53 smithi139 bash[43301]: cluster 2022-04-28T20:30:51.958853+0000 mgr.y (mgr.24961) 1445 : cluster [DBG] pgmap v1810: 105 pgs: 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:55 smithi059 bash[61152]: cluster 2022-04-28T20:30:53.959559+0000 mgr.y (mgr.24961) 1446 : cluster [DBG] pgmap v1811: 105 pgs: 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:55 smithi059 bash[59745]: cluster 2022-04-28T20:30:53.959559+0000 mgr.y (mgr.24961) 1446 : cluster [DBG] pgmap v1811: 105 pgs: 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:55 smithi139 bash[43301]: cluster 2022-04-28T20:30:53.959559+0000 mgr.y (mgr.24961) 1446 : cluster [DBG] pgmap v1811: 105 pgs: 105 active+clean; 24 KiB data, 703 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:57 smithi059 bash[61152]: cluster 2022-04-28T20:30:55.960796+0000 mgr.y (mgr.24961) 1447 : cluster [DBG] pgmap v1812: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:57 smithi059 bash[59745]: cluster 2022-04-28T20:30:55.960796+0000 mgr.y (mgr.24961) 1447 : cluster [DBG] pgmap v1812: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:30:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:57 smithi139 bash[43301]: cluster 2022-04-28T20:30:55.960796+0000 mgr.y (mgr.24961) 1447 : cluster [DBG] pgmap v1812: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:30:59 smithi059 bash[61152]: cluster 2022-04-28T20:30:57.961721+0000 mgr.y (mgr.24961) 1448 : cluster [DBG] pgmap v1813: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:30:59 smithi059 bash[59745]: cluster 2022-04-28T20:30:57.961721+0000 mgr.y (mgr.24961) 1448 : cluster [DBG] pgmap v1813: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:30:59 smithi139 bash[43301]: cluster 2022-04-28T20:30:57.961721+0000 mgr.y (mgr.24961) 1448 : cluster [DBG] pgmap v1813: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:01 smithi059 bash[61152]: cluster 2022-04-28T20:30:59.962848+0000 mgr.y (mgr.24961) 1449 : cluster [DBG] pgmap v1814: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:01.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:01 smithi059 bash[59745]: cluster 2022-04-28T20:30:59.962848+0000 mgr.y (mgr.24961) 1449 : cluster [DBG] pgmap v1814: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:01 smithi139 bash[43301]: cluster 2022-04-28T20:30:59.962848+0000 mgr.y (mgr.24961) 1449 : cluster [DBG] pgmap v1814: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:03 smithi059 bash[61152]: cluster 2022-04-28T20:31:01.964029+0000 mgr.y (mgr.24961) 1450 : cluster [DBG] pgmap v1815: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:04.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:03 smithi059 bash[59745]: cluster 2022-04-28T20:31:01.964029+0000 mgr.y (mgr.24961) 1450 : cluster [DBG] pgmap v1815: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:04.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:03.671Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:03 smithi139 bash[43301]: cluster 2022-04-28T20:31:01.964029+0000 mgr.y (mgr.24961) 1450 : cluster [DBG] pgmap v1815: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:05 smithi059 bash[59745]: cluster 2022-04-28T20:31:03.964730+0000 mgr.y (mgr.24961) 1451 : cluster [DBG] pgmap v1816: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:05 smithi059 bash[61152]: cluster 2022-04-28T20:31:03.964730+0000 mgr.y (mgr.24961) 1451 : cluster [DBG] pgmap v1816: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:05 smithi139 bash[43301]: cluster 2022-04-28T20:31:03.964730+0000 mgr.y (mgr.24961) 1451 : cluster [DBG] pgmap v1816: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:07 smithi059 bash[61152]: cluster 2022-04-28T20:31:05.965938+0000 mgr.y (mgr.24961) 1452 : cluster [DBG] pgmap v1817: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:07 smithi059 bash[59745]: cluster 2022-04-28T20:31:05.965938+0000 mgr.y (mgr.24961) 1452 : cluster [DBG] pgmap v1817: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:07 smithi139 bash[43301]: cluster 2022-04-28T20:31:05.965938+0000 mgr.y (mgr.24961) 1452 : cluster [DBG] pgmap v1817: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:09 smithi059 bash[61152]: cluster 2022-04-28T20:31:07.966850+0000 mgr.y (mgr.24961) 1453 : cluster [DBG] pgmap v1818: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:09 smithi059 bash[59745]: cluster 2022-04-28T20:31:07.966850+0000 mgr.y (mgr.24961) 1453 : cluster [DBG] pgmap v1818: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:09 smithi139 bash[43301]: cluster 2022-04-28T20:31:07.966850+0000 mgr.y (mgr.24961) 1453 : cluster [DBG] pgmap v1818: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:11 smithi059 bash[61152]: cluster 2022-04-28T20:31:09.967887+0000 mgr.y (mgr.24961) 1454 : cluster [DBG] pgmap v1819: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:11.782Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:11 smithi059 bash[59745]: cluster 2022-04-28T20:31:09.967887+0000 mgr.y (mgr.24961) 1454 : cluster [DBG] pgmap v1819: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:11 smithi139 bash[43301]: cluster 2022-04-28T20:31:09.967887+0000 mgr.y (mgr.24961) 1454 : cluster [DBG] pgmap v1819: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:13 smithi059 bash[61152]: cluster 2022-04-28T20:31:11.968975+0000 mgr.y (mgr.24961) 1455 : cluster [DBG] pgmap v1820: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:13.673Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:13 smithi059 bash[59745]: cluster 2022-04-28T20:31:11.968975+0000 mgr.y (mgr.24961) 1455 : cluster [DBG] pgmap v1820: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:13 smithi139 bash[43301]: cluster 2022-04-28T20:31:11.968975+0000 mgr.y (mgr.24961) 1455 : cluster [DBG] pgmap v1820: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:15 smithi059 bash[61152]: cluster 2022-04-28T20:31:13.969688+0000 mgr.y (mgr.24961) 1456 : cluster [DBG] pgmap v1821: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:15 smithi059 bash[59745]: cluster 2022-04-28T20:31:13.969688+0000 mgr.y (mgr.24961) 1456 : cluster [DBG] pgmap v1821: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:15 smithi139 bash[43301]: cluster 2022-04-28T20:31:13.969688+0000 mgr.y (mgr.24961) 1456 : cluster [DBG] pgmap v1821: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[59745]: cluster 2022-04-28T20:31:15.970915+0000 mgr.y (mgr.24961) 1457 : cluster [DBG] pgmap v1822: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[59745]: audit 2022-04-28T20:31:17.602036+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:31:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[59745]: audit 2022-04-28T20:31:17.602628+0000 mon.a (mon.0) 2025 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:31:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[59745]: audit 2022-04-28T20:31:17.613567+0000 mon.a (mon.0) 2026 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:31:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[59745]: audit 2022-04-28T20:31:17.614329+0000 mon.a (mon.0) 2027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:31:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[59745]: audit 2022-04-28T20:31:17.614914+0000 mon.a (mon.0) 2028 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:31:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[61152]: cluster 2022-04-28T20:31:15.970915+0000 mgr.y (mgr.24961) 1457 : cluster [DBG] pgmap v1822: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[61152]: audit 2022-04-28T20:31:17.602036+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:31:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[61152]: audit 2022-04-28T20:31:17.602628+0000 mon.a (mon.0) 2025 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:31:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[61152]: audit 2022-04-28T20:31:17.613567+0000 mon.a (mon.0) 2026 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:31:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[61152]: audit 2022-04-28T20:31:17.614329+0000 mon.a (mon.0) 2027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:31:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:17 smithi059 bash[61152]: audit 2022-04-28T20:31:17.614914+0000 mon.a (mon.0) 2028 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:31:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:17 smithi139 bash[43301]: cluster 2022-04-28T20:31:15.970915+0000 mgr.y (mgr.24961) 1457 : cluster [DBG] pgmap v1822: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:17 smithi139 bash[43301]: audit 2022-04-28T20:31:17.602036+0000 mon.a (mon.0) 2024 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:31:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:17 smithi139 bash[43301]: audit 2022-04-28T20:31:17.602628+0000 mon.a (mon.0) 2025 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:31:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:17 smithi139 bash[43301]: audit 2022-04-28T20:31:17.613567+0000 mon.a (mon.0) 2026 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:31:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:17 smithi139 bash[43301]: audit 2022-04-28T20:31:17.614329+0000 mon.a (mon.0) 2027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:31:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:17 smithi139 bash[43301]: audit 2022-04-28T20:31:17.614914+0000 mon.a (mon.0) 2028 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:31:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:19 smithi059 bash[61152]: cluster 2022-04-28T20:31:17.971842+0000 mgr.y (mgr.24961) 1458 : cluster [DBG] pgmap v1823: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:19 smithi059 bash[59745]: cluster 2022-04-28T20:31:17.971842+0000 mgr.y (mgr.24961) 1458 : cluster [DBG] pgmap v1823: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:19 smithi139 bash[43301]: cluster 2022-04-28T20:31:17.971842+0000 mgr.y (mgr.24961) 1458 : cluster [DBG] pgmap v1823: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:22.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:21 smithi059 bash[61152]: cluster 2022-04-28T20:31:19.972892+0000 mgr.y (mgr.24961) 1459 : cluster [DBG] pgmap v1824: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:22.096 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:21.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:22.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:21 smithi059 bash[59745]: cluster 2022-04-28T20:31:19.972892+0000 mgr.y (mgr.24961) 1459 : cluster [DBG] pgmap v1824: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:21 smithi139 bash[43301]: cluster 2022-04-28T20:31:19.972892+0000 mgr.y (mgr.24961) 1459 : cluster [DBG] pgmap v1824: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:23 smithi059 bash[61152]: cluster 2022-04-28T20:31:21.974002+0000 mgr.y (mgr.24961) 1460 : cluster [DBG] pgmap v1825: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:23.672Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:23 smithi059 bash[59745]: cluster 2022-04-28T20:31:21.974002+0000 mgr.y (mgr.24961) 1460 : cluster [DBG] pgmap v1825: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:23 smithi139 bash[43301]: cluster 2022-04-28T20:31:21.974002+0000 mgr.y (mgr.24961) 1460 : cluster [DBG] pgmap v1825: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:25 smithi059 bash[61152]: cluster 2022-04-28T20:31:23.974705+0000 mgr.y (mgr.24961) 1461 : cluster [DBG] pgmap v1826: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:25 smithi059 bash[59745]: cluster 2022-04-28T20:31:23.974705+0000 mgr.y (mgr.24961) 1461 : cluster [DBG] pgmap v1826: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:25 smithi139 bash[43301]: cluster 2022-04-28T20:31:23.974705+0000 mgr.y (mgr.24961) 1461 : cluster [DBG] pgmap v1826: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:27 smithi059 bash[61152]: cluster 2022-04-28T20:31:25.975930+0000 mgr.y (mgr.24961) 1462 : cluster [DBG] pgmap v1827: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:27 smithi059 bash[59745]: cluster 2022-04-28T20:31:25.975930+0000 mgr.y (mgr.24961) 1462 : cluster [DBG] pgmap v1827: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:27 smithi139 bash[43301]: cluster 2022-04-28T20:31:25.975930+0000 mgr.y (mgr.24961) 1462 : cluster [DBG] pgmap v1827: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:29 smithi059 bash[61152]: cluster 2022-04-28T20:31:27.976856+0000 mgr.y (mgr.24961) 1463 : cluster [DBG] pgmap v1828: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:29 smithi059 bash[59745]: cluster 2022-04-28T20:31:27.976856+0000 mgr.y (mgr.24961) 1463 : cluster [DBG] pgmap v1828: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:29 smithi139 bash[43301]: cluster 2022-04-28T20:31:27.976856+0000 mgr.y (mgr.24961) 1463 : cluster [DBG] pgmap v1828: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:30 smithi059 bash[59745]: audit 2022-04-28T20:31:30.652220+0000 mon.a (mon.0) 2029 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:31:31.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:30 smithi059 bash[59745]: audit 2022-04-28T20:31:30.655119+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:31:31.142 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:30 smithi059 bash[59745]: audit 2022-04-28T20:31:30.657149+0000 mon.a (mon.0) 2031 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:31:31.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:30 smithi059 bash[61152]: audit 2022-04-28T20:31:30.652220+0000 mon.a (mon.0) 2029 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:31:31.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:30 smithi059 bash[61152]: audit 2022-04-28T20:31:30.655119+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:31:31.143 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:30 smithi059 bash[61152]: audit 2022-04-28T20:31:30.657149+0000 mon.a (mon.0) 2031 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:31:31.144 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:30 smithi139 bash[43301]: audit 2022-04-28T20:31:30.652220+0000 mon.a (mon.0) 2029 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:31:31.145 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:30 smithi139 bash[43301]: audit 2022-04-28T20:31:30.655119+0000 mon.a (mon.0) 2030 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:31:31.145 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:30 smithi139 bash[43301]: audit 2022-04-28T20:31:30.657149+0000 mon.a (mon.0) 2031 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:31:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[61152]: cluster 2022-04-28T20:31:29.977918+0000 mgr.y (mgr.24961) 1464 : cluster [DBG] pgmap v1829: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[61152]: cephadm 2022-04-28T20:31:30.658942+0000 mgr.y (mgr.24961) 1465 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:31:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[61152]: audit 2022-04-28T20:31:30.990252+0000 mon.a (mon.0) 2032 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:31:32.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:31.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[59745]: cluster 2022-04-28T20:31:29.977918+0000 mgr.y (mgr.24961) 1464 : cluster [DBG] pgmap v1829: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[59745]: cephadm 2022-04-28T20:31:30.658942+0000 mgr.y (mgr.24961) 1465 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:31:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:31 smithi059 bash[59745]: audit 2022-04-28T20:31:30.990252+0000 mon.a (mon.0) 2032 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:31:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:31 smithi139 bash[43301]: cluster 2022-04-28T20:31:29.977918+0000 mgr.y (mgr.24961) 1464 : cluster [DBG] pgmap v1829: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:31 smithi139 bash[43301]: cephadm 2022-04-28T20:31:30.658942+0000 mgr.y (mgr.24961) 1465 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:31:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:31 smithi139 bash[43301]: audit 2022-04-28T20:31:30.990252+0000 mon.a (mon.0) 2032 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:31:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:33 smithi059 bash[61152]: cluster 2022-04-28T20:31:31.979009+0000 mgr.y (mgr.24961) 1466 : cluster [DBG] pgmap v1830: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:33.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:33 smithi059 bash[59745]: cluster 2022-04-28T20:31:31.979009+0000 mgr.y (mgr.24961) 1466 : cluster [DBG] pgmap v1830: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:33 smithi139 bash[43301]: cluster 2022-04-28T20:31:31.979009+0000 mgr.y (mgr.24961) 1466 : cluster [DBG] pgmap v1830: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:35 smithi059 bash[61152]: cluster 2022-04-28T20:31:33.979750+0000 mgr.y (mgr.24961) 1467 : cluster [DBG] pgmap v1831: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:35 smithi059 bash[59745]: cluster 2022-04-28T20:31:33.979750+0000 mgr.y (mgr.24961) 1467 : cluster [DBG] pgmap v1831: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:35 smithi139 bash[43301]: cluster 2022-04-28T20:31:33.979750+0000 mgr.y (mgr.24961) 1467 : cluster [DBG] pgmap v1831: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:37 smithi059 bash[61152]: cluster 2022-04-28T20:31:35.980958+0000 mgr.y (mgr.24961) 1468 : cluster [DBG] pgmap v1832: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:37 smithi059 bash[59745]: cluster 2022-04-28T20:31:35.980958+0000 mgr.y (mgr.24961) 1468 : cluster [DBG] pgmap v1832: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:37 smithi139 bash[43301]: cluster 2022-04-28T20:31:35.980958+0000 mgr.y (mgr.24961) 1468 : cluster [DBG] pgmap v1832: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:39 smithi059 bash[61152]: cluster 2022-04-28T20:31:37.981873+0000 mgr.y (mgr.24961) 1469 : cluster [DBG] pgmap v1833: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:39 smithi059 bash[59745]: cluster 2022-04-28T20:31:37.981873+0000 mgr.y (mgr.24961) 1469 : cluster [DBG] pgmap v1833: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:39 smithi139 bash[43301]: cluster 2022-04-28T20:31:37.981873+0000 mgr.y (mgr.24961) 1469 : cluster [DBG] pgmap v1833: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:41 smithi059 bash[61152]: cluster 2022-04-28T20:31:39.982954+0000 mgr.y (mgr.24961) 1470 : cluster [DBG] pgmap v1834: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:41.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:41 smithi059 bash[59745]: cluster 2022-04-28T20:31:39.982954+0000 mgr.y (mgr.24961) 1470 : cluster [DBG] pgmap v1834: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:41 smithi139 bash[43301]: cluster 2022-04-28T20:31:39.982954+0000 mgr.y (mgr.24961) 1470 : cluster [DBG] pgmap v1834: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:43 smithi059 bash[61152]: cluster 2022-04-28T20:31:41.984060+0000 mgr.y (mgr.24961) 1471 : cluster [DBG] pgmap v1835: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:43.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:43 smithi059 bash[59745]: cluster 2022-04-28T20:31:41.984060+0000 mgr.y (mgr.24961) 1471 : cluster [DBG] pgmap v1835: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:43 smithi139 bash[43301]: cluster 2022-04-28T20:31:41.984060+0000 mgr.y (mgr.24961) 1471 : cluster [DBG] pgmap v1835: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:45 smithi059 bash[61152]: cluster 2022-04-28T20:31:43.984805+0000 mgr.y (mgr.24961) 1472 : cluster [DBG] pgmap v1836: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:45 smithi059 bash[59745]: cluster 2022-04-28T20:31:43.984805+0000 mgr.y (mgr.24961) 1472 : cluster [DBG] pgmap v1836: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:45 smithi139 bash[43301]: cluster 2022-04-28T20:31:43.984805+0000 mgr.y (mgr.24961) 1472 : cluster [DBG] pgmap v1836: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:47 smithi059 bash[61152]: cluster 2022-04-28T20:31:45.986073+0000 mgr.y (mgr.24961) 1473 : cluster [DBG] pgmap v1837: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:47 smithi059 bash[59745]: cluster 2022-04-28T20:31:45.986073+0000 mgr.y (mgr.24961) 1473 : cluster [DBG] pgmap v1837: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:47 smithi139 bash[43301]: cluster 2022-04-28T20:31:45.986073+0000 mgr.y (mgr.24961) 1473 : cluster [DBG] pgmap v1837: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:49 smithi059 bash[61152]: cluster 2022-04-28T20:31:47.986917+0000 mgr.y (mgr.24961) 1474 : cluster [DBG] pgmap v1838: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:49 smithi059 bash[59745]: cluster 2022-04-28T20:31:47.986917+0000 mgr.y (mgr.24961) 1474 : cluster [DBG] pgmap v1838: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:49 smithi139 bash[43301]: cluster 2022-04-28T20:31:47.986917+0000 mgr.y (mgr.24961) 1474 : cluster [DBG] pgmap v1838: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:51 smithi059 bash[61152]: cluster 2022-04-28T20:31:49.988033+0000 mgr.y (mgr.24961) 1475 : cluster [DBG] pgmap v1839: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:51.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:51 smithi059 bash[59745]: cluster 2022-04-28T20:31:49.988033+0000 mgr.y (mgr.24961) 1475 : cluster [DBG] pgmap v1839: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:51 smithi139 bash[43301]: cluster 2022-04-28T20:31:49.988033+0000 mgr.y (mgr.24961) 1475 : cluster [DBG] pgmap v1839: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:53 smithi059 bash[61152]: cluster 2022-04-28T20:31:51.989129+0000 mgr.y (mgr.24961) 1476 : cluster [DBG] pgmap v1840: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:31:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:31:53.674Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:31:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:53 smithi059 bash[59745]: cluster 2022-04-28T20:31:51.989129+0000 mgr.y (mgr.24961) 1476 : cluster [DBG] pgmap v1840: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:53 smithi139 bash[43301]: cluster 2022-04-28T20:31:51.989129+0000 mgr.y (mgr.24961) 1476 : cluster [DBG] pgmap v1840: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:55 smithi059 bash[61152]: cluster 2022-04-28T20:31:53.990028+0000 mgr.y (mgr.24961) 1477 : cluster [DBG] pgmap v1841: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:55 smithi059 bash[59745]: cluster 2022-04-28T20:31:53.990028+0000 mgr.y (mgr.24961) 1477 : cluster [DBG] pgmap v1841: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:55 smithi139 bash[43301]: cluster 2022-04-28T20:31:53.990028+0000 mgr.y (mgr.24961) 1477 : cluster [DBG] pgmap v1841: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:56 smithi059 bash[61152]: cluster 2022-04-28T20:31:55.991218+0000 mgr.y (mgr.24961) 1478 : cluster [DBG] pgmap v1842: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:56 smithi059 bash[59745]: cluster 2022-04-28T20:31:55.991218+0000 mgr.y (mgr.24961) 1478 : cluster [DBG] pgmap v1842: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:56 smithi139 bash[43301]: cluster 2022-04-28T20:31:55.991218+0000 mgr.y (mgr.24961) 1478 : cluster [DBG] pgmap v1842: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:31:59 smithi059 bash[61152]: cluster 2022-04-28T20:31:57.992161+0000 mgr.y (mgr.24961) 1479 : cluster [DBG] pgmap v1843: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:31:59 smithi059 bash[59745]: cluster 2022-04-28T20:31:57.992161+0000 mgr.y (mgr.24961) 1479 : cluster [DBG] pgmap v1843: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:31:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:31:59 smithi139 bash[43301]: cluster 2022-04-28T20:31:57.992161+0000 mgr.y (mgr.24961) 1479 : cluster [DBG] pgmap v1843: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:01 smithi059 bash[61152]: cluster 2022-04-28T20:31:59.993335+0000 mgr.y (mgr.24961) 1480 : cluster [DBG] pgmap v1844: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:01.891 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:01.785Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:01.892 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:01 smithi059 bash[59745]: cluster 2022-04-28T20:31:59.993335+0000 mgr.y (mgr.24961) 1480 : cluster [DBG] pgmap v1844: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:01.892 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:01 smithi139 bash[43301]: cluster 2022-04-28T20:31:59.993335+0000 mgr.y (mgr.24961) 1480 : cluster [DBG] pgmap v1844: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:03 smithi059 bash[61152]: cluster 2022-04-28T20:32:01.994460+0000 mgr.y (mgr.24961) 1481 : cluster [DBG] pgmap v1845: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:03.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:03.676Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:03 smithi059 bash[59745]: cluster 2022-04-28T20:32:01.994460+0000 mgr.y (mgr.24961) 1481 : cluster [DBG] pgmap v1845: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:03 smithi139 bash[43301]: cluster 2022-04-28T20:32:01.994460+0000 mgr.y (mgr.24961) 1481 : cluster [DBG] pgmap v1845: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:05 smithi059 bash[61152]: cluster 2022-04-28T20:32:03.995288+0000 mgr.y (mgr.24961) 1482 : cluster [DBG] pgmap v1846: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:05 smithi059 bash[59745]: cluster 2022-04-28T20:32:03.995288+0000 mgr.y (mgr.24961) 1482 : cluster [DBG] pgmap v1846: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:05 smithi139 bash[43301]: cluster 2022-04-28T20:32:03.995288+0000 mgr.y (mgr.24961) 1482 : cluster [DBG] pgmap v1846: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:07 smithi059 bash[59745]: cluster 2022-04-28T20:32:05.996479+0000 mgr.y (mgr.24961) 1483 : cluster [DBG] pgmap v1847: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:07 smithi059 bash[61152]: cluster 2022-04-28T20:32:05.996479+0000 mgr.y (mgr.24961) 1483 : cluster [DBG] pgmap v1847: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:07 smithi139 bash[43301]: cluster 2022-04-28T20:32:05.996479+0000 mgr.y (mgr.24961) 1483 : cluster [DBG] pgmap v1847: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:09 smithi059 bash[61152]: cluster 2022-04-28T20:32:07.997333+0000 mgr.y (mgr.24961) 1484 : cluster [DBG] pgmap v1848: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:09 smithi059 bash[59745]: cluster 2022-04-28T20:32:07.997333+0000 mgr.y (mgr.24961) 1484 : cluster [DBG] pgmap v1848: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:09.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:09 smithi139 bash[43301]: cluster 2022-04-28T20:32:07.997333+0000 mgr.y (mgr.24961) 1484 : cluster [DBG] pgmap v1848: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:11 smithi059 bash[61152]: cluster 2022-04-28T20:32:09.998376+0000 mgr.y (mgr.24961) 1485 : cluster [DBG] pgmap v1849: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:11.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:11.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:11 smithi059 bash[59745]: cluster 2022-04-28T20:32:09.998376+0000 mgr.y (mgr.24961) 1485 : cluster [DBG] pgmap v1849: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:11.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:11 smithi139 bash[43301]: cluster 2022-04-28T20:32:09.998376+0000 mgr.y (mgr.24961) 1485 : cluster [DBG] pgmap v1849: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:13 smithi059 bash[61152]: cluster 2022-04-28T20:32:11.999464+0000 mgr.y (mgr.24961) 1486 : cluster [DBG] pgmap v1850: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:13.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:13.675Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:13 smithi059 bash[59745]: cluster 2022-04-28T20:32:11.999464+0000 mgr.y (mgr.24961) 1486 : cluster [DBG] pgmap v1850: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:13 smithi139 bash[43301]: cluster 2022-04-28T20:32:11.999464+0000 mgr.y (mgr.24961) 1486 : cluster [DBG] pgmap v1850: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:15 smithi059 bash[61152]: cluster 2022-04-28T20:32:14.000319+0000 mgr.y (mgr.24961) 1487 : cluster [DBG] pgmap v1851: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:15 smithi059 bash[59745]: cluster 2022-04-28T20:32:14.000319+0000 mgr.y (mgr.24961) 1487 : cluster [DBG] pgmap v1851: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:15 smithi139 bash[43301]: cluster 2022-04-28T20:32:14.000319+0000 mgr.y (mgr.24961) 1487 : cluster [DBG] pgmap v1851: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:17 smithi139 bash[43301]: cluster 2022-04-28T20:32:16.001498+0000 mgr.y (mgr.24961) 1488 : cluster [DBG] pgmap v1852: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:17 smithi059 bash[61152]: cluster 2022-04-28T20:32:16.001498+0000 mgr.y (mgr.24961) 1488 : cluster [DBG] pgmap v1852: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:17 smithi059 bash[59745]: cluster 2022-04-28T20:32:16.001498+0000 mgr.y (mgr.24961) 1488 : cluster [DBG] pgmap v1852: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:18 smithi139 bash[43301]: audit 2022-04-28T20:32:17.609032+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:32:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:18 smithi139 bash[43301]: audit 2022-04-28T20:32:17.609499+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:32:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:18 smithi139 bash[43301]: audit 2022-04-28T20:32:17.621743+0000 mon.a (mon.0) 2035 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:32:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:18 smithi139 bash[43301]: audit 2022-04-28T20:32:17.622171+0000 mon.a (mon.0) 2036 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:32:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:18 smithi139 bash[43301]: audit 2022-04-28T20:32:17.622615+0000 mon.a (mon.0) 2037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[59745]: audit 2022-04-28T20:32:17.609032+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:32:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[59745]: audit 2022-04-28T20:32:17.609499+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:32:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[59745]: audit 2022-04-28T20:32:17.621743+0000 mon.a (mon.0) 2035 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[59745]: audit 2022-04-28T20:32:17.622171+0000 mon.a (mon.0) 2036 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[59745]: audit 2022-04-28T20:32:17.622615+0000 mon.a (mon.0) 2037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[61152]: audit 2022-04-28T20:32:17.609032+0000 mon.a (mon.0) 2033 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:32:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[61152]: audit 2022-04-28T20:32:17.609499+0000 mon.a (mon.0) 2034 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:32:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[61152]: audit 2022-04-28T20:32:17.621743+0000 mon.a (mon.0) 2035 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[61152]: audit 2022-04-28T20:32:17.622171+0000 mon.a (mon.0) 2036 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:18 smithi059 bash[61152]: audit 2022-04-28T20:32:17.622615+0000 mon.a (mon.0) 2037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:32:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:19 smithi139 bash[43301]: cluster 2022-04-28T20:32:18.002307+0000 mgr.y (mgr.24961) 1489 : cluster [DBG] pgmap v1853: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:19 smithi059 bash[59745]: cluster 2022-04-28T20:32:18.002307+0000 mgr.y (mgr.24961) 1489 : cluster [DBG] pgmap v1853: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:19 smithi059 bash[61152]: cluster 2022-04-28T20:32:18.002307+0000 mgr.y (mgr.24961) 1489 : cluster [DBG] pgmap v1853: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:21 smithi139 bash[43301]: cluster 2022-04-28T20:32:20.003349+0000 mgr.y (mgr.24961) 1490 : cluster [DBG] pgmap v1854: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:21 smithi059 bash[61152]: cluster 2022-04-28T20:32:20.003349+0000 mgr.y (mgr.24961) 1490 : cluster [DBG] pgmap v1854: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:21.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:21 smithi059 bash[59745]: cluster 2022-04-28T20:32:20.003349+0000 mgr.y (mgr.24961) 1490 : cluster [DBG] pgmap v1854: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:23 smithi139 bash[43301]: cluster 2022-04-28T20:32:22.004519+0000 mgr.y (mgr.24961) 1491 : cluster [DBG] pgmap v1855: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:23 smithi059 bash[61152]: cluster 2022-04-28T20:32:22.004519+0000 mgr.y (mgr.24961) 1491 : cluster [DBG] pgmap v1855: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:24.187 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:23.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:24.217 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:23 smithi059 bash[59745]: cluster 2022-04-28T20:32:22.004519+0000 mgr.y (mgr.24961) 1491 : cluster [DBG] pgmap v1855: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:25 smithi059 bash[61152]: cluster 2022-04-28T20:32:24.005363+0000 mgr.y (mgr.24961) 1492 : cluster [DBG] pgmap v1856: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:25 smithi059 bash[59745]: cluster 2022-04-28T20:32:24.005363+0000 mgr.y (mgr.24961) 1492 : cluster [DBG] pgmap v1856: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:25 smithi139 bash[43301]: cluster 2022-04-28T20:32:24.005363+0000 mgr.y (mgr.24961) 1492 : cluster [DBG] pgmap v1856: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:27 smithi059 bash[61152]: cluster 2022-04-28T20:32:26.006611+0000 mgr.y (mgr.24961) 1493 : cluster [DBG] pgmap v1857: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:27 smithi059 bash[59745]: cluster 2022-04-28T20:32:26.006611+0000 mgr.y (mgr.24961) 1493 : cluster [DBG] pgmap v1857: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:27 smithi139 bash[43301]: cluster 2022-04-28T20:32:26.006611+0000 mgr.y (mgr.24961) 1493 : cluster [DBG] pgmap v1857: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:29 smithi059 bash[61152]: cluster 2022-04-28T20:32:28.007300+0000 mgr.y (mgr.24961) 1494 : cluster [DBG] pgmap v1858: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:29 smithi059 bash[59745]: cluster 2022-04-28T20:32:28.007300+0000 mgr.y (mgr.24961) 1494 : cluster [DBG] pgmap v1858: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:29 smithi139 bash[43301]: cluster 2022-04-28T20:32:28.007300+0000 mgr.y (mgr.24961) 1494 : cluster [DBG] pgmap v1858: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:31.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:32.131 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[59745]: cluster 2022-04-28T20:32:30.008355+0000 mgr.y (mgr.24961) 1495 : cluster [DBG] pgmap v1859: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:32.131 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[59745]: audit 2022-04-28T20:32:30.996630+0000 mon.a (mon.0) 2038 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:32:32.132 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[59745]: audit 2022-04-28T20:32:30.999612+0000 mon.a (mon.0) 2039 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:32:32.132 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[59745]: audit 2022-04-28T20:32:31.001616+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:32:32.132 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[59745]: audit 2022-04-28T20:32:31.325572+0000 mon.a (mon.0) 2041 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:32:32.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[61152]: cluster 2022-04-28T20:32:30.008355+0000 mgr.y (mgr.24961) 1495 : cluster [DBG] pgmap v1859: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:32.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[61152]: audit 2022-04-28T20:32:30.996630+0000 mon.a (mon.0) 2038 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:32:32.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[61152]: audit 2022-04-28T20:32:30.999612+0000 mon.a (mon.0) 2039 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:32:32.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[61152]: audit 2022-04-28T20:32:31.001616+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:32:32.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:31 smithi059 bash[61152]: audit 2022-04-28T20:32:31.325572+0000 mon.a (mon.0) 2041 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:32:32.134 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:31 smithi139 bash[43301]: cluster 2022-04-28T20:32:30.008355+0000 mgr.y (mgr.24961) 1495 : cluster [DBG] pgmap v1859: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:32.135 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:31 smithi139 bash[43301]: audit 2022-04-28T20:32:30.996630+0000 mon.a (mon.0) 2038 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:32:32.135 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:31 smithi139 bash[43301]: audit 2022-04-28T20:32:30.999612+0000 mon.a (mon.0) 2039 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:32:32.135 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:31 smithi139 bash[43301]: audit 2022-04-28T20:32:31.001616+0000 mon.a (mon.0) 2040 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:32:32.135 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:31 smithi139 bash[43301]: audit 2022-04-28T20:32:31.325572+0000 mon.a (mon.0) 2041 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:32:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:32 smithi059 bash[61152]: cephadm 2022-04-28T20:32:31.003471+0000 mgr.y (mgr.24961) 1496 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:32:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:32 smithi059 bash[59745]: cephadm 2022-04-28T20:32:31.003471+0000 mgr.y (mgr.24961) 1496 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:32:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:32 smithi139 bash[43301]: cephadm 2022-04-28T20:32:31.003471+0000 mgr.y (mgr.24961) 1496 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:32:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:33 smithi059 bash[61152]: cluster 2022-04-28T20:32:32.009425+0000 mgr.y (mgr.24961) 1497 : cluster [DBG] pgmap v1860: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:33.677Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:33 smithi059 bash[59745]: cluster 2022-04-28T20:32:32.009425+0000 mgr.y (mgr.24961) 1497 : cluster [DBG] pgmap v1860: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:33 smithi139 bash[43301]: cluster 2022-04-28T20:32:32.009425+0000 mgr.y (mgr.24961) 1497 : cluster [DBG] pgmap v1860: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:35 smithi059 bash[61152]: cluster 2022-04-28T20:32:34.010236+0000 mgr.y (mgr.24961) 1498 : cluster [DBG] pgmap v1861: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:35 smithi059 bash[59745]: cluster 2022-04-28T20:32:34.010236+0000 mgr.y (mgr.24961) 1498 : cluster [DBG] pgmap v1861: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:35 smithi139 bash[43301]: cluster 2022-04-28T20:32:34.010236+0000 mgr.y (mgr.24961) 1498 : cluster [DBG] pgmap v1861: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:37 smithi059 bash[59745]: cluster 2022-04-28T20:32:36.011521+0000 mgr.y (mgr.24961) 1499 : cluster [DBG] pgmap v1862: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:37 smithi059 bash[61152]: cluster 2022-04-28T20:32:36.011521+0000 mgr.y (mgr.24961) 1499 : cluster [DBG] pgmap v1862: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:37 smithi139 bash[43301]: cluster 2022-04-28T20:32:36.011521+0000 mgr.y (mgr.24961) 1499 : cluster [DBG] pgmap v1862: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:39 smithi059 bash[61152]: cluster 2022-04-28T20:32:38.012290+0000 mgr.y (mgr.24961) 1500 : cluster [DBG] pgmap v1863: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:39 smithi059 bash[59745]: cluster 2022-04-28T20:32:38.012290+0000 mgr.y (mgr.24961) 1500 : cluster [DBG] pgmap v1863: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:39 smithi139 bash[43301]: cluster 2022-04-28T20:32:38.012290+0000 mgr.y (mgr.24961) 1500 : cluster [DBG] pgmap v1863: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:41 smithi059 bash[61152]: cluster 2022-04-28T20:32:40.013386+0000 mgr.y (mgr.24961) 1501 : cluster [DBG] pgmap v1864: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:41.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:41 smithi059 bash[59745]: cluster 2022-04-28T20:32:40.013386+0000 mgr.y (mgr.24961) 1501 : cluster [DBG] pgmap v1864: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:41 smithi139 bash[43301]: cluster 2022-04-28T20:32:40.013386+0000 mgr.y (mgr.24961) 1501 : cluster [DBG] pgmap v1864: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:43 smithi059 bash[61152]: cluster 2022-04-28T20:32:42.014551+0000 mgr.y (mgr.24961) 1502 : cluster [DBG] pgmap v1865: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:43.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:43 smithi059 bash[59745]: cluster 2022-04-28T20:32:42.014551+0000 mgr.y (mgr.24961) 1502 : cluster [DBG] pgmap v1865: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:43 smithi139 bash[43301]: cluster 2022-04-28T20:32:42.014551+0000 mgr.y (mgr.24961) 1502 : cluster [DBG] pgmap v1865: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:45 smithi059 bash[59745]: cluster 2022-04-28T20:32:44.015393+0000 mgr.y (mgr.24961) 1503 : cluster [DBG] pgmap v1866: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:45 smithi059 bash[61152]: cluster 2022-04-28T20:32:44.015393+0000 mgr.y (mgr.24961) 1503 : cluster [DBG] pgmap v1866: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:45 smithi139 bash[43301]: cluster 2022-04-28T20:32:44.015393+0000 mgr.y (mgr.24961) 1503 : cluster [DBG] pgmap v1866: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:47 smithi059 bash[61152]: cluster 2022-04-28T20:32:46.016729+0000 mgr.y (mgr.24961) 1504 : cluster [DBG] pgmap v1867: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:47 smithi059 bash[59745]: cluster 2022-04-28T20:32:46.016729+0000 mgr.y (mgr.24961) 1504 : cluster [DBG] pgmap v1867: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:47 smithi139 bash[43301]: cluster 2022-04-28T20:32:46.016729+0000 mgr.y (mgr.24961) 1504 : cluster [DBG] pgmap v1867: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:49 smithi059 bash[61152]: cluster 2022-04-28T20:32:48.017457+0000 mgr.y (mgr.24961) 1505 : cluster [DBG] pgmap v1868: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:49 smithi059 bash[59745]: cluster 2022-04-28T20:32:48.017457+0000 mgr.y (mgr.24961) 1505 : cluster [DBG] pgmap v1868: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:49 smithi139 bash[43301]: cluster 2022-04-28T20:32:48.017457+0000 mgr.y (mgr.24961) 1505 : cluster [DBG] pgmap v1868: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:51 smithi059 bash[61152]: cluster 2022-04-28T20:32:50.018499+0000 mgr.y (mgr.24961) 1506 : cluster [DBG] pgmap v1869: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:52.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:51.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:51 smithi059 bash[59745]: cluster 2022-04-28T20:32:50.018499+0000 mgr.y (mgr.24961) 1506 : cluster [DBG] pgmap v1869: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:51 smithi139 bash[43301]: cluster 2022-04-28T20:32:50.018499+0000 mgr.y (mgr.24961) 1506 : cluster [DBG] pgmap v1869: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:53 smithi059 bash[61152]: cluster 2022-04-28T20:32:52.019607+0000 mgr.y (mgr.24961) 1507 : cluster [DBG] pgmap v1870: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:53 smithi059 bash[59745]: cluster 2022-04-28T20:32:52.019607+0000 mgr.y (mgr.24961) 1507 : cluster [DBG] pgmap v1870: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:32:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:32:53.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:32:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:53 smithi139 bash[43301]: cluster 2022-04-28T20:32:52.019607+0000 mgr.y (mgr.24961) 1507 : cluster [DBG] pgmap v1870: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:55 smithi059 bash[59745]: cluster 2022-04-28T20:32:54.020441+0000 mgr.y (mgr.24961) 1508 : cluster [DBG] pgmap v1871: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:56.177 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:55 smithi059 bash[61152]: cluster 2022-04-28T20:32:54.020441+0000 mgr.y (mgr.24961) 1508 : cluster [DBG] pgmap v1871: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:56.178 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:55 smithi139 bash[43301]: cluster 2022-04-28T20:32:54.020441+0000 mgr.y (mgr.24961) 1508 : cluster [DBG] pgmap v1871: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:57 smithi059 bash[61152]: cluster 2022-04-28T20:32:56.021700+0000 mgr.y (mgr.24961) 1509 : cluster [DBG] pgmap v1872: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:57 smithi059 bash[59745]: cluster 2022-04-28T20:32:56.021700+0000 mgr.y (mgr.24961) 1509 : cluster [DBG] pgmap v1872: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:32:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:57 smithi139 bash[43301]: cluster 2022-04-28T20:32:56.021700+0000 mgr.y (mgr.24961) 1509 : cluster [DBG] pgmap v1872: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:32:59 smithi059 bash[61152]: cluster 2022-04-28T20:32:58.022467+0000 mgr.y (mgr.24961) 1510 : cluster [DBG] pgmap v1873: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:32:59 smithi059 bash[59745]: cluster 2022-04-28T20:32:58.022467+0000 mgr.y (mgr.24961) 1510 : cluster [DBG] pgmap v1873: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:32:59 smithi139 bash[43301]: cluster 2022-04-28T20:32:58.022467+0000 mgr.y (mgr.24961) 1510 : cluster [DBG] pgmap v1873: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:01 smithi059 bash[61152]: cluster 2022-04-28T20:33:00.023684+0000 mgr.y (mgr.24961) 1511 : cluster [DBG] pgmap v1874: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:02.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:01.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:01 smithi059 bash[59745]: cluster 2022-04-28T20:33:00.023684+0000 mgr.y (mgr.24961) 1511 : cluster [DBG] pgmap v1874: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:01 smithi139 bash[43301]: cluster 2022-04-28T20:33:00.023684+0000 mgr.y (mgr.24961) 1511 : cluster [DBG] pgmap v1874: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:03 smithi059 bash[59745]: cluster 2022-04-28T20:33:02.024776+0000 mgr.y (mgr.24961) 1512 : cluster [DBG] pgmap v1875: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:03 smithi059 bash[61152]: cluster 2022-04-28T20:33:02.024776+0000 mgr.y (mgr.24961) 1512 : cluster [DBG] pgmap v1875: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:03.678Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:03 smithi139 bash[43301]: cluster 2022-04-28T20:33:02.024776+0000 mgr.y (mgr.24961) 1512 : cluster [DBG] pgmap v1875: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:05 smithi059 bash[59745]: cluster 2022-04-28T20:33:04.025588+0000 mgr.y (mgr.24961) 1513 : cluster [DBG] pgmap v1876: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:05 smithi059 bash[61152]: cluster 2022-04-28T20:33:04.025588+0000 mgr.y (mgr.24961) 1513 : cluster [DBG] pgmap v1876: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:05 smithi139 bash[43301]: cluster 2022-04-28T20:33:04.025588+0000 mgr.y (mgr.24961) 1513 : cluster [DBG] pgmap v1876: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:07 smithi059 bash[61152]: cluster 2022-04-28T20:33:06.026803+0000 mgr.y (mgr.24961) 1514 : cluster [DBG] pgmap v1877: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:07 smithi059 bash[59745]: cluster 2022-04-28T20:33:06.026803+0000 mgr.y (mgr.24961) 1514 : cluster [DBG] pgmap v1877: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:07 smithi139 bash[43301]: cluster 2022-04-28T20:33:06.026803+0000 mgr.y (mgr.24961) 1514 : cluster [DBG] pgmap v1877: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:09 smithi059 bash[61152]: cluster 2022-04-28T20:33:08.027514+0000 mgr.y (mgr.24961) 1515 : cluster [DBG] pgmap v1878: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:09 smithi059 bash[59745]: cluster 2022-04-28T20:33:08.027514+0000 mgr.y (mgr.24961) 1515 : cluster [DBG] pgmap v1878: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:09 smithi139 bash[43301]: cluster 2022-04-28T20:33:08.027514+0000 mgr.y (mgr.24961) 1515 : cluster [DBG] pgmap v1878: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:11 smithi059 bash[59745]: cluster 2022-04-28T20:33:10.028489+0000 mgr.y (mgr.24961) 1516 : cluster [DBG] pgmap v1879: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:11 smithi059 bash[61152]: cluster 2022-04-28T20:33:10.028489+0000 mgr.y (mgr.24961) 1516 : cluster [DBG] pgmap v1879: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:11.788Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:11 smithi139 bash[43301]: cluster 2022-04-28T20:33:10.028489+0000 mgr.y (mgr.24961) 1516 : cluster [DBG] pgmap v1879: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:13 smithi059 bash[61152]: cluster 2022-04-28T20:33:12.029637+0000 mgr.y (mgr.24961) 1517 : cluster [DBG] pgmap v1880: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:13.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:13 smithi059 bash[59745]: cluster 2022-04-28T20:33:12.029637+0000 mgr.y (mgr.24961) 1517 : cluster [DBG] pgmap v1880: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:13 smithi139 bash[43301]: cluster 2022-04-28T20:33:12.029637+0000 mgr.y (mgr.24961) 1517 : cluster [DBG] pgmap v1880: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:15 smithi059 bash[61152]: cluster 2022-04-28T20:33:14.030484+0000 mgr.y (mgr.24961) 1518 : cluster [DBG] pgmap v1881: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:15 smithi059 bash[59745]: cluster 2022-04-28T20:33:14.030484+0000 mgr.y (mgr.24961) 1518 : cluster [DBG] pgmap v1881: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:15 smithi139 bash[43301]: cluster 2022-04-28T20:33:14.030484+0000 mgr.y (mgr.24961) 1518 : cluster [DBG] pgmap v1881: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[61152]: cluster 2022-04-28T20:33:16.031765+0000 mgr.y (mgr.24961) 1519 : cluster [DBG] pgmap v1882: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[61152]: audit 2022-04-28T20:33:17.617698+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:33:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[61152]: audit 2022-04-28T20:33:17.618160+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:33:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[61152]: audit 2022-04-28T20:33:17.629090+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[61152]: audit 2022-04-28T20:33:17.629457+0000 mon.a (mon.0) 2045 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[61152]: audit 2022-04-28T20:33:17.629900+0000 mon.a (mon.0) 2046 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[59745]: cluster 2022-04-28T20:33:16.031765+0000 mgr.y (mgr.24961) 1519 : cluster [DBG] pgmap v1882: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[59745]: audit 2022-04-28T20:33:17.617698+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:33:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[59745]: audit 2022-04-28T20:33:17.618160+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:33:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[59745]: audit 2022-04-28T20:33:17.629090+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[59745]: audit 2022-04-28T20:33:17.629457+0000 mon.a (mon.0) 2045 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:17 smithi059 bash[59745]: audit 2022-04-28T20:33:17.629900+0000 mon.a (mon.0) 2046 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:17 smithi139 bash[43301]: cluster 2022-04-28T20:33:16.031765+0000 mgr.y (mgr.24961) 1519 : cluster [DBG] pgmap v1882: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:17 smithi139 bash[43301]: audit 2022-04-28T20:33:17.617698+0000 mon.a (mon.0) 2042 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:33:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:17 smithi139 bash[43301]: audit 2022-04-28T20:33:17.618160+0000 mon.a (mon.0) 2043 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:33:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:17 smithi139 bash[43301]: audit 2022-04-28T20:33:17.629090+0000 mon.a (mon.0) 2044 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:17 smithi139 bash[43301]: audit 2022-04-28T20:33:17.629457+0000 mon.a (mon.0) 2045 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:33:18.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:17 smithi139 bash[43301]: audit 2022-04-28T20:33:17.629900+0000 mon.a (mon.0) 2046 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:33:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:19 smithi059 bash[59745]: cluster 2022-04-28T20:33:18.032495+0000 mgr.y (mgr.24961) 1520 : cluster [DBG] pgmap v1883: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:19 smithi059 bash[61152]: cluster 2022-04-28T20:33:18.032495+0000 mgr.y (mgr.24961) 1520 : cluster [DBG] pgmap v1883: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:19 smithi139 bash[43301]: cluster 2022-04-28T20:33:18.032495+0000 mgr.y (mgr.24961) 1520 : cluster [DBG] pgmap v1883: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:21 smithi059 bash[61152]: cluster 2022-04-28T20:33:20.033638+0000 mgr.y (mgr.24961) 1521 : cluster [DBG] pgmap v1884: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:21.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:21 smithi059 bash[59745]: cluster 2022-04-28T20:33:20.033638+0000 mgr.y (mgr.24961) 1521 : cluster [DBG] pgmap v1884: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:21 smithi139 bash[43301]: cluster 2022-04-28T20:33:20.033638+0000 mgr.y (mgr.24961) 1521 : cluster [DBG] pgmap v1884: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:23.789 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:23.679Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:23 smithi059 bash[59745]: cluster 2022-04-28T20:33:22.034840+0000 mgr.y (mgr.24961) 1522 : cluster [DBG] pgmap v1885: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:23 smithi059 bash[61152]: cluster 2022-04-28T20:33:22.034840+0000 mgr.y (mgr.24961) 1522 : cluster [DBG] pgmap v1885: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:23 smithi139 bash[43301]: cluster 2022-04-28T20:33:22.034840+0000 mgr.y (mgr.24961) 1522 : cluster [DBG] pgmap v1885: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:25 smithi059 bash[59745]: cluster 2022-04-28T20:33:24.035702+0000 mgr.y (mgr.24961) 1523 : cluster [DBG] pgmap v1886: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:25 smithi059 bash[61152]: cluster 2022-04-28T20:33:24.035702+0000 mgr.y (mgr.24961) 1523 : cluster [DBG] pgmap v1886: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:25 smithi139 bash[43301]: cluster 2022-04-28T20:33:24.035702+0000 mgr.y (mgr.24961) 1523 : cluster [DBG] pgmap v1886: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:27 smithi059 bash[61152]: cluster 2022-04-28T20:33:26.036975+0000 mgr.y (mgr.24961) 1524 : cluster [DBG] pgmap v1887: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:27 smithi059 bash[59745]: cluster 2022-04-28T20:33:26.036975+0000 mgr.y (mgr.24961) 1524 : cluster [DBG] pgmap v1887: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:27 smithi139 bash[43301]: cluster 2022-04-28T20:33:26.036975+0000 mgr.y (mgr.24961) 1524 : cluster [DBG] pgmap v1887: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:29 smithi059 bash[59745]: cluster 2022-04-28T20:33:28.037645+0000 mgr.y (mgr.24961) 1525 : cluster [DBG] pgmap v1888: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:29 smithi059 bash[61152]: cluster 2022-04-28T20:33:28.037645+0000 mgr.y (mgr.24961) 1525 : cluster [DBG] pgmap v1888: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:29 smithi139 bash[43301]: cluster 2022-04-28T20:33:28.037645+0000 mgr.y (mgr.24961) 1525 : cluster [DBG] pgmap v1888: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[61152]: cluster 2022-04-28T20:33:30.038682+0000 mgr.y (mgr.24961) 1526 : cluster [DBG] pgmap v1889: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[61152]: audit 2022-04-28T20:33:31.334044+0000 mon.a (mon.0) 2047 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:33:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[61152]: audit 2022-04-28T20:33:31.337003+0000 mon.a (mon.0) 2048 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:33:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[61152]: audit 2022-04-28T20:33:31.338942+0000 mon.a (mon.0) 2049 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:33:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[61152]: audit 2022-04-28T20:33:31.668056+0000 mon.a (mon.0) 2050 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:33:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[59745]: cluster 2022-04-28T20:33:30.038682+0000 mgr.y (mgr.24961) 1526 : cluster [DBG] pgmap v1889: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[59745]: audit 2022-04-28T20:33:31.334044+0000 mon.a (mon.0) 2047 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:33:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[59745]: audit 2022-04-28T20:33:31.337003+0000 mon.a (mon.0) 2048 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:33:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[59745]: audit 2022-04-28T20:33:31.338942+0000 mon.a (mon.0) 2049 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:33:32.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[59745]: audit 2022-04-28T20:33:31.668056+0000 mon.a (mon.0) 2050 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:33:32.094 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:31.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:31 smithi139 bash[43301]: cluster 2022-04-28T20:33:30.038682+0000 mgr.y (mgr.24961) 1526 : cluster [DBG] pgmap v1889: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:31 smithi139 bash[43301]: audit 2022-04-28T20:33:31.334044+0000 mon.a (mon.0) 2047 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:33:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:31 smithi139 bash[43301]: audit 2022-04-28T20:33:31.337003+0000 mon.a (mon.0) 2048 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:33:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:31 smithi139 bash[43301]: audit 2022-04-28T20:33:31.338942+0000 mon.a (mon.0) 2049 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:33:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:31 smithi139 bash[43301]: audit 2022-04-28T20:33:31.668056+0000 mon.a (mon.0) 2050 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:33:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:32 smithi059 bash[61152]: cephadm 2022-04-28T20:33:31.340685+0000 mgr.y (mgr.24961) 1527 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:33:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:32 smithi059 bash[59745]: cephadm 2022-04-28T20:33:31.340685+0000 mgr.y (mgr.24961) 1527 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:33:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:32 smithi139 bash[43301]: cephadm 2022-04-28T20:33:31.340685+0000 mgr.y (mgr.24961) 1527 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:33:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:33.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:33 smithi059 bash[59745]: cluster 2022-04-28T20:33:32.039714+0000 mgr.y (mgr.24961) 1528 : cluster [DBG] pgmap v1890: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:34.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:33 smithi059 bash[61152]: cluster 2022-04-28T20:33:32.039714+0000 mgr.y (mgr.24961) 1528 : cluster [DBG] pgmap v1890: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:33 smithi139 bash[43301]: cluster 2022-04-28T20:33:32.039714+0000 mgr.y (mgr.24961) 1528 : cluster [DBG] pgmap v1890: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:35 smithi059 bash[59745]: cluster 2022-04-28T20:33:34.040580+0000 mgr.y (mgr.24961) 1529 : cluster [DBG] pgmap v1891: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:36.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:35 smithi059 bash[61152]: cluster 2022-04-28T20:33:34.040580+0000 mgr.y (mgr.24961) 1529 : cluster [DBG] pgmap v1891: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:35 smithi139 bash[43301]: cluster 2022-04-28T20:33:34.040580+0000 mgr.y (mgr.24961) 1529 : cluster [DBG] pgmap v1891: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:37 smithi059 bash[61152]: cluster 2022-04-28T20:33:36.041854+0000 mgr.y (mgr.24961) 1530 : cluster [DBG] pgmap v1892: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:37 smithi059 bash[59745]: cluster 2022-04-28T20:33:36.041854+0000 mgr.y (mgr.24961) 1530 : cluster [DBG] pgmap v1892: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:37 smithi139 bash[43301]: cluster 2022-04-28T20:33:36.041854+0000 mgr.y (mgr.24961) 1530 : cluster [DBG] pgmap v1892: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:39 smithi059 bash[61152]: cluster 2022-04-28T20:33:38.042564+0000 mgr.y (mgr.24961) 1531 : cluster [DBG] pgmap v1893: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:39 smithi059 bash[59745]: cluster 2022-04-28T20:33:38.042564+0000 mgr.y (mgr.24961) 1531 : cluster [DBG] pgmap v1893: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:39.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:39 smithi139 bash[43301]: cluster 2022-04-28T20:33:38.042564+0000 mgr.y (mgr.24961) 1531 : cluster [DBG] pgmap v1893: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:41 smithi059 bash[61152]: cluster 2022-04-28T20:33:40.043701+0000 mgr.y (mgr.24961) 1532 : cluster [DBG] pgmap v1894: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:41.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:41.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:41.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:41 smithi059 bash[59745]: cluster 2022-04-28T20:33:40.043701+0000 mgr.y (mgr.24961) 1532 : cluster [DBG] pgmap v1894: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:41 smithi139 bash[43301]: cluster 2022-04-28T20:33:40.043701+0000 mgr.y (mgr.24961) 1532 : cluster [DBG] pgmap v1894: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:43 smithi139 bash[43301]: cluster 2022-04-28T20:33:42.044830+0000 mgr.y (mgr.24961) 1533 : cluster [DBG] pgmap v1895: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:43 smithi059 bash[61152]: cluster 2022-04-28T20:33:42.044830+0000 mgr.y (mgr.24961) 1533 : cluster [DBG] pgmap v1895: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:43.680Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:43 smithi059 bash[59745]: cluster 2022-04-28T20:33:42.044830+0000 mgr.y (mgr.24961) 1533 : cluster [DBG] pgmap v1895: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:45 smithi139 bash[43301]: cluster 2022-04-28T20:33:44.045764+0000 mgr.y (mgr.24961) 1534 : cluster [DBG] pgmap v1896: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:45 smithi059 bash[61152]: cluster 2022-04-28T20:33:44.045764+0000 mgr.y (mgr.24961) 1534 : cluster [DBG] pgmap v1896: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:45 smithi059 bash[59745]: cluster 2022-04-28T20:33:44.045764+0000 mgr.y (mgr.24961) 1534 : cluster [DBG] pgmap v1896: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:47 smithi139 bash[43301]: cluster 2022-04-28T20:33:46.046925+0000 mgr.y (mgr.24961) 1535 : cluster [DBG] pgmap v1897: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:47 smithi059 bash[61152]: cluster 2022-04-28T20:33:46.046925+0000 mgr.y (mgr.24961) 1535 : cluster [DBG] pgmap v1897: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:47 smithi059 bash[59745]: cluster 2022-04-28T20:33:46.046925+0000 mgr.y (mgr.24961) 1535 : cluster [DBG] pgmap v1897: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:49 smithi139 bash[43301]: cluster 2022-04-28T20:33:48.047638+0000 mgr.y (mgr.24961) 1536 : cluster [DBG] pgmap v1898: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:49 smithi059 bash[61152]: cluster 2022-04-28T20:33:48.047638+0000 mgr.y (mgr.24961) 1536 : cluster [DBG] pgmap v1898: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:49 smithi059 bash[59745]: cluster 2022-04-28T20:33:48.047638+0000 mgr.y (mgr.24961) 1536 : cluster [DBG] pgmap v1898: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:51 smithi139 bash[43301]: cluster 2022-04-28T20:33:50.048680+0000 mgr.y (mgr.24961) 1537 : cluster [DBG] pgmap v1899: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:51 smithi059 bash[61152]: cluster 2022-04-28T20:33:50.048680+0000 mgr.y (mgr.24961) 1537 : cluster [DBG] pgmap v1899: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:52.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:51.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:52.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:51 smithi059 bash[59745]: cluster 2022-04-28T20:33:50.048680+0000 mgr.y (mgr.24961) 1537 : cluster [DBG] pgmap v1899: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:53 smithi059 bash[61152]: cluster 2022-04-28T20:33:52.049692+0000 mgr.y (mgr.24961) 1538 : cluster [DBG] pgmap v1900: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:33:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:33:53.682Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:33:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:53 smithi059 bash[59745]: cluster 2022-04-28T20:33:52.049692+0000 mgr.y (mgr.24961) 1538 : cluster [DBG] pgmap v1900: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:53 smithi139 bash[43301]: cluster 2022-04-28T20:33:52.049692+0000 mgr.y (mgr.24961) 1538 : cluster [DBG] pgmap v1900: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:55 smithi059 bash[61152]: cluster 2022-04-28T20:33:54.050617+0000 mgr.y (mgr.24961) 1539 : cluster [DBG] pgmap v1901: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:56.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:55 smithi059 bash[59745]: cluster 2022-04-28T20:33:54.050617+0000 mgr.y (mgr.24961) 1539 : cluster [DBG] pgmap v1901: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:56.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:55 smithi139 bash[43301]: cluster 2022-04-28T20:33:54.050617+0000 mgr.y (mgr.24961) 1539 : cluster [DBG] pgmap v1901: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:57 smithi059 bash[61152]: cluster 2022-04-28T20:33:56.051819+0000 mgr.y (mgr.24961) 1540 : cluster [DBG] pgmap v1902: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:57 smithi059 bash[59745]: cluster 2022-04-28T20:33:56.051819+0000 mgr.y (mgr.24961) 1540 : cluster [DBG] pgmap v1902: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:33:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:57 smithi139 bash[43301]: cluster 2022-04-28T20:33:56.051819+0000 mgr.y (mgr.24961) 1540 : cluster [DBG] pgmap v1902: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:33:59 smithi059 bash[61152]: cluster 2022-04-28T20:33:58.052605+0000 mgr.y (mgr.24961) 1541 : cluster [DBG] pgmap v1903: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:33:59 smithi059 bash[59745]: cluster 2022-04-28T20:33:58.052605+0000 mgr.y (mgr.24961) 1541 : cluster [DBG] pgmap v1903: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:33:59 smithi139 bash[43301]: cluster 2022-04-28T20:33:58.052605+0000 mgr.y (mgr.24961) 1541 : cluster [DBG] pgmap v1903: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:01 smithi059 bash[61152]: cluster 2022-04-28T20:34:00.053602+0000 mgr.y (mgr.24961) 1542 : cluster [DBG] pgmap v1904: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:01.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:01 smithi059 bash[59745]: cluster 2022-04-28T20:34:00.053602+0000 mgr.y (mgr.24961) 1542 : cluster [DBG] pgmap v1904: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:01 smithi139 bash[43301]: cluster 2022-04-28T20:34:00.053602+0000 mgr.y (mgr.24961) 1542 : cluster [DBG] pgmap v1904: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:03 smithi059 bash[61152]: cluster 2022-04-28T20:34:02.054638+0000 mgr.y (mgr.24961) 1543 : cluster [DBG] pgmap v1905: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:03.684Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:03 smithi059 bash[59745]: cluster 2022-04-28T20:34:02.054638+0000 mgr.y (mgr.24961) 1543 : cluster [DBG] pgmap v1905: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:03 smithi139 bash[43301]: cluster 2022-04-28T20:34:02.054638+0000 mgr.y (mgr.24961) 1543 : cluster [DBG] pgmap v1905: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:05 smithi059 bash[61152]: cluster 2022-04-28T20:34:04.055575+0000 mgr.y (mgr.24961) 1544 : cluster [DBG] pgmap v1906: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:05 smithi059 bash[59745]: cluster 2022-04-28T20:34:04.055575+0000 mgr.y (mgr.24961) 1544 : cluster [DBG] pgmap v1906: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:05 smithi139 bash[43301]: cluster 2022-04-28T20:34:04.055575+0000 mgr.y (mgr.24961) 1544 : cluster [DBG] pgmap v1906: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:07 smithi059 bash[61152]: cluster 2022-04-28T20:34:06.056723+0000 mgr.y (mgr.24961) 1545 : cluster [DBG] pgmap v1907: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:07 smithi059 bash[59745]: cluster 2022-04-28T20:34:06.056723+0000 mgr.y (mgr.24961) 1545 : cluster [DBG] pgmap v1907: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:07 smithi139 bash[43301]: cluster 2022-04-28T20:34:06.056723+0000 mgr.y (mgr.24961) 1545 : cluster [DBG] pgmap v1907: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:09 smithi059 bash[61152]: cluster 2022-04-28T20:34:08.057405+0000 mgr.y (mgr.24961) 1546 : cluster [DBG] pgmap v1908: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:09 smithi059 bash[59745]: cluster 2022-04-28T20:34:08.057405+0000 mgr.y (mgr.24961) 1546 : cluster [DBG] pgmap v1908: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:09 smithi139 bash[43301]: cluster 2022-04-28T20:34:08.057405+0000 mgr.y (mgr.24961) 1546 : cluster [DBG] pgmap v1908: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:11 smithi059 bash[61152]: cluster 2022-04-28T20:34:10.058441+0000 mgr.y (mgr.24961) 1547 : cluster [DBG] pgmap v1909: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:12.094 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:11.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:12.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:11 smithi059 bash[59745]: cluster 2022-04-28T20:34:10.058441+0000 mgr.y (mgr.24961) 1547 : cluster [DBG] pgmap v1909: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:11 smithi139 bash[43301]: cluster 2022-04-28T20:34:10.058441+0000 mgr.y (mgr.24961) 1547 : cluster [DBG] pgmap v1909: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:13.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:13 smithi059 bash[59745]: cluster 2022-04-28T20:34:12.059507+0000 mgr.y (mgr.24961) 1548 : cluster [DBG] pgmap v1910: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:13 smithi059 bash[61152]: cluster 2022-04-28T20:34:12.059507+0000 mgr.y (mgr.24961) 1548 : cluster [DBG] pgmap v1910: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:13 smithi139 bash[43301]: cluster 2022-04-28T20:34:12.059507+0000 mgr.y (mgr.24961) 1548 : cluster [DBG] pgmap v1910: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:15 smithi059 bash[61152]: cluster 2022-04-28T20:34:14.060436+0000 mgr.y (mgr.24961) 1549 : cluster [DBG] pgmap v1911: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:15 smithi059 bash[59745]: cluster 2022-04-28T20:34:14.060436+0000 mgr.y (mgr.24961) 1549 : cluster [DBG] pgmap v1911: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:15 smithi139 bash[43301]: cluster 2022-04-28T20:34:14.060436+0000 mgr.y (mgr.24961) 1549 : cluster [DBG] pgmap v1911: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[59745]: cluster 2022-04-28T20:34:16.061237+0000 mgr.y (mgr.24961) 1550 : cluster [DBG] pgmap v1912: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[59745]: cluster 2022-04-28T20:34:16.689791+0000 mon.a (mon.0) 2051 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2022-04-28T20:34:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[59745]: audit 2022-04-28T20:34:16.695957+0000 mon.c (mon.1) 30 : audit [INF] from='client.? 172.21.15.59:0/1718799588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:34:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[59745]: audit 2022-04-28T20:34:16.698265+0000 mon.a (mon.0) 2052 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:34:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[59745]: audit 2022-04-28T20:34:17.631681+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:34:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[59745]: audit 2022-04-28T20:34:17.632144+0000 mon.a (mon.0) 2054 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:34:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[61152]: cluster 2022-04-28T20:34:16.061237+0000 mgr.y (mgr.24961) 1550 : cluster [DBG] pgmap v1912: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[61152]: cluster 2022-04-28T20:34:16.689791+0000 mon.a (mon.0) 2051 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2022-04-28T20:34:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[61152]: audit 2022-04-28T20:34:16.695957+0000 mon.c (mon.1) 30 : audit [INF] from='client.? 172.21.15.59:0/1718799588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:34:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[61152]: audit 2022-04-28T20:34:16.698265+0000 mon.a (mon.0) 2052 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:34:18.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[61152]: audit 2022-04-28T20:34:17.631681+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:34:18.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:17 smithi059 bash[61152]: audit 2022-04-28T20:34:17.632144+0000 mon.a (mon.0) 2054 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:34:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:17 smithi139 bash[43301]: cluster 2022-04-28T20:34:16.061237+0000 mgr.y (mgr.24961) 1550 : cluster [DBG] pgmap v1912: 105 pgs: 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:17 smithi139 bash[43301]: cluster 2022-04-28T20:34:16.689791+0000 mon.a (mon.0) 2051 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2022-04-28T20:34:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:17 smithi139 bash[43301]: audit 2022-04-28T20:34:16.695957+0000 mon.c (mon.1) 30 : audit [INF] from='client.? 172.21.15.59:0/1718799588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:34:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:17 smithi139 bash[43301]: audit 2022-04-28T20:34:16.698265+0000 mon.a (mon.0) 2052 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:34:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:17 smithi139 bash[43301]: audit 2022-04-28T20:34:17.631681+0000 mon.a (mon.0) 2053 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:34:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:17 smithi139 bash[43301]: audit 2022-04-28T20:34:17.632144+0000 mon.a (mon.0) 2054 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:34:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:18 smithi059 bash[61152]: audit 2022-04-28T20:34:17.695985+0000 mon.a (mon.0) 2055 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:34:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:18 smithi059 bash[61152]: cluster 2022-04-28T20:34:17.696100+0000 mon.a (mon.0) 2056 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2022-04-28T20:34:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:18 smithi059 bash[59745]: audit 2022-04-28T20:34:17.695985+0000 mon.a (mon.0) 2055 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:34:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:18 smithi059 bash[59745]: cluster 2022-04-28T20:34:17.696100+0000 mon.a (mon.0) 2056 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2022-04-28T20:34:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:18 smithi139 bash[43301]: audit 2022-04-28T20:34:17.695985+0000 mon.a (mon.0) 2055 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-77440-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:34:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:18 smithi139 bash[43301]: cluster 2022-04-28T20:34:17.696100+0000 mon.a (mon.0) 2056 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2022-04-28T20:34:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:19 smithi059 bash[61152]: cluster 2022-04-28T20:34:18.061857+0000 mgr.y (mgr.24961) 1551 : cluster [DBG] pgmap v1915: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:19 smithi059 bash[61152]: cluster 2022-04-28T20:34:18.720595+0000 mon.a (mon.0) 2057 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2022-04-28T20:34:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:19 smithi059 bash[59745]: cluster 2022-04-28T20:34:18.061857+0000 mgr.y (mgr.24961) 1551 : cluster [DBG] pgmap v1915: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:19 smithi059 bash[59745]: cluster 2022-04-28T20:34:18.720595+0000 mon.a (mon.0) 2057 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2022-04-28T20:34:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:19 smithi139 bash[43301]: cluster 2022-04-28T20:34:18.061857+0000 mgr.y (mgr.24961) 1551 : cluster [DBG] pgmap v1915: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 708 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:19 smithi139 bash[43301]: cluster 2022-04-28T20:34:18.720595+0000 mon.a (mon.0) 2057 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2022-04-28T20:34:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:21 smithi059 bash[61152]: cluster 2022-04-28T20:34:20.062984+0000 mgr.y (mgr.24961) 1552 : cluster [DBG] pgmap v1917: 105 pgs: 105 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:21 smithi059 bash[59745]: cluster 2022-04-28T20:34:20.062984+0000 mgr.y (mgr.24961) 1552 : cluster [DBG] pgmap v1917: 105 pgs: 105 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:21.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:21 smithi139 bash[43301]: cluster 2022-04-28T20:34:20.062984+0000 mgr.y (mgr.24961) 1552 : cluster [DBG] pgmap v1917: 105 pgs: 105 active+clean; 24 KiB data, 709 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:23 smithi059 bash[59745]: cluster 2022-04-28T20:34:22.064221+0000 mgr.y (mgr.24961) 1553 : cluster [DBG] pgmap v1918: 105 pgs: 105 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:23 smithi059 bash[61152]: cluster 2022-04-28T20:34:22.064221+0000 mgr.y (mgr.24961) 1553 : cluster [DBG] pgmap v1918: 105 pgs: 105 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:23.686Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:23 smithi139 bash[43301]: cluster 2022-04-28T20:34:22.064221+0000 mgr.y (mgr.24961) 1553 : cluster [DBG] pgmap v1918: 105 pgs: 105 active+clean; 24 KiB data, 713 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:25 smithi059 bash[61152]: cluster 2022-04-28T20:34:24.065153+0000 mgr.y (mgr.24961) 1554 : cluster [DBG] pgmap v1919: 105 pgs: 105 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:25 smithi059 bash[59745]: cluster 2022-04-28T20:34:24.065153+0000 mgr.y (mgr.24961) 1554 : cluster [DBG] pgmap v1919: 105 pgs: 105 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:25 smithi139 bash[43301]: cluster 2022-04-28T20:34:24.065153+0000 mgr.y (mgr.24961) 1554 : cluster [DBG] pgmap v1919: 105 pgs: 105 active+clean; 24 KiB data, 717 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:27 smithi059 bash[61152]: cluster 2022-04-28T20:34:26.066333+0000 mgr.y (mgr.24961) 1555 : cluster [DBG] pgmap v1920: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:27 smithi059 bash[59745]: cluster 2022-04-28T20:34:26.066333+0000 mgr.y (mgr.24961) 1555 : cluster [DBG] pgmap v1920: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:27 smithi139 bash[43301]: cluster 2022-04-28T20:34:26.066333+0000 mgr.y (mgr.24961) 1555 : cluster [DBG] pgmap v1920: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:29 smithi059 bash[59745]: cluster 2022-04-28T20:34:28.067035+0000 mgr.y (mgr.24961) 1556 : cluster [DBG] pgmap v1921: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:29 smithi059 bash[61152]: cluster 2022-04-28T20:34:28.067035+0000 mgr.y (mgr.24961) 1556 : cluster [DBG] pgmap v1921: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:29 smithi139 bash[43301]: cluster 2022-04-28T20:34:28.067035+0000 mgr.y (mgr.24961) 1556 : cluster [DBG] pgmap v1921: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:31.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[61152]: cluster 2022-04-28T20:34:30.068121+0000 mgr.y (mgr.24961) 1557 : cluster [DBG] pgmap v1922: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[61152]: audit 2022-04-28T20:34:31.677024+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:34:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[61152]: audit 2022-04-28T20:34:31.680123+0000 mon.a (mon.0) 2059 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:34:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[61152]: audit 2022-04-28T20:34:31.681979+0000 mon.a (mon.0) 2060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:34:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[59745]: cluster 2022-04-28T20:34:30.068121+0000 mgr.y (mgr.24961) 1557 : cluster [DBG] pgmap v1922: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[59745]: audit 2022-04-28T20:34:31.677024+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:34:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[59745]: audit 2022-04-28T20:34:31.680123+0000 mon.a (mon.0) 2059 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:34:32.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:31 smithi059 bash[59745]: audit 2022-04-28T20:34:31.681979+0000 mon.a (mon.0) 2060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:34:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:31 smithi139 bash[43301]: cluster 2022-04-28T20:34:30.068121+0000 mgr.y (mgr.24961) 1557 : cluster [DBG] pgmap v1922: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:31 smithi139 bash[43301]: audit 2022-04-28T20:34:31.677024+0000 mon.a (mon.0) 2058 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:34:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:31 smithi139 bash[43301]: audit 2022-04-28T20:34:31.680123+0000 mon.a (mon.0) 2059 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:34:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:31 smithi139 bash[43301]: audit 2022-04-28T20:34:31.681979+0000 mon.a (mon.0) 2060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:34:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:32 smithi059 bash[59745]: cephadm 2022-04-28T20:34:31.683589+0000 mgr.y (mgr.24961) 1558 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:34:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:32 smithi059 bash[59745]: audit 2022-04-28T20:34:32.008244+0000 mon.a (mon.0) 2061 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:34:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:32 smithi059 bash[61152]: cephadm 2022-04-28T20:34:31.683589+0000 mgr.y (mgr.24961) 1558 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:34:33.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:32 smithi059 bash[61152]: audit 2022-04-28T20:34:32.008244+0000 mon.a (mon.0) 2061 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:34:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:32 smithi139 bash[43301]: cephadm 2022-04-28T20:34:31.683589+0000 mgr.y (mgr.24961) 1558 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:34:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:32 smithi139 bash[43301]: audit 2022-04-28T20:34:32.008244+0000 mon.a (mon.0) 2061 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:34:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:33 smithi059 bash[61152]: cluster 2022-04-28T20:34:32.069132+0000 mgr.y (mgr.24961) 1559 : cluster [DBG] pgmap v1923: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:33.683Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:33 smithi059 bash[59745]: cluster 2022-04-28T20:34:32.069132+0000 mgr.y (mgr.24961) 1559 : cluster [DBG] pgmap v1923: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:33 smithi139 bash[43301]: cluster 2022-04-28T20:34:32.069132+0000 mgr.y (mgr.24961) 1559 : cluster [DBG] pgmap v1923: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:35 smithi059 bash[61152]: cluster 2022-04-28T20:34:34.070036+0000 mgr.y (mgr.24961) 1560 : cluster [DBG] pgmap v1924: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:35 smithi059 bash[59745]: cluster 2022-04-28T20:34:34.070036+0000 mgr.y (mgr.24961) 1560 : cluster [DBG] pgmap v1924: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:35 smithi139 bash[43301]: cluster 2022-04-28T20:34:34.070036+0000 mgr.y (mgr.24961) 1560 : cluster [DBG] pgmap v1924: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:37 smithi059 bash[61152]: cluster 2022-04-28T20:34:36.071200+0000 mgr.y (mgr.24961) 1561 : cluster [DBG] pgmap v1925: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:37 smithi059 bash[59745]: cluster 2022-04-28T20:34:36.071200+0000 mgr.y (mgr.24961) 1561 : cluster [DBG] pgmap v1925: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:37 smithi139 bash[43301]: cluster 2022-04-28T20:34:36.071200+0000 mgr.y (mgr.24961) 1561 : cluster [DBG] pgmap v1925: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:39 smithi059 bash[61152]: cluster 2022-04-28T20:34:38.071871+0000 mgr.y (mgr.24961) 1562 : cluster [DBG] pgmap v1926: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:39 smithi059 bash[59745]: cluster 2022-04-28T20:34:38.071871+0000 mgr.y (mgr.24961) 1562 : cluster [DBG] pgmap v1926: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:39 smithi139 bash[43301]: cluster 2022-04-28T20:34:38.071871+0000 mgr.y (mgr.24961) 1562 : cluster [DBG] pgmap v1926: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:41 smithi059 bash[61152]: cluster 2022-04-28T20:34:40.072970+0000 mgr.y (mgr.24961) 1563 : cluster [DBG] pgmap v1927: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:42.111 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:41.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:42.112 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:41 smithi059 bash[59745]: cluster 2022-04-28T20:34:40.072970+0000 mgr.y (mgr.24961) 1563 : cluster [DBG] pgmap v1927: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:41 smithi139 bash[43301]: cluster 2022-04-28T20:34:40.072970+0000 mgr.y (mgr.24961) 1563 : cluster [DBG] pgmap v1927: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:43 smithi059 bash[61152]: cluster 2022-04-28T20:34:42.074022+0000 mgr.y (mgr.24961) 1564 : cluster [DBG] pgmap v1928: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:44.104 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:43.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:44.105 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:43 smithi059 bash[59745]: cluster 2022-04-28T20:34:42.074022+0000 mgr.y (mgr.24961) 1564 : cluster [DBG] pgmap v1928: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:43 smithi139 bash[43301]: cluster 2022-04-28T20:34:42.074022+0000 mgr.y (mgr.24961) 1564 : cluster [DBG] pgmap v1928: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:45 smithi059 bash[61152]: cluster 2022-04-28T20:34:44.074927+0000 mgr.y (mgr.24961) 1565 : cluster [DBG] pgmap v1929: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:45 smithi059 bash[59745]: cluster 2022-04-28T20:34:44.074927+0000 mgr.y (mgr.24961) 1565 : cluster [DBG] pgmap v1929: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:45 smithi139 bash[43301]: cluster 2022-04-28T20:34:44.074927+0000 mgr.y (mgr.24961) 1565 : cluster [DBG] pgmap v1929: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:47 smithi059 bash[61152]: cluster 2022-04-28T20:34:46.076227+0000 mgr.y (mgr.24961) 1566 : cluster [DBG] pgmap v1930: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:47 smithi059 bash[59745]: cluster 2022-04-28T20:34:46.076227+0000 mgr.y (mgr.24961) 1566 : cluster [DBG] pgmap v1930: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:47 smithi139 bash[43301]: cluster 2022-04-28T20:34:46.076227+0000 mgr.y (mgr.24961) 1566 : cluster [DBG] pgmap v1930: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:49 smithi059 bash[61152]: cluster 2022-04-28T20:34:48.076973+0000 mgr.y (mgr.24961) 1567 : cluster [DBG] pgmap v1931: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:49 smithi059 bash[59745]: cluster 2022-04-28T20:34:48.076973+0000 mgr.y (mgr.24961) 1567 : cluster [DBG] pgmap v1931: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:49 smithi139 bash[43301]: cluster 2022-04-28T20:34:48.076973+0000 mgr.y (mgr.24961) 1567 : cluster [DBG] pgmap v1931: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:51 smithi059 bash[59745]: cluster 2022-04-28T20:34:50.078072+0000 mgr.y (mgr.24961) 1568 : cluster [DBG] pgmap v1932: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:51 smithi059 bash[61152]: cluster 2022-04-28T20:34:50.078072+0000 mgr.y (mgr.24961) 1568 : cluster [DBG] pgmap v1932: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:52.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:51.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:51 smithi139 bash[43301]: cluster 2022-04-28T20:34:50.078072+0000 mgr.y (mgr.24961) 1568 : cluster [DBG] pgmap v1932: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:53 smithi059 bash[61152]: cluster 2022-04-28T20:34:52.079098+0000 mgr.y (mgr.24961) 1569 : cluster [DBG] pgmap v1933: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:34:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:34:53.690Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:34:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:53 smithi059 bash[59745]: cluster 2022-04-28T20:34:52.079098+0000 mgr.y (mgr.24961) 1569 : cluster [DBG] pgmap v1933: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:53 smithi139 bash[43301]: cluster 2022-04-28T20:34:52.079098+0000 mgr.y (mgr.24961) 1569 : cluster [DBG] pgmap v1933: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:55 smithi059 bash[61152]: cluster 2022-04-28T20:34:54.080047+0000 mgr.y (mgr.24961) 1570 : cluster [DBG] pgmap v1934: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:55 smithi059 bash[59745]: cluster 2022-04-28T20:34:54.080047+0000 mgr.y (mgr.24961) 1570 : cluster [DBG] pgmap v1934: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:55 smithi139 bash[43301]: cluster 2022-04-28T20:34:54.080047+0000 mgr.y (mgr.24961) 1570 : cluster [DBG] pgmap v1934: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:57 smithi059 bash[61152]: cluster 2022-04-28T20:34:56.081229+0000 mgr.y (mgr.24961) 1571 : cluster [DBG] pgmap v1935: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:57 smithi059 bash[59745]: cluster 2022-04-28T20:34:56.081229+0000 mgr.y (mgr.24961) 1571 : cluster [DBG] pgmap v1935: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:34:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:57 smithi139 bash[43301]: cluster 2022-04-28T20:34:56.081229+0000 mgr.y (mgr.24961) 1571 : cluster [DBG] pgmap v1935: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:34:59 smithi059 bash[61152]: cluster 2022-04-28T20:34:58.081914+0000 mgr.y (mgr.24961) 1572 : cluster [DBG] pgmap v1936: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:00.117 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:34:59 smithi059 bash[59745]: cluster 2022-04-28T20:34:58.081914+0000 mgr.y (mgr.24961) 1572 : cluster [DBG] pgmap v1936: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:34:59 smithi139 bash[43301]: cluster 2022-04-28T20:34:58.081914+0000 mgr.y (mgr.24961) 1572 : cluster [DBG] pgmap v1936: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:01 smithi059 bash[61152]: cluster 2022-04-28T20:35:00.082953+0000 mgr.y (mgr.24961) 1573 : cluster [DBG] pgmap v1937: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:02.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:01 smithi059 bash[59745]: cluster 2022-04-28T20:35:00.082953+0000 mgr.y (mgr.24961) 1573 : cluster [DBG] pgmap v1937: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:02.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:01.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:01 smithi139 bash[43301]: cluster 2022-04-28T20:35:00.082953+0000 mgr.y (mgr.24961) 1573 : cluster [DBG] pgmap v1937: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:03 smithi059 bash[61152]: cluster 2022-04-28T20:35:02.084006+0000 mgr.y (mgr.24961) 1574 : cluster [DBG] pgmap v1938: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:04.148 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:03.687Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:04.149 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:03 smithi059 bash[59745]: cluster 2022-04-28T20:35:02.084006+0000 mgr.y (mgr.24961) 1574 : cluster [DBG] pgmap v1938: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:04.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:03 smithi139 bash[43301]: cluster 2022-04-28T20:35:02.084006+0000 mgr.y (mgr.24961) 1574 : cluster [DBG] pgmap v1938: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:05 smithi139 bash[43301]: cluster 2022-04-28T20:35:04.084907+0000 mgr.y (mgr.24961) 1575 : cluster [DBG] pgmap v1939: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:05 smithi059 bash[61152]: cluster 2022-04-28T20:35:04.084907+0000 mgr.y (mgr.24961) 1575 : cluster [DBG] pgmap v1939: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:05 smithi059 bash[59745]: cluster 2022-04-28T20:35:04.084907+0000 mgr.y (mgr.24961) 1575 : cluster [DBG] pgmap v1939: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:07 smithi139 bash[43301]: cluster 2022-04-28T20:35:06.086082+0000 mgr.y (mgr.24961) 1576 : cluster [DBG] pgmap v1940: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:07 smithi059 bash[61152]: cluster 2022-04-28T20:35:06.086082+0000 mgr.y (mgr.24961) 1576 : cluster [DBG] pgmap v1940: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:08.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:07 smithi059 bash[59745]: cluster 2022-04-28T20:35:06.086082+0000 mgr.y (mgr.24961) 1576 : cluster [DBG] pgmap v1940: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:09.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:09 smithi139 bash[43301]: cluster 2022-04-28T20:35:08.086761+0000 mgr.y (mgr.24961) 1577 : cluster [DBG] pgmap v1941: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:09 smithi059 bash[61152]: cluster 2022-04-28T20:35:08.086761+0000 mgr.y (mgr.24961) 1577 : cluster [DBG] pgmap v1941: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:09 smithi059 bash[59745]: cluster 2022-04-28T20:35:08.086761+0000 mgr.y (mgr.24961) 1577 : cluster [DBG] pgmap v1941: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:11 smithi059 bash[61152]: cluster 2022-04-28T20:35:10.087855+0000 mgr.y (mgr.24961) 1578 : cluster [DBG] pgmap v1942: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:11.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:11 smithi059 bash[59745]: cluster 2022-04-28T20:35:10.087855+0000 mgr.y (mgr.24961) 1578 : cluster [DBG] pgmap v1942: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:11 smithi139 bash[43301]: cluster 2022-04-28T20:35:10.087855+0000 mgr.y (mgr.24961) 1578 : cluster [DBG] pgmap v1942: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:13 smithi059 bash[61152]: cluster 2022-04-28T20:35:12.088863+0000 mgr.y (mgr.24961) 1579 : cluster [DBG] pgmap v1943: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:13.688Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:13 smithi059 bash[59745]: cluster 2022-04-28T20:35:12.088863+0000 mgr.y (mgr.24961) 1579 : cluster [DBG] pgmap v1943: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:13 smithi139 bash[43301]: cluster 2022-04-28T20:35:12.088863+0000 mgr.y (mgr.24961) 1579 : cluster [DBG] pgmap v1943: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:15 smithi059 bash[61152]: cluster 2022-04-28T20:35:14.089786+0000 mgr.y (mgr.24961) 1580 : cluster [DBG] pgmap v1944: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:15 smithi059 bash[59745]: cluster 2022-04-28T20:35:14.089786+0000 mgr.y (mgr.24961) 1580 : cluster [DBG] pgmap v1944: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:15 smithi139 bash[43301]: cluster 2022-04-28T20:35:14.089786+0000 mgr.y (mgr.24961) 1580 : cluster [DBG] pgmap v1944: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:17 smithi059 bash[61152]: cluster 2022-04-28T20:35:16.091047+0000 mgr.y (mgr.24961) 1581 : cluster [DBG] pgmap v1945: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:17 smithi059 bash[59745]: cluster 2022-04-28T20:35:16.091047+0000 mgr.y (mgr.24961) 1581 : cluster [DBG] pgmap v1945: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:17 smithi139 bash[43301]: cluster 2022-04-28T20:35:16.091047+0000 mgr.y (mgr.24961) 1581 : cluster [DBG] pgmap v1945: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[61152]: audit 2022-04-28T20:35:17.646093+0000 mon.a (mon.0) 2062 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:35:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[61152]: audit 2022-04-28T20:35:17.646481+0000 mon.a (mon.0) 2063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:35:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[61152]: audit 2022-04-28T20:35:17.653100+0000 mon.a (mon.0) 2064 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:35:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[61152]: audit 2022-04-28T20:35:17.653508+0000 mon.a (mon.0) 2065 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:35:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[61152]: audit 2022-04-28T20:35:17.653941+0000 mon.a (mon.0) 2066 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:35:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[59745]: audit 2022-04-28T20:35:17.646093+0000 mon.a (mon.0) 2062 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:35:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[59745]: audit 2022-04-28T20:35:17.646481+0000 mon.a (mon.0) 2063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:35:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[59745]: audit 2022-04-28T20:35:17.653100+0000 mon.a (mon.0) 2064 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:35:19.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[59745]: audit 2022-04-28T20:35:17.653508+0000 mon.a (mon.0) 2065 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:35:19.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:18 smithi059 bash[59745]: audit 2022-04-28T20:35:17.653941+0000 mon.a (mon.0) 2066 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:35:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:18 smithi139 bash[43301]: audit 2022-04-28T20:35:17.646093+0000 mon.a (mon.0) 2062 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:35:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:18 smithi139 bash[43301]: audit 2022-04-28T20:35:17.646481+0000 mon.a (mon.0) 2063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:35:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:18 smithi139 bash[43301]: audit 2022-04-28T20:35:17.653100+0000 mon.a (mon.0) 2064 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:35:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:18 smithi139 bash[43301]: audit 2022-04-28T20:35:17.653508+0000 mon.a (mon.0) 2065 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:35:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:18 smithi139 bash[43301]: audit 2022-04-28T20:35:17.653941+0000 mon.a (mon.0) 2066 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:35:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:19 smithi059 bash[61152]: cluster 2022-04-28T20:35:18.092036+0000 mgr.y (mgr.24961) 1582 : cluster [DBG] pgmap v1946: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:19 smithi059 bash[59745]: cluster 2022-04-28T20:35:18.092036+0000 mgr.y (mgr.24961) 1582 : cluster [DBG] pgmap v1946: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:19 smithi139 bash[43301]: cluster 2022-04-28T20:35:18.092036+0000 mgr.y (mgr.24961) 1582 : cluster [DBG] pgmap v1946: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:21 smithi059 bash[61152]: cluster 2022-04-28T20:35:20.093172+0000 mgr.y (mgr.24961) 1583 : cluster [DBG] pgmap v1947: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:21.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:21 smithi059 bash[59745]: cluster 2022-04-28T20:35:20.093172+0000 mgr.y (mgr.24961) 1583 : cluster [DBG] pgmap v1947: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:21 smithi139 bash[43301]: cluster 2022-04-28T20:35:20.093172+0000 mgr.y (mgr.24961) 1583 : cluster [DBG] pgmap v1947: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:23 smithi059 bash[61152]: cluster 2022-04-28T20:35:22.094120+0000 mgr.y (mgr.24961) 1584 : cluster [DBG] pgmap v1948: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:23.689Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:23 smithi059 bash[59745]: cluster 2022-04-28T20:35:22.094120+0000 mgr.y (mgr.24961) 1584 : cluster [DBG] pgmap v1948: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:23 smithi139 bash[43301]: cluster 2022-04-28T20:35:22.094120+0000 mgr.y (mgr.24961) 1584 : cluster [DBG] pgmap v1948: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:25 smithi059 bash[59745]: cluster 2022-04-28T20:35:24.095121+0000 mgr.y (mgr.24961) 1585 : cluster [DBG] pgmap v1949: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:25 smithi059 bash[61152]: cluster 2022-04-28T20:35:24.095121+0000 mgr.y (mgr.24961) 1585 : cluster [DBG] pgmap v1949: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:25 smithi139 bash[43301]: cluster 2022-04-28T20:35:24.095121+0000 mgr.y (mgr.24961) 1585 : cluster [DBG] pgmap v1949: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:27 smithi059 bash[61152]: cluster 2022-04-28T20:35:26.096396+0000 mgr.y (mgr.24961) 1586 : cluster [DBG] pgmap v1950: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:27 smithi059 bash[59745]: cluster 2022-04-28T20:35:26.096396+0000 mgr.y (mgr.24961) 1586 : cluster [DBG] pgmap v1950: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:27 smithi139 bash[43301]: cluster 2022-04-28T20:35:26.096396+0000 mgr.y (mgr.24961) 1586 : cluster [DBG] pgmap v1950: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:29 smithi059 bash[61152]: cluster 2022-04-28T20:35:28.097127+0000 mgr.y (mgr.24961) 1587 : cluster [DBG] pgmap v1951: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:29 smithi059 bash[59745]: cluster 2022-04-28T20:35:28.097127+0000 mgr.y (mgr.24961) 1587 : cluster [DBG] pgmap v1951: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:29 smithi139 bash[43301]: cluster 2022-04-28T20:35:28.097127+0000 mgr.y (mgr.24961) 1587 : cluster [DBG] pgmap v1951: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:31 smithi059 bash[61152]: cluster 2022-04-28T20:35:30.098210+0000 mgr.y (mgr.24961) 1588 : cluster [DBG] pgmap v1952: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:31 smithi059 bash[59745]: cluster 2022-04-28T20:35:30.098210+0000 mgr.y (mgr.24961) 1588 : cluster [DBG] pgmap v1952: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:32.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:31.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:31 smithi139 bash[43301]: cluster 2022-04-28T20:35:30.098210+0000 mgr.y (mgr.24961) 1588 : cluster [DBG] pgmap v1952: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[59745]: audit 2022-04-28T20:35:32.014310+0000 mon.a (mon.0) 2067 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:35:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[59745]: audit 2022-04-28T20:35:32.017238+0000 mon.a (mon.0) 2068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:35:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[59745]: audit 2022-04-28T20:35:32.019163+0000 mon.a (mon.0) 2069 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:35:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[59745]: audit 2022-04-28T20:35:32.346316+0000 mon.a (mon.0) 2070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:35:33.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[61152]: audit 2022-04-28T20:35:32.014310+0000 mon.a (mon.0) 2067 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:35:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[61152]: audit 2022-04-28T20:35:32.017238+0000 mon.a (mon.0) 2068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:35:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[61152]: audit 2022-04-28T20:35:32.019163+0000 mon.a (mon.0) 2069 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:35:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:32 smithi059 bash[61152]: audit 2022-04-28T20:35:32.346316+0000 mon.a (mon.0) 2070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:35:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:32 smithi139 bash[43301]: audit 2022-04-28T20:35:32.014310+0000 mon.a (mon.0) 2067 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:35:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:32 smithi139 bash[43301]: audit 2022-04-28T20:35:32.017238+0000 mon.a (mon.0) 2068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:35:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:32 smithi139 bash[43301]: audit 2022-04-28T20:35:32.019163+0000 mon.a (mon.0) 2069 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:35:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:32 smithi139 bash[43301]: audit 2022-04-28T20:35:32.346316+0000 mon.a (mon.0) 2070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:35:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:33 smithi059 bash[61152]: cephadm 2022-04-28T20:35:32.020850+0000 mgr.y (mgr.24961) 1589 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:35:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:33 smithi059 bash[61152]: cluster 2022-04-28T20:35:32.099233+0000 mgr.y (mgr.24961) 1590 : cluster [DBG] pgmap v1953: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:33.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:33 smithi059 bash[59745]: cephadm 2022-04-28T20:35:32.020850+0000 mgr.y (mgr.24961) 1589 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:35:34.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:33 smithi059 bash[59745]: cluster 2022-04-28T20:35:32.099233+0000 mgr.y (mgr.24961) 1590 : cluster [DBG] pgmap v1953: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:33 smithi139 bash[43301]: cephadm 2022-04-28T20:35:32.020850+0000 mgr.y (mgr.24961) 1589 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:35:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:33 smithi139 bash[43301]: cluster 2022-04-28T20:35:32.099233+0000 mgr.y (mgr.24961) 1590 : cluster [DBG] pgmap v1953: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:35 smithi059 bash[61152]: cluster 2022-04-28T20:35:34.100279+0000 mgr.y (mgr.24961) 1591 : cluster [DBG] pgmap v1954: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:36.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:35 smithi059 bash[59745]: cluster 2022-04-28T20:35:34.100279+0000 mgr.y (mgr.24961) 1591 : cluster [DBG] pgmap v1954: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:35 smithi139 bash[43301]: cluster 2022-04-28T20:35:34.100279+0000 mgr.y (mgr.24961) 1591 : cluster [DBG] pgmap v1954: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:37 smithi059 bash[61152]: cluster 2022-04-28T20:35:36.101486+0000 mgr.y (mgr.24961) 1592 : cluster [DBG] pgmap v1955: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:37 smithi059 bash[59745]: cluster 2022-04-28T20:35:36.101486+0000 mgr.y (mgr.24961) 1592 : cluster [DBG] pgmap v1955: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:37 smithi139 bash[43301]: cluster 2022-04-28T20:35:36.101486+0000 mgr.y (mgr.24961) 1592 : cluster [DBG] pgmap v1955: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:39 smithi059 bash[61152]: cluster 2022-04-28T20:35:38.102185+0000 mgr.y (mgr.24961) 1593 : cluster [DBG] pgmap v1956: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:39 smithi059 bash[59745]: cluster 2022-04-28T20:35:38.102185+0000 mgr.y (mgr.24961) 1593 : cluster [DBG] pgmap v1956: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:39 smithi139 bash[43301]: cluster 2022-04-28T20:35:38.102185+0000 mgr.y (mgr.24961) 1593 : cluster [DBG] pgmap v1956: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:41 smithi059 bash[59745]: cluster 2022-04-28T20:35:40.103278+0000 mgr.y (mgr.24961) 1594 : cluster [DBG] pgmap v1957: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:41 smithi059 bash[61152]: cluster 2022-04-28T20:35:40.103278+0000 mgr.y (mgr.24961) 1594 : cluster [DBG] pgmap v1957: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:42.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:41.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:41 smithi139 bash[43301]: cluster 2022-04-28T20:35:40.103278+0000 mgr.y (mgr.24961) 1594 : cluster [DBG] pgmap v1957: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:43 smithi059 bash[61152]: cluster 2022-04-28T20:35:42.104297+0000 mgr.y (mgr.24961) 1595 : cluster [DBG] pgmap v1958: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:43.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:43 smithi059 bash[59745]: cluster 2022-04-28T20:35:42.104297+0000 mgr.y (mgr.24961) 1595 : cluster [DBG] pgmap v1958: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:43 smithi139 bash[43301]: cluster 2022-04-28T20:35:42.104297+0000 mgr.y (mgr.24961) 1595 : cluster [DBG] pgmap v1958: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:45 smithi059 bash[61152]: cluster 2022-04-28T20:35:44.105225+0000 mgr.y (mgr.24961) 1596 : cluster [DBG] pgmap v1959: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:45 smithi059 bash[59745]: cluster 2022-04-28T20:35:44.105225+0000 mgr.y (mgr.24961) 1596 : cluster [DBG] pgmap v1959: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:45 smithi139 bash[43301]: cluster 2022-04-28T20:35:44.105225+0000 mgr.y (mgr.24961) 1596 : cluster [DBG] pgmap v1959: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:47 smithi059 bash[61152]: cluster 2022-04-28T20:35:46.106434+0000 mgr.y (mgr.24961) 1597 : cluster [DBG] pgmap v1960: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:47 smithi059 bash[59745]: cluster 2022-04-28T20:35:46.106434+0000 mgr.y (mgr.24961) 1597 : cluster [DBG] pgmap v1960: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:47 smithi139 bash[43301]: cluster 2022-04-28T20:35:46.106434+0000 mgr.y (mgr.24961) 1597 : cluster [DBG] pgmap v1960: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:49 smithi059 bash[61152]: cluster 2022-04-28T20:35:48.107125+0000 mgr.y (mgr.24961) 1598 : cluster [DBG] pgmap v1961: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:49 smithi059 bash[59745]: cluster 2022-04-28T20:35:48.107125+0000 mgr.y (mgr.24961) 1598 : cluster [DBG] pgmap v1961: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:49 smithi139 bash[43301]: cluster 2022-04-28T20:35:48.107125+0000 mgr.y (mgr.24961) 1598 : cluster [DBG] pgmap v1961: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:51 smithi059 bash[61152]: cluster 2022-04-28T20:35:50.108254+0000 mgr.y (mgr.24961) 1599 : cluster [DBG] pgmap v1962: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:51.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:51 smithi059 bash[59745]: cluster 2022-04-28T20:35:50.108254+0000 mgr.y (mgr.24961) 1599 : cluster [DBG] pgmap v1962: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:51 smithi139 bash[43301]: cluster 2022-04-28T20:35:50.108254+0000 mgr.y (mgr.24961) 1599 : cluster [DBG] pgmap v1962: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:53 smithi059 bash[61152]: cluster 2022-04-28T20:35:52.109280+0000 mgr.y (mgr.24961) 1600 : cluster [DBG] pgmap v1963: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:35:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:35:53.691Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:35:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:53 smithi059 bash[59745]: cluster 2022-04-28T20:35:52.109280+0000 mgr.y (mgr.24961) 1600 : cluster [DBG] pgmap v1963: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:53 smithi139 bash[43301]: cluster 2022-04-28T20:35:52.109280+0000 mgr.y (mgr.24961) 1600 : cluster [DBG] pgmap v1963: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:56.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:55 smithi059 bash[59745]: cluster 2022-04-28T20:35:54.110231+0000 mgr.y (mgr.24961) 1601 : cluster [DBG] pgmap v1964: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:56.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:55 smithi059 bash[61152]: cluster 2022-04-28T20:35:54.110231+0000 mgr.y (mgr.24961) 1601 : cluster [DBG] pgmap v1964: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:55 smithi139 bash[43301]: cluster 2022-04-28T20:35:54.110231+0000 mgr.y (mgr.24961) 1601 : cluster [DBG] pgmap v1964: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:57 smithi059 bash[61152]: cluster 2022-04-28T20:35:56.111362+0000 mgr.y (mgr.24961) 1602 : cluster [DBG] pgmap v1965: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:57 smithi059 bash[59745]: cluster 2022-04-28T20:35:56.111362+0000 mgr.y (mgr.24961) 1602 : cluster [DBG] pgmap v1965: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:35:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:57 smithi139 bash[43301]: cluster 2022-04-28T20:35:56.111362+0000 mgr.y (mgr.24961) 1602 : cluster [DBG] pgmap v1965: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:35:59 smithi059 bash[61152]: cluster 2022-04-28T20:35:58.112040+0000 mgr.y (mgr.24961) 1603 : cluster [DBG] pgmap v1966: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:35:59 smithi059 bash[59745]: cluster 2022-04-28T20:35:58.112040+0000 mgr.y (mgr.24961) 1603 : cluster [DBG] pgmap v1966: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:35:59 smithi139 bash[43301]: cluster 2022-04-28T20:35:58.112040+0000 mgr.y (mgr.24961) 1603 : cluster [DBG] pgmap v1966: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:01 smithi059 bash[61152]: cluster 2022-04-28T20:36:00.113080+0000 mgr.y (mgr.24961) 1604 : cluster [DBG] pgmap v1967: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:01.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:01 smithi059 bash[59745]: cluster 2022-04-28T20:36:00.113080+0000 mgr.y (mgr.24961) 1604 : cluster [DBG] pgmap v1967: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:01 smithi139 bash[43301]: cluster 2022-04-28T20:36:00.113080+0000 mgr.y (mgr.24961) 1604 : cluster [DBG] pgmap v1967: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:03 smithi059 bash[61152]: cluster 2022-04-28T20:36:02.114075+0000 mgr.y (mgr.24961) 1605 : cluster [DBG] pgmap v1968: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:03.693Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:03 smithi059 bash[59745]: cluster 2022-04-28T20:36:02.114075+0000 mgr.y (mgr.24961) 1605 : cluster [DBG] pgmap v1968: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:03 smithi139 bash[43301]: cluster 2022-04-28T20:36:02.114075+0000 mgr.y (mgr.24961) 1605 : cluster [DBG] pgmap v1968: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:05 smithi059 bash[61152]: cluster 2022-04-28T20:36:04.114992+0000 mgr.y (mgr.24961) 1606 : cluster [DBG] pgmap v1969: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:05 smithi059 bash[59745]: cluster 2022-04-28T20:36:04.114992+0000 mgr.y (mgr.24961) 1606 : cluster [DBG] pgmap v1969: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:05 smithi139 bash[43301]: cluster 2022-04-28T20:36:04.114992+0000 mgr.y (mgr.24961) 1606 : cluster [DBG] pgmap v1969: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:07 smithi059 bash[61152]: cluster 2022-04-28T20:36:06.116155+0000 mgr.y (mgr.24961) 1607 : cluster [DBG] pgmap v1970: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:07 smithi059 bash[59745]: cluster 2022-04-28T20:36:06.116155+0000 mgr.y (mgr.24961) 1607 : cluster [DBG] pgmap v1970: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:07 smithi139 bash[43301]: cluster 2022-04-28T20:36:06.116155+0000 mgr.y (mgr.24961) 1607 : cluster [DBG] pgmap v1970: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:09 smithi059 bash[61152]: cluster 2022-04-28T20:36:08.116851+0000 mgr.y (mgr.24961) 1608 : cluster [DBG] pgmap v1971: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:09 smithi059 bash[59745]: cluster 2022-04-28T20:36:08.116851+0000 mgr.y (mgr.24961) 1608 : cluster [DBG] pgmap v1971: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:09 smithi139 bash[43301]: cluster 2022-04-28T20:36:08.116851+0000 mgr.y (mgr.24961) 1608 : cluster [DBG] pgmap v1971: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:11 smithi059 bash[59745]: cluster 2022-04-28T20:36:10.117948+0000 mgr.y (mgr.24961) 1609 : cluster [DBG] pgmap v1972: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:11 smithi059 bash[61152]: cluster 2022-04-28T20:36:10.117948+0000 mgr.y (mgr.24961) 1609 : cluster [DBG] pgmap v1972: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:11.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:11 smithi139 bash[43301]: cluster 2022-04-28T20:36:10.117948+0000 mgr.y (mgr.24961) 1609 : cluster [DBG] pgmap v1972: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:13 smithi059 bash[61152]: cluster 2022-04-28T20:36:12.118970+0000 mgr.y (mgr.24961) 1610 : cluster [DBG] pgmap v1973: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:13.692Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:13 smithi059 bash[59745]: cluster 2022-04-28T20:36:12.118970+0000 mgr.y (mgr.24961) 1610 : cluster [DBG] pgmap v1973: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:13 smithi139 bash[43301]: cluster 2022-04-28T20:36:12.118970+0000 mgr.y (mgr.24961) 1610 : cluster [DBG] pgmap v1973: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:15 smithi059 bash[61152]: cluster 2022-04-28T20:36:14.119902+0000 mgr.y (mgr.24961) 1611 : cluster [DBG] pgmap v1974: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:15 smithi059 bash[59745]: cluster 2022-04-28T20:36:14.119902+0000 mgr.y (mgr.24961) 1611 : cluster [DBG] pgmap v1974: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:15 smithi139 bash[43301]: cluster 2022-04-28T20:36:14.119902+0000 mgr.y (mgr.24961) 1611 : cluster [DBG] pgmap v1974: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[59745]: cluster 2022-04-28T20:36:16.121040+0000 mgr.y (mgr.24961) 1612 : cluster [DBG] pgmap v1975: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[59745]: audit 2022-04-28T20:36:17.654406+0000 mon.a (mon.0) 2071 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:36:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[59745]: audit 2022-04-28T20:36:17.662602+0000 mon.a (mon.0) 2072 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:36:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[59745]: audit 2022-04-28T20:36:17.672650+0000 mon.a (mon.0) 2073 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T20:36:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[59745]: audit 2022-04-28T20:36:17.673406+0000 mon.a (mon.0) 2074 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:36:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[59745]: audit 2022-04-28T20:36:17.674026+0000 mon.a (mon.0) 2075 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:36:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[61152]: cluster 2022-04-28T20:36:16.121040+0000 mgr.y (mgr.24961) 1612 : cluster [DBG] pgmap v1975: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[61152]: audit 2022-04-28T20:36:17.654406+0000 mon.a (mon.0) 2071 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:36:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[61152]: audit 2022-04-28T20:36:17.662602+0000 mon.a (mon.0) 2072 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:36:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[61152]: audit 2022-04-28T20:36:17.672650+0000 mon.a (mon.0) 2073 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T20:36:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[61152]: audit 2022-04-28T20:36:17.673406+0000 mon.a (mon.0) 2074 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:36:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:17 smithi059 bash[61152]: audit 2022-04-28T20:36:17.674026+0000 mon.a (mon.0) 2075 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:36:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:17 smithi139 bash[43301]: cluster 2022-04-28T20:36:16.121040+0000 mgr.y (mgr.24961) 1612 : cluster [DBG] pgmap v1975: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:17 smithi139 bash[43301]: audit 2022-04-28T20:36:17.654406+0000 mon.a (mon.0) 2071 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:36:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:17 smithi139 bash[43301]: audit 2022-04-28T20:36:17.662602+0000 mon.a (mon.0) 2072 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:36:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:17 smithi139 bash[43301]: audit 2022-04-28T20:36:17.672650+0000 mon.a (mon.0) 2073 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T20:36:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:17 smithi139 bash[43301]: audit 2022-04-28T20:36:17.673406+0000 mon.a (mon.0) 2074 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:36:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:17 smithi139 bash[43301]: audit 2022-04-28T20:36:17.674026+0000 mon.a (mon.0) 2075 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:36:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:19 smithi059 bash[59745]: cluster 2022-04-28T20:36:18.121890+0000 mgr.y (mgr.24961) 1613 : cluster [DBG] pgmap v1976: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:19 smithi059 bash[61152]: cluster 2022-04-28T20:36:18.121890+0000 mgr.y (mgr.24961) 1613 : cluster [DBG] pgmap v1976: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:19 smithi139 bash[43301]: cluster 2022-04-28T20:36:18.121890+0000 mgr.y (mgr.24961) 1613 : cluster [DBG] pgmap v1976: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:21 smithi059 bash[61152]: cluster 2022-04-28T20:36:20.123056+0000 mgr.y (mgr.24961) 1614 : cluster [DBG] pgmap v1977: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:21.801Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:21 smithi059 bash[59745]: cluster 2022-04-28T20:36:20.123056+0000 mgr.y (mgr.24961) 1614 : cluster [DBG] pgmap v1977: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:21 smithi139 bash[43301]: cluster 2022-04-28T20:36:20.123056+0000 mgr.y (mgr.24961) 1614 : cluster [DBG] pgmap v1977: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:23 smithi059 bash[61152]: cluster 2022-04-28T20:36:22.124081+0000 mgr.y (mgr.24961) 1615 : cluster [DBG] pgmap v1978: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:23.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:23 smithi059 bash[59745]: cluster 2022-04-28T20:36:22.124081+0000 mgr.y (mgr.24961) 1615 : cluster [DBG] pgmap v1978: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:23 smithi139 bash[43301]: cluster 2022-04-28T20:36:22.124081+0000 mgr.y (mgr.24961) 1615 : cluster [DBG] pgmap v1978: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:25 smithi139 bash[43301]: cluster 2022-04-28T20:36:24.124986+0000 mgr.y (mgr.24961) 1616 : cluster [DBG] pgmap v1979: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:25 smithi059 bash[61152]: cluster 2022-04-28T20:36:24.124986+0000 mgr.y (mgr.24961) 1616 : cluster [DBG] pgmap v1979: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:25 smithi059 bash[59745]: cluster 2022-04-28T20:36:24.124986+0000 mgr.y (mgr.24961) 1616 : cluster [DBG] pgmap v1979: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:27 smithi139 bash[43301]: cluster 2022-04-28T20:36:26.126261+0000 mgr.y (mgr.24961) 1617 : cluster [DBG] pgmap v1980: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:27 smithi059 bash[61152]: cluster 2022-04-28T20:36:26.126261+0000 mgr.y (mgr.24961) 1617 : cluster [DBG] pgmap v1980: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:27 smithi059 bash[59745]: cluster 2022-04-28T20:36:26.126261+0000 mgr.y (mgr.24961) 1617 : cluster [DBG] pgmap v1980: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:29 smithi139 bash[43301]: cluster 2022-04-28T20:36:28.127012+0000 mgr.y (mgr.24961) 1618 : cluster [DBG] pgmap v1981: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:29 smithi059 bash[61152]: cluster 2022-04-28T20:36:28.127012+0000 mgr.y (mgr.24961) 1618 : cluster [DBG] pgmap v1981: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:29 smithi059 bash[59745]: cluster 2022-04-28T20:36:28.127012+0000 mgr.y (mgr.24961) 1618 : cluster [DBG] pgmap v1981: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:31 smithi059 bash[61152]: cluster 2022-04-28T20:36:30.128122+0000 mgr.y (mgr.24961) 1619 : cluster [DBG] pgmap v1982: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:31.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:31 smithi059 bash[59745]: cluster 2022-04-28T20:36:30.128122+0000 mgr.y (mgr.24961) 1619 : cluster [DBG] pgmap v1982: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:31 smithi139 bash[43301]: cluster 2022-04-28T20:36:30.128122+0000 mgr.y (mgr.24961) 1619 : cluster [DBG] pgmap v1982: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[59745]: audit 2022-04-28T20:36:32.354873+0000 mon.a (mon.0) 2076 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:36:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[59745]: audit 2022-04-28T20:36:32.357650+0000 mon.a (mon.0) 2077 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:36:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[59745]: audit 2022-04-28T20:36:32.359460+0000 mon.a (mon.0) 2078 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:36:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[59745]: audit 2022-04-28T20:36:32.362638+0000 mon.a (mon.0) 2079 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:36:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[59745]: audit 2022-04-28T20:36:32.363374+0000 mon.a (mon.0) 2080 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:36:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[61152]: audit 2022-04-28T20:36:32.354873+0000 mon.a (mon.0) 2076 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:36:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[61152]: audit 2022-04-28T20:36:32.357650+0000 mon.a (mon.0) 2077 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:36:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[61152]: audit 2022-04-28T20:36:32.359460+0000 mon.a (mon.0) 2078 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:36:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[61152]: audit 2022-04-28T20:36:32.362638+0000 mon.a (mon.0) 2079 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:36:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:32 smithi059 bash[61152]: audit 2022-04-28T20:36:32.363374+0000 mon.a (mon.0) 2080 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:32 smithi139 bash[43301]: audit 2022-04-28T20:36:32.354873+0000 mon.a (mon.0) 2076 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:32 smithi139 bash[43301]: audit 2022-04-28T20:36:32.357650+0000 mon.a (mon.0) 2077 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:32 smithi139 bash[43301]: audit 2022-04-28T20:36:32.359460+0000 mon.a (mon.0) 2078 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:36:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:32 smithi139 bash[43301]: audit 2022-04-28T20:36:32.362638+0000 mon.a (mon.0) 2079 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:36:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:32 smithi139 bash[43301]: audit 2022-04-28T20:36:32.363374+0000 mon.a (mon.0) 2080 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:36:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:33 smithi059 bash[61152]: cluster 2022-04-28T20:36:32.129140+0000 mgr.y (mgr.24961) 1620 : cluster [DBG] pgmap v1983: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:33 smithi059 bash[61152]: cephadm 2022-04-28T20:36:32.361112+0000 mgr.y (mgr.24961) 1621 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:36:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:33.694Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:33 smithi059 bash[59745]: cluster 2022-04-28T20:36:32.129140+0000 mgr.y (mgr.24961) 1620 : cluster [DBG] pgmap v1983: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:33 smithi059 bash[59745]: cephadm 2022-04-28T20:36:32.361112+0000 mgr.y (mgr.24961) 1621 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:36:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:33 smithi139 bash[43301]: cluster 2022-04-28T20:36:32.129140+0000 mgr.y (mgr.24961) 1620 : cluster [DBG] pgmap v1983: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:34.388 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:33 smithi139 bash[43301]: cephadm 2022-04-28T20:36:32.361112+0000 mgr.y (mgr.24961) 1621 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:36:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:34 smithi059 bash[61152]: cluster 2022-04-28T20:36:34.130071+0000 mgr.y (mgr.24961) 1622 : cluster [DBG] pgmap v1984: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:34 smithi059 bash[59745]: cluster 2022-04-28T20:36:34.130071+0000 mgr.y (mgr.24961) 1622 : cluster [DBG] pgmap v1984: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:34 smithi139 bash[43301]: cluster 2022-04-28T20:36:34.130071+0000 mgr.y (mgr.24961) 1622 : cluster [DBG] pgmap v1984: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[59745]: cluster 2022-04-28T20:36:36.130999+0000 mgr.y (mgr.24961) 1623 : cluster [DBG] pgmap v1985: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[59745]: audit 2022-04-28T20:36:36.463310+0000 mon.a (mon.0) 2081 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[59745]: audit 2022-04-28T20:36:36.664427+0000 mon.a (mon.0) 2082 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[59745]: audit 2022-04-28T20:36:36.947758+0000 mon.a (mon.0) 2083 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[61152]: cluster 2022-04-28T20:36:36.130999+0000 mgr.y (mgr.24961) 1623 : cluster [DBG] pgmap v1985: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[61152]: audit 2022-04-28T20:36:36.463310+0000 mon.a (mon.0) 2081 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[61152]: audit 2022-04-28T20:36:36.664427+0000 mon.a (mon.0) 2082 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:37 smithi059 bash[61152]: audit 2022-04-28T20:36:36.947758+0000 mon.a (mon.0) 2083 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:37 smithi139 bash[43301]: cluster 2022-04-28T20:36:36.130999+0000 mgr.y (mgr.24961) 1623 : cluster [DBG] pgmap v1985: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:37 smithi139 bash[43301]: audit 2022-04-28T20:36:36.463310+0000 mon.a (mon.0) 2081 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:37 smithi139 bash[43301]: audit 2022-04-28T20:36:36.664427+0000 mon.a (mon.0) 2082 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:37 smithi139 bash[43301]: audit 2022-04-28T20:36:36.947758+0000 mon.a (mon.0) 2083 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:36:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:39 smithi059 bash[61152]: cluster 2022-04-28T20:36:38.131716+0000 mgr.y (mgr.24961) 1624 : cluster [DBG] pgmap v1986: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:39 smithi059 bash[59745]: cluster 2022-04-28T20:36:38.131716+0000 mgr.y (mgr.24961) 1624 : cluster [DBG] pgmap v1986: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:39 smithi139 bash[43301]: cluster 2022-04-28T20:36:38.131716+0000 mgr.y (mgr.24961) 1624 : cluster [DBG] pgmap v1986: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:41 smithi059 bash[61152]: cluster 2022-04-28T20:36:40.132891+0000 mgr.y (mgr.24961) 1625 : cluster [DBG] pgmap v1987: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:42.136 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:41.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:42.136 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:41 smithi059 bash[59745]: cluster 2022-04-28T20:36:40.132891+0000 mgr.y (mgr.24961) 1625 : cluster [DBG] pgmap v1987: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:42.137 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:41 smithi139 bash[43301]: cluster 2022-04-28T20:36:40.132891+0000 mgr.y (mgr.24961) 1625 : cluster [DBG] pgmap v1987: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:43 smithi059 bash[61152]: cluster 2022-04-28T20:36:42.133857+0000 mgr.y (mgr.24961) 1626 : cluster [DBG] pgmap v1988: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:43.695Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:43 smithi059 bash[59745]: cluster 2022-04-28T20:36:42.133857+0000 mgr.y (mgr.24961) 1626 : cluster [DBG] pgmap v1988: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:43 smithi139 bash[43301]: cluster 2022-04-28T20:36:42.133857+0000 mgr.y (mgr.24961) 1626 : cluster [DBG] pgmap v1988: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:45 smithi059 bash[61152]: cluster 2022-04-28T20:36:44.134792+0000 mgr.y (mgr.24961) 1627 : cluster [DBG] pgmap v1989: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:45 smithi059 bash[59745]: cluster 2022-04-28T20:36:44.134792+0000 mgr.y (mgr.24961) 1627 : cluster [DBG] pgmap v1989: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:45 smithi139 bash[43301]: cluster 2022-04-28T20:36:44.134792+0000 mgr.y (mgr.24961) 1627 : cluster [DBG] pgmap v1989: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:47 smithi059 bash[61152]: cluster 2022-04-28T20:36:46.136028+0000 mgr.y (mgr.24961) 1628 : cluster [DBG] pgmap v1990: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:47 smithi059 bash[59745]: cluster 2022-04-28T20:36:46.136028+0000 mgr.y (mgr.24961) 1628 : cluster [DBG] pgmap v1990: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:47 smithi139 bash[43301]: cluster 2022-04-28T20:36:46.136028+0000 mgr.y (mgr.24961) 1628 : cluster [DBG] pgmap v1990: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:49 smithi059 bash[59745]: cluster 2022-04-28T20:36:48.136729+0000 mgr.y (mgr.24961) 1629 : cluster [DBG] pgmap v1991: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:50.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:49 smithi059 bash[61152]: cluster 2022-04-28T20:36:48.136729+0000 mgr.y (mgr.24961) 1629 : cluster [DBG] pgmap v1991: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:49 smithi139 bash[43301]: cluster 2022-04-28T20:36:48.136729+0000 mgr.y (mgr.24961) 1629 : cluster [DBG] pgmap v1991: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:51 smithi059 bash[61152]: cluster 2022-04-28T20:36:50.137929+0000 mgr.y (mgr.24961) 1630 : cluster [DBG] pgmap v1992: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:51.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:51 smithi059 bash[59745]: cluster 2022-04-28T20:36:50.137929+0000 mgr.y (mgr.24961) 1630 : cluster [DBG] pgmap v1992: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:51 smithi139 bash[43301]: cluster 2022-04-28T20:36:50.137929+0000 mgr.y (mgr.24961) 1630 : cluster [DBG] pgmap v1992: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:53 smithi059 bash[59745]: cluster 2022-04-28T20:36:52.138827+0000 mgr.y (mgr.24961) 1631 : cluster [DBG] pgmap v1993: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:53 smithi059 bash[61152]: cluster 2022-04-28T20:36:52.138827+0000 mgr.y (mgr.24961) 1631 : cluster [DBG] pgmap v1993: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:36:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:36:53.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:36:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:53 smithi139 bash[43301]: cluster 2022-04-28T20:36:52.138827+0000 mgr.y (mgr.24961) 1631 : cluster [DBG] pgmap v1993: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:55 smithi059 bash[61152]: cluster 2022-04-28T20:36:54.139898+0000 mgr.y (mgr.24961) 1632 : cluster [DBG] pgmap v1994: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:56.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:55 smithi059 bash[59745]: cluster 2022-04-28T20:36:54.139898+0000 mgr.y (mgr.24961) 1632 : cluster [DBG] pgmap v1994: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:55 smithi139 bash[43301]: cluster 2022-04-28T20:36:54.139898+0000 mgr.y (mgr.24961) 1632 : cluster [DBG] pgmap v1994: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:57 smithi059 bash[61152]: cluster 2022-04-28T20:36:56.141098+0000 mgr.y (mgr.24961) 1633 : cluster [DBG] pgmap v1995: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:57 smithi059 bash[59745]: cluster 2022-04-28T20:36:56.141098+0000 mgr.y (mgr.24961) 1633 : cluster [DBG] pgmap v1995: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:36:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:57 smithi139 bash[43301]: cluster 2022-04-28T20:36:56.141098+0000 mgr.y (mgr.24961) 1633 : cluster [DBG] pgmap v1995: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:36:59 smithi059 bash[61152]: cluster 2022-04-28T20:36:58.141788+0000 mgr.y (mgr.24961) 1634 : cluster [DBG] pgmap v1996: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:36:59 smithi059 bash[59745]: cluster 2022-04-28T20:36:58.141788+0000 mgr.y (mgr.24961) 1634 : cluster [DBG] pgmap v1996: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:36:59 smithi139 bash[43301]: cluster 2022-04-28T20:36:58.141788+0000 mgr.y (mgr.24961) 1634 : cluster [DBG] pgmap v1996: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:01 smithi059 bash[61152]: cluster 2022-04-28T20:37:00.142939+0000 mgr.y (mgr.24961) 1635 : cluster [DBG] pgmap v1997: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:01.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:01 smithi059 bash[59745]: cluster 2022-04-28T20:37:00.142939+0000 mgr.y (mgr.24961) 1635 : cluster [DBG] pgmap v1997: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:01 smithi139 bash[43301]: cluster 2022-04-28T20:37:00.142939+0000 mgr.y (mgr.24961) 1635 : cluster [DBG] pgmap v1997: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:03 smithi059 bash[61152]: cluster 2022-04-28T20:37:02.143986+0000 mgr.y (mgr.24961) 1636 : cluster [DBG] pgmap v1998: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:03.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:03 smithi059 bash[59745]: cluster 2022-04-28T20:37:02.143986+0000 mgr.y (mgr.24961) 1636 : cluster [DBG] pgmap v1998: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:03 smithi139 bash[43301]: cluster 2022-04-28T20:37:02.143986+0000 mgr.y (mgr.24961) 1636 : cluster [DBG] pgmap v1998: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:05 smithi059 bash[61152]: cluster 2022-04-28T20:37:04.144933+0000 mgr.y (mgr.24961) 1637 : cluster [DBG] pgmap v1999: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:05 smithi059 bash[59745]: cluster 2022-04-28T20:37:04.144933+0000 mgr.y (mgr.24961) 1637 : cluster [DBG] pgmap v1999: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:05 smithi139 bash[43301]: cluster 2022-04-28T20:37:04.144933+0000 mgr.y (mgr.24961) 1637 : cluster [DBG] pgmap v1999: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:07 smithi059 bash[61152]: cluster 2022-04-28T20:37:06.146066+0000 mgr.y (mgr.24961) 1638 : cluster [DBG] pgmap v2000: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:07 smithi059 bash[59745]: cluster 2022-04-28T20:37:06.146066+0000 mgr.y (mgr.24961) 1638 : cluster [DBG] pgmap v2000: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:07 smithi139 bash[43301]: cluster 2022-04-28T20:37:06.146066+0000 mgr.y (mgr.24961) 1638 : cluster [DBG] pgmap v2000: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:09 smithi059 bash[59745]: cluster 2022-04-28T20:37:08.146775+0000 mgr.y (mgr.24961) 1639 : cluster [DBG] pgmap v2001: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:09 smithi059 bash[61152]: cluster 2022-04-28T20:37:08.146775+0000 mgr.y (mgr.24961) 1639 : cluster [DBG] pgmap v2001: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:09 smithi139 bash[43301]: cluster 2022-04-28T20:37:08.146775+0000 mgr.y (mgr.24961) 1639 : cluster [DBG] pgmap v2001: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:11 smithi059 bash[61152]: cluster 2022-04-28T20:37:10.148142+0000 mgr.y (mgr.24961) 1640 : cluster [DBG] pgmap v2002: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:11.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:11 smithi059 bash[59745]: cluster 2022-04-28T20:37:10.148142+0000 mgr.y (mgr.24961) 1640 : cluster [DBG] pgmap v2002: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:11 smithi139 bash[43301]: cluster 2022-04-28T20:37:10.148142+0000 mgr.y (mgr.24961) 1640 : cluster [DBG] pgmap v2002: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:13 smithi059 bash[61152]: cluster 2022-04-28T20:37:12.149078+0000 mgr.y (mgr.24961) 1641 : cluster [DBG] pgmap v2003: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:13.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:13 smithi059 bash[59745]: cluster 2022-04-28T20:37:12.149078+0000 mgr.y (mgr.24961) 1641 : cluster [DBG] pgmap v2003: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:13 smithi139 bash[43301]: cluster 2022-04-28T20:37:12.149078+0000 mgr.y (mgr.24961) 1641 : cluster [DBG] pgmap v2003: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:15 smithi059 bash[61152]: cluster 2022-04-28T20:37:14.150065+0000 mgr.y (mgr.24961) 1642 : cluster [DBG] pgmap v2004: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:15 smithi059 bash[59745]: cluster 2022-04-28T20:37:14.150065+0000 mgr.y (mgr.24961) 1642 : cluster [DBG] pgmap v2004: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:15 smithi139 bash[43301]: cluster 2022-04-28T20:37:14.150065+0000 mgr.y (mgr.24961) 1642 : cluster [DBG] pgmap v2004: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[61152]: cluster 2022-04-28T20:37:16.151297+0000 mgr.y (mgr.24961) 1643 : cluster [DBG] pgmap v2005: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[61152]: audit 2022-04-28T20:37:17.675019+0000 mon.a (mon.0) 2084 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:37:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[61152]: audit 2022-04-28T20:37:17.675258+0000 mon.a (mon.0) 2085 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:37:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[61152]: audit 2022-04-28T20:37:17.684868+0000 mon.a (mon.0) 2086 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:37:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[61152]: audit 2022-04-28T20:37:17.685362+0000 mon.a (mon.0) 2087 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:37:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[61152]: audit 2022-04-28T20:37:17.685730+0000 mon.a (mon.0) 2088 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:37:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[59745]: cluster 2022-04-28T20:37:16.151297+0000 mgr.y (mgr.24961) 1643 : cluster [DBG] pgmap v2005: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[59745]: audit 2022-04-28T20:37:17.675019+0000 mon.a (mon.0) 2084 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:37:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[59745]: audit 2022-04-28T20:37:17.675258+0000 mon.a (mon.0) 2085 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:37:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[59745]: audit 2022-04-28T20:37:17.684868+0000 mon.a (mon.0) 2086 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:37:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[59745]: audit 2022-04-28T20:37:17.685362+0000 mon.a (mon.0) 2087 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:37:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:17 smithi059 bash[59745]: audit 2022-04-28T20:37:17.685730+0000 mon.a (mon.0) 2088 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:37:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:17 smithi139 bash[43301]: cluster 2022-04-28T20:37:16.151297+0000 mgr.y (mgr.24961) 1643 : cluster [DBG] pgmap v2005: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:17 smithi139 bash[43301]: audit 2022-04-28T20:37:17.675019+0000 mon.a (mon.0) 2084 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:37:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:17 smithi139 bash[43301]: audit 2022-04-28T20:37:17.675258+0000 mon.a (mon.0) 2085 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:37:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:17 smithi139 bash[43301]: audit 2022-04-28T20:37:17.684868+0000 mon.a (mon.0) 2086 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:37:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:17 smithi139 bash[43301]: audit 2022-04-28T20:37:17.685362+0000 mon.a (mon.0) 2087 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:37:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:17 smithi139 bash[43301]: audit 2022-04-28T20:37:17.685730+0000 mon.a (mon.0) 2088 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:37:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:19 smithi059 bash[61152]: cluster 2022-04-28T20:37:18.151973+0000 mgr.y (mgr.24961) 1644 : cluster [DBG] pgmap v2006: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:19 smithi059 bash[59745]: cluster 2022-04-28T20:37:18.151973+0000 mgr.y (mgr.24961) 1644 : cluster [DBG] pgmap v2006: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:19 smithi139 bash[43301]: cluster 2022-04-28T20:37:18.151973+0000 mgr.y (mgr.24961) 1644 : cluster [DBG] pgmap v2006: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:21 smithi059 bash[61152]: cluster 2022-04-28T20:37:20.153132+0000 mgr.y (mgr.24961) 1645 : cluster [DBG] pgmap v2007: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:21 smithi059 bash[59745]: cluster 2022-04-28T20:37:20.153132+0000 mgr.y (mgr.24961) 1645 : cluster [DBG] pgmap v2007: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:21.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:21 smithi139 bash[43301]: cluster 2022-04-28T20:37:20.153132+0000 mgr.y (mgr.24961) 1645 : cluster [DBG] pgmap v2007: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:23 smithi059 bash[61152]: cluster 2022-04-28T20:37:22.154032+0000 mgr.y (mgr.24961) 1646 : cluster [DBG] pgmap v2008: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:23.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:23 smithi059 bash[59745]: cluster 2022-04-28T20:37:22.154032+0000 mgr.y (mgr.24961) 1646 : cluster [DBG] pgmap v2008: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:23 smithi139 bash[43301]: cluster 2022-04-28T20:37:22.154032+0000 mgr.y (mgr.24961) 1646 : cluster [DBG] pgmap v2008: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:25 smithi059 bash[61152]: cluster 2022-04-28T20:37:24.154915+0000 mgr.y (mgr.24961) 1647 : cluster [DBG] pgmap v2009: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:25 smithi059 bash[59745]: cluster 2022-04-28T20:37:24.154915+0000 mgr.y (mgr.24961) 1647 : cluster [DBG] pgmap v2009: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:25 smithi139 bash[43301]: cluster 2022-04-28T20:37:24.154915+0000 mgr.y (mgr.24961) 1647 : cluster [DBG] pgmap v2009: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:27 smithi059 bash[61152]: cluster 2022-04-28T20:37:26.156077+0000 mgr.y (mgr.24961) 1648 : cluster [DBG] pgmap v2010: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:27 smithi059 bash[59745]: cluster 2022-04-28T20:37:26.156077+0000 mgr.y (mgr.24961) 1648 : cluster [DBG] pgmap v2010: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:27 smithi139 bash[43301]: cluster 2022-04-28T20:37:26.156077+0000 mgr.y (mgr.24961) 1648 : cluster [DBG] pgmap v2010: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:29 smithi059 bash[61152]: cluster 2022-04-28T20:37:28.156755+0000 mgr.y (mgr.24961) 1649 : cluster [DBG] pgmap v2011: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:29 smithi059 bash[59745]: cluster 2022-04-28T20:37:28.156755+0000 mgr.y (mgr.24961) 1649 : cluster [DBG] pgmap v2011: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:29 smithi139 bash[43301]: cluster 2022-04-28T20:37:28.156755+0000 mgr.y (mgr.24961) 1649 : cluster [DBG] pgmap v2011: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:31 smithi059 bash[61152]: cluster 2022-04-28T20:37:30.157902+0000 mgr.y (mgr.24961) 1650 : cluster [DBG] pgmap v2012: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:31.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:31 smithi059 bash[59745]: cluster 2022-04-28T20:37:30.157902+0000 mgr.y (mgr.24961) 1650 : cluster [DBG] pgmap v2012: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:31 smithi139 bash[43301]: cluster 2022-04-28T20:37:30.157902+0000 mgr.y (mgr.24961) 1650 : cluster [DBG] pgmap v2012: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:33 smithi059 bash[59745]: cluster 2022-04-28T20:37:32.158892+0000 mgr.y (mgr.24961) 1651 : cluster [DBG] pgmap v2013: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:33 smithi059 bash[61152]: cluster 2022-04-28T20:37:32.158892+0000 mgr.y (mgr.24961) 1651 : cluster [DBG] pgmap v2013: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:33.698Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:33 smithi139 bash[43301]: cluster 2022-04-28T20:37:32.158892+0000 mgr.y (mgr.24961) 1651 : cluster [DBG] pgmap v2013: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:35 smithi059 bash[61152]: cluster 2022-04-28T20:37:34.159841+0000 mgr.y (mgr.24961) 1652 : cluster [DBG] pgmap v2014: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:36.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:35 smithi059 bash[59745]: cluster 2022-04-28T20:37:34.159841+0000 mgr.y (mgr.24961) 1652 : cluster [DBG] pgmap v2014: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:35 smithi139 bash[43301]: cluster 2022-04-28T20:37:34.159841+0000 mgr.y (mgr.24961) 1652 : cluster [DBG] pgmap v2014: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[59745]: cluster 2022-04-28T20:37:36.161040+0000 mgr.y (mgr.24961) 1653 : cluster [DBG] pgmap v2015: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[59745]: audit 2022-04-28T20:37:36.957288+0000 mon.a (mon.0) 2089 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:37:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[59745]: audit 2022-04-28T20:37:36.960018+0000 mon.a (mon.0) 2090 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:37:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[59745]: audit 2022-04-28T20:37:36.961829+0000 mon.a (mon.0) 2091 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:37:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[59745]: audit 2022-04-28T20:37:37.288062+0000 mon.a (mon.0) 2092 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:37:38.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[61152]: cluster 2022-04-28T20:37:36.161040+0000 mgr.y (mgr.24961) 1653 : cluster [DBG] pgmap v2015: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:38.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[61152]: audit 2022-04-28T20:37:36.957288+0000 mon.a (mon.0) 2089 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:37:38.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[61152]: audit 2022-04-28T20:37:36.960018+0000 mon.a (mon.0) 2090 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:37:38.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[61152]: audit 2022-04-28T20:37:36.961829+0000 mon.a (mon.0) 2091 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:37:38.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:37 smithi059 bash[61152]: audit 2022-04-28T20:37:37.288062+0000 mon.a (mon.0) 2092 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:37:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:37 smithi139 bash[43301]: cluster 2022-04-28T20:37:36.161040+0000 mgr.y (mgr.24961) 1653 : cluster [DBG] pgmap v2015: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:37 smithi139 bash[43301]: audit 2022-04-28T20:37:36.957288+0000 mon.a (mon.0) 2089 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:37:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:37 smithi139 bash[43301]: audit 2022-04-28T20:37:36.960018+0000 mon.a (mon.0) 2090 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:37:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:37 smithi139 bash[43301]: audit 2022-04-28T20:37:36.961829+0000 mon.a (mon.0) 2091 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:37:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:37 smithi139 bash[43301]: audit 2022-04-28T20:37:37.288062+0000 mon.a (mon.0) 2092 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:37:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:38 smithi059 bash[61152]: cephadm 2022-04-28T20:37:36.963432+0000 mgr.y (mgr.24961) 1654 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:37:39.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:38 smithi059 bash[59745]: cephadm 2022-04-28T20:37:36.963432+0000 mgr.y (mgr.24961) 1654 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:37:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:38 smithi139 bash[43301]: cephadm 2022-04-28T20:37:36.963432+0000 mgr.y (mgr.24961) 1654 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:37:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:39 smithi059 bash[61152]: cluster 2022-04-28T20:37:38.161728+0000 mgr.y (mgr.24961) 1655 : cluster [DBG] pgmap v2016: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:39 smithi059 bash[59745]: cluster 2022-04-28T20:37:38.161728+0000 mgr.y (mgr.24961) 1655 : cluster [DBG] pgmap v2016: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:39 smithi139 bash[43301]: cluster 2022-04-28T20:37:38.161728+0000 mgr.y (mgr.24961) 1655 : cluster [DBG] pgmap v2016: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:41 smithi059 bash[61152]: cluster 2022-04-28T20:37:40.162930+0000 mgr.y (mgr.24961) 1656 : cluster [DBG] pgmap v2017: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:41.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:41 smithi059 bash[59745]: cluster 2022-04-28T20:37:40.162930+0000 mgr.y (mgr.24961) 1656 : cluster [DBG] pgmap v2017: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:41 smithi139 bash[43301]: cluster 2022-04-28T20:37:40.162930+0000 mgr.y (mgr.24961) 1656 : cluster [DBG] pgmap v2017: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:43 smithi059 bash[61152]: cluster 2022-04-28T20:37:42.163993+0000 mgr.y (mgr.24961) 1657 : cluster [DBG] pgmap v2018: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:43 smithi059 bash[59745]: cluster 2022-04-28T20:37:42.163993+0000 mgr.y (mgr.24961) 1657 : cluster [DBG] pgmap v2018: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:43.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:43 smithi139 bash[43301]: cluster 2022-04-28T20:37:42.163993+0000 mgr.y (mgr.24961) 1657 : cluster [DBG] pgmap v2018: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:45 smithi059 bash[61152]: cluster 2022-04-28T20:37:44.164939+0000 mgr.y (mgr.24961) 1658 : cluster [DBG] pgmap v2019: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:45 smithi059 bash[59745]: cluster 2022-04-28T20:37:44.164939+0000 mgr.y (mgr.24961) 1658 : cluster [DBG] pgmap v2019: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:45 smithi139 bash[43301]: cluster 2022-04-28T20:37:44.164939+0000 mgr.y (mgr.24961) 1658 : cluster [DBG] pgmap v2019: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:47 smithi059 bash[61152]: cluster 2022-04-28T20:37:46.166149+0000 mgr.y (mgr.24961) 1659 : cluster [DBG] pgmap v2020: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:48.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:47 smithi059 bash[59745]: cluster 2022-04-28T20:37:46.166149+0000 mgr.y (mgr.24961) 1659 : cluster [DBG] pgmap v2020: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:47 smithi139 bash[43301]: cluster 2022-04-28T20:37:46.166149+0000 mgr.y (mgr.24961) 1659 : cluster [DBG] pgmap v2020: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:49 smithi059 bash[61152]: cluster 2022-04-28T20:37:48.166819+0000 mgr.y (mgr.24961) 1660 : cluster [DBG] pgmap v2021: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:49 smithi059 bash[59745]: cluster 2022-04-28T20:37:48.166819+0000 mgr.y (mgr.24961) 1660 : cluster [DBG] pgmap v2021: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:49 smithi139 bash[43301]: cluster 2022-04-28T20:37:48.166819+0000 mgr.y (mgr.24961) 1660 : cluster [DBG] pgmap v2021: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:51 smithi059 bash[61152]: cluster 2022-04-28T20:37:50.168031+0000 mgr.y (mgr.24961) 1661 : cluster [DBG] pgmap v2022: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:52.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:51.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:52.099 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:51 smithi059 bash[59745]: cluster 2022-04-28T20:37:50.168031+0000 mgr.y (mgr.24961) 1661 : cluster [DBG] pgmap v2022: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:51 smithi139 bash[43301]: cluster 2022-04-28T20:37:50.168031+0000 mgr.y (mgr.24961) 1661 : cluster [DBG] pgmap v2022: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:53 smithi059 bash[59745]: cluster 2022-04-28T20:37:52.169004+0000 mgr.y (mgr.24961) 1662 : cluster [DBG] pgmap v2023: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:53 smithi059 bash[61152]: cluster 2022-04-28T20:37:52.169004+0000 mgr.y (mgr.24961) 1662 : cluster [DBG] pgmap v2023: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:37:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:37:53.697Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:37:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:53 smithi139 bash[43301]: cluster 2022-04-28T20:37:52.169004+0000 mgr.y (mgr.24961) 1662 : cluster [DBG] pgmap v2023: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:55 smithi139 bash[43301]: cluster 2022-04-28T20:37:54.169951+0000 mgr.y (mgr.24961) 1663 : cluster [DBG] pgmap v2024: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:55 smithi059 bash[59745]: cluster 2022-04-28T20:37:54.169951+0000 mgr.y (mgr.24961) 1663 : cluster [DBG] pgmap v2024: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:55 smithi059 bash[61152]: cluster 2022-04-28T20:37:54.169951+0000 mgr.y (mgr.24961) 1663 : cluster [DBG] pgmap v2024: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:57 smithi139 bash[43301]: cluster 2022-04-28T20:37:56.171140+0000 mgr.y (mgr.24961) 1664 : cluster [DBG] pgmap v2025: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:57 smithi059 bash[59745]: cluster 2022-04-28T20:37:56.171140+0000 mgr.y (mgr.24961) 1664 : cluster [DBG] pgmap v2025: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:37:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:57 smithi059 bash[61152]: cluster 2022-04-28T20:37:56.171140+0000 mgr.y (mgr.24961) 1664 : cluster [DBG] pgmap v2025: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:37:59 smithi139 bash[43301]: cluster 2022-04-28T20:37:58.171832+0000 mgr.y (mgr.24961) 1665 : cluster [DBG] pgmap v2026: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:37:59 smithi059 bash[59745]: cluster 2022-04-28T20:37:58.171832+0000 mgr.y (mgr.24961) 1665 : cluster [DBG] pgmap v2026: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:37:59 smithi059 bash[61152]: cluster 2022-04-28T20:37:58.171832+0000 mgr.y (mgr.24961) 1665 : cluster [DBG] pgmap v2026: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:01 smithi059 bash[61152]: cluster 2022-04-28T20:38:00.173000+0000 mgr.y (mgr.24961) 1666 : cluster [DBG] pgmap v2027: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:01.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:01 smithi059 bash[59745]: cluster 2022-04-28T20:38:00.173000+0000 mgr.y (mgr.24961) 1666 : cluster [DBG] pgmap v2027: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:01 smithi139 bash[43301]: cluster 2022-04-28T20:38:00.173000+0000 mgr.y (mgr.24961) 1666 : cluster [DBG] pgmap v2027: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:03 smithi059 bash[61152]: cluster 2022-04-28T20:38:02.173961+0000 mgr.y (mgr.24961) 1667 : cluster [DBG] pgmap v2028: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:03.700Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:03 smithi059 bash[59745]: cluster 2022-04-28T20:38:02.173961+0000 mgr.y (mgr.24961) 1667 : cluster [DBG] pgmap v2028: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:04.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:03 smithi139 bash[43301]: cluster 2022-04-28T20:38:02.173961+0000 mgr.y (mgr.24961) 1667 : cluster [DBG] pgmap v2028: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:05 smithi059 bash[61152]: cluster 2022-04-28T20:38:04.174873+0000 mgr.y (mgr.24961) 1668 : cluster [DBG] pgmap v2029: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:05 smithi059 bash[59745]: cluster 2022-04-28T20:38:04.174873+0000 mgr.y (mgr.24961) 1668 : cluster [DBG] pgmap v2029: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:05 smithi139 bash[43301]: cluster 2022-04-28T20:38:04.174873+0000 mgr.y (mgr.24961) 1668 : cluster [DBG] pgmap v2029: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:07 smithi059 bash[61152]: cluster 2022-04-28T20:38:06.176062+0000 mgr.y (mgr.24961) 1669 : cluster [DBG] pgmap v2030: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:07 smithi059 bash[59745]: cluster 2022-04-28T20:38:06.176062+0000 mgr.y (mgr.24961) 1669 : cluster [DBG] pgmap v2030: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:07 smithi139 bash[43301]: cluster 2022-04-28T20:38:06.176062+0000 mgr.y (mgr.24961) 1669 : cluster [DBG] pgmap v2030: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:09 smithi059 bash[61152]: cluster 2022-04-28T20:38:08.176748+0000 mgr.y (mgr.24961) 1670 : cluster [DBG] pgmap v2031: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:10.100 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:09 smithi059 bash[59745]: cluster 2022-04-28T20:38:08.176748+0000 mgr.y (mgr.24961) 1670 : cluster [DBG] pgmap v2031: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:09 smithi139 bash[43301]: cluster 2022-04-28T20:38:08.176748+0000 mgr.y (mgr.24961) 1670 : cluster [DBG] pgmap v2031: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:11 smithi059 bash[61152]: cluster 2022-04-28T20:38:10.177986+0000 mgr.y (mgr.24961) 1671 : cluster [DBG] pgmap v2032: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:11.808Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:11 smithi059 bash[59745]: cluster 2022-04-28T20:38:10.177986+0000 mgr.y (mgr.24961) 1671 : cluster [DBG] pgmap v2032: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:11 smithi139 bash[43301]: cluster 2022-04-28T20:38:10.177986+0000 mgr.y (mgr.24961) 1671 : cluster [DBG] pgmap v2032: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:13 smithi059 bash[61152]: cluster 2022-04-28T20:38:12.178906+0000 mgr.y (mgr.24961) 1672 : cluster [DBG] pgmap v2033: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:13.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:13 smithi059 bash[59745]: cluster 2022-04-28T20:38:12.178906+0000 mgr.y (mgr.24961) 1672 : cluster [DBG] pgmap v2033: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:13 smithi139 bash[43301]: cluster 2022-04-28T20:38:12.178906+0000 mgr.y (mgr.24961) 1672 : cluster [DBG] pgmap v2033: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:15 smithi059 bash[61152]: cluster 2022-04-28T20:38:14.179884+0000 mgr.y (mgr.24961) 1673 : cluster [DBG] pgmap v2034: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:15 smithi059 bash[59745]: cluster 2022-04-28T20:38:14.179884+0000 mgr.y (mgr.24961) 1673 : cluster [DBG] pgmap v2034: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:15 smithi139 bash[43301]: cluster 2022-04-28T20:38:14.179884+0000 mgr.y (mgr.24961) 1673 : cluster [DBG] pgmap v2034: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:17 smithi059 bash[61152]: cluster 2022-04-28T20:38:16.181101+0000 mgr.y (mgr.24961) 1674 : cluster [DBG] pgmap v2035: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:17 smithi059 bash[61152]: audit 2022-04-28T20:38:17.688445+0000 mon.a (mon.0) 2093 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:38:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:17 smithi059 bash[61152]: audit 2022-04-28T20:38:17.689197+0000 mon.a (mon.0) 2094 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:38:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:17 smithi059 bash[59745]: cluster 2022-04-28T20:38:16.181101+0000 mgr.y (mgr.24961) 1674 : cluster [DBG] pgmap v2035: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:17 smithi059 bash[59745]: audit 2022-04-28T20:38:17.688445+0000 mon.a (mon.0) 2093 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:38:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:17 smithi059 bash[59745]: audit 2022-04-28T20:38:17.689197+0000 mon.a (mon.0) 2094 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:38:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:17 smithi139 bash[43301]: cluster 2022-04-28T20:38:16.181101+0000 mgr.y (mgr.24961) 1674 : cluster [DBG] pgmap v2035: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:17 smithi139 bash[43301]: audit 2022-04-28T20:38:17.688445+0000 mon.a (mon.0) 2093 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:38:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:17 smithi139 bash[43301]: audit 2022-04-28T20:38:17.689197+0000 mon.a (mon.0) 2094 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:38:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[61152]: audit 2022-04-28T20:38:17.701440+0000 mon.a (mon.0) 2095 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[61152]: audit 2022-04-28T20:38:17.702214+0000 mon.a (mon.0) 2096 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[61152]: audit 2022-04-28T20:38:17.702641+0000 mon.a (mon.0) 2097 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2022-04-28T20:38:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[61152]: audit 2022-04-28T20:38:17.703038+0000 mon.a (mon.0) 2098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[59745]: audit 2022-04-28T20:38:17.701440+0000 mon.a (mon.0) 2095 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[59745]: audit 2022-04-28T20:38:17.702214+0000 mon.a (mon.0) 2096 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[59745]: audit 2022-04-28T20:38:17.702641+0000 mon.a (mon.0) 2097 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2022-04-28T20:38:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:18 smithi059 bash[59745]: audit 2022-04-28T20:38:17.703038+0000 mon.a (mon.0) 2098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:18 smithi139 bash[43301]: audit 2022-04-28T20:38:17.701440+0000 mon.a (mon.0) 2095 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:18 smithi139 bash[43301]: audit 2022-04-28T20:38:17.702214+0000 mon.a (mon.0) 2096 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:38:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:18 smithi139 bash[43301]: audit 2022-04-28T20:38:17.702641+0000 mon.a (mon.0) 2097 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2022-04-28T20:38:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:18 smithi139 bash[43301]: audit 2022-04-28T20:38:17.703038+0000 mon.a (mon.0) 2098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:38:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:19 smithi059 bash[61152]: cluster 2022-04-28T20:38:18.181764+0000 mgr.y (mgr.24961) 1675 : cluster [DBG] pgmap v2036: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:19 smithi059 bash[59745]: cluster 2022-04-28T20:38:18.181764+0000 mgr.y (mgr.24961) 1675 : cluster [DBG] pgmap v2036: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:19 smithi139 bash[43301]: cluster 2022-04-28T20:38:18.181764+0000 mgr.y (mgr.24961) 1675 : cluster [DBG] pgmap v2036: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:21 smithi059 bash[61152]: cluster 2022-04-28T20:38:20.182961+0000 mgr.y (mgr.24961) 1676 : cluster [DBG] pgmap v2037: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:22.136 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:21.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:22.137 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:21 smithi059 bash[59745]: cluster 2022-04-28T20:38:20.182961+0000 mgr.y (mgr.24961) 1676 : cluster [DBG] pgmap v2037: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:22.137 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:21 smithi139 bash[43301]: cluster 2022-04-28T20:38:20.182961+0000 mgr.y (mgr.24961) 1676 : cluster [DBG] pgmap v2037: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:23 smithi059 bash[61152]: cluster 2022-04-28T20:38:22.183906+0000 mgr.y (mgr.24961) 1677 : cluster [DBG] pgmap v2038: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:23.701Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:23 smithi059 bash[59745]: cluster 2022-04-28T20:38:22.183906+0000 mgr.y (mgr.24961) 1677 : cluster [DBG] pgmap v2038: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:23 smithi139 bash[43301]: cluster 2022-04-28T20:38:22.183906+0000 mgr.y (mgr.24961) 1677 : cluster [DBG] pgmap v2038: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:25 smithi059 bash[61152]: cluster 2022-04-28T20:38:24.184829+0000 mgr.y (mgr.24961) 1678 : cluster [DBG] pgmap v2039: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:25 smithi059 bash[59745]: cluster 2022-04-28T20:38:24.184829+0000 mgr.y (mgr.24961) 1678 : cluster [DBG] pgmap v2039: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:25 smithi139 bash[43301]: cluster 2022-04-28T20:38:24.184829+0000 mgr.y (mgr.24961) 1678 : cluster [DBG] pgmap v2039: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:27 smithi059 bash[61152]: cluster 2022-04-28T20:38:26.186033+0000 mgr.y (mgr.24961) 1679 : cluster [DBG] pgmap v2040: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:27 smithi059 bash[59745]: cluster 2022-04-28T20:38:26.186033+0000 mgr.y (mgr.24961) 1679 : cluster [DBG] pgmap v2040: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:27 smithi139 bash[43301]: cluster 2022-04-28T20:38:26.186033+0000 mgr.y (mgr.24961) 1679 : cluster [DBG] pgmap v2040: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:29 smithi059 bash[59745]: cluster 2022-04-28T20:38:28.186694+0000 mgr.y (mgr.24961) 1680 : cluster [DBG] pgmap v2041: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:29 smithi059 bash[61152]: cluster 2022-04-28T20:38:28.186694+0000 mgr.y (mgr.24961) 1680 : cluster [DBG] pgmap v2041: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:29 smithi139 bash[43301]: cluster 2022-04-28T20:38:28.186694+0000 mgr.y (mgr.24961) 1680 : cluster [DBG] pgmap v2041: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:31 smithi059 bash[59745]: cluster 2022-04-28T20:38:30.187851+0000 mgr.y (mgr.24961) 1681 : cluster [DBG] pgmap v2042: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:31.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:31 smithi059 bash[61152]: cluster 2022-04-28T20:38:30.187851+0000 mgr.y (mgr.24961) 1681 : cluster [DBG] pgmap v2042: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:31 smithi139 bash[43301]: cluster 2022-04-28T20:38:30.187851+0000 mgr.y (mgr.24961) 1681 : cluster [DBG] pgmap v2042: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:33 smithi059 bash[61152]: cluster 2022-04-28T20:38:32.188720+0000 mgr.y (mgr.24961) 1682 : cluster [DBG] pgmap v2043: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:33.704Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:33 smithi059 bash[59745]: cluster 2022-04-28T20:38:32.188720+0000 mgr.y (mgr.24961) 1682 : cluster [DBG] pgmap v2043: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:33 smithi139 bash[43301]: cluster 2022-04-28T20:38:32.188720+0000 mgr.y (mgr.24961) 1682 : cluster [DBG] pgmap v2043: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:35 smithi059 bash[61152]: cluster 2022-04-28T20:38:34.189633+0000 mgr.y (mgr.24961) 1683 : cluster [DBG] pgmap v2044: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:36.228 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:35 smithi059 bash[59745]: cluster 2022-04-28T20:38:34.189633+0000 mgr.y (mgr.24961) 1683 : cluster [DBG] pgmap v2044: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:36.229 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:35 smithi139 bash[43301]: cluster 2022-04-28T20:38:34.189633+0000 mgr.y (mgr.24961) 1683 : cluster [DBG] pgmap v2044: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[61152]: cluster 2022-04-28T20:38:36.190810+0000 mgr.y (mgr.24961) 1684 : cluster [DBG] pgmap v2045: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[61152]: audit 2022-04-28T20:38:37.294285+0000 mon.a (mon.0) 2099 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:38:38.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[61152]: audit 2022-04-28T20:38:37.297261+0000 mon.a (mon.0) 2100 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:38:38.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[61152]: audit 2022-04-28T20:38:37.299167+0000 mon.a (mon.0) 2101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:38:38.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[61152]: audit 2022-04-28T20:38:37.602965+0000 mon.a (mon.0) 2102 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:38.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[61152]: audit 2022-04-28T20:38:37.610361+0000 mon.a (mon.0) 2103 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[59745]: cluster 2022-04-28T20:38:36.190810+0000 mgr.y (mgr.24961) 1684 : cluster [DBG] pgmap v2045: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:38.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[59745]: audit 2022-04-28T20:38:37.294285+0000 mon.a (mon.0) 2099 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:38:38.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[59745]: audit 2022-04-28T20:38:37.297261+0000 mon.a (mon.0) 2100 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:38:38.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[59745]: audit 2022-04-28T20:38:37.299167+0000 mon.a (mon.0) 2101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:38:38.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[59745]: audit 2022-04-28T20:38:37.602965+0000 mon.a (mon.0) 2102 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:38.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:37 smithi059 bash[59745]: audit 2022-04-28T20:38:37.610361+0000 mon.a (mon.0) 2103 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:37 smithi139 bash[43301]: cluster 2022-04-28T20:38:36.190810+0000 mgr.y (mgr.24961) 1684 : cluster [DBG] pgmap v2045: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:37 smithi139 bash[43301]: audit 2022-04-28T20:38:37.294285+0000 mon.a (mon.0) 2099 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:38:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:37 smithi139 bash[43301]: audit 2022-04-28T20:38:37.297261+0000 mon.a (mon.0) 2100 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:38:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:37 smithi139 bash[43301]: audit 2022-04-28T20:38:37.299167+0000 mon.a (mon.0) 2101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:38:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:37 smithi139 bash[43301]: audit 2022-04-28T20:38:37.602965+0000 mon.a (mon.0) 2102 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:37 smithi139 bash[43301]: audit 2022-04-28T20:38:37.610361+0000 mon.a (mon.0) 2103 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:38 smithi059 bash[61152]: cephadm 2022-04-28T20:38:37.300961+0000 mgr.y (mgr.24961) 1685 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:38:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:38 smithi059 bash[61152]: audit 2022-04-28T20:38:37.920259+0000 mon.a (mon.0) 2104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:39.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:38 smithi059 bash[59745]: cephadm 2022-04-28T20:38:37.300961+0000 mgr.y (mgr.24961) 1685 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:38:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:38 smithi059 bash[59745]: audit 2022-04-28T20:38:37.920259+0000 mon.a (mon.0) 2104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:38 smithi139 bash[43301]: cephadm 2022-04-28T20:38:37.300961+0000 mgr.y (mgr.24961) 1685 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:38:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:38 smithi139 bash[43301]: audit 2022-04-28T20:38:37.920259+0000 mon.a (mon.0) 2104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:38:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:39 smithi059 bash[61152]: cluster 2022-04-28T20:38:38.191456+0000 mgr.y (mgr.24961) 1686 : cluster [DBG] pgmap v2046: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:39 smithi059 bash[59745]: cluster 2022-04-28T20:38:38.191456+0000 mgr.y (mgr.24961) 1686 : cluster [DBG] pgmap v2046: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:39 smithi139 bash[43301]: cluster 2022-04-28T20:38:38.191456+0000 mgr.y (mgr.24961) 1686 : cluster [DBG] pgmap v2046: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:41 smithi059 bash[61152]: cluster 2022-04-28T20:38:40.192691+0000 mgr.y (mgr.24961) 1687 : cluster [DBG] pgmap v2047: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:41.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:41 smithi059 bash[59745]: cluster 2022-04-28T20:38:40.192691+0000 mgr.y (mgr.24961) 1687 : cluster [DBG] pgmap v2047: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:41 smithi139 bash[43301]: cluster 2022-04-28T20:38:40.192691+0000 mgr.y (mgr.24961) 1687 : cluster [DBG] pgmap v2047: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:43 smithi059 bash[61152]: cluster 2022-04-28T20:38:42.193647+0000 mgr.y (mgr.24961) 1688 : cluster [DBG] pgmap v2048: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:43.703Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:43 smithi059 bash[59745]: cluster 2022-04-28T20:38:42.193647+0000 mgr.y (mgr.24961) 1688 : cluster [DBG] pgmap v2048: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:43 smithi139 bash[43301]: cluster 2022-04-28T20:38:42.193647+0000 mgr.y (mgr.24961) 1688 : cluster [DBG] pgmap v2048: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:45 smithi059 bash[61152]: cluster 2022-04-28T20:38:44.194524+0000 mgr.y (mgr.24961) 1689 : cluster [DBG] pgmap v2049: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:45 smithi059 bash[59745]: cluster 2022-04-28T20:38:44.194524+0000 mgr.y (mgr.24961) 1689 : cluster [DBG] pgmap v2049: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:45 smithi139 bash[43301]: cluster 2022-04-28T20:38:44.194524+0000 mgr.y (mgr.24961) 1689 : cluster [DBG] pgmap v2049: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:47 smithi059 bash[61152]: cluster 2022-04-28T20:38:46.195789+0000 mgr.y (mgr.24961) 1690 : cluster [DBG] pgmap v2050: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:47 smithi059 bash[59745]: cluster 2022-04-28T20:38:46.195789+0000 mgr.y (mgr.24961) 1690 : cluster [DBG] pgmap v2050: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:47 smithi139 bash[43301]: cluster 2022-04-28T20:38:46.195789+0000 mgr.y (mgr.24961) 1690 : cluster [DBG] pgmap v2050: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:49 smithi059 bash[61152]: cluster 2022-04-28T20:38:48.196432+0000 mgr.y (mgr.24961) 1691 : cluster [DBG] pgmap v2051: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:49 smithi059 bash[59745]: cluster 2022-04-28T20:38:48.196432+0000 mgr.y (mgr.24961) 1691 : cluster [DBG] pgmap v2051: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:49 smithi139 bash[43301]: cluster 2022-04-28T20:38:48.196432+0000 mgr.y (mgr.24961) 1691 : cluster [DBG] pgmap v2051: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:51 smithi059 bash[61152]: cluster 2022-04-28T20:38:50.197597+0000 mgr.y (mgr.24961) 1692 : cluster [DBG] pgmap v2052: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:52.144 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:51.811Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:52.144 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:51 smithi059 bash[59745]: cluster 2022-04-28T20:38:50.197597+0000 mgr.y (mgr.24961) 1692 : cluster [DBG] pgmap v2052: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:52.145 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:51 smithi139 bash[43301]: cluster 2022-04-28T20:38:50.197597+0000 mgr.y (mgr.24961) 1692 : cluster [DBG] pgmap v2052: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:53 smithi059 bash[61152]: cluster 2022-04-28T20:38:52.198574+0000 mgr.y (mgr.24961) 1693 : cluster [DBG] pgmap v2053: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:53 smithi059 bash[59745]: cluster 2022-04-28T20:38:52.198574+0000 mgr.y (mgr.24961) 1693 : cluster [DBG] pgmap v2053: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:38:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:38:53.705Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:38:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:53 smithi139 bash[43301]: cluster 2022-04-28T20:38:52.198574+0000 mgr.y (mgr.24961) 1693 : cluster [DBG] pgmap v2053: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:55 smithi059 bash[61152]: cluster 2022-04-28T20:38:54.199549+0000 mgr.y (mgr.24961) 1694 : cluster [DBG] pgmap v2054: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:56.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:55 smithi059 bash[59745]: cluster 2022-04-28T20:38:54.199549+0000 mgr.y (mgr.24961) 1694 : cluster [DBG] pgmap v2054: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:55 smithi139 bash[43301]: cluster 2022-04-28T20:38:54.199549+0000 mgr.y (mgr.24961) 1694 : cluster [DBG] pgmap v2054: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:57 smithi059 bash[61152]: cluster 2022-04-28T20:38:56.200671+0000 mgr.y (mgr.24961) 1695 : cluster [DBG] pgmap v2055: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:57 smithi059 bash[59745]: cluster 2022-04-28T20:38:56.200671+0000 mgr.y (mgr.24961) 1695 : cluster [DBG] pgmap v2055: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:38:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:57 smithi139 bash[43301]: cluster 2022-04-28T20:38:56.200671+0000 mgr.y (mgr.24961) 1695 : cluster [DBG] pgmap v2055: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:38:59 smithi059 bash[61152]: cluster 2022-04-28T20:38:58.201351+0000 mgr.y (mgr.24961) 1696 : cluster [DBG] pgmap v2056: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:38:59 smithi059 bash[59745]: cluster 2022-04-28T20:38:58.201351+0000 mgr.y (mgr.24961) 1696 : cluster [DBG] pgmap v2056: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:38:59 smithi139 bash[43301]: cluster 2022-04-28T20:38:58.201351+0000 mgr.y (mgr.24961) 1696 : cluster [DBG] pgmap v2056: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:01 smithi059 bash[61152]: cluster 2022-04-28T20:39:00.202534+0000 mgr.y (mgr.24961) 1697 : cluster [DBG] pgmap v2057: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:02.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:01.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:01 smithi059 bash[59745]: cluster 2022-04-28T20:39:00.202534+0000 mgr.y (mgr.24961) 1697 : cluster [DBG] pgmap v2057: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:01 smithi139 bash[43301]: cluster 2022-04-28T20:39:00.202534+0000 mgr.y (mgr.24961) 1697 : cluster [DBG] pgmap v2057: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:03 smithi059 bash[61152]: cluster 2022-04-28T20:39:02.203622+0000 mgr.y (mgr.24961) 1698 : cluster [DBG] pgmap v2058: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:03.706Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:03 smithi059 bash[59745]: cluster 2022-04-28T20:39:02.203622+0000 mgr.y (mgr.24961) 1698 : cluster [DBG] pgmap v2058: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:03 smithi139 bash[43301]: cluster 2022-04-28T20:39:02.203622+0000 mgr.y (mgr.24961) 1698 : cluster [DBG] pgmap v2058: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:05 smithi059 bash[61152]: cluster 2022-04-28T20:39:04.204604+0000 mgr.y (mgr.24961) 1699 : cluster [DBG] pgmap v2059: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:05 smithi059 bash[59745]: cluster 2022-04-28T20:39:04.204604+0000 mgr.y (mgr.24961) 1699 : cluster [DBG] pgmap v2059: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:05 smithi139 bash[43301]: cluster 2022-04-28T20:39:04.204604+0000 mgr.y (mgr.24961) 1699 : cluster [DBG] pgmap v2059: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:07 smithi139 bash[43301]: cluster 2022-04-28T20:39:06.205810+0000 mgr.y (mgr.24961) 1700 : cluster [DBG] pgmap v2060: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:07 smithi059 bash[61152]: cluster 2022-04-28T20:39:06.205810+0000 mgr.y (mgr.24961) 1700 : cluster [DBG] pgmap v2060: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:07 smithi059 bash[59745]: cluster 2022-04-28T20:39:06.205810+0000 mgr.y (mgr.24961) 1700 : cluster [DBG] pgmap v2060: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:09 smithi139 bash[43301]: cluster 2022-04-28T20:39:08.206503+0000 mgr.y (mgr.24961) 1701 : cluster [DBG] pgmap v2061: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:09 smithi059 bash[61152]: cluster 2022-04-28T20:39:08.206503+0000 mgr.y (mgr.24961) 1701 : cluster [DBG] pgmap v2061: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:09 smithi059 bash[59745]: cluster 2022-04-28T20:39:08.206503+0000 mgr.y (mgr.24961) 1701 : cluster [DBG] pgmap v2061: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:11 smithi059 bash[61152]: cluster 2022-04-28T20:39:10.207786+0000 mgr.y (mgr.24961) 1702 : cluster [DBG] pgmap v2062: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:11.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:11 smithi059 bash[59745]: cluster 2022-04-28T20:39:10.207786+0000 mgr.y (mgr.24961) 1702 : cluster [DBG] pgmap v2062: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:11 smithi139 bash[43301]: cluster 2022-04-28T20:39:10.207786+0000 mgr.y (mgr.24961) 1702 : cluster [DBG] pgmap v2062: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:13 smithi059 bash[61152]: cluster 2022-04-28T20:39:12.208749+0000 mgr.y (mgr.24961) 1703 : cluster [DBG] pgmap v2063: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:13.708Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:13 smithi059 bash[59745]: cluster 2022-04-28T20:39:12.208749+0000 mgr.y (mgr.24961) 1703 : cluster [DBG] pgmap v2063: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:13 smithi139 bash[43301]: cluster 2022-04-28T20:39:12.208749+0000 mgr.y (mgr.24961) 1703 : cluster [DBG] pgmap v2063: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:15 smithi059 bash[61152]: cluster 2022-04-28T20:39:14.209736+0000 mgr.y (mgr.24961) 1704 : cluster [DBG] pgmap v2064: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:15 smithi059 bash[59745]: cluster 2022-04-28T20:39:14.209736+0000 mgr.y (mgr.24961) 1704 : cluster [DBG] pgmap v2064: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:15 smithi139 bash[43301]: cluster 2022-04-28T20:39:14.209736+0000 mgr.y (mgr.24961) 1704 : cluster [DBG] pgmap v2064: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[59745]: cluster 2022-04-28T20:39:16.210833+0000 mgr.y (mgr.24961) 1705 : cluster [DBG] pgmap v2065: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[59745]: audit 2022-04-28T20:39:17.702822+0000 mon.a (mon.0) 2105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:39:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[59745]: audit 2022-04-28T20:39:17.703586+0000 mon.a (mon.0) 2106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:39:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[59745]: audit 2022-04-28T20:39:17.713703+0000 mon.a (mon.0) 2107 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:39:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[59745]: audit 2022-04-28T20:39:17.714529+0000 mon.a (mon.0) 2108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:39:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[59745]: audit 2022-04-28T20:39:17.715323+0000 mon.a (mon.0) 2109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:39:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[61152]: cluster 2022-04-28T20:39:16.210833+0000 mgr.y (mgr.24961) 1705 : cluster [DBG] pgmap v2065: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[61152]: audit 2022-04-28T20:39:17.702822+0000 mon.a (mon.0) 2105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:39:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[61152]: audit 2022-04-28T20:39:17.703586+0000 mon.a (mon.0) 2106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:39:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[61152]: audit 2022-04-28T20:39:17.713703+0000 mon.a (mon.0) 2107 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:39:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[61152]: audit 2022-04-28T20:39:17.714529+0000 mon.a (mon.0) 2108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:39:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:17 smithi059 bash[61152]: audit 2022-04-28T20:39:17.715323+0000 mon.a (mon.0) 2109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:39:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:17 smithi139 bash[43301]: cluster 2022-04-28T20:39:16.210833+0000 mgr.y (mgr.24961) 1705 : cluster [DBG] pgmap v2065: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:17 smithi139 bash[43301]: audit 2022-04-28T20:39:17.702822+0000 mon.a (mon.0) 2105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:39:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:17 smithi139 bash[43301]: audit 2022-04-28T20:39:17.703586+0000 mon.a (mon.0) 2106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:39:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:17 smithi139 bash[43301]: audit 2022-04-28T20:39:17.713703+0000 mon.a (mon.0) 2107 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:39:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:17 smithi139 bash[43301]: audit 2022-04-28T20:39:17.714529+0000 mon.a (mon.0) 2108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:39:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:17 smithi139 bash[43301]: audit 2022-04-28T20:39:17.715323+0000 mon.a (mon.0) 2109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:39:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:19 smithi059 bash[61152]: cluster 2022-04-28T20:39:18.211532+0000 mgr.y (mgr.24961) 1706 : cluster [DBG] pgmap v2066: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:19 smithi059 bash[59745]: cluster 2022-04-28T20:39:18.211532+0000 mgr.y (mgr.24961) 1706 : cluster [DBG] pgmap v2066: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:19 smithi139 bash[43301]: cluster 2022-04-28T20:39:18.211532+0000 mgr.y (mgr.24961) 1706 : cluster [DBG] pgmap v2066: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:21 smithi059 bash[61152]: cluster 2022-04-28T20:39:20.212748+0000 mgr.y (mgr.24961) 1707 : cluster [DBG] pgmap v2067: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:22.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:21.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:22.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:21 smithi059 bash[59745]: cluster 2022-04-28T20:39:20.212748+0000 mgr.y (mgr.24961) 1707 : cluster [DBG] pgmap v2067: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:21 smithi139 bash[43301]: cluster 2022-04-28T20:39:20.212748+0000 mgr.y (mgr.24961) 1707 : cluster [DBG] pgmap v2067: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:23 smithi059 bash[61152]: cluster 2022-04-28T20:39:22.213672+0000 mgr.y (mgr.24961) 1708 : cluster [DBG] pgmap v2068: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:23.707Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:23 smithi059 bash[59745]: cluster 2022-04-28T20:39:22.213672+0000 mgr.y (mgr.24961) 1708 : cluster [DBG] pgmap v2068: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:23 smithi139 bash[43301]: cluster 2022-04-28T20:39:22.213672+0000 mgr.y (mgr.24961) 1708 : cluster [DBG] pgmap v2068: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:25 smithi059 bash[61152]: cluster 2022-04-28T20:39:24.214669+0000 mgr.y (mgr.24961) 1709 : cluster [DBG] pgmap v2069: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:25 smithi059 bash[59745]: cluster 2022-04-28T20:39:24.214669+0000 mgr.y (mgr.24961) 1709 : cluster [DBG] pgmap v2069: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:25 smithi139 bash[43301]: cluster 2022-04-28T20:39:24.214669+0000 mgr.y (mgr.24961) 1709 : cluster [DBG] pgmap v2069: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:27 smithi059 bash[61152]: cluster 2022-04-28T20:39:26.215798+0000 mgr.y (mgr.24961) 1710 : cluster [DBG] pgmap v2070: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:27 smithi059 bash[59745]: cluster 2022-04-28T20:39:26.215798+0000 mgr.y (mgr.24961) 1710 : cluster [DBG] pgmap v2070: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:27 smithi139 bash[43301]: cluster 2022-04-28T20:39:26.215798+0000 mgr.y (mgr.24961) 1710 : cluster [DBG] pgmap v2070: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:29 smithi059 bash[61152]: cluster 2022-04-28T20:39:28.216497+0000 mgr.y (mgr.24961) 1711 : cluster [DBG] pgmap v2071: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:30.160 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:29 smithi059 bash[59745]: cluster 2022-04-28T20:39:28.216497+0000 mgr.y (mgr.24961) 1711 : cluster [DBG] pgmap v2071: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:30.161 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:29 smithi139 bash[43301]: cluster 2022-04-28T20:39:28.216497+0000 mgr.y (mgr.24961) 1711 : cluster [DBG] pgmap v2071: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:31 smithi059 bash[61152]: cluster 2022-04-28T20:39:30.217686+0000 mgr.y (mgr.24961) 1712 : cluster [DBG] pgmap v2072: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:31.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:31 smithi059 bash[59745]: cluster 2022-04-28T20:39:30.217686+0000 mgr.y (mgr.24961) 1712 : cluster [DBG] pgmap v2072: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:31 smithi139 bash[43301]: cluster 2022-04-28T20:39:30.217686+0000 mgr.y (mgr.24961) 1712 : cluster [DBG] pgmap v2072: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:33 smithi059 bash[61152]: cluster 2022-04-28T20:39:32.218600+0000 mgr.y (mgr.24961) 1713 : cluster [DBG] pgmap v2073: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:33.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:33 smithi059 bash[59745]: cluster 2022-04-28T20:39:32.218600+0000 mgr.y (mgr.24961) 1713 : cluster [DBG] pgmap v2073: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:33 smithi139 bash[43301]: cluster 2022-04-28T20:39:32.218600+0000 mgr.y (mgr.24961) 1713 : cluster [DBG] pgmap v2073: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:36.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:35 smithi059 bash[61152]: cluster 2022-04-28T20:39:34.219736+0000 mgr.y (mgr.24961) 1714 : cluster [DBG] pgmap v2074: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:36.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:35 smithi059 bash[59745]: cluster 2022-04-28T20:39:34.219736+0000 mgr.y (mgr.24961) 1714 : cluster [DBG] pgmap v2074: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:35 smithi139 bash[43301]: cluster 2022-04-28T20:39:34.219736+0000 mgr.y (mgr.24961) 1714 : cluster [DBG] pgmap v2074: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:37 smithi059 bash[61152]: cluster 2022-04-28T20:39:36.220870+0000 mgr.y (mgr.24961) 1715 : cluster [DBG] pgmap v2075: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:37 smithi059 bash[59745]: cluster 2022-04-28T20:39:36.220870+0000 mgr.y (mgr.24961) 1715 : cluster [DBG] pgmap v2075: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:37 smithi139 bash[43301]: cluster 2022-04-28T20:39:36.220870+0000 mgr.y (mgr.24961) 1715 : cluster [DBG] pgmap v2075: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:39.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[59745]: audit 2022-04-28T20:39:37.927276+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:39:39.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[59745]: audit 2022-04-28T20:39:37.930160+0000 mon.a (mon.0) 2111 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:39:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[59745]: audit 2022-04-28T20:39:37.932089+0000 mon.a (mon.0) 2112 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:39:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[59745]: audit 2022-04-28T20:39:37.935237+0000 mon.a (mon.0) 2113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:39:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[61152]: audit 2022-04-28T20:39:37.927276+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:39:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[61152]: audit 2022-04-28T20:39:37.930160+0000 mon.a (mon.0) 2111 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:39:39.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[61152]: audit 2022-04-28T20:39:37.932089+0000 mon.a (mon.0) 2112 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:39:39.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:38 smithi059 bash[61152]: audit 2022-04-28T20:39:37.935237+0000 mon.a (mon.0) 2113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:39:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:38 smithi139 bash[43301]: audit 2022-04-28T20:39:37.927276+0000 mon.a (mon.0) 2110 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:39:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:38 smithi139 bash[43301]: audit 2022-04-28T20:39:37.930160+0000 mon.a (mon.0) 2111 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:39:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:38 smithi139 bash[43301]: audit 2022-04-28T20:39:37.932089+0000 mon.a (mon.0) 2112 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:39:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:38 smithi139 bash[43301]: audit 2022-04-28T20:39:37.935237+0000 mon.a (mon.0) 2113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:39:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:39 smithi059 bash[61152]: cephadm 2022-04-28T20:39:37.933745+0000 mgr.y (mgr.24961) 1716 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:39:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:39 smithi059 bash[61152]: cluster 2022-04-28T20:39:38.221679+0000 mgr.y (mgr.24961) 1717 : cluster [DBG] pgmap v2076: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:39 smithi059 bash[59745]: cephadm 2022-04-28T20:39:37.933745+0000 mgr.y (mgr.24961) 1716 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:39:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:39 smithi059 bash[59745]: cluster 2022-04-28T20:39:38.221679+0000 mgr.y (mgr.24961) 1717 : cluster [DBG] pgmap v2076: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:39 smithi139 bash[43301]: cephadm 2022-04-28T20:39:37.933745+0000 mgr.y (mgr.24961) 1716 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:39:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:39 smithi139 bash[43301]: cluster 2022-04-28T20:39:38.221679+0000 mgr.y (mgr.24961) 1717 : cluster [DBG] pgmap v2076: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:42.037 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:41 smithi059 bash[61152]: cluster 2022-04-28T20:39:40.223256+0000 mgr.y (mgr.24961) 1718 : cluster [DBG] pgmap v2077: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:42.037 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:41 smithi059 bash[59745]: cluster 2022-04-28T20:39:40.223256+0000 mgr.y (mgr.24961) 1718 : cluster [DBG] pgmap v2077: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:42.038 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:41.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:41 smithi139 bash[43301]: cluster 2022-04-28T20:39:40.223256+0000 mgr.y (mgr.24961) 1718 : cluster [DBG] pgmap v2077: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:43.742 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:43.709Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:43 smithi059 bash[61152]: cluster 2022-04-28T20:39:42.224323+0000 mgr.y (mgr.24961) 1719 : cluster [DBG] pgmap v2078: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:43 smithi059 bash[59745]: cluster 2022-04-28T20:39:42.224323+0000 mgr.y (mgr.24961) 1719 : cluster [DBG] pgmap v2078: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:44.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:43 smithi139 bash[43301]: cluster 2022-04-28T20:39:42.224323+0000 mgr.y (mgr.24961) 1719 : cluster [DBG] pgmap v2078: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:46.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[61152]: cluster 2022-04-28T20:39:44.225116+0000 mgr.y (mgr.24961) 1720 : cluster [DBG] pgmap v2079: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:46.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[61152]: audit 2022-04-28T20:39:45.055696+0000 mon.a (mon.0) 2114 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:39:46.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[61152]: audit 2022-04-28T20:39:45.283256+0000 mon.a (mon.0) 2115 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:39:46.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[61152]: audit 2022-04-28T20:39:45.566860+0000 mon.a (mon.0) 2116 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:39:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[59745]: cluster 2022-04-28T20:39:44.225116+0000 mgr.y (mgr.24961) 1720 : cluster [DBG] pgmap v2079: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[59745]: audit 2022-04-28T20:39:45.055696+0000 mon.a (mon.0) 2114 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:39:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[59745]: audit 2022-04-28T20:39:45.283256+0000 mon.a (mon.0) 2115 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:39:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:45 smithi059 bash[59745]: audit 2022-04-28T20:39:45.566860+0000 mon.a (mon.0) 2116 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:39:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:45 smithi139 bash[43301]: cluster 2022-04-28T20:39:44.225116+0000 mgr.y (mgr.24961) 1720 : cluster [DBG] pgmap v2079: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:45 smithi139 bash[43301]: audit 2022-04-28T20:39:45.055696+0000 mon.a (mon.0) 2114 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:39:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:45 smithi139 bash[43301]: audit 2022-04-28T20:39:45.283256+0000 mon.a (mon.0) 2115 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:39:46.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:45 smithi139 bash[43301]: audit 2022-04-28T20:39:45.566860+0000 mon.a (mon.0) 2116 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:39:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:46 smithi059 bash[61152]: cephadm 2022-04-28T20:39:45.275056+0000 mgr.y (mgr.24961) 1721 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T20:39:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:46 smithi059 bash[59745]: cephadm 2022-04-28T20:39:45.275056+0000 mgr.y (mgr.24961) 1721 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T20:39:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:46 smithi139 bash[43301]: cephadm 2022-04-28T20:39:45.275056+0000 mgr.y (mgr.24961) 1721 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T20:39:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:47 smithi059 bash[59745]: cluster 2022-04-28T20:39:46.226190+0000 mgr.y (mgr.24961) 1722 : cluster [DBG] pgmap v2080: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:47 smithi059 bash[61152]: cluster 2022-04-28T20:39:46.226190+0000 mgr.y (mgr.24961) 1722 : cluster [DBG] pgmap v2080: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:47 smithi139 bash[43301]: cluster 2022-04-28T20:39:46.226190+0000 mgr.y (mgr.24961) 1722 : cluster [DBG] pgmap v2080: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:49 smithi059 bash[61152]: cluster 2022-04-28T20:39:48.226903+0000 mgr.y (mgr.24961) 1723 : cluster [DBG] pgmap v2081: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:49 smithi059 bash[59745]: cluster 2022-04-28T20:39:48.226903+0000 mgr.y (mgr.24961) 1723 : cluster [DBG] pgmap v2081: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:49 smithi139 bash[43301]: cluster 2022-04-28T20:39:48.226903+0000 mgr.y (mgr.24961) 1723 : cluster [DBG] pgmap v2081: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:51 smithi059 bash[61152]: cluster 2022-04-28T20:39:50.228155+0000 mgr.y (mgr.24961) 1724 : cluster [DBG] pgmap v2082: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:52.123 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:51.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:52.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:51 smithi059 bash[59745]: cluster 2022-04-28T20:39:50.228155+0000 mgr.y (mgr.24961) 1724 : cluster [DBG] pgmap v2082: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:52.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:51 smithi139 bash[43301]: cluster 2022-04-28T20:39:50.228155+0000 mgr.y (mgr.24961) 1724 : cluster [DBG] pgmap v2082: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:53 smithi059 bash[61152]: cluster 2022-04-28T20:39:52.229104+0000 mgr.y (mgr.24961) 1725 : cluster [DBG] pgmap v2083: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:39:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:39:53.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:39:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:53 smithi059 bash[59745]: cluster 2022-04-28T20:39:52.229104+0000 mgr.y (mgr.24961) 1725 : cluster [DBG] pgmap v2083: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:53 smithi139 bash[43301]: cluster 2022-04-28T20:39:52.229104+0000 mgr.y (mgr.24961) 1725 : cluster [DBG] pgmap v2083: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:55 smithi059 bash[61152]: cluster 2022-04-28T20:39:54.230104+0000 mgr.y (mgr.24961) 1726 : cluster [DBG] pgmap v2084: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:55 smithi059 bash[59745]: cluster 2022-04-28T20:39:54.230104+0000 mgr.y (mgr.24961) 1726 : cluster [DBG] pgmap v2084: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:55 smithi139 bash[43301]: cluster 2022-04-28T20:39:54.230104+0000 mgr.y (mgr.24961) 1726 : cluster [DBG] pgmap v2084: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:57 smithi059 bash[61152]: cluster 2022-04-28T20:39:56.231244+0000 mgr.y (mgr.24961) 1727 : cluster [DBG] pgmap v2085: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:57 smithi059 bash[59745]: cluster 2022-04-28T20:39:56.231244+0000 mgr.y (mgr.24961) 1727 : cluster [DBG] pgmap v2085: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:39:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:57 smithi139 bash[43301]: cluster 2022-04-28T20:39:56.231244+0000 mgr.y (mgr.24961) 1727 : cluster [DBG] pgmap v2085: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:39:59 smithi059 bash[61152]: cluster 2022-04-28T20:39:58.231901+0000 mgr.y (mgr.24961) 1728 : cluster [DBG] pgmap v2086: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:39:59 smithi059 bash[59745]: cluster 2022-04-28T20:39:58.231901+0000 mgr.y (mgr.24961) 1728 : cluster [DBG] pgmap v2086: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:39:59 smithi139 bash[43301]: cluster 2022-04-28T20:39:58.231901+0000 mgr.y (mgr.24961) 1728 : cluster [DBG] pgmap v2086: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:00 smithi059 bash[61152]: cluster 2022-04-28T20:40:00.000162+0000 mon.a (mon.0) 2117 : cluster [INF] overall HEALTH_OK 2022-04-28T20:40:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:00 smithi059 bash[59745]: cluster 2022-04-28T20:40:00.000162+0000 mon.a (mon.0) 2117 : cluster [INF] overall HEALTH_OK 2022-04-28T20:40:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:00 smithi139 bash[43301]: cluster 2022-04-28T20:40:00.000162+0000 mon.a (mon.0) 2117 : cluster [INF] overall HEALTH_OK 2022-04-28T20:40:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:01 smithi059 bash[61152]: cluster 2022-04-28T20:40:00.233137+0000 mgr.y (mgr.24961) 1729 : cluster [DBG] pgmap v2087: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:01.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:01 smithi059 bash[59745]: cluster 2022-04-28T20:40:00.233137+0000 mgr.y (mgr.24961) 1729 : cluster [DBG] pgmap v2087: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:01 smithi139 bash[43301]: cluster 2022-04-28T20:40:00.233137+0000 mgr.y (mgr.24961) 1729 : cluster [DBG] pgmap v2087: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:03 smithi059 bash[59745]: cluster 2022-04-28T20:40:02.234057+0000 mgr.y (mgr.24961) 1730 : cluster [DBG] pgmap v2088: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:03 smithi059 bash[61152]: cluster 2022-04-28T20:40:02.234057+0000 mgr.y (mgr.24961) 1730 : cluster [DBG] pgmap v2088: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:03.712Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:03 smithi139 bash[43301]: cluster 2022-04-28T20:40:02.234057+0000 mgr.y (mgr.24961) 1730 : cluster [DBG] pgmap v2088: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:05 smithi059 bash[61152]: cluster 2022-04-28T20:40:04.234998+0000 mgr.y (mgr.24961) 1731 : cluster [DBG] pgmap v2089: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:05 smithi059 bash[59745]: cluster 2022-04-28T20:40:04.234998+0000 mgr.y (mgr.24961) 1731 : cluster [DBG] pgmap v2089: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:05 smithi139 bash[43301]: cluster 2022-04-28T20:40:04.234998+0000 mgr.y (mgr.24961) 1731 : cluster [DBG] pgmap v2089: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:07 smithi059 bash[61152]: cluster 2022-04-28T20:40:06.236182+0000 mgr.y (mgr.24961) 1732 : cluster [DBG] pgmap v2090: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:07 smithi059 bash[59745]: cluster 2022-04-28T20:40:06.236182+0000 mgr.y (mgr.24961) 1732 : cluster [DBG] pgmap v2090: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:07 smithi139 bash[43301]: cluster 2022-04-28T20:40:06.236182+0000 mgr.y (mgr.24961) 1732 : cluster [DBG] pgmap v2090: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:09 smithi059 bash[61152]: cluster 2022-04-28T20:40:08.236873+0000 mgr.y (mgr.24961) 1733 : cluster [DBG] pgmap v2091: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:09 smithi059 bash[59745]: cluster 2022-04-28T20:40:08.236873+0000 mgr.y (mgr.24961) 1733 : cluster [DBG] pgmap v2091: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:09 smithi139 bash[43301]: cluster 2022-04-28T20:40:08.236873+0000 mgr.y (mgr.24961) 1733 : cluster [DBG] pgmap v2091: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:11 smithi059 bash[61152]: cluster 2022-04-28T20:40:10.238118+0000 mgr.y (mgr.24961) 1734 : cluster [DBG] pgmap v2092: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:11.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:11 smithi059 bash[59745]: cluster 2022-04-28T20:40:10.238118+0000 mgr.y (mgr.24961) 1734 : cluster [DBG] pgmap v2092: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:11 smithi139 bash[43301]: cluster 2022-04-28T20:40:10.238118+0000 mgr.y (mgr.24961) 1734 : cluster [DBG] pgmap v2092: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:13 smithi059 bash[61152]: cluster 2022-04-28T20:40:12.239024+0000 mgr.y (mgr.24961) 1735 : cluster [DBG] pgmap v2093: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:13.711Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:13 smithi059 bash[59745]: cluster 2022-04-28T20:40:12.239024+0000 mgr.y (mgr.24961) 1735 : cluster [DBG] pgmap v2093: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:13 smithi139 bash[43301]: cluster 2022-04-28T20:40:12.239024+0000 mgr.y (mgr.24961) 1735 : cluster [DBG] pgmap v2093: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:15 smithi139 bash[43301]: cluster 2022-04-28T20:40:14.240031+0000 mgr.y (mgr.24961) 1736 : cluster [DBG] pgmap v2094: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:15 smithi059 bash[61152]: cluster 2022-04-28T20:40:14.240031+0000 mgr.y (mgr.24961) 1736 : cluster [DBG] pgmap v2094: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:15 smithi059 bash[59745]: cluster 2022-04-28T20:40:14.240031+0000 mgr.y (mgr.24961) 1736 : cluster [DBG] pgmap v2094: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:17 smithi139 bash[43301]: cluster 2022-04-28T20:40:16.241184+0000 mgr.y (mgr.24961) 1737 : cluster [DBG] pgmap v2095: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:17 smithi139 bash[43301]: audit 2022-04-28T20:40:17.716166+0000 mon.a (mon.0) 2118 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:40:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:17 smithi139 bash[43301]: audit 2022-04-28T20:40:17.716670+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:40:18.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:17 smithi139 bash[43301]: audit 2022-04-28T20:40:17.723455+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:40:18.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:17 smithi139 bash[43301]: audit 2022-04-28T20:40:17.724085+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:40:18.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:17 smithi139 bash[43301]: audit 2022-04-28T20:40:17.724644+0000 mon.a (mon.0) 2122 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:40:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[59745]: cluster 2022-04-28T20:40:16.241184+0000 mgr.y (mgr.24961) 1737 : cluster [DBG] pgmap v2095: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[59745]: audit 2022-04-28T20:40:17.716166+0000 mon.a (mon.0) 2118 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:40:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[59745]: audit 2022-04-28T20:40:17.716670+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:40:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[59745]: audit 2022-04-28T20:40:17.723455+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:40:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[59745]: audit 2022-04-28T20:40:17.724085+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:40:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[59745]: audit 2022-04-28T20:40:17.724644+0000 mon.a (mon.0) 2122 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:40:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[61152]: cluster 2022-04-28T20:40:16.241184+0000 mgr.y (mgr.24961) 1737 : cluster [DBG] pgmap v2095: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[61152]: audit 2022-04-28T20:40:17.716166+0000 mon.a (mon.0) 2118 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:40:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[61152]: audit 2022-04-28T20:40:17.716670+0000 mon.a (mon.0) 2119 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:40:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[61152]: audit 2022-04-28T20:40:17.723455+0000 mon.a (mon.0) 2120 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:40:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[61152]: audit 2022-04-28T20:40:17.724085+0000 mon.a (mon.0) 2121 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:40:18.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:17 smithi059 bash[61152]: audit 2022-04-28T20:40:17.724644+0000 mon.a (mon.0) 2122 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:40:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:19 smithi139 bash[43301]: cluster 2022-04-28T20:40:18.241880+0000 mgr.y (mgr.24961) 1738 : cluster [DBG] pgmap v2096: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:19 smithi059 bash[61152]: cluster 2022-04-28T20:40:18.241880+0000 mgr.y (mgr.24961) 1738 : cluster [DBG] pgmap v2096: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:19 smithi059 bash[59745]: cluster 2022-04-28T20:40:18.241880+0000 mgr.y (mgr.24961) 1738 : cluster [DBG] pgmap v2096: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:21 smithi059 bash[61152]: cluster 2022-04-28T20:40:20.243023+0000 mgr.y (mgr.24961) 1739 : cluster [DBG] pgmap v2097: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:21.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:21 smithi059 bash[59745]: cluster 2022-04-28T20:40:20.243023+0000 mgr.y (mgr.24961) 1739 : cluster [DBG] pgmap v2097: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:21 smithi139 bash[43301]: cluster 2022-04-28T20:40:20.243023+0000 mgr.y (mgr.24961) 1739 : cluster [DBG] pgmap v2097: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:23 smithi059 bash[61152]: cluster 2022-04-28T20:40:22.243985+0000 mgr.y (mgr.24961) 1740 : cluster [DBG] pgmap v2098: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:23.713Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:23 smithi059 bash[59745]: cluster 2022-04-28T20:40:22.243985+0000 mgr.y (mgr.24961) 1740 : cluster [DBG] pgmap v2098: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:23 smithi139 bash[43301]: cluster 2022-04-28T20:40:22.243985+0000 mgr.y (mgr.24961) 1740 : cluster [DBG] pgmap v2098: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:25 smithi059 bash[61152]: cluster 2022-04-28T20:40:24.245004+0000 mgr.y (mgr.24961) 1741 : cluster [DBG] pgmap v2099: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:25 smithi059 bash[59745]: cluster 2022-04-28T20:40:24.245004+0000 mgr.y (mgr.24961) 1741 : cluster [DBG] pgmap v2099: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:25 smithi139 bash[43301]: cluster 2022-04-28T20:40:24.245004+0000 mgr.y (mgr.24961) 1741 : cluster [DBG] pgmap v2099: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:27 smithi059 bash[61152]: cluster 2022-04-28T20:40:26.246131+0000 mgr.y (mgr.24961) 1742 : cluster [DBG] pgmap v2100: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:27 smithi059 bash[59745]: cluster 2022-04-28T20:40:26.246131+0000 mgr.y (mgr.24961) 1742 : cluster [DBG] pgmap v2100: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:27 smithi139 bash[43301]: cluster 2022-04-28T20:40:26.246131+0000 mgr.y (mgr.24961) 1742 : cluster [DBG] pgmap v2100: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:29 smithi059 bash[61152]: cluster 2022-04-28T20:40:28.246877+0000 mgr.y (mgr.24961) 1743 : cluster [DBG] pgmap v2101: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:29 smithi059 bash[59745]: cluster 2022-04-28T20:40:28.246877+0000 mgr.y (mgr.24961) 1743 : cluster [DBG] pgmap v2101: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:29 smithi139 bash[43301]: cluster 2022-04-28T20:40:28.246877+0000 mgr.y (mgr.24961) 1743 : cluster [DBG] pgmap v2101: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:31 smithi059 bash[61152]: cluster 2022-04-28T20:40:30.248083+0000 mgr.y (mgr.24961) 1744 : cluster [DBG] pgmap v2102: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:31.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:31 smithi059 bash[59745]: cluster 2022-04-28T20:40:30.248083+0000 mgr.y (mgr.24961) 1744 : cluster [DBG] pgmap v2102: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:31 smithi139 bash[43301]: cluster 2022-04-28T20:40:30.248083+0000 mgr.y (mgr.24961) 1744 : cluster [DBG] pgmap v2102: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:33 smithi059 bash[59745]: cluster 2022-04-28T20:40:32.248973+0000 mgr.y (mgr.24961) 1745 : cluster [DBG] pgmap v2103: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:34.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:33 smithi059 bash[61152]: cluster 2022-04-28T20:40:32.248973+0000 mgr.y (mgr.24961) 1745 : cluster [DBG] pgmap v2103: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:34.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:33.714Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:33 smithi139 bash[43301]: cluster 2022-04-28T20:40:32.248973+0000 mgr.y (mgr.24961) 1745 : cluster [DBG] pgmap v2103: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:35 smithi059 bash[61152]: cluster 2022-04-28T20:40:34.249934+0000 mgr.y (mgr.24961) 1746 : cluster [DBG] pgmap v2104: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:35 smithi059 bash[59745]: cluster 2022-04-28T20:40:34.249934+0000 mgr.y (mgr.24961) 1746 : cluster [DBG] pgmap v2104: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:35 smithi139 bash[43301]: cluster 2022-04-28T20:40:34.249934+0000 mgr.y (mgr.24961) 1746 : cluster [DBG] pgmap v2104: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:37 smithi059 bash[61152]: cluster 2022-04-28T20:40:36.251100+0000 mgr.y (mgr.24961) 1747 : cluster [DBG] pgmap v2105: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:37 smithi059 bash[59745]: cluster 2022-04-28T20:40:36.251100+0000 mgr.y (mgr.24961) 1747 : cluster [DBG] pgmap v2105: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:37 smithi139 bash[43301]: cluster 2022-04-28T20:40:36.251100+0000 mgr.y (mgr.24961) 1747 : cluster [DBG] pgmap v2105: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:39 smithi059 bash[61152]: cluster 2022-04-28T20:40:38.251849+0000 mgr.y (mgr.24961) 1748 : cluster [DBG] pgmap v2106: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:39 smithi059 bash[59745]: cluster 2022-04-28T20:40:38.251849+0000 mgr.y (mgr.24961) 1748 : cluster [DBG] pgmap v2106: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:39 smithi139 bash[43301]: cluster 2022-04-28T20:40:38.251849+0000 mgr.y (mgr.24961) 1748 : cluster [DBG] pgmap v2106: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:41 smithi059 bash[61152]: cluster 2022-04-28T20:40:40.253069+0000 mgr.y (mgr.24961) 1749 : cluster [DBG] pgmap v2107: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:41.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:41 smithi059 bash[59745]: cluster 2022-04-28T20:40:40.253069+0000 mgr.y (mgr.24961) 1749 : cluster [DBG] pgmap v2107: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:41 smithi139 bash[43301]: cluster 2022-04-28T20:40:40.253069+0000 mgr.y (mgr.24961) 1749 : cluster [DBG] pgmap v2107: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:42 smithi059 bash[61152]: cluster 2022-04-28T20:40:42.254030+0000 mgr.y (mgr.24961) 1750 : cluster [DBG] pgmap v2108: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:43.368 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:42 smithi059 bash[59745]: cluster 2022-04-28T20:40:42.254030+0000 mgr.y (mgr.24961) 1750 : cluster [DBG] pgmap v2108: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:42 smithi139 bash[43301]: cluster 2022-04-28T20:40:42.254030+0000 mgr.y (mgr.24961) 1750 : cluster [DBG] pgmap v2108: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:43.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[61152]: cluster 2022-04-28T20:40:44.255004+0000 mgr.y (mgr.24961) 1751 : cluster [DBG] pgmap v2109: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[61152]: audit 2022-04-28T20:40:45.575101+0000 mon.a (mon.0) 2123 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:40:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[61152]: audit 2022-04-28T20:40:45.578083+0000 mon.a (mon.0) 2124 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:40:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[61152]: audit 2022-04-28T20:40:45.580190+0000 mon.a (mon.0) 2125 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:40:46.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[61152]: audit 2022-04-28T20:40:45.584350+0000 mon.a (mon.0) 2126 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:40:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[59745]: cluster 2022-04-28T20:40:44.255004+0000 mgr.y (mgr.24961) 1751 : cluster [DBG] pgmap v2109: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[59745]: audit 2022-04-28T20:40:45.575101+0000 mon.a (mon.0) 2123 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:40:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[59745]: audit 2022-04-28T20:40:45.578083+0000 mon.a (mon.0) 2124 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:40:46.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[59745]: audit 2022-04-28T20:40:45.580190+0000 mon.a (mon.0) 2125 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:40:46.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:45 smithi059 bash[59745]: audit 2022-04-28T20:40:45.584350+0000 mon.a (mon.0) 2126 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:40:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:45 smithi139 bash[43301]: cluster 2022-04-28T20:40:44.255004+0000 mgr.y (mgr.24961) 1751 : cluster [DBG] pgmap v2109: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:45 smithi139 bash[43301]: audit 2022-04-28T20:40:45.575101+0000 mon.a (mon.0) 2123 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:40:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:45 smithi139 bash[43301]: audit 2022-04-28T20:40:45.578083+0000 mon.a (mon.0) 2124 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:40:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:45 smithi139 bash[43301]: audit 2022-04-28T20:40:45.580190+0000 mon.a (mon.0) 2125 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:40:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:45 smithi139 bash[43301]: audit 2022-04-28T20:40:45.584350+0000 mon.a (mon.0) 2126 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T20:40:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: cephadm 2022-04-28T20:40:45.582104+0000 mgr.y (mgr.24961) 1752 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:40:47.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.873492+0000 mon.a (mon.0) 2127 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.874459+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.875397+0000 mon.a (mon.0) 2129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.876250+0000 mon.a (mon.0) 2130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.877143+0000 mon.a (mon.0) 2131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.877987+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.878818+0000 mon.a (mon.0) 2133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.879670+0000 mon.a (mon.0) 2134 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.880560+0000 mon.a (mon.0) 2135 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:40:47.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[61152]: audit 2022-04-28T20:40:45.881461+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: cephadm 2022-04-28T20:40:45.582104+0000 mgr.y (mgr.24961) 1752 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:40:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.873492+0000 mon.a (mon.0) 2127 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.874459+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.875397+0000 mon.a (mon.0) 2129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.876250+0000 mon.a (mon.0) 2130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.877143+0000 mon.a (mon.0) 2131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.877987+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.878818+0000 mon.a (mon.0) 2133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.879670+0000 mon.a (mon.0) 2134 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.880560+0000 mon.a (mon.0) 2135 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:40:47.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:46 smithi059 bash[59745]: audit 2022-04-28T20:40:45.881461+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: cephadm 2022-04-28T20:40:45.582104+0000 mgr.y (mgr.24961) 1752 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:40:47.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.873492+0000 mon.a (mon.0) 2127 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.874459+0000 mon.a (mon.0) 2128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.875397+0000 mon.a (mon.0) 2129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.876250+0000 mon.a (mon.0) 2130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.877143+0000 mon.a (mon.0) 2131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.877987+0000 mon.a (mon.0) 2132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.878818+0000 mon.a (mon.0) 2133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:47.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.879670+0000 mon.a (mon.0) 2134 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:47.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.880560+0000 mon.a (mon.0) 2135 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:40:47.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:46 smithi139 bash[43301]: audit 2022-04-28T20:40:45.881461+0000 mon.a (mon.0) 2136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:40:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:47 smithi059 bash[61152]: cluster 2022-04-28T20:40:46.256181+0000 mgr.y (mgr.24961) 1753 : cluster [DBG] pgmap v2110: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:47 smithi059 bash[59745]: cluster 2022-04-28T20:40:46.256181+0000 mgr.y (mgr.24961) 1753 : cluster [DBG] pgmap v2110: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:47 smithi139 bash[43301]: cluster 2022-04-28T20:40:46.256181+0000 mgr.y (mgr.24961) 1753 : cluster [DBG] pgmap v2110: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:49 smithi139 bash[43301]: cluster 2022-04-28T20:40:48.256865+0000 mgr.y (mgr.24961) 1754 : cluster [DBG] pgmap v2111: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:49 smithi059 bash[61152]: cluster 2022-04-28T20:40:48.256865+0000 mgr.y (mgr.24961) 1754 : cluster [DBG] pgmap v2111: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:49 smithi059 bash[59745]: cluster 2022-04-28T20:40:48.256865+0000 mgr.y (mgr.24961) 1754 : cluster [DBG] pgmap v2111: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:51 smithi059 bash[61152]: cluster 2022-04-28T20:40:50.258044+0000 mgr.y (mgr.24961) 1755 : cluster [DBG] pgmap v2112: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:51.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:51 smithi059 bash[59745]: cluster 2022-04-28T20:40:50.258044+0000 mgr.y (mgr.24961) 1755 : cluster [DBG] pgmap v2112: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:51 smithi139 bash[43301]: cluster 2022-04-28T20:40:50.258044+0000 mgr.y (mgr.24961) 1755 : cluster [DBG] pgmap v2112: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:52 smithi059 bash[61152]: audit 2022-04-28T20:40:52.423377+0000 mon.a (mon.0) 2137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:40:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:52 smithi059 bash[61152]: audit 2022-04-28T20:40:52.650650+0000 mon.a (mon.0) 2138 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:40:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:52 smithi059 bash[59745]: audit 2022-04-28T20:40:52.423377+0000 mon.a (mon.0) 2137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:40:53.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:52 smithi059 bash[59745]: audit 2022-04-28T20:40:52.650650+0000 mon.a (mon.0) 2138 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:40:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:52 smithi139 bash[43301]: audit 2022-04-28T20:40:52.423377+0000 mon.a (mon.0) 2137 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:40:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:52 smithi139 bash[43301]: audit 2022-04-28T20:40:52.650650+0000 mon.a (mon.0) 2138 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:40:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:40:53.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:40:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: cluster 2022-04-28T20:40:52.259053+0000 mgr.y (mgr.24961) 1756 : cluster [DBG] pgmap v2113: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: cephadm 2022-04-28T20:40:52.638948+0000 mgr.y (mgr.24961) 1757 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T20:40:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.928360+0000 mon.a (mon.0) 2139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.929482+0000 mon.a (mon.0) 2140 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.930468+0000 mon.a (mon.0) 2141 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.931422+0000 mon.a (mon.0) 2142 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.932481+0000 mon.a (mon.0) 2143 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.933415+0000 mon.a (mon.0) 2144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.934409+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.935373+0000 mon.a (mon.0) 2146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.936362+0000 mon.a (mon.0) 2147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:40:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.937304+0000 mon.a (mon.0) 2148 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[61152]: audit 2022-04-28T20:40:52.946229+0000 mon.a (mon.0) 2149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:40:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: cluster 2022-04-28T20:40:52.259053+0000 mgr.y (mgr.24961) 1756 : cluster [DBG] pgmap v2113: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: cephadm 2022-04-28T20:40:52.638948+0000 mgr.y (mgr.24961) 1757 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T20:40:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.928360+0000 mon.a (mon.0) 2139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.929482+0000 mon.a (mon.0) 2140 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.930468+0000 mon.a (mon.0) 2141 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.931422+0000 mon.a (mon.0) 2142 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.932481+0000 mon.a (mon.0) 2143 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.933415+0000 mon.a (mon.0) 2144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.934409+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.935373+0000 mon.a (mon.0) 2146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.936362+0000 mon.a (mon.0) 2147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:40:54.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.937304+0000 mon.a (mon.0) 2148 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:53 smithi059 bash[59745]: audit 2022-04-28T20:40:52.946229+0000 mon.a (mon.0) 2149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:40:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: cluster 2022-04-28T20:40:52.259053+0000 mgr.y (mgr.24961) 1756 : cluster [DBG] pgmap v2113: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: cephadm 2022-04-28T20:40:52.638948+0000 mgr.y (mgr.24961) 1757 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T20:40:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.928360+0000 mon.a (mon.0) 2139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.929482+0000 mon.a (mon.0) 2140 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.930468+0000 mon.a (mon.0) 2141 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.931422+0000 mon.a (mon.0) 2142 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.932481+0000 mon.a (mon.0) 2143 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.933415+0000 mon.a (mon.0) 2144 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.934409+0000 mon.a (mon.0) 2145 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:40:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.935373+0000 mon.a (mon.0) 2146 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.936362+0000 mon.a (mon.0) 2147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:40:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.937304+0000 mon.a (mon.0) 2148 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:40:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:53 smithi139 bash[43301]: audit 2022-04-28T20:40:52.946229+0000 mon.a (mon.0) 2149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:40:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:55 smithi059 bash[61152]: cluster 2022-04-28T20:40:54.260096+0000 mgr.y (mgr.24961) 1758 : cluster [DBG] pgmap v2114: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:55 smithi059 bash[59745]: cluster 2022-04-28T20:40:54.260096+0000 mgr.y (mgr.24961) 1758 : cluster [DBG] pgmap v2114: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:56.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:55 smithi139 bash[43301]: cluster 2022-04-28T20:40:54.260096+0000 mgr.y (mgr.24961) 1758 : cluster [DBG] pgmap v2114: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:57 smithi059 bash[61152]: cluster 2022-04-28T20:40:56.261174+0000 mgr.y (mgr.24961) 1759 : cluster [DBG] pgmap v2115: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:57 smithi059 bash[59745]: cluster 2022-04-28T20:40:56.261174+0000 mgr.y (mgr.24961) 1759 : cluster [DBG] pgmap v2115: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:40:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:57 smithi139 bash[43301]: cluster 2022-04-28T20:40:56.261174+0000 mgr.y (mgr.24961) 1759 : cluster [DBG] pgmap v2115: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:40:59 smithi059 bash[61152]: cluster 2022-04-28T20:40:58.261893+0000 mgr.y (mgr.24961) 1760 : cluster [DBG] pgmap v2116: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:40:59 smithi059 bash[59745]: cluster 2022-04-28T20:40:58.261893+0000 mgr.y (mgr.24961) 1760 : cluster [DBG] pgmap v2116: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:40:59 smithi139 bash[43301]: cluster 2022-04-28T20:40:58.261893+0000 mgr.y (mgr.24961) 1760 : cluster [DBG] pgmap v2116: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:01 smithi059 bash[61152]: cluster 2022-04-28T20:41:00.263137+0000 mgr.y (mgr.24961) 1761 : cluster [DBG] pgmap v2117: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:01.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:01 smithi059 bash[59745]: cluster 2022-04-28T20:41:00.263137+0000 mgr.y (mgr.24961) 1761 : cluster [DBG] pgmap v2117: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:01 smithi139 bash[43301]: cluster 2022-04-28T20:41:00.263137+0000 mgr.y (mgr.24961) 1761 : cluster [DBG] pgmap v2117: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:03 smithi059 bash[59745]: cluster 2022-04-28T20:41:02.264107+0000 mgr.y (mgr.24961) 1762 : cluster [DBG] pgmap v2118: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:03 smithi059 bash[61152]: cluster 2022-04-28T20:41:02.264107+0000 mgr.y (mgr.24961) 1762 : cluster [DBG] pgmap v2118: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:03.716Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:03 smithi139 bash[43301]: cluster 2022-04-28T20:41:02.264107+0000 mgr.y (mgr.24961) 1762 : cluster [DBG] pgmap v2118: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:05 smithi059 bash[61152]: cluster 2022-04-28T20:41:04.265101+0000 mgr.y (mgr.24961) 1763 : cluster [DBG] pgmap v2119: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:05 smithi059 bash[59745]: cluster 2022-04-28T20:41:04.265101+0000 mgr.y (mgr.24961) 1763 : cluster [DBG] pgmap v2119: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:05 smithi139 bash[43301]: cluster 2022-04-28T20:41:04.265101+0000 mgr.y (mgr.24961) 1763 : cluster [DBG] pgmap v2119: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:07 smithi059 bash[61152]: cluster 2022-04-28T20:41:06.266254+0000 mgr.y (mgr.24961) 1764 : cluster [DBG] pgmap v2120: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:07 smithi059 bash[59745]: cluster 2022-04-28T20:41:06.266254+0000 mgr.y (mgr.24961) 1764 : cluster [DBG] pgmap v2120: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:07 smithi139 bash[43301]: cluster 2022-04-28T20:41:06.266254+0000 mgr.y (mgr.24961) 1764 : cluster [DBG] pgmap v2120: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:09 smithi059 bash[61152]: cluster 2022-04-28T20:41:08.267005+0000 mgr.y (mgr.24961) 1765 : cluster [DBG] pgmap v2121: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:09 smithi059 bash[59745]: cluster 2022-04-28T20:41:08.267005+0000 mgr.y (mgr.24961) 1765 : cluster [DBG] pgmap v2121: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:09 smithi139 bash[43301]: cluster 2022-04-28T20:41:08.267005+0000 mgr.y (mgr.24961) 1765 : cluster [DBG] pgmap v2121: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:11 smithi059 bash[61152]: cluster 2022-04-28T20:41:10.268240+0000 mgr.y (mgr.24961) 1766 : cluster [DBG] pgmap v2122: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:11.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:11 smithi059 bash[59745]: cluster 2022-04-28T20:41:10.268240+0000 mgr.y (mgr.24961) 1766 : cluster [DBG] pgmap v2122: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:11 smithi139 bash[43301]: cluster 2022-04-28T20:41:10.268240+0000 mgr.y (mgr.24961) 1766 : cluster [DBG] pgmap v2122: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:13 smithi059 bash[59745]: cluster 2022-04-28T20:41:12.269224+0000 mgr.y (mgr.24961) 1767 : cluster [DBG] pgmap v2123: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:13 smithi059 bash[61152]: cluster 2022-04-28T20:41:12.269224+0000 mgr.y (mgr.24961) 1767 : cluster [DBG] pgmap v2123: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:13.715Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:13 smithi139 bash[43301]: cluster 2022-04-28T20:41:12.269224+0000 mgr.y (mgr.24961) 1767 : cluster [DBG] pgmap v2123: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:15 smithi059 bash[61152]: cluster 2022-04-28T20:41:14.270194+0000 mgr.y (mgr.24961) 1768 : cluster [DBG] pgmap v2124: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:15 smithi059 bash[59745]: cluster 2022-04-28T20:41:14.270194+0000 mgr.y (mgr.24961) 1768 : cluster [DBG] pgmap v2124: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:15 smithi139 bash[43301]: cluster 2022-04-28T20:41:14.270194+0000 mgr.y (mgr.24961) 1768 : cluster [DBG] pgmap v2124: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[59745]: cluster 2022-04-28T20:41:16.271281+0000 mgr.y (mgr.24961) 1769 : cluster [DBG] pgmap v2125: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[59745]: audit 2022-04-28T20:41:17.730051+0000 mon.a (mon.0) 2150 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:41:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[59745]: audit 2022-04-28T20:41:17.730524+0000 mon.a (mon.0) 2151 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:41:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[59745]: audit 2022-04-28T20:41:17.741089+0000 mon.a (mon.0) 2152 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:41:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[59745]: audit 2022-04-28T20:41:17.741450+0000 mon.a (mon.0) 2153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:41:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[59745]: audit 2022-04-28T20:41:17.741793+0000 mon.a (mon.0) 2154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T20:41:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[61152]: cluster 2022-04-28T20:41:16.271281+0000 mgr.y (mgr.24961) 1769 : cluster [DBG] pgmap v2125: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[61152]: audit 2022-04-28T20:41:17.730051+0000 mon.a (mon.0) 2150 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:41:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[61152]: audit 2022-04-28T20:41:17.730524+0000 mon.a (mon.0) 2151 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:41:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[61152]: audit 2022-04-28T20:41:17.741089+0000 mon.a (mon.0) 2152 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:41:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[61152]: audit 2022-04-28T20:41:17.741450+0000 mon.a (mon.0) 2153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:41:18.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:17 smithi059 bash[61152]: audit 2022-04-28T20:41:17.741793+0000 mon.a (mon.0) 2154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T20:41:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:17 smithi139 bash[43301]: cluster 2022-04-28T20:41:16.271281+0000 mgr.y (mgr.24961) 1769 : cluster [DBG] pgmap v2125: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:17 smithi139 bash[43301]: audit 2022-04-28T20:41:17.730051+0000 mon.a (mon.0) 2150 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:41:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:17 smithi139 bash[43301]: audit 2022-04-28T20:41:17.730524+0000 mon.a (mon.0) 2151 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:41:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:17 smithi139 bash[43301]: audit 2022-04-28T20:41:17.741089+0000 mon.a (mon.0) 2152 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:41:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:17 smithi139 bash[43301]: audit 2022-04-28T20:41:17.741450+0000 mon.a (mon.0) 2153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:41:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:17 smithi139 bash[43301]: audit 2022-04-28T20:41:17.741793+0000 mon.a (mon.0) 2154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T20:41:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:19 smithi059 bash[59745]: cluster 2022-04-28T20:41:18.272069+0000 mgr.y (mgr.24961) 1770 : cluster [DBG] pgmap v2126: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:19 smithi059 bash[61152]: cluster 2022-04-28T20:41:18.272069+0000 mgr.y (mgr.24961) 1770 : cluster [DBG] pgmap v2126: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:19 smithi139 bash[43301]: cluster 2022-04-28T20:41:18.272069+0000 mgr.y (mgr.24961) 1770 : cluster [DBG] pgmap v2126: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:21 smithi059 bash[61152]: cluster 2022-04-28T20:41:20.273283+0000 mgr.y (mgr.24961) 1771 : cluster [DBG] pgmap v2127: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:21.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:21 smithi059 bash[59745]: cluster 2022-04-28T20:41:20.273283+0000 mgr.y (mgr.24961) 1771 : cluster [DBG] pgmap v2127: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:21 smithi139 bash[43301]: cluster 2022-04-28T20:41:20.273283+0000 mgr.y (mgr.24961) 1771 : cluster [DBG] pgmap v2127: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:23 smithi059 bash[61152]: cluster 2022-04-28T20:41:22.274280+0000 mgr.y (mgr.24961) 1772 : cluster [DBG] pgmap v2128: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:23.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:23 smithi059 bash[59745]: cluster 2022-04-28T20:41:22.274280+0000 mgr.y (mgr.24961) 1772 : cluster [DBG] pgmap v2128: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:23 smithi139 bash[43301]: cluster 2022-04-28T20:41:22.274280+0000 mgr.y (mgr.24961) 1772 : cluster [DBG] pgmap v2128: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:25 smithi139 bash[43301]: cluster 2022-04-28T20:41:24.275273+0000 mgr.y (mgr.24961) 1773 : cluster [DBG] pgmap v2129: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:25 smithi059 bash[61152]: cluster 2022-04-28T20:41:24.275273+0000 mgr.y (mgr.24961) 1773 : cluster [DBG] pgmap v2129: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:25 smithi059 bash[59745]: cluster 2022-04-28T20:41:24.275273+0000 mgr.y (mgr.24961) 1773 : cluster [DBG] pgmap v2129: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:27 smithi139 bash[43301]: cluster 2022-04-28T20:41:26.276411+0000 mgr.y (mgr.24961) 1774 : cluster [DBG] pgmap v2130: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:27 smithi059 bash[61152]: cluster 2022-04-28T20:41:26.276411+0000 mgr.y (mgr.24961) 1774 : cluster [DBG] pgmap v2130: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:27 smithi059 bash[59745]: cluster 2022-04-28T20:41:26.276411+0000 mgr.y (mgr.24961) 1774 : cluster [DBG] pgmap v2130: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:29 smithi139 bash[43301]: cluster 2022-04-28T20:41:28.277079+0000 mgr.y (mgr.24961) 1775 : cluster [DBG] pgmap v2131: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:29 smithi059 bash[61152]: cluster 2022-04-28T20:41:28.277079+0000 mgr.y (mgr.24961) 1775 : cluster [DBG] pgmap v2131: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:29 smithi059 bash[59745]: cluster 2022-04-28T20:41:28.277079+0000 mgr.y (mgr.24961) 1775 : cluster [DBG] pgmap v2131: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:31 smithi059 bash[61152]: cluster 2022-04-28T20:41:30.278195+0000 mgr.y (mgr.24961) 1776 : cluster [DBG] pgmap v2132: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:31.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:31 smithi059 bash[59745]: cluster 2022-04-28T20:41:30.278195+0000 mgr.y (mgr.24961) 1776 : cluster [DBG] pgmap v2132: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:31 smithi139 bash[43301]: cluster 2022-04-28T20:41:30.278195+0000 mgr.y (mgr.24961) 1776 : cluster [DBG] pgmap v2132: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:33 smithi059 bash[61152]: cluster 2022-04-28T20:41:32.279112+0000 mgr.y (mgr.24961) 1777 : cluster [DBG] pgmap v2133: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:33.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:33 smithi059 bash[59745]: cluster 2022-04-28T20:41:32.279112+0000 mgr.y (mgr.24961) 1777 : cluster [DBG] pgmap v2133: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:33 smithi139 bash[43301]: cluster 2022-04-28T20:41:32.279112+0000 mgr.y (mgr.24961) 1777 : cluster [DBG] pgmap v2133: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:35 smithi059 bash[61152]: cluster 2022-04-28T20:41:34.280155+0000 mgr.y (mgr.24961) 1778 : cluster [DBG] pgmap v2134: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:35 smithi059 bash[59745]: cluster 2022-04-28T20:41:34.280155+0000 mgr.y (mgr.24961) 1778 : cluster [DBG] pgmap v2134: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:35 smithi139 bash[43301]: cluster 2022-04-28T20:41:34.280155+0000 mgr.y (mgr.24961) 1778 : cluster [DBG] pgmap v2134: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:37 smithi059 bash[61152]: cluster 2022-04-28T20:41:36.281281+0000 mgr.y (mgr.24961) 1779 : cluster [DBG] pgmap v2135: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:37 smithi059 bash[59745]: cluster 2022-04-28T20:41:36.281281+0000 mgr.y (mgr.24961) 1779 : cluster [DBG] pgmap v2135: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:37 smithi139 bash[43301]: cluster 2022-04-28T20:41:36.281281+0000 mgr.y (mgr.24961) 1779 : cluster [DBG] pgmap v2135: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:39 smithi059 bash[61152]: cluster 2022-04-28T20:41:38.281946+0000 mgr.y (mgr.24961) 1780 : cluster [DBG] pgmap v2136: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:39 smithi059 bash[59745]: cluster 2022-04-28T20:41:38.281946+0000 mgr.y (mgr.24961) 1780 : cluster [DBG] pgmap v2136: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:39 smithi139 bash[43301]: cluster 2022-04-28T20:41:38.281946+0000 mgr.y (mgr.24961) 1780 : cluster [DBG] pgmap v2136: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:41 smithi059 bash[61152]: cluster 2022-04-28T20:41:40.283178+0000 mgr.y (mgr.24961) 1781 : cluster [DBG] pgmap v2137: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:41.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:41 smithi059 bash[59745]: cluster 2022-04-28T20:41:40.283178+0000 mgr.y (mgr.24961) 1781 : cluster [DBG] pgmap v2137: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:41 smithi139 bash[43301]: cluster 2022-04-28T20:41:40.283178+0000 mgr.y (mgr.24961) 1781 : cluster [DBG] pgmap v2137: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:43 smithi059 bash[61152]: cluster 2022-04-28T20:41:42.284052+0000 mgr.y (mgr.24961) 1782 : cluster [DBG] pgmap v2138: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:43.717Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:43 smithi059 bash[59745]: cluster 2022-04-28T20:41:42.284052+0000 mgr.y (mgr.24961) 1782 : cluster [DBG] pgmap v2138: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:43 smithi139 bash[43301]: cluster 2022-04-28T20:41:42.284052+0000 mgr.y (mgr.24961) 1782 : cluster [DBG] pgmap v2138: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:45 smithi059 bash[61152]: cluster 2022-04-28T20:41:44.285066+0000 mgr.y (mgr.24961) 1783 : cluster [DBG] pgmap v2139: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:45 smithi059 bash[59745]: cluster 2022-04-28T20:41:44.285066+0000 mgr.y (mgr.24961) 1783 : cluster [DBG] pgmap v2139: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:45 smithi139 bash[43301]: cluster 2022-04-28T20:41:44.285066+0000 mgr.y (mgr.24961) 1783 : cluster [DBG] pgmap v2139: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:47 smithi059 bash[61152]: cluster 2022-04-28T20:41:46.286207+0000 mgr.y (mgr.24961) 1784 : cluster [DBG] pgmap v2140: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:47 smithi059 bash[59745]: cluster 2022-04-28T20:41:46.286207+0000 mgr.y (mgr.24961) 1784 : cluster [DBG] pgmap v2140: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:47 smithi139 bash[43301]: cluster 2022-04-28T20:41:46.286207+0000 mgr.y (mgr.24961) 1784 : cluster [DBG] pgmap v2140: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:50.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:49 smithi059 bash[59745]: cluster 2022-04-28T20:41:48.286882+0000 mgr.y (mgr.24961) 1785 : cluster [DBG] pgmap v2141: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:49 smithi059 bash[61152]: cluster 2022-04-28T20:41:48.286882+0000 mgr.y (mgr.24961) 1785 : cluster [DBG] pgmap v2141: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:49 smithi139 bash[43301]: cluster 2022-04-28T20:41:48.286882+0000 mgr.y (mgr.24961) 1785 : cluster [DBG] pgmap v2141: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:51 smithi059 bash[61152]: cluster 2022-04-28T20:41:50.288123+0000 mgr.y (mgr.24961) 1786 : cluster [DBG] pgmap v2142: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:51 smithi059 bash[59745]: cluster 2022-04-28T20:41:50.288123+0000 mgr.y (mgr.24961) 1786 : cluster [DBG] pgmap v2142: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:52.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:51.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:51 smithi139 bash[43301]: cluster 2022-04-28T20:41:50.288123+0000 mgr.y (mgr.24961) 1786 : cluster [DBG] pgmap v2142: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[59745]: cluster 2022-04-28T20:41:52.289029+0000 mgr.y (mgr.24961) 1787 : cluster [DBG] pgmap v2143: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[59745]: audit 2022-04-28T20:41:52.954594+0000 mon.a (mon.0) 2155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:41:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[59745]: audit 2022-04-28T20:41:52.957524+0000 mon.a (mon.0) 2156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:41:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[59745]: audit 2022-04-28T20:41:52.959612+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:41:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[59745]: audit 2022-04-28T20:41:53.263520+0000 mon.a (mon.0) 2158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:41:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[61152]: cluster 2022-04-28T20:41:52.289029+0000 mgr.y (mgr.24961) 1787 : cluster [DBG] pgmap v2143: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[61152]: audit 2022-04-28T20:41:52.954594+0000 mon.a (mon.0) 2155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:41:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[61152]: audit 2022-04-28T20:41:52.957524+0000 mon.a (mon.0) 2156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:41:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[61152]: audit 2022-04-28T20:41:52.959612+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:41:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[61152]: audit 2022-04-28T20:41:53.263520+0000 mon.a (mon.0) 2158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:41:54.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:41:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:41:53.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:41:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:53 smithi139 bash[43301]: cluster 2022-04-28T20:41:52.289029+0000 mgr.y (mgr.24961) 1787 : cluster [DBG] pgmap v2143: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:53 smithi139 bash[43301]: audit 2022-04-28T20:41:52.954594+0000 mon.a (mon.0) 2155 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:41:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:53 smithi139 bash[43301]: audit 2022-04-28T20:41:52.957524+0000 mon.a (mon.0) 2156 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:41:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:53 smithi139 bash[43301]: audit 2022-04-28T20:41:52.959612+0000 mon.a (mon.0) 2157 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:41:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:53 smithi139 bash[43301]: audit 2022-04-28T20:41:53.263520+0000 mon.a (mon.0) 2158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:41:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:54 smithi059 bash[61152]: cephadm 2022-04-28T20:41:52.961324+0000 mgr.y (mgr.24961) 1788 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:41:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:54 smithi059 bash[59745]: cephadm 2022-04-28T20:41:52.961324+0000 mgr.y (mgr.24961) 1788 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:41:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:54 smithi139 bash[43301]: cephadm 2022-04-28T20:41:52.961324+0000 mgr.y (mgr.24961) 1788 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:41:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:55 smithi059 bash[61152]: cluster 2022-04-28T20:41:54.289998+0000 mgr.y (mgr.24961) 1789 : cluster [DBG] pgmap v2144: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:55 smithi059 bash[59745]: cluster 2022-04-28T20:41:54.289998+0000 mgr.y (mgr.24961) 1789 : cluster [DBG] pgmap v2144: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:55 smithi139 bash[43301]: cluster 2022-04-28T20:41:54.289998+0000 mgr.y (mgr.24961) 1789 : cluster [DBG] pgmap v2144: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:56 smithi059 bash[61152]: cluster 2022-04-28T20:41:56.291091+0000 mgr.y (mgr.24961) 1790 : cluster [DBG] pgmap v2145: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:56 smithi059 bash[59745]: cluster 2022-04-28T20:41:56.291091+0000 mgr.y (mgr.24961) 1790 : cluster [DBG] pgmap v2145: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:41:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:56 smithi139 bash[43301]: cluster 2022-04-28T20:41:56.291091+0000 mgr.y (mgr.24961) 1790 : cluster [DBG] pgmap v2145: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:41:59 smithi059 bash[61152]: cluster 2022-04-28T20:41:58.291835+0000 mgr.y (mgr.24961) 1791 : cluster [DBG] pgmap v2146: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:41:59 smithi059 bash[59745]: cluster 2022-04-28T20:41:58.291835+0000 mgr.y (mgr.24961) 1791 : cluster [DBG] pgmap v2146: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:41:59 smithi139 bash[43301]: cluster 2022-04-28T20:41:58.291835+0000 mgr.y (mgr.24961) 1791 : cluster [DBG] pgmap v2146: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:01 smithi059 bash[61152]: cluster 2022-04-28T20:42:00.293029+0000 mgr.y (mgr.24961) 1792 : cluster [DBG] pgmap v2147: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:01.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:01 smithi059 bash[59745]: cluster 2022-04-28T20:42:00.293029+0000 mgr.y (mgr.24961) 1792 : cluster [DBG] pgmap v2147: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:01 smithi139 bash[43301]: cluster 2022-04-28T20:42:00.293029+0000 mgr.y (mgr.24961) 1792 : cluster [DBG] pgmap v2147: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:03 smithi059 bash[61152]: cluster 2022-04-28T20:42:02.293972+0000 mgr.y (mgr.24961) 1793 : cluster [DBG] pgmap v2148: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:03.718Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:03 smithi059 bash[59745]: cluster 2022-04-28T20:42:02.293972+0000 mgr.y (mgr.24961) 1793 : cluster [DBG] pgmap v2148: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:03 smithi139 bash[43301]: cluster 2022-04-28T20:42:02.293972+0000 mgr.y (mgr.24961) 1793 : cluster [DBG] pgmap v2148: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:05 smithi059 bash[61152]: cluster 2022-04-28T20:42:04.295023+0000 mgr.y (mgr.24961) 1794 : cluster [DBG] pgmap v2149: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:05 smithi059 bash[59745]: cluster 2022-04-28T20:42:04.295023+0000 mgr.y (mgr.24961) 1794 : cluster [DBG] pgmap v2149: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:05 smithi139 bash[43301]: cluster 2022-04-28T20:42:04.295023+0000 mgr.y (mgr.24961) 1794 : cluster [DBG] pgmap v2149: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:07 smithi059 bash[61152]: cluster 2022-04-28T20:42:06.296195+0000 mgr.y (mgr.24961) 1795 : cluster [DBG] pgmap v2150: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:07 smithi059 bash[59745]: cluster 2022-04-28T20:42:06.296195+0000 mgr.y (mgr.24961) 1795 : cluster [DBG] pgmap v2150: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:07 smithi139 bash[43301]: cluster 2022-04-28T20:42:06.296195+0000 mgr.y (mgr.24961) 1795 : cluster [DBG] pgmap v2150: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:09 smithi059 bash[61152]: cluster 2022-04-28T20:42:08.296884+0000 mgr.y (mgr.24961) 1796 : cluster [DBG] pgmap v2151: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:09 smithi059 bash[59745]: cluster 2022-04-28T20:42:08.296884+0000 mgr.y (mgr.24961) 1796 : cluster [DBG] pgmap v2151: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:09 smithi139 bash[43301]: cluster 2022-04-28T20:42:08.296884+0000 mgr.y (mgr.24961) 1796 : cluster [DBG] pgmap v2151: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:11 smithi059 bash[61152]: cluster 2022-04-28T20:42:10.298028+0000 mgr.y (mgr.24961) 1797 : cluster [DBG] pgmap v2152: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:11.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:11 smithi059 bash[59745]: cluster 2022-04-28T20:42:10.298028+0000 mgr.y (mgr.24961) 1797 : cluster [DBG] pgmap v2152: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:11 smithi139 bash[43301]: cluster 2022-04-28T20:42:10.298028+0000 mgr.y (mgr.24961) 1797 : cluster [DBG] pgmap v2152: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:13 smithi059 bash[61152]: cluster 2022-04-28T20:42:12.298924+0000 mgr.y (mgr.24961) 1798 : cluster [DBG] pgmap v2153: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:13.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:13 smithi059 bash[59745]: cluster 2022-04-28T20:42:12.298924+0000 mgr.y (mgr.24961) 1798 : cluster [DBG] pgmap v2153: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:14.206 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:13 smithi139 bash[43301]: cluster 2022-04-28T20:42:12.298924+0000 mgr.y (mgr.24961) 1798 : cluster [DBG] pgmap v2153: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:15 smithi059 bash[61152]: cluster 2022-04-28T20:42:14.299967+0000 mgr.y (mgr.24961) 1799 : cluster [DBG] pgmap v2154: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:15 smithi059 bash[59745]: cluster 2022-04-28T20:42:14.299967+0000 mgr.y (mgr.24961) 1799 : cluster [DBG] pgmap v2154: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:15 smithi139 bash[43301]: cluster 2022-04-28T20:42:14.299967+0000 mgr.y (mgr.24961) 1799 : cluster [DBG] pgmap v2154: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[61152]: cluster 2022-04-28T20:42:16.301079+0000 mgr.y (mgr.24961) 1800 : cluster [DBG] pgmap v2155: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[61152]: audit 2022-04-28T20:42:17.742740+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:42:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[61152]: audit 2022-04-28T20:42:17.743420+0000 mon.a (mon.0) 2160 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:42:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[61152]: audit 2022-04-28T20:42:17.754254+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:42:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[61152]: audit 2022-04-28T20:42:17.754801+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:42:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[61152]: audit 2022-04-28T20:42:17.755267+0000 mon.a (mon.0) 2163 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:42:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[59745]: cluster 2022-04-28T20:42:16.301079+0000 mgr.y (mgr.24961) 1800 : cluster [DBG] pgmap v2155: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[59745]: audit 2022-04-28T20:42:17.742740+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:42:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[59745]: audit 2022-04-28T20:42:17.743420+0000 mon.a (mon.0) 2160 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:42:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[59745]: audit 2022-04-28T20:42:17.754254+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:42:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[59745]: audit 2022-04-28T20:42:17.754801+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:42:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:17 smithi059 bash[59745]: audit 2022-04-28T20:42:17.755267+0000 mon.a (mon.0) 2163 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:42:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:17 smithi139 bash[43301]: cluster 2022-04-28T20:42:16.301079+0000 mgr.y (mgr.24961) 1800 : cluster [DBG] pgmap v2155: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:17 smithi139 bash[43301]: audit 2022-04-28T20:42:17.742740+0000 mon.a (mon.0) 2159 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:42:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:17 smithi139 bash[43301]: audit 2022-04-28T20:42:17.743420+0000 mon.a (mon.0) 2160 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:42:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:17 smithi139 bash[43301]: audit 2022-04-28T20:42:17.754254+0000 mon.a (mon.0) 2161 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:42:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:17 smithi139 bash[43301]: audit 2022-04-28T20:42:17.754801+0000 mon.a (mon.0) 2162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T20:42:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:17 smithi139 bash[43301]: audit 2022-04-28T20:42:17.755267+0000 mon.a (mon.0) 2163 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:42:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:19 smithi059 bash[61152]: cluster 2022-04-28T20:42:18.301778+0000 mgr.y (mgr.24961) 1801 : cluster [DBG] pgmap v2156: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:20.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:19 smithi059 bash[59745]: cluster 2022-04-28T20:42:18.301778+0000 mgr.y (mgr.24961) 1801 : cluster [DBG] pgmap v2156: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:19 smithi139 bash[43301]: cluster 2022-04-28T20:42:18.301778+0000 mgr.y (mgr.24961) 1801 : cluster [DBG] pgmap v2156: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:21 smithi059 bash[59745]: cluster 2022-04-28T20:42:20.302968+0000 mgr.y (mgr.24961) 1802 : cluster [DBG] pgmap v2157: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:21 smithi059 bash[61152]: cluster 2022-04-28T20:42:20.302968+0000 mgr.y (mgr.24961) 1802 : cluster [DBG] pgmap v2157: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:21.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:22.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:21 smithi139 bash[43301]: cluster 2022-04-28T20:42:20.302968+0000 mgr.y (mgr.24961) 1802 : cluster [DBG] pgmap v2157: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:23 smithi059 bash[61152]: cluster 2022-04-28T20:42:22.303870+0000 mgr.y (mgr.24961) 1803 : cluster [DBG] pgmap v2158: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:23 smithi059 bash[59745]: cluster 2022-04-28T20:42:22.303870+0000 mgr.y (mgr.24961) 1803 : cluster [DBG] pgmap v2158: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:23.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:23 smithi139 bash[43301]: cluster 2022-04-28T20:42:22.303870+0000 mgr.y (mgr.24961) 1803 : cluster [DBG] pgmap v2158: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:25 smithi059 bash[59745]: cluster 2022-04-28T20:42:24.304802+0000 mgr.y (mgr.24961) 1804 : cluster [DBG] pgmap v2159: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:26.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:25 smithi059 bash[61152]: cluster 2022-04-28T20:42:24.304802+0000 mgr.y (mgr.24961) 1804 : cluster [DBG] pgmap v2159: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:25 smithi139 bash[43301]: cluster 2022-04-28T20:42:24.304802+0000 mgr.y (mgr.24961) 1804 : cluster [DBG] pgmap v2159: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:27 smithi059 bash[59745]: cluster 2022-04-28T20:42:26.305939+0000 mgr.y (mgr.24961) 1805 : cluster [DBG] pgmap v2160: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:28.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:27 smithi059 bash[61152]: cluster 2022-04-28T20:42:26.305939+0000 mgr.y (mgr.24961) 1805 : cluster [DBG] pgmap v2160: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:27 smithi139 bash[43301]: cluster 2022-04-28T20:42:26.305939+0000 mgr.y (mgr.24961) 1805 : cluster [DBG] pgmap v2160: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:29 smithi139 bash[43301]: cluster 2022-04-28T20:42:28.306664+0000 mgr.y (mgr.24961) 1806 : cluster [DBG] pgmap v2161: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:29 smithi059 bash[61152]: cluster 2022-04-28T20:42:28.306664+0000 mgr.y (mgr.24961) 1806 : cluster [DBG] pgmap v2161: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:29 smithi059 bash[59745]: cluster 2022-04-28T20:42:28.306664+0000 mgr.y (mgr.24961) 1806 : cluster [DBG] pgmap v2161: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:31 smithi059 bash[61152]: cluster 2022-04-28T20:42:30.307807+0000 mgr.y (mgr.24961) 1807 : cluster [DBG] pgmap v2162: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:32.124 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:31.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:32.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:31 smithi059 bash[59745]: cluster 2022-04-28T20:42:30.307807+0000 mgr.y (mgr.24961) 1807 : cluster [DBG] pgmap v2162: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:32.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:31 smithi139 bash[43301]: cluster 2022-04-28T20:42:30.307807+0000 mgr.y (mgr.24961) 1807 : cluster [DBG] pgmap v2162: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:33 smithi059 bash[61152]: cluster 2022-04-28T20:42:32.308750+0000 mgr.y (mgr.24961) 1808 : cluster [DBG] pgmap v2163: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:33.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:33 smithi059 bash[59745]: cluster 2022-04-28T20:42:32.308750+0000 mgr.y (mgr.24961) 1808 : cluster [DBG] pgmap v2163: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:34.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:33 smithi139 bash[43301]: cluster 2022-04-28T20:42:32.308750+0000 mgr.y (mgr.24961) 1808 : cluster [DBG] pgmap v2163: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:35 smithi139 bash[43301]: cluster 2022-04-28T20:42:34.309806+0000 mgr.y (mgr.24961) 1809 : cluster [DBG] pgmap v2164: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:35 smithi059 bash[61152]: cluster 2022-04-28T20:42:34.309806+0000 mgr.y (mgr.24961) 1809 : cluster [DBG] pgmap v2164: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:35 smithi059 bash[59745]: cluster 2022-04-28T20:42:34.309806+0000 mgr.y (mgr.24961) 1809 : cluster [DBG] pgmap v2164: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:38.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:37 smithi059 bash[61152]: cluster 2022-04-28T20:42:36.310897+0000 mgr.y (mgr.24961) 1810 : cluster [DBG] pgmap v2165: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:37 smithi059 bash[59745]: cluster 2022-04-28T20:42:36.310897+0000 mgr.y (mgr.24961) 1810 : cluster [DBG] pgmap v2165: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:37 smithi139 bash[43301]: cluster 2022-04-28T20:42:36.310897+0000 mgr.y (mgr.24961) 1810 : cluster [DBG] pgmap v2165: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:39 smithi059 bash[61152]: cluster 2022-04-28T20:42:38.311612+0000 mgr.y (mgr.24961) 1811 : cluster [DBG] pgmap v2166: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:39 smithi059 bash[59745]: cluster 2022-04-28T20:42:38.311612+0000 mgr.y (mgr.24961) 1811 : cluster [DBG] pgmap v2166: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:39 smithi139 bash[43301]: cluster 2022-04-28T20:42:38.311612+0000 mgr.y (mgr.24961) 1811 : cluster [DBG] pgmap v2166: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:41 smithi059 bash[61152]: cluster 2022-04-28T20:42:40.312814+0000 mgr.y (mgr.24961) 1812 : cluster [DBG] pgmap v2167: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:41.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:41 smithi059 bash[59745]: cluster 2022-04-28T20:42:40.312814+0000 mgr.y (mgr.24961) 1812 : cluster [DBG] pgmap v2167: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:41 smithi139 bash[43301]: cluster 2022-04-28T20:42:40.312814+0000 mgr.y (mgr.24961) 1812 : cluster [DBG] pgmap v2167: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:43 smithi059 bash[61152]: cluster 2022-04-28T20:42:42.313693+0000 mgr.y (mgr.24961) 1813 : cluster [DBG] pgmap v2168: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:43.719Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:43 smithi059 bash[59745]: cluster 2022-04-28T20:42:42.313693+0000 mgr.y (mgr.24961) 1813 : cluster [DBG] pgmap v2168: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:43 smithi139 bash[43301]: cluster 2022-04-28T20:42:42.313693+0000 mgr.y (mgr.24961) 1813 : cluster [DBG] pgmap v2168: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:46.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:45 smithi059 bash[61152]: cluster 2022-04-28T20:42:44.314744+0000 mgr.y (mgr.24961) 1814 : cluster [DBG] pgmap v2169: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:45 smithi059 bash[59745]: cluster 2022-04-28T20:42:44.314744+0000 mgr.y (mgr.24961) 1814 : cluster [DBG] pgmap v2169: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:45 smithi139 bash[43301]: cluster 2022-04-28T20:42:44.314744+0000 mgr.y (mgr.24961) 1814 : cluster [DBG] pgmap v2169: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:47 smithi059 bash[61152]: cluster 2022-04-28T20:42:46.315895+0000 mgr.y (mgr.24961) 1815 : cluster [DBG] pgmap v2170: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:47 smithi059 bash[59745]: cluster 2022-04-28T20:42:46.315895+0000 mgr.y (mgr.24961) 1815 : cluster [DBG] pgmap v2170: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:47 smithi139 bash[43301]: cluster 2022-04-28T20:42:46.315895+0000 mgr.y (mgr.24961) 1815 : cluster [DBG] pgmap v2170: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:49 smithi059 bash[61152]: cluster 2022-04-28T20:42:48.316592+0000 mgr.y (mgr.24961) 1816 : cluster [DBG] pgmap v2171: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:49 smithi059 bash[59745]: cluster 2022-04-28T20:42:48.316592+0000 mgr.y (mgr.24961) 1816 : cluster [DBG] pgmap v2171: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:49 smithi139 bash[43301]: cluster 2022-04-28T20:42:48.316592+0000 mgr.y (mgr.24961) 1816 : cluster [DBG] pgmap v2171: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:51 smithi059 bash[61152]: cluster 2022-04-28T20:42:50.317772+0000 mgr.y (mgr.24961) 1817 : cluster [DBG] pgmap v2172: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:51 smithi059 bash[59745]: cluster 2022-04-28T20:42:50.317772+0000 mgr.y (mgr.24961) 1817 : cluster [DBG] pgmap v2172: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:52.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:51.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:51 smithi139 bash[43301]: cluster 2022-04-28T20:42:50.317772+0000 mgr.y (mgr.24961) 1817 : cluster [DBG] pgmap v2172: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:42:53.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:42:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[59745]: cluster 2022-04-28T20:42:52.318701+0000 mgr.y (mgr.24961) 1818 : cluster [DBG] pgmap v2173: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[59745]: audit 2022-04-28T20:42:53.269043+0000 mon.a (mon.0) 2164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:42:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[59745]: audit 2022-04-28T20:42:53.271889+0000 mon.a (mon.0) 2165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:42:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[59745]: audit 2022-04-28T20:42:53.273682+0000 mon.a (mon.0) 2166 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:42:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[59745]: audit 2022-04-28T20:42:53.581342+0000 mon.a (mon.0) 2167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:42:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[61152]: cluster 2022-04-28T20:42:52.318701+0000 mgr.y (mgr.24961) 1818 : cluster [DBG] pgmap v2173: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[61152]: audit 2022-04-28T20:42:53.269043+0000 mon.a (mon.0) 2164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:42:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[61152]: audit 2022-04-28T20:42:53.271889+0000 mon.a (mon.0) 2165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:42:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[61152]: audit 2022-04-28T20:42:53.273682+0000 mon.a (mon.0) 2166 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:42:54.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:53 smithi059 bash[61152]: audit 2022-04-28T20:42:53.581342+0000 mon.a (mon.0) 2167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:42:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:53 smithi139 bash[43301]: cluster 2022-04-28T20:42:52.318701+0000 mgr.y (mgr.24961) 1818 : cluster [DBG] pgmap v2173: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:53 smithi139 bash[43301]: audit 2022-04-28T20:42:53.269043+0000 mon.a (mon.0) 2164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:42:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:53 smithi139 bash[43301]: audit 2022-04-28T20:42:53.271889+0000 mon.a (mon.0) 2165 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:42:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:53 smithi139 bash[43301]: audit 2022-04-28T20:42:53.273682+0000 mon.a (mon.0) 2166 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:42:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:53 smithi139 bash[43301]: audit 2022-04-28T20:42:53.581342+0000 mon.a (mon.0) 2167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:42:55.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:54 smithi059 bash[59745]: cephadm 2022-04-28T20:42:53.275312+0000 mgr.y (mgr.24961) 1819 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:42:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:54 smithi059 bash[61152]: cephadm 2022-04-28T20:42:53.275312+0000 mgr.y (mgr.24961) 1819 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:42:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:54 smithi139 bash[43301]: cephadm 2022-04-28T20:42:53.275312+0000 mgr.y (mgr.24961) 1819 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:42:56.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:56 smithi059 bash[59745]: cluster 2022-04-28T20:42:54.319723+0000 mgr.y (mgr.24961) 1820 : cluster [DBG] pgmap v2174: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:56 smithi059 bash[61152]: cluster 2022-04-28T20:42:54.319723+0000 mgr.y (mgr.24961) 1820 : cluster [DBG] pgmap v2174: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:56 smithi139 bash[43301]: cluster 2022-04-28T20:42:54.319723+0000 mgr.y (mgr.24961) 1820 : cluster [DBG] pgmap v2174: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:57 smithi059 bash[61152]: cluster 2022-04-28T20:42:56.309389+0000 mon.a (mon.0) 2168 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2022-04-28T20:42:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:57 smithi059 bash[61152]: audit 2022-04-28T20:42:56.313336+0000 mon.a (mon.0) 2169 : audit [INF] from='client.? 172.21.15.59:0/1152971669' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78266-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:42:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:57 smithi059 bash[61152]: cluster 2022-04-28T20:42:56.320148+0000 mgr.y (mgr.24961) 1821 : cluster [DBG] pgmap v2176: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:57 smithi059 bash[59745]: cluster 2022-04-28T20:42:56.309389+0000 mon.a (mon.0) 2168 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2022-04-28T20:42:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:57 smithi059 bash[59745]: audit 2022-04-28T20:42:56.313336+0000 mon.a (mon.0) 2169 : audit [INF] from='client.? 172.21.15.59:0/1152971669' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78266-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:42:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:57 smithi059 bash[59745]: cluster 2022-04-28T20:42:56.320148+0000 mgr.y (mgr.24961) 1821 : cluster [DBG] pgmap v2176: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:57 smithi139 bash[43301]: cluster 2022-04-28T20:42:56.309389+0000 mon.a (mon.0) 2168 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2022-04-28T20:42:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:57 smithi139 bash[43301]: audit 2022-04-28T20:42:56.313336+0000 mon.a (mon.0) 2169 : audit [INF] from='client.? 172.21.15.59:0/1152971669' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78266-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:42:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:57 smithi139 bash[43301]: cluster 2022-04-28T20:42:56.320148+0000 mgr.y (mgr.24961) 1821 : cluster [DBG] pgmap v2176: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:58 smithi059 bash[61152]: audit 2022-04-28T20:42:57.321985+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? 172.21.15.59:0/1152971669' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78266-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:42:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:58 smithi059 bash[61152]: cluster 2022-04-28T20:42:57.322421+0000 mon.a (mon.0) 2171 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2022-04-28T20:42:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:58 smithi059 bash[59745]: audit 2022-04-28T20:42:57.321985+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? 172.21.15.59:0/1152971669' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78266-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:42:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:58 smithi059 bash[59745]: cluster 2022-04-28T20:42:57.322421+0000 mon.a (mon.0) 2171 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2022-04-28T20:42:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:58 smithi139 bash[43301]: audit 2022-04-28T20:42:57.321985+0000 mon.a (mon.0) 2170 : audit [INF] from='client.? 172.21.15.59:0/1152971669' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78266-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:42:58.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:58 smithi139 bash[43301]: cluster 2022-04-28T20:42:57.322421+0000 mon.a (mon.0) 2171 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2022-04-28T20:42:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:59 smithi059 bash[61152]: cluster 2022-04-28T20:42:58.320861+0000 mgr.y (mgr.24961) 1822 : cluster [DBG] pgmap v2179: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:42:59 smithi059 bash[61152]: cluster 2022-04-28T20:42:58.323146+0000 mon.a (mon.0) 2172 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2022-04-28T20:42:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:59 smithi059 bash[59745]: cluster 2022-04-28T20:42:58.320861+0000 mgr.y (mgr.24961) 1822 : cluster [DBG] pgmap v2179: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:42:59 smithi059 bash[59745]: cluster 2022-04-28T20:42:58.323146+0000 mon.a (mon.0) 2172 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2022-04-28T20:42:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:59 smithi139 bash[43301]: cluster 2022-04-28T20:42:58.320861+0000 mgr.y (mgr.24961) 1822 : cluster [DBG] pgmap v2179: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:42:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:42:59 smithi139 bash[43301]: cluster 2022-04-28T20:42:58.323146+0000 mon.a (mon.0) 2172 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2022-04-28T20:43:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:01 smithi059 bash[61152]: cluster 2022-04-28T20:43:00.322198+0000 mgr.y (mgr.24961) 1823 : cluster [DBG] pgmap v2180: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:01 smithi059 bash[59745]: cluster 2022-04-28T20:43:00.322198+0000 mgr.y (mgr.24961) 1823 : cluster [DBG] pgmap v2180: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:01 smithi139 bash[43301]: cluster 2022-04-28T20:43:00.322198+0000 mgr.y (mgr.24961) 1823 : cluster [DBG] pgmap v2180: 105 pgs: 105 active+clean; 24 KiB data, 718 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:02.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:01.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:03 smithi139 bash[43301]: cluster 2022-04-28T20:43:02.323179+0000 mgr.y (mgr.24961) 1824 : cluster [DBG] pgmap v2181: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:03.721 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:03 smithi059 bash[61152]: cluster 2022-04-28T20:43:02.323179+0000 mgr.y (mgr.24961) 1824 : cluster [DBG] pgmap v2181: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:03.721 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:03 smithi059 bash[59745]: cluster 2022-04-28T20:43:02.323179+0000 mgr.y (mgr.24961) 1824 : cluster [DBG] pgmap v2181: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:03.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:05 smithi059 bash[61152]: cluster 2022-04-28T20:43:04.324190+0000 mgr.y (mgr.24961) 1825 : cluster [DBG] pgmap v2182: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:06.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:05 smithi059 bash[59745]: cluster 2022-04-28T20:43:04.324190+0000 mgr.y (mgr.24961) 1825 : cluster [DBG] pgmap v2182: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:06.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:05 smithi139 bash[43301]: cluster 2022-04-28T20:43:04.324190+0000 mgr.y (mgr.24961) 1825 : cluster [DBG] pgmap v2182: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:07 smithi059 bash[59745]: cluster 2022-04-28T20:43:06.325228+0000 mgr.y (mgr.24961) 1826 : cluster [DBG] pgmap v2183: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:07 smithi059 bash[61152]: cluster 2022-04-28T20:43:06.325228+0000 mgr.y (mgr.24961) 1826 : cluster [DBG] pgmap v2183: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:07 smithi139 bash[43301]: cluster 2022-04-28T20:43:06.325228+0000 mgr.y (mgr.24961) 1826 : cluster [DBG] pgmap v2183: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:09 smithi059 bash[61152]: cluster 2022-04-28T20:43:08.325912+0000 mgr.y (mgr.24961) 1827 : cluster [DBG] pgmap v2184: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:09 smithi059 bash[59745]: cluster 2022-04-28T20:43:08.325912+0000 mgr.y (mgr.24961) 1827 : cluster [DBG] pgmap v2184: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:09 smithi139 bash[43301]: cluster 2022-04-28T20:43:08.325912+0000 mgr.y (mgr.24961) 1827 : cluster [DBG] pgmap v2184: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[61152]: cluster 2022-04-28T20:43:09.806040+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2022-04-28T20:43:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[61152]: audit 2022-04-28T20:43:09.810920+0000 mon.a (mon.0) 2174 : audit [INF] from='client.? 172.21.15.59:0/2032905356' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78289-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:43:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[61152]: audit 2022-04-28T20:43:10.811468+0000 mon.a (mon.0) 2175 : audit [INF] from='client.? 172.21.15.59:0/2032905356' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78289-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:43:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[61152]: cluster 2022-04-28T20:43:10.811845+0000 mon.a (mon.0) 2176 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2022-04-28T20:43:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[59745]: cluster 2022-04-28T20:43:09.806040+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2022-04-28T20:43:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[59745]: audit 2022-04-28T20:43:09.810920+0000 mon.a (mon.0) 2174 : audit [INF] from='client.? 172.21.15.59:0/2032905356' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78289-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:43:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[59745]: audit 2022-04-28T20:43:10.811468+0000 mon.a (mon.0) 2175 : audit [INF] from='client.? 172.21.15.59:0/2032905356' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78289-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:43:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:10 smithi059 bash[59745]: cluster 2022-04-28T20:43:10.811845+0000 mon.a (mon.0) 2176 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2022-04-28T20:43:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:10 smithi139 bash[43301]: cluster 2022-04-28T20:43:09.806040+0000 mon.a (mon.0) 2173 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2022-04-28T20:43:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:10 smithi139 bash[43301]: audit 2022-04-28T20:43:09.810920+0000 mon.a (mon.0) 2174 : audit [INF] from='client.? 172.21.15.59:0/2032905356' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78289-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:43:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:10 smithi139 bash[43301]: audit 2022-04-28T20:43:10.811468+0000 mon.a (mon.0) 2175 : audit [INF] from='client.? 172.21.15.59:0/2032905356' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78289-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:43:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:10 smithi139 bash[43301]: cluster 2022-04-28T20:43:10.811845+0000 mon.a (mon.0) 2176 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2022-04-28T20:43:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:11.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:11 smithi059 bash[61152]: cluster 2022-04-28T20:43:10.326941+0000 mgr.y (mgr.24961) 1828 : cluster [DBG] pgmap v2186: 137 pgs: 13 creating+peering, 19 unknown, 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:11 smithi059 bash[61152]: cluster 2022-04-28T20:43:11.828203+0000 mon.a (mon.0) 2177 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2022-04-28T20:43:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:11 smithi059 bash[59745]: cluster 2022-04-28T20:43:10.326941+0000 mgr.y (mgr.24961) 1828 : cluster [DBG] pgmap v2186: 137 pgs: 13 creating+peering, 19 unknown, 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:11 smithi059 bash[59745]: cluster 2022-04-28T20:43:11.828203+0000 mon.a (mon.0) 2177 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2022-04-28T20:43:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:11 smithi139 bash[43301]: cluster 2022-04-28T20:43:10.326941+0000 mgr.y (mgr.24961) 1828 : cluster [DBG] pgmap v2186: 137 pgs: 13 creating+peering, 19 unknown, 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:11 smithi139 bash[43301]: cluster 2022-04-28T20:43:11.828203+0000 mon.a (mon.0) 2177 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2022-04-28T20:43:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:13 smithi059 bash[61152]: cluster 2022-04-28T20:43:12.327659+0000 mgr.y (mgr.24961) 1829 : cluster [DBG] pgmap v2189: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:13.721Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:13 smithi059 bash[59745]: cluster 2022-04-28T20:43:12.327659+0000 mgr.y (mgr.24961) 1829 : cluster [DBG] pgmap v2189: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:13 smithi139 bash[43301]: cluster 2022-04-28T20:43:12.327659+0000 mgr.y (mgr.24961) 1829 : cluster [DBG] pgmap v2189: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:15 smithi139 bash[43301]: cluster 2022-04-28T20:43:14.328635+0000 mgr.y (mgr.24961) 1830 : cluster [DBG] pgmap v2190: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:15 smithi059 bash[59745]: cluster 2022-04-28T20:43:14.328635+0000 mgr.y (mgr.24961) 1830 : cluster [DBG] pgmap v2190: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:15 smithi059 bash[61152]: cluster 2022-04-28T20:43:14.328635+0000 mgr.y (mgr.24961) 1830 : cluster [DBG] pgmap v2190: 105 pgs: 105 active+clean; 24 KiB data, 719 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:17 smithi139 bash[43301]: cluster 2022-04-28T20:43:16.329846+0000 mgr.y (mgr.24961) 1831 : cluster [DBG] pgmap v2191: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:17 smithi139 bash[43301]: audit 2022-04-28T20:43:17.755535+0000 mon.a (mon.0) 2178 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:17 smithi139 bash[43301]: audit 2022-04-28T20:43:17.756051+0000 mon.a (mon.0) 2179 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:17 smithi139 bash[43301]: audit 2022-04-28T20:43:17.767002+0000 mon.a (mon.0) 2180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:17 smithi139 bash[43301]: audit 2022-04-28T20:43:17.767829+0000 mon.a (mon.0) 2181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:17 smithi139 bash[43301]: audit 2022-04-28T20:43:17.768310+0000 mon.a (mon.0) 2182 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:43:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[59745]: cluster 2022-04-28T20:43:16.329846+0000 mgr.y (mgr.24961) 1831 : cluster [DBG] pgmap v2191: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[59745]: audit 2022-04-28T20:43:17.755535+0000 mon.a (mon.0) 2178 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:43:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[59745]: audit 2022-04-28T20:43:17.756051+0000 mon.a (mon.0) 2179 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:43:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[59745]: audit 2022-04-28T20:43:17.767002+0000 mon.a (mon.0) 2180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:43:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[59745]: audit 2022-04-28T20:43:17.767829+0000 mon.a (mon.0) 2181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:43:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[59745]: audit 2022-04-28T20:43:17.768310+0000 mon.a (mon.0) 2182 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:43:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[61152]: cluster 2022-04-28T20:43:16.329846+0000 mgr.y (mgr.24961) 1831 : cluster [DBG] pgmap v2191: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[61152]: audit 2022-04-28T20:43:17.755535+0000 mon.a (mon.0) 2178 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:43:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[61152]: audit 2022-04-28T20:43:17.756051+0000 mon.a (mon.0) 2179 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:43:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[61152]: audit 2022-04-28T20:43:17.767002+0000 mon.a (mon.0) 2180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:43:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[61152]: audit 2022-04-28T20:43:17.767829+0000 mon.a (mon.0) 2181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T20:43:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:17 smithi059 bash[61152]: audit 2022-04-28T20:43:17.768310+0000 mon.a (mon.0) 2182 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:43:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:19 smithi139 bash[43301]: cluster 2022-04-28T20:43:18.330547+0000 mgr.y (mgr.24961) 1832 : cluster [DBG] pgmap v2192: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:20.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:19 smithi059 bash[59745]: cluster 2022-04-28T20:43:18.330547+0000 mgr.y (mgr.24961) 1832 : cluster [DBG] pgmap v2192: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:19 smithi059 bash[61152]: cluster 2022-04-28T20:43:18.330547+0000 mgr.y (mgr.24961) 1832 : cluster [DBG] pgmap v2192: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:22.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:21 smithi059 bash[61152]: cluster 2022-04-28T20:43:20.331767+0000 mgr.y (mgr.24961) 1833 : cluster [DBG] pgmap v2193: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:21 smithi059 bash[59745]: cluster 2022-04-28T20:43:20.331767+0000 mgr.y (mgr.24961) 1833 : cluster [DBG] pgmap v2193: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:21.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:21 smithi139 bash[43301]: cluster 2022-04-28T20:43:20.331767+0000 mgr.y (mgr.24961) 1833 : cluster [DBG] pgmap v2193: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:23 smithi059 bash[61152]: cluster 2022-04-28T20:43:22.332734+0000 mgr.y (mgr.24961) 1834 : cluster [DBG] pgmap v2194: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:23 smithi059 bash[59745]: cluster 2022-04-28T20:43:22.332734+0000 mgr.y (mgr.24961) 1834 : cluster [DBG] pgmap v2194: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:23.722Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:23 smithi139 bash[43301]: cluster 2022-04-28T20:43:22.332734+0000 mgr.y (mgr.24961) 1834 : cluster [DBG] pgmap v2194: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:26.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:25 smithi059 bash[59745]: cluster 2022-04-28T20:43:24.333699+0000 mgr.y (mgr.24961) 1835 : cluster [DBG] pgmap v2195: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:25 smithi059 bash[61152]: cluster 2022-04-28T20:43:24.333699+0000 mgr.y (mgr.24961) 1835 : cluster [DBG] pgmap v2195: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:25 smithi139 bash[43301]: cluster 2022-04-28T20:43:24.333699+0000 mgr.y (mgr.24961) 1835 : cluster [DBG] pgmap v2195: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:27 smithi059 bash[59745]: cluster 2022-04-28T20:43:26.334797+0000 mgr.y (mgr.24961) 1836 : cluster [DBG] pgmap v2196: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:27 smithi059 bash[61152]: cluster 2022-04-28T20:43:26.334797+0000 mgr.y (mgr.24961) 1836 : cluster [DBG] pgmap v2196: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:27 smithi139 bash[43301]: cluster 2022-04-28T20:43:26.334797+0000 mgr.y (mgr.24961) 1836 : cluster [DBG] pgmap v2196: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:29 smithi059 bash[61152]: cluster 2022-04-28T20:43:28.335477+0000 mgr.y (mgr.24961) 1837 : cluster [DBG] pgmap v2197: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:29 smithi059 bash[59745]: cluster 2022-04-28T20:43:28.335477+0000 mgr.y (mgr.24961) 1837 : cluster [DBG] pgmap v2197: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:29 smithi139 bash[43301]: cluster 2022-04-28T20:43:28.335477+0000 mgr.y (mgr.24961) 1837 : cluster [DBG] pgmap v2197: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:32.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:31 smithi059 bash[61152]: cluster 2022-04-28T20:43:30.336825+0000 mgr.y (mgr.24961) 1838 : cluster [DBG] pgmap v2198: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:31.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:31 smithi059 bash[59745]: cluster 2022-04-28T20:43:30.336825+0000 mgr.y (mgr.24961) 1838 : cluster [DBG] pgmap v2198: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:31 smithi139 bash[43301]: cluster 2022-04-28T20:43:30.336825+0000 mgr.y (mgr.24961) 1838 : cluster [DBG] pgmap v2198: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:34.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:33 smithi059 bash[61152]: cluster 2022-04-28T20:43:32.337816+0000 mgr.y (mgr.24961) 1839 : cluster [DBG] pgmap v2199: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:33.723Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:33 smithi059 bash[59745]: cluster 2022-04-28T20:43:32.337816+0000 mgr.y (mgr.24961) 1839 : cluster [DBG] pgmap v2199: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:33 smithi139 bash[43301]: cluster 2022-04-28T20:43:32.337816+0000 mgr.y (mgr.24961) 1839 : cluster [DBG] pgmap v2199: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:36.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:35 smithi059 bash[61152]: cluster 2022-04-28T20:43:34.338859+0000 mgr.y (mgr.24961) 1840 : cluster [DBG] pgmap v2200: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:35 smithi059 bash[59745]: cluster 2022-04-28T20:43:34.338859+0000 mgr.y (mgr.24961) 1840 : cluster [DBG] pgmap v2200: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:35 smithi139 bash[43301]: cluster 2022-04-28T20:43:34.338859+0000 mgr.y (mgr.24961) 1840 : cluster [DBG] pgmap v2200: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:38.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:37 smithi059 bash[61152]: cluster 2022-04-28T20:43:36.340001+0000 mgr.y (mgr.24961) 1841 : cluster [DBG] pgmap v2201: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:37 smithi059 bash[59745]: cluster 2022-04-28T20:43:36.340001+0000 mgr.y (mgr.24961) 1841 : cluster [DBG] pgmap v2201: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:37 smithi139 bash[43301]: cluster 2022-04-28T20:43:36.340001+0000 mgr.y (mgr.24961) 1841 : cluster [DBG] pgmap v2201: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:39 smithi059 bash[61152]: cluster 2022-04-28T20:43:38.340695+0000 mgr.y (mgr.24961) 1842 : cluster [DBG] pgmap v2202: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:39 smithi059 bash[59745]: cluster 2022-04-28T20:43:38.340695+0000 mgr.y (mgr.24961) 1842 : cluster [DBG] pgmap v2202: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:39 smithi139 bash[43301]: cluster 2022-04-28T20:43:38.340695+0000 mgr.y (mgr.24961) 1842 : cluster [DBG] pgmap v2202: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:40 smithi059 bash[61152]: cluster 2022-04-28T20:43:40.341982+0000 mgr.y (mgr.24961) 1843 : cluster [DBG] pgmap v2203: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:40 smithi059 bash[59745]: cluster 2022-04-28T20:43:40.341982+0000 mgr.y (mgr.24961) 1843 : cluster [DBG] pgmap v2203: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:40 smithi139 bash[43301]: cluster 2022-04-28T20:43:40.341982+0000 mgr.y (mgr.24961) 1843 : cluster [DBG] pgmap v2203: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:42.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:41.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:43.724 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:43 smithi059 bash[61152]: cluster 2022-04-28T20:43:42.342909+0000 mgr.y (mgr.24961) 1844 : cluster [DBG] pgmap v2204: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:43.725 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:43.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:43.725 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:43 smithi059 bash[59745]: cluster 2022-04-28T20:43:42.342909+0000 mgr.y (mgr.24961) 1844 : cluster [DBG] pgmap v2204: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:43.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:43 smithi139 bash[43301]: cluster 2022-04-28T20:43:42.342909+0000 mgr.y (mgr.24961) 1844 : cluster [DBG] pgmap v2204: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:45 smithi059 bash[61152]: cluster 2022-04-28T20:43:44.343920+0000 mgr.y (mgr.24961) 1845 : cluster [DBG] pgmap v2205: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:45 smithi059 bash[59745]: cluster 2022-04-28T20:43:44.343920+0000 mgr.y (mgr.24961) 1845 : cluster [DBG] pgmap v2205: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:45 smithi139 bash[43301]: cluster 2022-04-28T20:43:44.343920+0000 mgr.y (mgr.24961) 1845 : cluster [DBG] pgmap v2205: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:48.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:47 smithi059 bash[61152]: cluster 2022-04-28T20:43:46.345010+0000 mgr.y (mgr.24961) 1846 : cluster [DBG] pgmap v2206: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:48.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:47 smithi059 bash[59745]: cluster 2022-04-28T20:43:46.345010+0000 mgr.y (mgr.24961) 1846 : cluster [DBG] pgmap v2206: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:47 smithi139 bash[43301]: cluster 2022-04-28T20:43:46.345010+0000 mgr.y (mgr.24961) 1846 : cluster [DBG] pgmap v2206: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:50.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:49 smithi059 bash[61152]: cluster 2022-04-28T20:43:48.345703+0000 mgr.y (mgr.24961) 1847 : cluster [DBG] pgmap v2207: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:50.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:49 smithi059 bash[59745]: cluster 2022-04-28T20:43:48.345703+0000 mgr.y (mgr.24961) 1847 : cluster [DBG] pgmap v2207: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:50.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:49 smithi139 bash[43301]: cluster 2022-04-28T20:43:48.345703+0000 mgr.y (mgr.24961) 1847 : cluster [DBG] pgmap v2207: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:52.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:51 smithi059 bash[61152]: cluster 2022-04-28T20:43:50.346915+0000 mgr.y (mgr.24961) 1848 : cluster [DBG] pgmap v2208: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:51.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:51 smithi059 bash[59745]: cluster 2022-04-28T20:43:50.346915+0000 mgr.y (mgr.24961) 1848 : cluster [DBG] pgmap v2208: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:52.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:51 smithi139 bash[43301]: cluster 2022-04-28T20:43:50.346915+0000 mgr.y (mgr.24961) 1848 : cluster [DBG] pgmap v2208: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[59745]: cluster 2022-04-28T20:43:52.347866+0000 mgr.y (mgr.24961) 1849 : cluster [DBG] pgmap v2209: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[59745]: audit 2022-04-28T20:43:53.587853+0000 mon.a (mon.0) 2183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:43:54.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[59745]: audit 2022-04-28T20:43:53.590842+0000 mon.a (mon.0) 2184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:43:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[59745]: audit 2022-04-28T20:43:53.592890+0000 mon.a (mon.0) 2185 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:43:54.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:43:53.724Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:43:54.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[61152]: cluster 2022-04-28T20:43:52.347866+0000 mgr.y (mgr.24961) 1849 : cluster [DBG] pgmap v2209: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:54.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[61152]: audit 2022-04-28T20:43:53.587853+0000 mon.a (mon.0) 2183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:43:54.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[61152]: audit 2022-04-28T20:43:53.590842+0000 mon.a (mon.0) 2184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:43:54.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:53 smithi059 bash[61152]: audit 2022-04-28T20:43:53.592890+0000 mon.a (mon.0) 2185 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:43:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:53 smithi139 bash[43301]: cluster 2022-04-28T20:43:52.347866+0000 mgr.y (mgr.24961) 1849 : cluster [DBG] pgmap v2209: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:53 smithi139 bash[43301]: audit 2022-04-28T20:43:53.587853+0000 mon.a (mon.0) 2183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:43:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:53 smithi139 bash[43301]: audit 2022-04-28T20:43:53.590842+0000 mon.a (mon.0) 2184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:43:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:53 smithi139 bash[43301]: audit 2022-04-28T20:43:53.592890+0000 mon.a (mon.0) 2185 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:43:55.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:54 smithi059 bash[61152]: cephadm 2022-04-28T20:43:53.594569+0000 mgr.y (mgr.24961) 1850 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:43:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:54 smithi059 bash[61152]: audit 2022-04-28T20:43:53.896955+0000 mon.a (mon.0) 2186 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:43:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:54 smithi059 bash[59745]: cephadm 2022-04-28T20:43:53.594569+0000 mgr.y (mgr.24961) 1850 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:43:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:54 smithi059 bash[59745]: audit 2022-04-28T20:43:53.896955+0000 mon.a (mon.0) 2186 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:43:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:54 smithi139 bash[43301]: cephadm 2022-04-28T20:43:53.594569+0000 mgr.y (mgr.24961) 1850 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:43:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:54 smithi139 bash[43301]: audit 2022-04-28T20:43:53.896955+0000 mon.a (mon.0) 2186 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:43:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:55 smithi059 bash[61152]: cluster 2022-04-28T20:43:54.348791+0000 mgr.y (mgr.24961) 1851 : cluster [DBG] pgmap v2210: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:55 smithi059 bash[59745]: cluster 2022-04-28T20:43:54.348791+0000 mgr.y (mgr.24961) 1851 : cluster [DBG] pgmap v2210: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:55 smithi139 bash[43301]: cluster 2022-04-28T20:43:54.348791+0000 mgr.y (mgr.24961) 1851 : cluster [DBG] pgmap v2210: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:57 smithi059 bash[61152]: cluster 2022-04-28T20:43:56.349889+0000 mgr.y (mgr.24961) 1852 : cluster [DBG] pgmap v2211: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:57 smithi059 bash[59745]: cluster 2022-04-28T20:43:56.349889+0000 mgr.y (mgr.24961) 1852 : cluster [DBG] pgmap v2211: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:43:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:57 smithi139 bash[43301]: cluster 2022-04-28T20:43:56.349889+0000 mgr.y (mgr.24961) 1852 : cluster [DBG] pgmap v2211: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:00.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:43:59 smithi059 bash[61152]: cluster 2022-04-28T20:43:58.350587+0000 mgr.y (mgr.24961) 1853 : cluster [DBG] pgmap v2212: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:43:59 smithi059 bash[59745]: cluster 2022-04-28T20:43:58.350587+0000 mgr.y (mgr.24961) 1853 : cluster [DBG] pgmap v2212: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:43:59 smithi139 bash[43301]: cluster 2022-04-28T20:43:58.350587+0000 mgr.y (mgr.24961) 1853 : cluster [DBG] pgmap v2212: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:01 smithi059 bash[61152]: cluster 2022-04-28T20:44:00.351915+0000 mgr.y (mgr.24961) 1854 : cluster [DBG] pgmap v2213: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:01.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:02.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:01 smithi059 bash[59745]: cluster 2022-04-28T20:44:00.351915+0000 mgr.y (mgr.24961) 1854 : cluster [DBG] pgmap v2213: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:01 smithi139 bash[43301]: cluster 2022-04-28T20:44:00.351915+0000 mgr.y (mgr.24961) 1854 : cluster [DBG] pgmap v2213: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:04.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:03 smithi059 bash[61152]: cluster 2022-04-28T20:44:02.352972+0000 mgr.y (mgr.24961) 1855 : cluster [DBG] pgmap v2214: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:03.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:03 smithi059 bash[59745]: cluster 2022-04-28T20:44:02.352972+0000 mgr.y (mgr.24961) 1855 : cluster [DBG] pgmap v2214: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:03 smithi139 bash[43301]: cluster 2022-04-28T20:44:02.352972+0000 mgr.y (mgr.24961) 1855 : cluster [DBG] pgmap v2214: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:05 smithi059 bash[61152]: cluster 2022-04-28T20:44:04.354022+0000 mgr.y (mgr.24961) 1856 : cluster [DBG] pgmap v2215: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:05 smithi059 bash[59745]: cluster 2022-04-28T20:44:04.354022+0000 mgr.y (mgr.24961) 1856 : cluster [DBG] pgmap v2215: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:05 smithi139 bash[43301]: cluster 2022-04-28T20:44:04.354022+0000 mgr.y (mgr.24961) 1856 : cluster [DBG] pgmap v2215: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:07 smithi059 bash[61152]: cluster 2022-04-28T20:44:06.355131+0000 mgr.y (mgr.24961) 1857 : cluster [DBG] pgmap v2216: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:08.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:07 smithi059 bash[59745]: cluster 2022-04-28T20:44:06.355131+0000 mgr.y (mgr.24961) 1857 : cluster [DBG] pgmap v2216: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:07 smithi139 bash[43301]: cluster 2022-04-28T20:44:06.355131+0000 mgr.y (mgr.24961) 1857 : cluster [DBG] pgmap v2216: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:09 smithi059 bash[61152]: cluster 2022-04-28T20:44:08.355864+0000 mgr.y (mgr.24961) 1858 : cluster [DBG] pgmap v2217: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:10.415 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:09 smithi059 bash[59745]: cluster 2022-04-28T20:44:08.355864+0000 mgr.y (mgr.24961) 1858 : cluster [DBG] pgmap v2217: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:10.419 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:09 smithi139 bash[43301]: cluster 2022-04-28T20:44:08.355864+0000 mgr.y (mgr.24961) 1858 : cluster [DBG] pgmap v2217: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:12.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:11 smithi059 bash[61152]: cluster 2022-04-28T20:44:10.357017+0000 mgr.y (mgr.24961) 1859 : cluster [DBG] pgmap v2218: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:11.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:11 smithi059 bash[59745]: cluster 2022-04-28T20:44:10.357017+0000 mgr.y (mgr.24961) 1859 : cluster [DBG] pgmap v2218: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:11 smithi139 bash[43301]: cluster 2022-04-28T20:44:10.357017+0000 mgr.y (mgr.24961) 1859 : cluster [DBG] pgmap v2218: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:14.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:13 smithi059 bash[61152]: cluster 2022-04-28T20:44:12.357955+0000 mgr.y (mgr.24961) 1860 : cluster [DBG] pgmap v2219: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:13 smithi059 bash[59745]: cluster 2022-04-28T20:44:12.357955+0000 mgr.y (mgr.24961) 1860 : cluster [DBG] pgmap v2219: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:13.725Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:13 smithi139 bash[43301]: cluster 2022-04-28T20:44:12.357955+0000 mgr.y (mgr.24961) 1860 : cluster [DBG] pgmap v2219: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:15 smithi139 bash[43301]: cluster 2022-04-28T20:44:14.358885+0000 mgr.y (mgr.24961) 1861 : cluster [DBG] pgmap v2220: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:15.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:15 smithi059 bash[59745]: cluster 2022-04-28T20:44:14.358885+0000 mgr.y (mgr.24961) 1861 : cluster [DBG] pgmap v2220: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:15 smithi059 bash[61152]: cluster 2022-04-28T20:44:14.358885+0000 mgr.y (mgr.24961) 1861 : cluster [DBG] pgmap v2220: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:16 smithi139 bash[43301]: cluster 2022-04-28T20:44:15.350629+0000 mon.a (mon.0) 2187 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2022-04-28T20:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:16 smithi139 bash[43301]: audit 2022-04-28T20:44:15.356989+0000 mon.a (mon.0) 2188 : audit [INF] from='client.? 172.21.15.59:0/3811941518' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78321-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:16 smithi139 bash[43301]: audit 2022-04-28T20:44:16.354792+0000 mon.a (mon.0) 2189 : audit [INF] from='client.? 172.21.15.59:0/3811941518' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78321-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:44:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:16 smithi139 bash[43301]: cluster 2022-04-28T20:44:16.355436+0000 mon.a (mon.0) 2190 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2022-04-28T20:44:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[59745]: cluster 2022-04-28T20:44:15.350629+0000 mon.a (mon.0) 2187 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2022-04-28T20:44:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[59745]: audit 2022-04-28T20:44:15.356989+0000 mon.a (mon.0) 2188 : audit [INF] from='client.? 172.21.15.59:0/3811941518' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78321-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:44:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[59745]: audit 2022-04-28T20:44:16.354792+0000 mon.a (mon.0) 2189 : audit [INF] from='client.? 172.21.15.59:0/3811941518' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78321-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:44:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[59745]: cluster 2022-04-28T20:44:16.355436+0000 mon.a (mon.0) 2190 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2022-04-28T20:44:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[61152]: cluster 2022-04-28T20:44:15.350629+0000 mon.a (mon.0) 2187 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2022-04-28T20:44:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[61152]: audit 2022-04-28T20:44:15.356989+0000 mon.a (mon.0) 2188 : audit [INF] from='client.? 172.21.15.59:0/3811941518' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78321-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:44:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[61152]: audit 2022-04-28T20:44:16.354792+0000 mon.a (mon.0) 2189 : audit [INF] from='client.? 172.21.15.59:0/3811941518' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78321-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:44:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:16 smithi059 bash[61152]: cluster 2022-04-28T20:44:16.355436+0000 mon.a (mon.0) 2190 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2022-04-28T20:44:17.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:17 smithi059 bash[61152]: cluster 2022-04-28T20:44:16.359458+0000 mgr.y (mgr.24961) 1862 : cluster [DBG] pgmap v2223: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:17 smithi059 bash[59745]: cluster 2022-04-28T20:44:16.359458+0000 mgr.y (mgr.24961) 1862 : cluster [DBG] pgmap v2223: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:17.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:17 smithi139 bash[43301]: cluster 2022-04-28T20:44:16.359458+0000 mgr.y (mgr.24961) 1862 : cluster [DBG] pgmap v2223: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[59745]: cluster 2022-04-28T20:44:17.369888+0000 mon.a (mon.0) 2191 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2022-04-28T20:44:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[59745]: audit 2022-04-28T20:44:17.767197+0000 mon.a (mon.0) 2192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:44:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[59745]: audit 2022-04-28T20:44:17.768439+0000 mon.a (mon.0) 2193 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:44:18.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[59745]: audit 2022-04-28T20:44:17.779264+0000 mon.a (mon.0) 2194 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:44:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[59745]: audit 2022-04-28T20:44:17.779878+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:44:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[59745]: audit 2022-04-28T20:44:17.780105+0000 mon.a (mon.0) 2196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:44:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[61152]: cluster 2022-04-28T20:44:17.369888+0000 mon.a (mon.0) 2191 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2022-04-28T20:44:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[61152]: audit 2022-04-28T20:44:17.767197+0000 mon.a (mon.0) 2192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:44:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[61152]: audit 2022-04-28T20:44:17.768439+0000 mon.a (mon.0) 2193 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:44:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[61152]: audit 2022-04-28T20:44:17.779264+0000 mon.a (mon.0) 2194 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:44:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[61152]: audit 2022-04-28T20:44:17.779878+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:44:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:18 smithi059 bash[61152]: audit 2022-04-28T20:44:17.780105+0000 mon.a (mon.0) 2196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:44:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:18 smithi139 bash[43301]: cluster 2022-04-28T20:44:17.369888+0000 mon.a (mon.0) 2191 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2022-04-28T20:44:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:18 smithi139 bash[43301]: audit 2022-04-28T20:44:17.767197+0000 mon.a (mon.0) 2192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:44:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:18 smithi139 bash[43301]: audit 2022-04-28T20:44:17.768439+0000 mon.a (mon.0) 2193 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:44:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:18 smithi139 bash[43301]: audit 2022-04-28T20:44:17.779264+0000 mon.a (mon.0) 2194 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:44:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:18 smithi139 bash[43301]: audit 2022-04-28T20:44:17.779878+0000 mon.a (mon.0) 2195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:44:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:18 smithi139 bash[43301]: audit 2022-04-28T20:44:17.780105+0000 mon.a (mon.0) 2196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:44:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:19 smithi059 bash[61152]: cluster 2022-04-28T20:44:18.360117+0000 mgr.y (mgr.24961) 1863 : cluster [DBG] pgmap v2225: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:19 smithi059 bash[59745]: cluster 2022-04-28T20:44:18.360117+0000 mgr.y (mgr.24961) 1863 : cluster [DBG] pgmap v2225: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:19 smithi139 bash[43301]: cluster 2022-04-28T20:44:18.360117+0000 mgr.y (mgr.24961) 1863 : cluster [DBG] pgmap v2225: 105 pgs: 105 active+clean; 24 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:21 smithi059 bash[61152]: cluster 2022-04-28T20:44:20.361313+0000 mgr.y (mgr.24961) 1864 : cluster [DBG] pgmap v2226: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:21.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:21 smithi059 bash[59745]: cluster 2022-04-28T20:44:20.361313+0000 mgr.y (mgr.24961) 1864 : cluster [DBG] pgmap v2226: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:22.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:21 smithi139 bash[43301]: cluster 2022-04-28T20:44:20.361313+0000 mgr.y (mgr.24961) 1864 : cluster [DBG] pgmap v2226: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:24.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:23 smithi059 bash[59745]: cluster 2022-04-28T20:44:22.362346+0000 mgr.y (mgr.24961) 1865 : cluster [DBG] pgmap v2227: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:23 smithi059 bash[61152]: cluster 2022-04-28T20:44:22.362346+0000 mgr.y (mgr.24961) 1865 : cluster [DBG] pgmap v2227: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:23.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:24.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:23 smithi139 bash[43301]: cluster 2022-04-28T20:44:22.362346+0000 mgr.y (mgr.24961) 1865 : cluster [DBG] pgmap v2227: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:26.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:25 smithi059 bash[61152]: cluster 2022-04-28T20:44:24.363396+0000 mgr.y (mgr.24961) 1866 : cluster [DBG] pgmap v2228: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:25 smithi059 bash[59745]: cluster 2022-04-28T20:44:24.363396+0000 mgr.y (mgr.24961) 1866 : cluster [DBG] pgmap v2228: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:26.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:25 smithi139 bash[43301]: cluster 2022-04-28T20:44:24.363396+0000 mgr.y (mgr.24961) 1866 : cluster [DBG] pgmap v2228: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:28.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:27 smithi059 bash[61152]: cluster 2022-04-28T20:44:26.364502+0000 mgr.y (mgr.24961) 1867 : cluster [DBG] pgmap v2229: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:27 smithi059 bash[59745]: cluster 2022-04-28T20:44:26.364502+0000 mgr.y (mgr.24961) 1867 : cluster [DBG] pgmap v2229: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:28.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:27 smithi139 bash[43301]: cluster 2022-04-28T20:44:26.364502+0000 mgr.y (mgr.24961) 1867 : cluster [DBG] pgmap v2229: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:29 smithi139 bash[43301]: cluster 2022-04-28T20:44:28.365175+0000 mgr.y (mgr.24961) 1868 : cluster [DBG] pgmap v2230: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:30.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:29 smithi059 bash[61152]: cluster 2022-04-28T20:44:28.365175+0000 mgr.y (mgr.24961) 1868 : cluster [DBG] pgmap v2230: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:29 smithi059 bash[59745]: cluster 2022-04-28T20:44:28.365175+0000 mgr.y (mgr.24961) 1868 : cluster [DBG] pgmap v2230: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:32.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:31 smithi059 bash[61152]: cluster 2022-04-28T20:44:30.366355+0000 mgr.y (mgr.24961) 1869 : cluster [DBG] pgmap v2231: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:31.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:31 smithi059 bash[59745]: cluster 2022-04-28T20:44:30.366355+0000 mgr.y (mgr.24961) 1869 : cluster [DBG] pgmap v2231: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:31 smithi139 bash[43301]: cluster 2022-04-28T20:44:30.366355+0000 mgr.y (mgr.24961) 1869 : cluster [DBG] pgmap v2231: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:34.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:33 smithi059 bash[61152]: cluster 2022-04-28T20:44:32.367368+0000 mgr.y (mgr.24961) 1870 : cluster [DBG] pgmap v2232: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:33.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:33 smithi059 bash[59745]: cluster 2022-04-28T20:44:32.367368+0000 mgr.y (mgr.24961) 1870 : cluster [DBG] pgmap v2232: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:34.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:33 smithi139 bash[43301]: cluster 2022-04-28T20:44:32.367368+0000 mgr.y (mgr.24961) 1870 : cluster [DBG] pgmap v2232: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:36.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:35 smithi139 bash[43301]: cluster 2022-04-28T20:44:34.368408+0000 mgr.y (mgr.24961) 1871 : cluster [DBG] pgmap v2233: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:36.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:35 smithi059 bash[61152]: cluster 2022-04-28T20:44:34.368408+0000 mgr.y (mgr.24961) 1871 : cluster [DBG] pgmap v2233: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:35 smithi059 bash[59745]: cluster 2022-04-28T20:44:34.368408+0000 mgr.y (mgr.24961) 1871 : cluster [DBG] pgmap v2233: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:37 smithi139 bash[43301]: cluster 2022-04-28T20:44:36.369506+0000 mgr.y (mgr.24961) 1872 : cluster [DBG] pgmap v2234: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:38.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:37 smithi059 bash[61152]: cluster 2022-04-28T20:44:36.369506+0000 mgr.y (mgr.24961) 1872 : cluster [DBG] pgmap v2234: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:37 smithi059 bash[59745]: cluster 2022-04-28T20:44:36.369506+0000 mgr.y (mgr.24961) 1872 : cluster [DBG] pgmap v2234: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:39 smithi059 bash[61152]: cluster 2022-04-28T20:44:38.370184+0000 mgr.y (mgr.24961) 1873 : cluster [DBG] pgmap v2235: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:39 smithi059 bash[59745]: cluster 2022-04-28T20:44:38.370184+0000 mgr.y (mgr.24961) 1873 : cluster [DBG] pgmap v2235: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:39 smithi139 bash[43301]: cluster 2022-04-28T20:44:38.370184+0000 mgr.y (mgr.24961) 1873 : cluster [DBG] pgmap v2235: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:41 smithi059 bash[61152]: cluster 2022-04-28T20:44:40.371392+0000 mgr.y (mgr.24961) 1874 : cluster [DBG] pgmap v2236: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:41.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:41 smithi059 bash[59745]: cluster 2022-04-28T20:44:40.371392+0000 mgr.y (mgr.24961) 1874 : cluster [DBG] pgmap v2236: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:41 smithi139 bash[43301]: cluster 2022-04-28T20:44:40.371392+0000 mgr.y (mgr.24961) 1874 : cluster [DBG] pgmap v2236: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:44.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:43 smithi059 bash[61152]: cluster 2022-04-28T20:44:42.372338+0000 mgr.y (mgr.24961) 1875 : cluster [DBG] pgmap v2237: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:43.726Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:43 smithi059 bash[59745]: cluster 2022-04-28T20:44:42.372338+0000 mgr.y (mgr.24961) 1875 : cluster [DBG] pgmap v2237: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:43 smithi139 bash[43301]: cluster 2022-04-28T20:44:42.372338+0000 mgr.y (mgr.24961) 1875 : cluster [DBG] pgmap v2237: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:46.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:45 smithi059 bash[61152]: cluster 2022-04-28T20:44:44.373335+0000 mgr.y (mgr.24961) 1876 : cluster [DBG] pgmap v2238: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:45 smithi059 bash[59745]: cluster 2022-04-28T20:44:44.373335+0000 mgr.y (mgr.24961) 1876 : cluster [DBG] pgmap v2238: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:45 smithi139 bash[43301]: cluster 2022-04-28T20:44:44.373335+0000 mgr.y (mgr.24961) 1876 : cluster [DBG] pgmap v2238: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:47 smithi059 bash[61152]: cluster 2022-04-28T20:44:46.374477+0000 mgr.y (mgr.24961) 1877 : cluster [DBG] pgmap v2239: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:47 smithi059 bash[59745]: cluster 2022-04-28T20:44:46.374477+0000 mgr.y (mgr.24961) 1877 : cluster [DBG] pgmap v2239: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:47 smithi139 bash[43301]: cluster 2022-04-28T20:44:46.374477+0000 mgr.y (mgr.24961) 1877 : cluster [DBG] pgmap v2239: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:49 smithi059 bash[61152]: cluster 2022-04-28T20:44:48.375262+0000 mgr.y (mgr.24961) 1878 : cluster [DBG] pgmap v2240: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:49 smithi059 bash[59745]: cluster 2022-04-28T20:44:48.375262+0000 mgr.y (mgr.24961) 1878 : cluster [DBG] pgmap v2240: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:49 smithi139 bash[43301]: cluster 2022-04-28T20:44:48.375262+0000 mgr.y (mgr.24961) 1878 : cluster [DBG] pgmap v2240: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:52.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:51 smithi059 bash[61152]: cluster 2022-04-28T20:44:50.376472+0000 mgr.y (mgr.24961) 1879 : cluster [DBG] pgmap v2241: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:52.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:51.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:51 smithi059 bash[59745]: cluster 2022-04-28T20:44:50.376472+0000 mgr.y (mgr.24961) 1879 : cluster [DBG] pgmap v2241: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:51 smithi139 bash[43301]: cluster 2022-04-28T20:44:50.376472+0000 mgr.y (mgr.24961) 1879 : cluster [DBG] pgmap v2241: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:54.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:44:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:44:53.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:44:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:53 smithi059 bash[61152]: cluster 2022-04-28T20:44:52.377395+0000 mgr.y (mgr.24961) 1880 : cluster [DBG] pgmap v2242: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:53 smithi059 bash[61152]: audit 2022-04-28T20:44:53.904418+0000 mon.a (mon.0) 2197 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:44:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:53 smithi059 bash[59745]: cluster 2022-04-28T20:44:52.377395+0000 mgr.y (mgr.24961) 1880 : cluster [DBG] pgmap v2242: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:53 smithi059 bash[59745]: audit 2022-04-28T20:44:53.904418+0000 mon.a (mon.0) 2197 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:44:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:53 smithi139 bash[43301]: cluster 2022-04-28T20:44:52.377395+0000 mgr.y (mgr.24961) 1880 : cluster [DBG] pgmap v2242: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:53 smithi139 bash[43301]: audit 2022-04-28T20:44:53.904418+0000 mon.a (mon.0) 2197 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:44:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[59745]: audit 2022-04-28T20:44:53.907657+0000 mon.a (mon.0) 2198 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:44:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[59745]: audit 2022-04-28T20:44:53.916399+0000 mon.a (mon.0) 2199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:44:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[59745]: cephadm 2022-04-28T20:44:53.917732+0000 mgr.y (mgr.24961) 1881 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:44:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[59745]: audit 2022-04-28T20:44:54.217406+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:44:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[61152]: audit 2022-04-28T20:44:53.907657+0000 mon.a (mon.0) 2198 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:44:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[61152]: audit 2022-04-28T20:44:53.916399+0000 mon.a (mon.0) 2199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:44:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[61152]: cephadm 2022-04-28T20:44:53.917732+0000 mgr.y (mgr.24961) 1881 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:44:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:54 smithi059 bash[61152]: audit 2022-04-28T20:44:54.217406+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:44:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:54 smithi139 bash[43301]: audit 2022-04-28T20:44:53.907657+0000 mon.a (mon.0) 2198 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:44:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:54 smithi139 bash[43301]: audit 2022-04-28T20:44:53.916399+0000 mon.a (mon.0) 2199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:44:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:54 smithi139 bash[43301]: cephadm 2022-04-28T20:44:53.917732+0000 mgr.y (mgr.24961) 1881 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:44:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:54 smithi139 bash[43301]: audit 2022-04-28T20:44:54.217406+0000 mon.a (mon.0) 2200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:44:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:55 smithi059 bash[61152]: cluster 2022-04-28T20:44:54.378402+0000 mgr.y (mgr.24961) 1882 : cluster [DBG] pgmap v2243: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:55 smithi059 bash[59745]: cluster 2022-04-28T20:44:54.378402+0000 mgr.y (mgr.24961) 1882 : cluster [DBG] pgmap v2243: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:55 smithi139 bash[43301]: cluster 2022-04-28T20:44:54.378402+0000 mgr.y (mgr.24961) 1882 : cluster [DBG] pgmap v2243: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:57 smithi059 bash[61152]: cluster 2022-04-28T20:44:56.379607+0000 mgr.y (mgr.24961) 1883 : cluster [DBG] pgmap v2244: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:57 smithi059 bash[59745]: cluster 2022-04-28T20:44:56.379607+0000 mgr.y (mgr.24961) 1883 : cluster [DBG] pgmap v2244: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:44:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:57 smithi139 bash[43301]: cluster 2022-04-28T20:44:56.379607+0000 mgr.y (mgr.24961) 1883 : cluster [DBG] pgmap v2244: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:44:59 smithi059 bash[61152]: cluster 2022-04-28T20:44:58.380314+0000 mgr.y (mgr.24961) 1884 : cluster [DBG] pgmap v2245: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:44:59 smithi059 bash[59745]: cluster 2022-04-28T20:44:58.380314+0000 mgr.y (mgr.24961) 1884 : cluster [DBG] pgmap v2245: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:44:59 smithi139 bash[43301]: cluster 2022-04-28T20:44:58.380314+0000 mgr.y (mgr.24961) 1884 : cluster [DBG] pgmap v2245: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:01 smithi059 bash[61152]: cluster 2022-04-28T20:45:00.381507+0000 mgr.y (mgr.24961) 1885 : cluster [DBG] pgmap v2246: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:01.835Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:02.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:01 smithi059 bash[59745]: cluster 2022-04-28T20:45:00.381507+0000 mgr.y (mgr.24961) 1885 : cluster [DBG] pgmap v2246: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:01 smithi139 bash[43301]: cluster 2022-04-28T20:45:00.381507+0000 mgr.y (mgr.24961) 1885 : cluster [DBG] pgmap v2246: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:04.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:03 smithi059 bash[59745]: cluster 2022-04-28T20:45:02.382428+0000 mgr.y (mgr.24961) 1886 : cluster [DBG] pgmap v2247: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:03 smithi059 bash[61152]: cluster 2022-04-28T20:45:02.382428+0000 mgr.y (mgr.24961) 1886 : cluster [DBG] pgmap v2247: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:03.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:03 smithi139 bash[43301]: cluster 2022-04-28T20:45:02.382428+0000 mgr.y (mgr.24961) 1886 : cluster [DBG] pgmap v2247: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:06.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:05 smithi059 bash[61152]: cluster 2022-04-28T20:45:04.383397+0000 mgr.y (mgr.24961) 1887 : cluster [DBG] pgmap v2248: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:05 smithi059 bash[59745]: cluster 2022-04-28T20:45:04.383397+0000 mgr.y (mgr.24961) 1887 : cluster [DBG] pgmap v2248: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:05 smithi139 bash[43301]: cluster 2022-04-28T20:45:04.383397+0000 mgr.y (mgr.24961) 1887 : cluster [DBG] pgmap v2248: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:07 smithi059 bash[61152]: cluster 2022-04-28T20:45:06.384481+0000 mgr.y (mgr.24961) 1888 : cluster [DBG] pgmap v2249: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:07 smithi059 bash[59745]: cluster 2022-04-28T20:45:06.384481+0000 mgr.y (mgr.24961) 1888 : cluster [DBG] pgmap v2249: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:07 smithi139 bash[43301]: cluster 2022-04-28T20:45:06.384481+0000 mgr.y (mgr.24961) 1888 : cluster [DBG] pgmap v2249: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:09 smithi059 bash[61152]: cluster 2022-04-28T20:45:08.385255+0000 mgr.y (mgr.24961) 1889 : cluster [DBG] pgmap v2250: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:09 smithi059 bash[59745]: cluster 2022-04-28T20:45:08.385255+0000 mgr.y (mgr.24961) 1889 : cluster [DBG] pgmap v2250: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:09 smithi139 bash[43301]: cluster 2022-04-28T20:45:08.385255+0000 mgr.y (mgr.24961) 1889 : cluster [DBG] pgmap v2250: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:12.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:11 smithi059 bash[61152]: cluster 2022-04-28T20:45:10.386480+0000 mgr.y (mgr.24961) 1890 : cluster [DBG] pgmap v2251: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:12.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:11.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:11 smithi059 bash[59745]: cluster 2022-04-28T20:45:10.386480+0000 mgr.y (mgr.24961) 1890 : cluster [DBG] pgmap v2251: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:11 smithi139 bash[43301]: cluster 2022-04-28T20:45:10.386480+0000 mgr.y (mgr.24961) 1890 : cluster [DBG] pgmap v2251: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:14.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:13 smithi059 bash[59745]: cluster 2022-04-28T20:45:12.387445+0000 mgr.y (mgr.24961) 1891 : cluster [DBG] pgmap v2252: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:13 smithi059 bash[61152]: cluster 2022-04-28T20:45:12.387445+0000 mgr.y (mgr.24961) 1891 : cluster [DBG] pgmap v2252: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:13.728Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:13 smithi139 bash[43301]: cluster 2022-04-28T20:45:12.387445+0000 mgr.y (mgr.24961) 1891 : cluster [DBG] pgmap v2252: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:15 smithi059 bash[61152]: cluster 2022-04-28T20:45:14.388486+0000 mgr.y (mgr.24961) 1892 : cluster [DBG] pgmap v2253: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:16.363 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:15 smithi059 bash[59745]: cluster 2022-04-28T20:45:14.388486+0000 mgr.y (mgr.24961) 1892 : cluster [DBG] pgmap v2253: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:15 smithi139 bash[43301]: cluster 2022-04-28T20:45:14.388486+0000 mgr.y (mgr.24961) 1892 : cluster [DBG] pgmap v2253: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: cluster 2022-04-28T20:45:16.389695+0000 mgr.y (mgr.24961) 1893 : cluster [DBG] pgmap v2254: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.779479+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:45:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.780039+0000 mon.a (mon.0) 2202 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:45:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.791445+0000 mon.a (mon.0) 2203 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.792048+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.792554+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]: dispatch 2022-04-28T20:45:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.793071+0000 mon.a (mon.0) 2206 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[61152]: audit 2022-04-28T20:45:17.793579+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2022-04-28T20:45:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: cluster 2022-04-28T20:45:16.389695+0000 mgr.y (mgr.24961) 1893 : cluster [DBG] pgmap v2254: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.779479+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:45:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.780039+0000 mon.a (mon.0) 2202 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:45:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.791445+0000 mon.a (mon.0) 2203 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.792048+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.792554+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]: dispatch 2022-04-28T20:45:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.793071+0000 mon.a (mon.0) 2206 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:17 smithi059 bash[59745]: audit 2022-04-28T20:45:17.793579+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2022-04-28T20:45:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: cluster 2022-04-28T20:45:16.389695+0000 mgr.y (mgr.24961) 1893 : cluster [DBG] pgmap v2254: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.779479+0000 mon.a (mon.0) 2201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:45:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.780039+0000 mon.a (mon.0) 2202 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:45:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.791445+0000 mon.a (mon.0) 2203 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.792048+0000 mon.a (mon.0) 2204 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.792554+0000 mon.a (mon.0) 2205 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]: dispatch 2022-04-28T20:45:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.793071+0000 mon.a (mon.0) 2206 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:45:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:17 smithi139 bash[43301]: audit 2022-04-28T20:45:17.793579+0000 mon.a (mon.0) 2207 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 2]}]: dispatch 2022-04-28T20:45:20.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:19 smithi059 bash[59745]: cluster 2022-04-28T20:45:18.390399+0000 mgr.y (mgr.24961) 1894 : cluster [DBG] pgmap v2255: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:19 smithi059 bash[61152]: cluster 2022-04-28T20:45:18.390399+0000 mgr.y (mgr.24961) 1894 : cluster [DBG] pgmap v2255: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:20.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:19 smithi139 bash[43301]: cluster 2022-04-28T20:45:18.390399+0000 mgr.y (mgr.24961) 1894 : cluster [DBG] pgmap v2255: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:22.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:21 smithi059 bash[61152]: cluster 2022-04-28T20:45:20.391683+0000 mgr.y (mgr.24961) 1895 : cluster [DBG] pgmap v2256: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:22.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:21.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:21 smithi059 bash[59745]: cluster 2022-04-28T20:45:20.391683+0000 mgr.y (mgr.24961) 1895 : cluster [DBG] pgmap v2256: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:22.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:21 smithi139 bash[43301]: cluster 2022-04-28T20:45:20.391683+0000 mgr.y (mgr.24961) 1895 : cluster [DBG] pgmap v2256: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:24.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:23 smithi059 bash[61152]: cluster 2022-04-28T20:45:22.392689+0000 mgr.y (mgr.24961) 1896 : cluster [DBG] pgmap v2257: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:23.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:23 smithi059 bash[59745]: cluster 2022-04-28T20:45:22.392689+0000 mgr.y (mgr.24961) 1896 : cluster [DBG] pgmap v2257: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:24.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:23 smithi139 bash[43301]: cluster 2022-04-28T20:45:22.392689+0000 mgr.y (mgr.24961) 1896 : cluster [DBG] pgmap v2257: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:26.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:25 smithi139 bash[43301]: cluster 2022-04-28T20:45:24.393711+0000 mgr.y (mgr.24961) 1897 : cluster [DBG] pgmap v2258: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:25 smithi059 bash[61152]: cluster 2022-04-28T20:45:24.393711+0000 mgr.y (mgr.24961) 1897 : cluster [DBG] pgmap v2258: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:25 smithi059 bash[59745]: cluster 2022-04-28T20:45:24.393711+0000 mgr.y (mgr.24961) 1897 : cluster [DBG] pgmap v2258: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:27 smithi139 bash[43301]: cluster 2022-04-28T20:45:26.394829+0000 mgr.y (mgr.24961) 1898 : cluster [DBG] pgmap v2259: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:28.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:27 smithi059 bash[61152]: cluster 2022-04-28T20:45:26.394829+0000 mgr.y (mgr.24961) 1898 : cluster [DBG] pgmap v2259: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:27 smithi059 bash[59745]: cluster 2022-04-28T20:45:26.394829+0000 mgr.y (mgr.24961) 1898 : cluster [DBG] pgmap v2259: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:29 smithi139 bash[43301]: cluster 2022-04-28T20:45:28.395569+0000 mgr.y (mgr.24961) 1899 : cluster [DBG] pgmap v2260: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:30.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:29 smithi059 bash[61152]: cluster 2022-04-28T20:45:28.395569+0000 mgr.y (mgr.24961) 1899 : cluster [DBG] pgmap v2260: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:29 smithi059 bash[59745]: cluster 2022-04-28T20:45:28.395569+0000 mgr.y (mgr.24961) 1899 : cluster [DBG] pgmap v2260: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:31 smithi059 bash[61152]: cluster 2022-04-28T20:45:30.396733+0000 mgr.y (mgr.24961) 1900 : cluster [DBG] pgmap v2261: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:32.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:31.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:32.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:31 smithi059 bash[59745]: cluster 2022-04-28T20:45:30.396733+0000 mgr.y (mgr.24961) 1900 : cluster [DBG] pgmap v2261: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:31 smithi139 bash[43301]: cluster 2022-04-28T20:45:30.396733+0000 mgr.y (mgr.24961) 1900 : cluster [DBG] pgmap v2261: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:34.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:33 smithi059 bash[61152]: cluster 2022-04-28T20:45:32.397582+0000 mgr.y (mgr.24961) 1901 : cluster [DBG] pgmap v2262: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:33.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:33 smithi059 bash[59745]: cluster 2022-04-28T20:45:32.397582+0000 mgr.y (mgr.24961) 1901 : cluster [DBG] pgmap v2262: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:33 smithi139 bash[43301]: cluster 2022-04-28T20:45:32.397582+0000 mgr.y (mgr.24961) 1901 : cluster [DBG] pgmap v2262: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:35 smithi059 bash[61152]: cluster 2022-04-28T20:45:34.398503+0000 mgr.y (mgr.24961) 1902 : cluster [DBG] pgmap v2263: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:36.400 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:35 smithi059 bash[59745]: cluster 2022-04-28T20:45:34.398503+0000 mgr.y (mgr.24961) 1902 : cluster [DBG] pgmap v2263: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:36.401 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:35 smithi139 bash[43301]: cluster 2022-04-28T20:45:34.398503+0000 mgr.y (mgr.24961) 1902 : cluster [DBG] pgmap v2263: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:37 smithi059 bash[61152]: cluster 2022-04-28T20:45:36.399654+0000 mgr.y (mgr.24961) 1903 : cluster [DBG] pgmap v2264: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:37 smithi059 bash[59745]: cluster 2022-04-28T20:45:36.399654+0000 mgr.y (mgr.24961) 1903 : cluster [DBG] pgmap v2264: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:37 smithi139 bash[43301]: cluster 2022-04-28T20:45:36.399654+0000 mgr.y (mgr.24961) 1903 : cluster [DBG] pgmap v2264: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:39 smithi059 bash[61152]: cluster 2022-04-28T20:45:38.400316+0000 mgr.y (mgr.24961) 1904 : cluster [DBG] pgmap v2265: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:39 smithi059 bash[59745]: cluster 2022-04-28T20:45:38.400316+0000 mgr.y (mgr.24961) 1904 : cluster [DBG] pgmap v2265: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:39 smithi139 bash[43301]: cluster 2022-04-28T20:45:38.400316+0000 mgr.y (mgr.24961) 1904 : cluster [DBG] pgmap v2265: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:42.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:41 smithi059 bash[61152]: cluster 2022-04-28T20:45:40.401510+0000 mgr.y (mgr.24961) 1905 : cluster [DBG] pgmap v2266: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:41.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:41 smithi059 bash[59745]: cluster 2022-04-28T20:45:40.401510+0000 mgr.y (mgr.24961) 1905 : cluster [DBG] pgmap v2266: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:41 smithi139 bash[43301]: cluster 2022-04-28T20:45:40.401510+0000 mgr.y (mgr.24961) 1905 : cluster [DBG] pgmap v2266: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:44.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:43 smithi059 bash[61152]: cluster 2022-04-28T20:45:42.402433+0000 mgr.y (mgr.24961) 1906 : cluster [DBG] pgmap v2267: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:43 smithi059 bash[59745]: cluster 2022-04-28T20:45:42.402433+0000 mgr.y (mgr.24961) 1906 : cluster [DBG] pgmap v2267: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:43.731Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:43 smithi139 bash[43301]: cluster 2022-04-28T20:45:42.402433+0000 mgr.y (mgr.24961) 1906 : cluster [DBG] pgmap v2267: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:45 smithi059 bash[59745]: cluster 2022-04-28T20:45:44.403405+0000 mgr.y (mgr.24961) 1907 : cluster [DBG] pgmap v2268: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:46.363 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:45 smithi059 bash[61152]: cluster 2022-04-28T20:45:44.403405+0000 mgr.y (mgr.24961) 1907 : cluster [DBG] pgmap v2268: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:45 smithi139 bash[43301]: cluster 2022-04-28T20:45:44.403405+0000 mgr.y (mgr.24961) 1907 : cluster [DBG] pgmap v2268: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:48.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:47 smithi059 bash[61152]: cluster 2022-04-28T20:45:46.404558+0000 mgr.y (mgr.24961) 1908 : cluster [DBG] pgmap v2269: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:47 smithi059 bash[59745]: cluster 2022-04-28T20:45:46.404558+0000 mgr.y (mgr.24961) 1908 : cluster [DBG] pgmap v2269: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:47 smithi139 bash[43301]: cluster 2022-04-28T20:45:46.404558+0000 mgr.y (mgr.24961) 1908 : cluster [DBG] pgmap v2269: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:50.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:49 smithi059 bash[61152]: cluster 2022-04-28T20:45:48.405236+0000 mgr.y (mgr.24961) 1909 : cluster [DBG] pgmap v2270: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:49 smithi059 bash[59745]: cluster 2022-04-28T20:45:48.405236+0000 mgr.y (mgr.24961) 1909 : cluster [DBG] pgmap v2270: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:49 smithi139 bash[43301]: cluster 2022-04-28T20:45:48.405236+0000 mgr.y (mgr.24961) 1909 : cluster [DBG] pgmap v2270: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:52.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:51.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:51 smithi059 bash[61152]: cluster 2022-04-28T20:45:50.406471+0000 mgr.y (mgr.24961) 1910 : cluster [DBG] pgmap v2271: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:52.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:51 smithi059 bash[59745]: cluster 2022-04-28T20:45:50.406471+0000 mgr.y (mgr.24961) 1910 : cluster [DBG] pgmap v2271: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:51 smithi139 bash[43301]: cluster 2022-04-28T20:45:50.406471+0000 mgr.y (mgr.24961) 1910 : cluster [DBG] pgmap v2271: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:54.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:53 smithi059 bash[61152]: cluster 2022-04-28T20:45:52.407309+0000 mgr.y (mgr.24961) 1911 : cluster [DBG] pgmap v2272: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:53 smithi059 bash[59745]: cluster 2022-04-28T20:45:52.407309+0000 mgr.y (mgr.24961) 1911 : cluster [DBG] pgmap v2272: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:45:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:45:53.729Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:45:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:53 smithi139 bash[43301]: cluster 2022-04-28T20:45:52.407309+0000 mgr.y (mgr.24961) 1911 : cluster [DBG] pgmap v2272: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[61152]: audit 2022-04-28T20:45:54.224911+0000 mon.a (mon.0) 2208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:45:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[61152]: audit 2022-04-28T20:45:54.227629+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:45:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[61152]: audit 2022-04-28T20:45:54.229307+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:45:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[61152]: audit 2022-04-28T20:45:54.529786+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:45:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[59745]: audit 2022-04-28T20:45:54.224911+0000 mon.a (mon.0) 2208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:45:55.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[59745]: audit 2022-04-28T20:45:54.227629+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:45:55.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[59745]: audit 2022-04-28T20:45:54.229307+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:45:55.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:54 smithi059 bash[59745]: audit 2022-04-28T20:45:54.529786+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:45:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:54 smithi139 bash[43301]: audit 2022-04-28T20:45:54.224911+0000 mon.a (mon.0) 2208 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:45:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:54 smithi139 bash[43301]: audit 2022-04-28T20:45:54.227629+0000 mon.a (mon.0) 2209 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:45:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:54 smithi139 bash[43301]: audit 2022-04-28T20:45:54.229307+0000 mon.a (mon.0) 2210 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:45:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:54 smithi139 bash[43301]: audit 2022-04-28T20:45:54.529786+0000 mon.a (mon.0) 2211 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:45:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:55 smithi059 bash[61152]: cephadm 2022-04-28T20:45:54.230886+0000 mgr.y (mgr.24961) 1912 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:45:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:55 smithi059 bash[61152]: cluster 2022-04-28T20:45:54.408240+0000 mgr.y (mgr.24961) 1913 : cluster [DBG] pgmap v2273: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:55 smithi059 bash[59745]: cephadm 2022-04-28T20:45:54.230886+0000 mgr.y (mgr.24961) 1912 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:45:56.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:55 smithi059 bash[59745]: cluster 2022-04-28T20:45:54.408240+0000 mgr.y (mgr.24961) 1913 : cluster [DBG] pgmap v2273: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:55 smithi139 bash[43301]: cephadm 2022-04-28T20:45:54.230886+0000 mgr.y (mgr.24961) 1912 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:45:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:55 smithi139 bash[43301]: cluster 2022-04-28T20:45:54.408240+0000 mgr.y (mgr.24961) 1913 : cluster [DBG] pgmap v2273: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:58.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:57 smithi059 bash[59745]: cluster 2022-04-28T20:45:56.409393+0000 mgr.y (mgr.24961) 1914 : cluster [DBG] pgmap v2274: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:57 smithi059 bash[61152]: cluster 2022-04-28T20:45:56.409393+0000 mgr.y (mgr.24961) 1914 : cluster [DBG] pgmap v2274: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:45:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:57 smithi139 bash[43301]: cluster 2022-04-28T20:45:56.409393+0000 mgr.y (mgr.24961) 1914 : cluster [DBG] pgmap v2274: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:00.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:45:59 smithi059 bash[61152]: cluster 2022-04-28T20:45:58.410033+0000 mgr.y (mgr.24961) 1915 : cluster [DBG] pgmap v2275: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:45:59 smithi059 bash[59745]: cluster 2022-04-28T20:45:58.410033+0000 mgr.y (mgr.24961) 1915 : cluster [DBG] pgmap v2275: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:45:59 smithi139 bash[43301]: cluster 2022-04-28T20:45:58.410033+0000 mgr.y (mgr.24961) 1915 : cluster [DBG] pgmap v2275: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:01 smithi059 bash[61152]: cluster 2022-04-28T20:46:00.411160+0000 mgr.y (mgr.24961) 1916 : cluster [DBG] pgmap v2276: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:02.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:01 smithi059 bash[59745]: cluster 2022-04-28T20:46:00.411160+0000 mgr.y (mgr.24961) 1916 : cluster [DBG] pgmap v2276: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:01.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:01 smithi139 bash[43301]: cluster 2022-04-28T20:46:00.411160+0000 mgr.y (mgr.24961) 1916 : cluster [DBG] pgmap v2276: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:03 smithi059 bash[61152]: cluster 2022-04-28T20:46:02.412097+0000 mgr.y (mgr.24961) 1917 : cluster [DBG] pgmap v2277: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:03 smithi059 bash[59745]: cluster 2022-04-28T20:46:02.412097+0000 mgr.y (mgr.24961) 1917 : cluster [DBG] pgmap v2277: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:03.733Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:03 smithi139 bash[43301]: cluster 2022-04-28T20:46:02.412097+0000 mgr.y (mgr.24961) 1917 : cluster [DBG] pgmap v2277: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:06.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:05 smithi059 bash[61152]: cluster 2022-04-28T20:46:04.413076+0000 mgr.y (mgr.24961) 1918 : cluster [DBG] pgmap v2278: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:05 smithi059 bash[59745]: cluster 2022-04-28T20:46:04.413076+0000 mgr.y (mgr.24961) 1918 : cluster [DBG] pgmap v2278: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:05 smithi139 bash[43301]: cluster 2022-04-28T20:46:04.413076+0000 mgr.y (mgr.24961) 1918 : cluster [DBG] pgmap v2278: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:07 smithi059 bash[61152]: cluster 2022-04-28T20:46:06.414200+0000 mgr.y (mgr.24961) 1919 : cluster [DBG] pgmap v2279: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:07 smithi059 bash[59745]: cluster 2022-04-28T20:46:06.414200+0000 mgr.y (mgr.24961) 1919 : cluster [DBG] pgmap v2279: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:07 smithi139 bash[43301]: cluster 2022-04-28T20:46:06.414200+0000 mgr.y (mgr.24961) 1919 : cluster [DBG] pgmap v2279: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:09 smithi139 bash[43301]: cluster 2022-04-28T20:46:08.414910+0000 mgr.y (mgr.24961) 1920 : cluster [DBG] pgmap v2280: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:10.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:09 smithi059 bash[61152]: cluster 2022-04-28T20:46:08.414910+0000 mgr.y (mgr.24961) 1920 : cluster [DBG] pgmap v2280: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:09 smithi059 bash[59745]: cluster 2022-04-28T20:46:08.414910+0000 mgr.y (mgr.24961) 1920 : cluster [DBG] pgmap v2280: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:11 smithi059 bash[61152]: cluster 2022-04-28T20:46:10.416125+0000 mgr.y (mgr.24961) 1921 : cluster [DBG] pgmap v2281: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:12.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:11.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:11 smithi059 bash[59745]: cluster 2022-04-28T20:46:10.416125+0000 mgr.y (mgr.24961) 1921 : cluster [DBG] pgmap v2281: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:12.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:11 smithi139 bash[43301]: cluster 2022-04-28T20:46:10.416125+0000 mgr.y (mgr.24961) 1921 : cluster [DBG] pgmap v2281: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:14.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:13 smithi059 bash[61152]: cluster 2022-04-28T20:46:12.417051+0000 mgr.y (mgr.24961) 1922 : cluster [DBG] pgmap v2282: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:13.732Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:13 smithi059 bash[59745]: cluster 2022-04-28T20:46:12.417051+0000 mgr.y (mgr.24961) 1922 : cluster [DBG] pgmap v2282: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:13 smithi139 bash[43301]: cluster 2022-04-28T20:46:12.417051+0000 mgr.y (mgr.24961) 1922 : cluster [DBG] pgmap v2282: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:15 smithi139 bash[43301]: cluster 2022-04-28T20:46:14.418064+0000 mgr.y (mgr.24961) 1923 : cluster [DBG] pgmap v2283: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:15 smithi059 bash[61152]: cluster 2022-04-28T20:46:14.418064+0000 mgr.y (mgr.24961) 1923 : cluster [DBG] pgmap v2283: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:15 smithi059 bash[59745]: cluster 2022-04-28T20:46:14.418064+0000 mgr.y (mgr.24961) 1923 : cluster [DBG] pgmap v2283: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:17 smithi139 bash[43301]: cluster 2022-04-28T20:46:16.419161+0000 mgr.y (mgr.24961) 1924 : cluster [DBG] pgmap v2284: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:17 smithi139 bash[43301]: audit 2022-04-28T20:46:17.794277+0000 mon.a (mon.0) 2212 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:46:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:17 smithi139 bash[43301]: audit 2022-04-28T20:46:17.794793+0000 mon.a (mon.0) 2213 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:46:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:17 smithi139 bash[43301]: audit 2022-04-28T20:46:17.805056+0000 mon.a (mon.0) 2214 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:46:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:17 smithi139 bash[43301]: audit 2022-04-28T20:46:17.805773+0000 mon.a (mon.0) 2215 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:46:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:17 smithi139 bash[43301]: audit 2022-04-28T20:46:17.806334+0000 mon.a (mon.0) 2216 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:46:18.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[59745]: cluster 2022-04-28T20:46:16.419161+0000 mgr.y (mgr.24961) 1924 : cluster [DBG] pgmap v2284: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[59745]: audit 2022-04-28T20:46:17.794277+0000 mon.a (mon.0) 2212 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:46:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[59745]: audit 2022-04-28T20:46:17.794793+0000 mon.a (mon.0) 2213 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:46:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[59745]: audit 2022-04-28T20:46:17.805056+0000 mon.a (mon.0) 2214 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:46:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[59745]: audit 2022-04-28T20:46:17.805773+0000 mon.a (mon.0) 2215 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:46:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[59745]: audit 2022-04-28T20:46:17.806334+0000 mon.a (mon.0) 2216 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:46:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[61152]: cluster 2022-04-28T20:46:16.419161+0000 mgr.y (mgr.24961) 1924 : cluster [DBG] pgmap v2284: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[61152]: audit 2022-04-28T20:46:17.794277+0000 mon.a (mon.0) 2212 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:46:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[61152]: audit 2022-04-28T20:46:17.794793+0000 mon.a (mon.0) 2213 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:46:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[61152]: audit 2022-04-28T20:46:17.805056+0000 mon.a (mon.0) 2214 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:46:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[61152]: audit 2022-04-28T20:46:17.805773+0000 mon.a (mon.0) 2215 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:46:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:17 smithi059 bash[61152]: audit 2022-04-28T20:46:17.806334+0000 mon.a (mon.0) 2216 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:46:20.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:19 smithi059 bash[61152]: cluster 2022-04-28T20:46:18.419891+0000 mgr.y (mgr.24961) 1925 : cluster [DBG] pgmap v2285: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:19 smithi059 bash[59745]: cluster 2022-04-28T20:46:18.419891+0000 mgr.y (mgr.24961) 1925 : cluster [DBG] pgmap v2285: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:19 smithi139 bash[43301]: cluster 2022-04-28T20:46:18.419891+0000 mgr.y (mgr.24961) 1925 : cluster [DBG] pgmap v2285: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:22.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:21 smithi059 bash[61152]: cluster 2022-04-28T20:46:20.421039+0000 mgr.y (mgr.24961) 1926 : cluster [DBG] pgmap v2286: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:21.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:21 smithi059 bash[59745]: cluster 2022-04-28T20:46:20.421039+0000 mgr.y (mgr.24961) 1926 : cluster [DBG] pgmap v2286: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:21 smithi139 bash[43301]: cluster 2022-04-28T20:46:20.421039+0000 mgr.y (mgr.24961) 1926 : cluster [DBG] pgmap v2286: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:24.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:23 smithi059 bash[61152]: cluster 2022-04-28T20:46:22.421958+0000 mgr.y (mgr.24961) 1927 : cluster [DBG] pgmap v2287: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:23.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:23 smithi059 bash[59745]: cluster 2022-04-28T20:46:22.421958+0000 mgr.y (mgr.24961) 1927 : cluster [DBG] pgmap v2287: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:23 smithi139 bash[43301]: cluster 2022-04-28T20:46:22.421958+0000 mgr.y (mgr.24961) 1927 : cluster [DBG] pgmap v2287: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:25 smithi059 bash[61152]: cluster 2022-04-28T20:46:24.422935+0000 mgr.y (mgr.24961) 1928 : cluster [DBG] pgmap v2288: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:25 smithi059 bash[59745]: cluster 2022-04-28T20:46:24.422935+0000 mgr.y (mgr.24961) 1928 : cluster [DBG] pgmap v2288: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:25 smithi139 bash[43301]: cluster 2022-04-28T20:46:24.422935+0000 mgr.y (mgr.24961) 1928 : cluster [DBG] pgmap v2288: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:27 smithi059 bash[59745]: cluster 2022-04-28T20:46:26.424062+0000 mgr.y (mgr.24961) 1929 : cluster [DBG] pgmap v2289: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:27 smithi059 bash[61152]: cluster 2022-04-28T20:46:26.424062+0000 mgr.y (mgr.24961) 1929 : cluster [DBG] pgmap v2289: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:27 smithi139 bash[43301]: cluster 2022-04-28T20:46:26.424062+0000 mgr.y (mgr.24961) 1929 : cluster [DBG] pgmap v2289: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:30.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:29 smithi059 bash[61152]: cluster 2022-04-28T20:46:28.424748+0000 mgr.y (mgr.24961) 1930 : cluster [DBG] pgmap v2290: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:29 smithi059 bash[59745]: cluster 2022-04-28T20:46:28.424748+0000 mgr.y (mgr.24961) 1930 : cluster [DBG] pgmap v2290: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:29 smithi139 bash[43301]: cluster 2022-04-28T20:46:28.424748+0000 mgr.y (mgr.24961) 1930 : cluster [DBG] pgmap v2290: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:32.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:31 smithi059 bash[61152]: cluster 2022-04-28T20:46:30.425917+0000 mgr.y (mgr.24961) 1931 : cluster [DBG] pgmap v2291: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:31.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:31 smithi059 bash[59745]: cluster 2022-04-28T20:46:30.425917+0000 mgr.y (mgr.24961) 1931 : cluster [DBG] pgmap v2291: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:31 smithi139 bash[43301]: cluster 2022-04-28T20:46:30.425917+0000 mgr.y (mgr.24961) 1931 : cluster [DBG] pgmap v2291: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:33 smithi059 bash[61152]: cluster 2022-04-28T20:46:32.426842+0000 mgr.y (mgr.24961) 1932 : cluster [DBG] pgmap v2292: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:33.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:33 smithi059 bash[59745]: cluster 2022-04-28T20:46:32.426842+0000 mgr.y (mgr.24961) 1932 : cluster [DBG] pgmap v2292: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:33 smithi139 bash[43301]: cluster 2022-04-28T20:46:32.426842+0000 mgr.y (mgr.24961) 1932 : cluster [DBG] pgmap v2292: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:35 smithi059 bash[61152]: cluster 2022-04-28T20:46:34.427928+0000 mgr.y (mgr.24961) 1933 : cluster [DBG] pgmap v2293: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:36.368 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:35 smithi059 bash[59745]: cluster 2022-04-28T20:46:34.427928+0000 mgr.y (mgr.24961) 1933 : cluster [DBG] pgmap v2293: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:36.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:35 smithi139 bash[43301]: cluster 2022-04-28T20:46:34.427928+0000 mgr.y (mgr.24961) 1933 : cluster [DBG] pgmap v2293: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:38.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:37 smithi059 bash[61152]: cluster 2022-04-28T20:46:36.429012+0000 mgr.y (mgr.24961) 1934 : cluster [DBG] pgmap v2294: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:37 smithi059 bash[59745]: cluster 2022-04-28T20:46:36.429012+0000 mgr.y (mgr.24961) 1934 : cluster [DBG] pgmap v2294: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:37 smithi139 bash[43301]: cluster 2022-04-28T20:46:36.429012+0000 mgr.y (mgr.24961) 1934 : cluster [DBG] pgmap v2294: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:39 smithi059 bash[61152]: cluster 2022-04-28T20:46:38.429716+0000 mgr.y (mgr.24961) 1935 : cluster [DBG] pgmap v2295: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:39 smithi059 bash[59745]: cluster 2022-04-28T20:46:38.429716+0000 mgr.y (mgr.24961) 1935 : cluster [DBG] pgmap v2295: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:39 smithi139 bash[43301]: cluster 2022-04-28T20:46:38.429716+0000 mgr.y (mgr.24961) 1935 : cluster [DBG] pgmap v2295: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:42.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:41 smithi059 bash[61152]: cluster 2022-04-28T20:46:40.430877+0000 mgr.y (mgr.24961) 1936 : cluster [DBG] pgmap v2296: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:41.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:41 smithi059 bash[59745]: cluster 2022-04-28T20:46:40.430877+0000 mgr.y (mgr.24961) 1936 : cluster [DBG] pgmap v2296: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:41 smithi139 bash[43301]: cluster 2022-04-28T20:46:40.430877+0000 mgr.y (mgr.24961) 1936 : cluster [DBG] pgmap v2296: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:44.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:43 smithi059 bash[61152]: cluster 2022-04-28T20:46:42.431930+0000 mgr.y (mgr.24961) 1937 : cluster [DBG] pgmap v2297: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:43.734Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:43 smithi059 bash[59745]: cluster 2022-04-28T20:46:42.431930+0000 mgr.y (mgr.24961) 1937 : cluster [DBG] pgmap v2297: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:43 smithi139 bash[43301]: cluster 2022-04-28T20:46:42.431930+0000 mgr.y (mgr.24961) 1937 : cluster [DBG] pgmap v2297: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:45 smithi059 bash[59745]: cluster 2022-04-28T20:46:44.432952+0000 mgr.y (mgr.24961) 1938 : cluster [DBG] pgmap v2298: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:45 smithi059 bash[61152]: cluster 2022-04-28T20:46:44.432952+0000 mgr.y (mgr.24961) 1938 : cluster [DBG] pgmap v2298: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:45 smithi139 bash[43301]: cluster 2022-04-28T20:46:44.432952+0000 mgr.y (mgr.24961) 1938 : cluster [DBG] pgmap v2298: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:47 smithi059 bash[61152]: cluster 2022-04-28T20:46:46.434170+0000 mgr.y (mgr.24961) 1939 : cluster [DBG] pgmap v2299: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:47 smithi059 bash[59745]: cluster 2022-04-28T20:46:46.434170+0000 mgr.y (mgr.24961) 1939 : cluster [DBG] pgmap v2299: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:47 smithi139 bash[43301]: cluster 2022-04-28T20:46:46.434170+0000 mgr.y (mgr.24961) 1939 : cluster [DBG] pgmap v2299: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:50.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:49 smithi059 bash[61152]: cluster 2022-04-28T20:46:48.434911+0000 mgr.y (mgr.24961) 1940 : cluster [DBG] pgmap v2300: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:49 smithi059 bash[59745]: cluster 2022-04-28T20:46:48.434911+0000 mgr.y (mgr.24961) 1940 : cluster [DBG] pgmap v2300: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:49 smithi139 bash[43301]: cluster 2022-04-28T20:46:48.434911+0000 mgr.y (mgr.24961) 1940 : cluster [DBG] pgmap v2300: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:52.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:51 smithi059 bash[61152]: cluster 2022-04-28T20:46:50.436250+0000 mgr.y (mgr.24961) 1941 : cluster [DBG] pgmap v2301: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:51.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:51 smithi059 bash[59745]: cluster 2022-04-28T20:46:50.436250+0000 mgr.y (mgr.24961) 1941 : cluster [DBG] pgmap v2301: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:51 smithi139 bash[43301]: cluster 2022-04-28T20:46:50.436250+0000 mgr.y (mgr.24961) 1941 : cluster [DBG] pgmap v2301: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:53 smithi059 bash[61152]: cluster 2022-04-28T20:46:52.437323+0000 mgr.y (mgr.24961) 1942 : cluster [DBG] pgmap v2302: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:46:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:46:53.737Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:46:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:53 smithi059 bash[59745]: cluster 2022-04-28T20:46:52.437323+0000 mgr.y (mgr.24961) 1942 : cluster [DBG] pgmap v2302: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:53 smithi139 bash[43301]: cluster 2022-04-28T20:46:52.437323+0000 mgr.y (mgr.24961) 1942 : cluster [DBG] pgmap v2302: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[59745]: audit 2022-04-28T20:46:54.535880+0000 mon.a (mon.0) 2217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:46:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[59745]: audit 2022-04-28T20:46:54.538780+0000 mon.a (mon.0) 2218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:46:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[59745]: audit 2022-04-28T20:46:54.540740+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:46:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[59745]: audit 2022-04-28T20:46:54.544801+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:46:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[59745]: audit 2022-04-28T20:46:54.545320+0000 mon.a (mon.0) 2221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:46:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[61152]: audit 2022-04-28T20:46:54.535880+0000 mon.a (mon.0) 2217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:46:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[61152]: audit 2022-04-28T20:46:54.538780+0000 mon.a (mon.0) 2218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:46:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[61152]: audit 2022-04-28T20:46:54.540740+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:46:55.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[61152]: audit 2022-04-28T20:46:54.544801+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:46:55.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:54 smithi059 bash[61152]: audit 2022-04-28T20:46:54.545320+0000 mon.a (mon.0) 2221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:46:55.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:54 smithi139 bash[43301]: audit 2022-04-28T20:46:54.535880+0000 mon.a (mon.0) 2217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:46:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:54 smithi139 bash[43301]: audit 2022-04-28T20:46:54.538780+0000 mon.a (mon.0) 2218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:46:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:54 smithi139 bash[43301]: audit 2022-04-28T20:46:54.540740+0000 mon.a (mon.0) 2219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:46:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:54 smithi139 bash[43301]: audit 2022-04-28T20:46:54.544801+0000 mon.a (mon.0) 2220 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:46:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:54 smithi139 bash[43301]: audit 2022-04-28T20:46:54.545320+0000 mon.a (mon.0) 2221 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:46:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:55 smithi059 bash[61152]: cluster 2022-04-28T20:46:54.438286+0000 mgr.y (mgr.24961) 1943 : cluster [DBG] pgmap v2303: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:55 smithi059 bash[61152]: cephadm 2022-04-28T20:46:54.542451+0000 mgr.y (mgr.24961) 1944 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:46:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:55 smithi059 bash[59745]: cluster 2022-04-28T20:46:54.438286+0000 mgr.y (mgr.24961) 1943 : cluster [DBG] pgmap v2303: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:56.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:55 smithi059 bash[59745]: cephadm 2022-04-28T20:46:54.542451+0000 mgr.y (mgr.24961) 1944 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:46:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:55 smithi139 bash[43301]: cluster 2022-04-28T20:46:54.438286+0000 mgr.y (mgr.24961) 1943 : cluster [DBG] pgmap v2303: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:55 smithi139 bash[43301]: cephadm 2022-04-28T20:46:54.542451+0000 mgr.y (mgr.24961) 1944 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:46:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:57 smithi059 bash[61152]: cluster 2022-04-28T20:46:56.439379+0000 mgr.y (mgr.24961) 1945 : cluster [DBG] pgmap v2304: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:57 smithi059 bash[59745]: cluster 2022-04-28T20:46:56.439379+0000 mgr.y (mgr.24961) 1945 : cluster [DBG] pgmap v2304: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:46:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:57 smithi139 bash[43301]: cluster 2022-04-28T20:46:56.439379+0000 mgr.y (mgr.24961) 1945 : cluster [DBG] pgmap v2304: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[61152]: cluster 2022-04-28T20:46:58.439999+0000 mgr.y (mgr.24961) 1946 : cluster [DBG] pgmap v2305: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[61152]: audit 2022-04-28T20:46:58.687789+0000 mon.a (mon.0) 2222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[61152]: audit 2022-04-28T20:46:58.908277+0000 mon.a (mon.0) 2223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[61152]: audit 2022-04-28T20:46:59.196764+0000 mon.a (mon.0) 2224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[59745]: cluster 2022-04-28T20:46:58.439999+0000 mgr.y (mgr.24961) 1946 : cluster [DBG] pgmap v2305: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[59745]: audit 2022-04-28T20:46:58.687789+0000 mon.a (mon.0) 2222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[59745]: audit 2022-04-28T20:46:58.908277+0000 mon.a (mon.0) 2223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:46:59 smithi059 bash[59745]: audit 2022-04-28T20:46:59.196764+0000 mon.a (mon.0) 2224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:59 smithi139 bash[43301]: cluster 2022-04-28T20:46:58.439999+0000 mgr.y (mgr.24961) 1946 : cluster [DBG] pgmap v2305: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:59 smithi139 bash[43301]: audit 2022-04-28T20:46:58.687789+0000 mon.a (mon.0) 2222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:59 smithi139 bash[43301]: audit 2022-04-28T20:46:58.908277+0000 mon.a (mon.0) 2223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:00.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:46:59 smithi139 bash[43301]: audit 2022-04-28T20:46:59.196764+0000 mon.a (mon.0) 2224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:47:02.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:01 smithi059 bash[61152]: cluster 2022-04-28T20:47:00.441259+0000 mgr.y (mgr.24961) 1947 : cluster [DBG] pgmap v2306: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:02.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:01 smithi059 bash[59745]: cluster 2022-04-28T20:47:00.441259+0000 mgr.y (mgr.24961) 1947 : cluster [DBG] pgmap v2306: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:02.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:01.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:02.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:01 smithi139 bash[43301]: cluster 2022-04-28T20:47:00.441259+0000 mgr.y (mgr.24961) 1947 : cluster [DBG] pgmap v2306: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:04.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:03 smithi059 bash[61152]: cluster 2022-04-28T20:47:02.442183+0000 mgr.y (mgr.24961) 1948 : cluster [DBG] pgmap v2307: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:03.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:03 smithi059 bash[59745]: cluster 2022-04-28T20:47:02.442183+0000 mgr.y (mgr.24961) 1948 : cluster [DBG] pgmap v2307: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:04.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:03 smithi139 bash[43301]: cluster 2022-04-28T20:47:02.442183+0000 mgr.y (mgr.24961) 1948 : cluster [DBG] pgmap v2307: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:06.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:05 smithi059 bash[61152]: cluster 2022-04-28T20:47:04.443149+0000 mgr.y (mgr.24961) 1949 : cluster [DBG] pgmap v2308: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:05 smithi059 bash[59745]: cluster 2022-04-28T20:47:04.443149+0000 mgr.y (mgr.24961) 1949 : cluster [DBG] pgmap v2308: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:05 smithi139 bash[43301]: cluster 2022-04-28T20:47:04.443149+0000 mgr.y (mgr.24961) 1949 : cluster [DBG] pgmap v2308: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:08.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:07 smithi059 bash[61152]: cluster 2022-04-28T20:47:06.444308+0000 mgr.y (mgr.24961) 1950 : cluster [DBG] pgmap v2309: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:07 smithi059 bash[59745]: cluster 2022-04-28T20:47:06.444308+0000 mgr.y (mgr.24961) 1950 : cluster [DBG] pgmap v2309: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:07 smithi139 bash[43301]: cluster 2022-04-28T20:47:06.444308+0000 mgr.y (mgr.24961) 1950 : cluster [DBG] pgmap v2309: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:10.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:09 smithi059 bash[61152]: cluster 2022-04-28T20:47:08.445091+0000 mgr.y (mgr.24961) 1951 : cluster [DBG] pgmap v2310: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:09 smithi059 bash[59745]: cluster 2022-04-28T20:47:08.445091+0000 mgr.y (mgr.24961) 1951 : cluster [DBG] pgmap v2310: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:09 smithi139 bash[43301]: cluster 2022-04-28T20:47:08.445091+0000 mgr.y (mgr.24961) 1951 : cluster [DBG] pgmap v2310: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:11.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:11 smithi059 bash[61152]: cluster 2022-04-28T20:47:10.446278+0000 mgr.y (mgr.24961) 1952 : cluster [DBG] pgmap v2311: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:11 smithi059 bash[59745]: cluster 2022-04-28T20:47:10.446278+0000 mgr.y (mgr.24961) 1952 : cluster [DBG] pgmap v2311: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:11 smithi139 bash[43301]: cluster 2022-04-28T20:47:10.446278+0000 mgr.y (mgr.24961) 1952 : cluster [DBG] pgmap v2311: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:14.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:13 smithi059 bash[61152]: cluster 2022-04-28T20:47:12.447139+0000 mgr.y (mgr.24961) 1953 : cluster [DBG] pgmap v2312: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:13.736Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:13 smithi059 bash[59745]: cluster 2022-04-28T20:47:12.447139+0000 mgr.y (mgr.24961) 1953 : cluster [DBG] pgmap v2312: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:13 smithi139 bash[43301]: cluster 2022-04-28T20:47:12.447139+0000 mgr.y (mgr.24961) 1953 : cluster [DBG] pgmap v2312: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:15 smithi059 bash[61152]: cluster 2022-04-28T20:47:14.448152+0000 mgr.y (mgr.24961) 1954 : cluster [DBG] pgmap v2313: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:15 smithi059 bash[59745]: cluster 2022-04-28T20:47:14.448152+0000 mgr.y (mgr.24961) 1954 : cluster [DBG] pgmap v2313: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:15 smithi139 bash[43301]: cluster 2022-04-28T20:47:14.448152+0000 mgr.y (mgr.24961) 1954 : cluster [DBG] pgmap v2313: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: cluster 2022-04-28T20:47:16.449217+0000 mgr.y (mgr.24961) 1955 : cluster [DBG] pgmap v2314: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.806270+0000 mon.a (mon.0) 2225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:47:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.808620+0000 mon.a (mon.0) 2226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:47:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.816171+0000 mon.a (mon.0) 2227 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:47:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.816475+0000 mon.a (mon.0) 2228 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:47:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.816768+0000 mon.a (mon.0) 2229 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2022-04-28T20:47:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.817079+0000 mon.a (mon.0) 2230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2022-04-28T20:47:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[59745]: audit 2022-04-28T20:47:17.817385+0000 mon.a (mon.0) 2231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 0, 3, 2]}]: dispatch 2022-04-28T20:47:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: cluster 2022-04-28T20:47:16.449217+0000 mgr.y (mgr.24961) 1955 : cluster [DBG] pgmap v2314: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.806270+0000 mon.a (mon.0) 2225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:47:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.808620+0000 mon.a (mon.0) 2226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:47:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.816171+0000 mon.a (mon.0) 2227 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:47:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.816475+0000 mon.a (mon.0) 2228 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:47:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.816768+0000 mon.a (mon.0) 2229 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2022-04-28T20:47:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.817079+0000 mon.a (mon.0) 2230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2022-04-28T20:47:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:17 smithi059 bash[61152]: audit 2022-04-28T20:47:17.817385+0000 mon.a (mon.0) 2231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 0, 3, 2]}]: dispatch 2022-04-28T20:47:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: cluster 2022-04-28T20:47:16.449217+0000 mgr.y (mgr.24961) 1955 : cluster [DBG] pgmap v2314: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.806270+0000 mon.a (mon.0) 2225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:47:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.808620+0000 mon.a (mon.0) 2226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:47:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.816171+0000 mon.a (mon.0) 2227 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:47:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.816475+0000 mon.a (mon.0) 2228 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:47:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.816768+0000 mon.a (mon.0) 2229 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2022-04-28T20:47:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.817079+0000 mon.a (mon.0) 2230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 0]}]: dispatch 2022-04-28T20:47:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:17 smithi139 bash[43301]: audit 2022-04-28T20:47:17.817385+0000 mon.a (mon.0) 2231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 0, 3, 2]}]: dispatch 2022-04-28T20:47:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:19 smithi059 bash[61152]: cluster 2022-04-28T20:47:18.449989+0000 mgr.y (mgr.24961) 1956 : cluster [DBG] pgmap v2315: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:19 smithi059 bash[59745]: cluster 2022-04-28T20:47:18.449989+0000 mgr.y (mgr.24961) 1956 : cluster [DBG] pgmap v2315: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:19 smithi139 bash[43301]: cluster 2022-04-28T20:47:18.449989+0000 mgr.y (mgr.24961) 1956 : cluster [DBG] pgmap v2315: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:21 smithi059 bash[59745]: cluster 2022-04-28T20:47:20.451256+0000 mgr.y (mgr.24961) 1957 : cluster [DBG] pgmap v2316: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:21 smithi059 bash[61152]: cluster 2022-04-28T20:47:20.451256+0000 mgr.y (mgr.24961) 1957 : cluster [DBG] pgmap v2316: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:21.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:21 smithi139 bash[43301]: cluster 2022-04-28T20:47:20.451256+0000 mgr.y (mgr.24961) 1957 : cluster [DBG] pgmap v2316: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:24.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:23 smithi059 bash[59745]: cluster 2022-04-28T20:47:22.452182+0000 mgr.y (mgr.24961) 1958 : cluster [DBG] pgmap v2317: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:23 smithi059 bash[61152]: cluster 2022-04-28T20:47:22.452182+0000 mgr.y (mgr.24961) 1958 : cluster [DBG] pgmap v2317: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:23.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:23 smithi139 bash[43301]: cluster 2022-04-28T20:47:22.452182+0000 mgr.y (mgr.24961) 1958 : cluster [DBG] pgmap v2317: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:25 smithi059 bash[61152]: cluster 2022-04-28T20:47:24.453253+0000 mgr.y (mgr.24961) 1959 : cluster [DBG] pgmap v2318: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:25 smithi059 bash[59745]: cluster 2022-04-28T20:47:24.453253+0000 mgr.y (mgr.24961) 1959 : cluster [DBG] pgmap v2318: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:25 smithi139 bash[43301]: cluster 2022-04-28T20:47:24.453253+0000 mgr.y (mgr.24961) 1959 : cluster [DBG] pgmap v2318: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:27 smithi059 bash[61152]: cluster 2022-04-28T20:47:26.454347+0000 mgr.y (mgr.24961) 1960 : cluster [DBG] pgmap v2319: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:27 smithi059 bash[59745]: cluster 2022-04-28T20:47:26.454347+0000 mgr.y (mgr.24961) 1960 : cluster [DBG] pgmap v2319: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:27 smithi139 bash[43301]: cluster 2022-04-28T20:47:26.454347+0000 mgr.y (mgr.24961) 1960 : cluster [DBG] pgmap v2319: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:29 smithi059 bash[61152]: cluster 2022-04-28T20:47:28.454802+0000 mgr.y (mgr.24961) 1961 : cluster [DBG] pgmap v2320: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:29 smithi059 bash[59745]: cluster 2022-04-28T20:47:28.454802+0000 mgr.y (mgr.24961) 1961 : cluster [DBG] pgmap v2320: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:29 smithi139 bash[43301]: cluster 2022-04-28T20:47:28.454802+0000 mgr.y (mgr.24961) 1961 : cluster [DBG] pgmap v2320: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:32.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:31 smithi059 bash[59745]: cluster 2022-04-28T20:47:30.455890+0000 mgr.y (mgr.24961) 1962 : cluster [DBG] pgmap v2321: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:31 smithi059 bash[61152]: cluster 2022-04-28T20:47:30.455890+0000 mgr.y (mgr.24961) 1962 : cluster [DBG] pgmap v2321: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:31.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:31 smithi139 bash[43301]: cluster 2022-04-28T20:47:30.455890+0000 mgr.y (mgr.24961) 1962 : cluster [DBG] pgmap v2321: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:34.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:33 smithi059 bash[59745]: cluster 2022-04-28T20:47:32.456797+0000 mgr.y (mgr.24961) 1963 : cluster [DBG] pgmap v2322: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:33 smithi059 bash[61152]: cluster 2022-04-28T20:47:32.456797+0000 mgr.y (mgr.24961) 1963 : cluster [DBG] pgmap v2322: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:33.738Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:33 smithi139 bash[43301]: cluster 2022-04-28T20:47:32.456797+0000 mgr.y (mgr.24961) 1963 : cluster [DBG] pgmap v2322: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:36.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:35 smithi059 bash[61152]: cluster 2022-04-28T20:47:34.457758+0000 mgr.y (mgr.24961) 1964 : cluster [DBG] pgmap v2323: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:35 smithi059 bash[59745]: cluster 2022-04-28T20:47:34.457758+0000 mgr.y (mgr.24961) 1964 : cluster [DBG] pgmap v2323: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:35 smithi139 bash[43301]: cluster 2022-04-28T20:47:34.457758+0000 mgr.y (mgr.24961) 1964 : cluster [DBG] pgmap v2323: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:38.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:37 smithi059 bash[59745]: cluster 2022-04-28T20:47:36.458865+0000 mgr.y (mgr.24961) 1965 : cluster [DBG] pgmap v2324: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:37 smithi059 bash[61152]: cluster 2022-04-28T20:47:36.458865+0000 mgr.y (mgr.24961) 1965 : cluster [DBG] pgmap v2324: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:37 smithi139 bash[43301]: cluster 2022-04-28T20:47:36.458865+0000 mgr.y (mgr.24961) 1965 : cluster [DBG] pgmap v2324: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:39 smithi059 bash[61152]: cluster 2022-04-28T20:47:38.459510+0000 mgr.y (mgr.24961) 1966 : cluster [DBG] pgmap v2325: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:39 smithi059 bash[59745]: cluster 2022-04-28T20:47:38.459510+0000 mgr.y (mgr.24961) 1966 : cluster [DBG] pgmap v2325: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:39 smithi139 bash[43301]: cluster 2022-04-28T20:47:38.459510+0000 mgr.y (mgr.24961) 1966 : cluster [DBG] pgmap v2325: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:42.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:41 smithi059 bash[61152]: cluster 2022-04-28T20:47:40.460157+0000 mgr.y (mgr.24961) 1967 : cluster [DBG] pgmap v2326: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:41 smithi059 bash[59745]: cluster 2022-04-28T20:47:40.460157+0000 mgr.y (mgr.24961) 1967 : cluster [DBG] pgmap v2326: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:41.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:41 smithi139 bash[43301]: cluster 2022-04-28T20:47:40.460157+0000 mgr.y (mgr.24961) 1967 : cluster [DBG] pgmap v2326: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:44.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:43 smithi059 bash[61152]: cluster 2022-04-28T20:47:42.461001+0000 mgr.y (mgr.24961) 1968 : cluster [DBG] pgmap v2327: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:43 smithi059 bash[59745]: cluster 2022-04-28T20:47:42.461001+0000 mgr.y (mgr.24961) 1968 : cluster [DBG] pgmap v2327: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:43.739Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:43 smithi139 bash[43301]: cluster 2022-04-28T20:47:42.461001+0000 mgr.y (mgr.24961) 1968 : cluster [DBG] pgmap v2327: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:45 smithi059 bash[61152]: cluster 2022-04-28T20:47:44.461913+0000 mgr.y (mgr.24961) 1969 : cluster [DBG] pgmap v2328: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:45 smithi059 bash[59745]: cluster 2022-04-28T20:47:44.461913+0000 mgr.y (mgr.24961) 1969 : cluster [DBG] pgmap v2328: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:45 smithi139 bash[43301]: cluster 2022-04-28T20:47:44.461913+0000 mgr.y (mgr.24961) 1969 : cluster [DBG] pgmap v2328: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:48.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:47 smithi059 bash[59745]: cluster 2022-04-28T20:47:46.462910+0000 mgr.y (mgr.24961) 1970 : cluster [DBG] pgmap v2329: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:47 smithi059 bash[61152]: cluster 2022-04-28T20:47:46.462910+0000 mgr.y (mgr.24961) 1970 : cluster [DBG] pgmap v2329: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:47 smithi139 bash[43301]: cluster 2022-04-28T20:47:46.462910+0000 mgr.y (mgr.24961) 1970 : cluster [DBG] pgmap v2329: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:50.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:49 smithi059 bash[59745]: cluster 2022-04-28T20:47:48.463566+0000 mgr.y (mgr.24961) 1971 : cluster [DBG] pgmap v2330: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:49 smithi059 bash[61152]: cluster 2022-04-28T20:47:48.463566+0000 mgr.y (mgr.24961) 1971 : cluster [DBG] pgmap v2330: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:49 smithi139 bash[43301]: cluster 2022-04-28T20:47:48.463566+0000 mgr.y (mgr.24961) 1971 : cluster [DBG] pgmap v2330: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:52.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:51 smithi059 bash[59745]: cluster 2022-04-28T20:47:50.464784+0000 mgr.y (mgr.24961) 1972 : cluster [DBG] pgmap v2331: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:51 smithi059 bash[61152]: cluster 2022-04-28T20:47:50.464784+0000 mgr.y (mgr.24961) 1972 : cluster [DBG] pgmap v2331: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:51.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:51 smithi139 bash[43301]: cluster 2022-04-28T20:47:50.464784+0000 mgr.y (mgr.24961) 1972 : cluster [DBG] pgmap v2331: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:53.997 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:47:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:47:53.740Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:47:53.998 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:53 smithi059 bash[59745]: cluster 2022-04-28T20:47:52.465755+0000 mgr.y (mgr.24961) 1973 : cluster [DBG] pgmap v2332: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:53 smithi059 bash[61152]: cluster 2022-04-28T20:47:52.465755+0000 mgr.y (mgr.24961) 1973 : cluster [DBG] pgmap v2332: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:53 smithi139 bash[43301]: cluster 2022-04-28T20:47:52.465755+0000 mgr.y (mgr.24961) 1973 : cluster [DBG] pgmap v2332: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:56.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:56 smithi059 bash[61152]: cluster 2022-04-28T20:47:54.466826+0000 mgr.y (mgr.24961) 1974 : cluster [DBG] pgmap v2333: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:56 smithi059 bash[59745]: cluster 2022-04-28T20:47:54.466826+0000 mgr.y (mgr.24961) 1974 : cluster [DBG] pgmap v2333: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:56 smithi139 bash[43301]: cluster 2022-04-28T20:47:54.466826+0000 mgr.y (mgr.24961) 1974 : cluster [DBG] pgmap v2333: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:57.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:57 smithi059 bash[59745]: cluster 2022-04-28T20:47:56.467981+0000 mgr.y (mgr.24961) 1975 : cluster [DBG] pgmap v2334: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:57 smithi059 bash[61152]: cluster 2022-04-28T20:47:56.467981+0000 mgr.y (mgr.24961) 1975 : cluster [DBG] pgmap v2334: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:47:57.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:57 smithi139 bash[43301]: cluster 2022-04-28T20:47:56.467981+0000 mgr.y (mgr.24961) 1975 : cluster [DBG] pgmap v2334: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[59745]: cluster 2022-04-28T20:47:58.468692+0000 mgr.y (mgr.24961) 1976 : cluster [DBG] pgmap v2335: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[59745]: audit 2022-04-28T20:47:59.206056+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:48:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[59745]: audit 2022-04-28T20:47:59.208471+0000 mon.a (mon.0) 2233 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:48:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[59745]: audit 2022-04-28T20:47:59.209951+0000 mon.a (mon.0) 2234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:48:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[59745]: audit 2022-04-28T20:47:59.516581+0000 mon.a (mon.0) 2235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:48:00.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[61152]: cluster 2022-04-28T20:47:58.468692+0000 mgr.y (mgr.24961) 1976 : cluster [DBG] pgmap v2335: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:00.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[61152]: audit 2022-04-28T20:47:59.206056+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:48:00.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[61152]: audit 2022-04-28T20:47:59.208471+0000 mon.a (mon.0) 2233 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:48:00.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[61152]: audit 2022-04-28T20:47:59.209951+0000 mon.a (mon.0) 2234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:48:00.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:47:59 smithi059 bash[61152]: audit 2022-04-28T20:47:59.516581+0000 mon.a (mon.0) 2235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:48:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:59 smithi139 bash[43301]: cluster 2022-04-28T20:47:58.468692+0000 mgr.y (mgr.24961) 1976 : cluster [DBG] pgmap v2335: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:59 smithi139 bash[43301]: audit 2022-04-28T20:47:59.206056+0000 mon.a (mon.0) 2232 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:48:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:59 smithi139 bash[43301]: audit 2022-04-28T20:47:59.208471+0000 mon.a (mon.0) 2233 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:48:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:59 smithi139 bash[43301]: audit 2022-04-28T20:47:59.209951+0000 mon.a (mon.0) 2234 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:48:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:47:59 smithi139 bash[43301]: audit 2022-04-28T20:47:59.516581+0000 mon.a (mon.0) 2235 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:48:01.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:00 smithi059 bash[61152]: cephadm 2022-04-28T20:47:59.211362+0000 mgr.y (mgr.24961) 1977 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:48:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:00 smithi059 bash[59745]: cephadm 2022-04-28T20:47:59.211362+0000 mgr.y (mgr.24961) 1977 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:48:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:00 smithi139 bash[43301]: cephadm 2022-04-28T20:47:59.211362+0000 mgr.y (mgr.24961) 1977 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:48:02.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:01 smithi059 bash[59745]: cluster 2022-04-28T20:48:00.469836+0000 mgr.y (mgr.24961) 1978 : cluster [DBG] pgmap v2336: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:01 smithi059 bash[61152]: cluster 2022-04-28T20:48:00.469836+0000 mgr.y (mgr.24961) 1978 : cluster [DBG] pgmap v2336: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:01.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:01 smithi139 bash[43301]: cluster 2022-04-28T20:48:00.469836+0000 mgr.y (mgr.24961) 1978 : cluster [DBG] pgmap v2336: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:04.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:03 smithi059 bash[59745]: cluster 2022-04-28T20:48:02.470643+0000 mgr.y (mgr.24961) 1979 : cluster [DBG] pgmap v2337: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:03 smithi059 bash[61152]: cluster 2022-04-28T20:48:02.470643+0000 mgr.y (mgr.24961) 1979 : cluster [DBG] pgmap v2337: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:03.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:03 smithi139 bash[43301]: cluster 2022-04-28T20:48:02.470643+0000 mgr.y (mgr.24961) 1979 : cluster [DBG] pgmap v2337: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:06.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:05 smithi059 bash[59745]: cluster 2022-04-28T20:48:04.471597+0000 mgr.y (mgr.24961) 1980 : cluster [DBG] pgmap v2338: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:05 smithi059 bash[61152]: cluster 2022-04-28T20:48:04.471597+0000 mgr.y (mgr.24961) 1980 : cluster [DBG] pgmap v2338: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:05 smithi139 bash[43301]: cluster 2022-04-28T20:48:04.471597+0000 mgr.y (mgr.24961) 1980 : cluster [DBG] pgmap v2338: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:08.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:07 smithi059 bash[61152]: cluster 2022-04-28T20:48:06.472644+0000 mgr.y (mgr.24961) 1981 : cluster [DBG] pgmap v2339: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:07 smithi059 bash[59745]: cluster 2022-04-28T20:48:06.472644+0000 mgr.y (mgr.24961) 1981 : cluster [DBG] pgmap v2339: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:07 smithi139 bash[43301]: cluster 2022-04-28T20:48:06.472644+0000 mgr.y (mgr.24961) 1981 : cluster [DBG] pgmap v2339: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:10.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:09 smithi059 bash[61152]: cluster 2022-04-28T20:48:08.473316+0000 mgr.y (mgr.24961) 1982 : cluster [DBG] pgmap v2340: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:09 smithi059 bash[59745]: cluster 2022-04-28T20:48:08.473316+0000 mgr.y (mgr.24961) 1982 : cluster [DBG] pgmap v2340: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:09 smithi139 bash[43301]: cluster 2022-04-28T20:48:08.473316+0000 mgr.y (mgr.24961) 1982 : cluster [DBG] pgmap v2340: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:11 smithi059 bash[61152]: cluster 2022-04-28T20:48:10.474499+0000 mgr.y (mgr.24961) 1983 : cluster [DBG] pgmap v2341: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:11.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:11 smithi059 bash[59745]: cluster 2022-04-28T20:48:10.474499+0000 mgr.y (mgr.24961) 1983 : cluster [DBG] pgmap v2341: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:11 smithi139 bash[43301]: cluster 2022-04-28T20:48:10.474499+0000 mgr.y (mgr.24961) 1983 : cluster [DBG] pgmap v2341: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:14.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:13 smithi059 bash[61152]: cluster 2022-04-28T20:48:12.475421+0000 mgr.y (mgr.24961) 1984 : cluster [DBG] pgmap v2342: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:13.741Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:13 smithi059 bash[59745]: cluster 2022-04-28T20:48:12.475421+0000 mgr.y (mgr.24961) 1984 : cluster [DBG] pgmap v2342: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:13 smithi139 bash[43301]: cluster 2022-04-28T20:48:12.475421+0000 mgr.y (mgr.24961) 1984 : cluster [DBG] pgmap v2342: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:16.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:15 smithi059 bash[61152]: cluster 2022-04-28T20:48:14.476439+0000 mgr.y (mgr.24961) 1985 : cluster [DBG] pgmap v2343: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:15 smithi059 bash[59745]: cluster 2022-04-28T20:48:14.476439+0000 mgr.y (mgr.24961) 1985 : cluster [DBG] pgmap v2343: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:15 smithi139 bash[43301]: cluster 2022-04-28T20:48:14.476439+0000 mgr.y (mgr.24961) 1985 : cluster [DBG] pgmap v2343: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[61152]: cluster 2022-04-28T20:48:16.477553+0000 mgr.y (mgr.24961) 1986 : cluster [DBG] pgmap v2344: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[61152]: audit 2022-04-28T20:48:17.821566+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:48:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[61152]: audit 2022-04-28T20:48:17.822429+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:48:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[61152]: audit 2022-04-28T20:48:17.830668+0000 mon.a (mon.0) 2238 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:48:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[61152]: audit 2022-04-28T20:48:17.831217+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:48:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[61152]: audit 2022-04-28T20:48:17.831830+0000 mon.a (mon.0) 2240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:48:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[59745]: cluster 2022-04-28T20:48:16.477553+0000 mgr.y (mgr.24961) 1986 : cluster [DBG] pgmap v2344: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[59745]: audit 2022-04-28T20:48:17.821566+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:48:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[59745]: audit 2022-04-28T20:48:17.822429+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:48:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[59745]: audit 2022-04-28T20:48:17.830668+0000 mon.a (mon.0) 2238 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:48:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[59745]: audit 2022-04-28T20:48:17.831217+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:48:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:17 smithi059 bash[59745]: audit 2022-04-28T20:48:17.831830+0000 mon.a (mon.0) 2240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:48:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:17 smithi139 bash[43301]: cluster 2022-04-28T20:48:16.477553+0000 mgr.y (mgr.24961) 1986 : cluster [DBG] pgmap v2344: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:17 smithi139 bash[43301]: audit 2022-04-28T20:48:17.821566+0000 mon.a (mon.0) 2236 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:48:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:17 smithi139 bash[43301]: audit 2022-04-28T20:48:17.822429+0000 mon.a (mon.0) 2237 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:48:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:17 smithi139 bash[43301]: audit 2022-04-28T20:48:17.830668+0000 mon.a (mon.0) 2238 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:48:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:17 smithi139 bash[43301]: audit 2022-04-28T20:48:17.831217+0000 mon.a (mon.0) 2239 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T20:48:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:17 smithi139 bash[43301]: audit 2022-04-28T20:48:17.831830+0000 mon.a (mon.0) 2240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:48:20.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:19 smithi059 bash[61152]: cluster 2022-04-28T20:48:18.478239+0000 mgr.y (mgr.24961) 1987 : cluster [DBG] pgmap v2345: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:19 smithi059 bash[59745]: cluster 2022-04-28T20:48:18.478239+0000 mgr.y (mgr.24961) 1987 : cluster [DBG] pgmap v2345: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:19 smithi139 bash[43301]: cluster 2022-04-28T20:48:18.478239+0000 mgr.y (mgr.24961) 1987 : cluster [DBG] pgmap v2345: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:21 smithi059 bash[61152]: cluster 2022-04-28T20:48:20.479371+0000 mgr.y (mgr.24961) 1988 : cluster [DBG] pgmap v2346: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:21.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:21 smithi059 bash[59745]: cluster 2022-04-28T20:48:20.479371+0000 mgr.y (mgr.24961) 1988 : cluster [DBG] pgmap v2346: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:21 smithi139 bash[43301]: cluster 2022-04-28T20:48:20.479371+0000 mgr.y (mgr.24961) 1988 : cluster [DBG] pgmap v2346: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:23 smithi059 bash[61152]: cluster 2022-04-28T20:48:22.480566+0000 mgr.y (mgr.24961) 1989 : cluster [DBG] pgmap v2347: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:23.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:23 smithi059 bash[59745]: cluster 2022-04-28T20:48:22.480566+0000 mgr.y (mgr.24961) 1989 : cluster [DBG] pgmap v2347: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:23 smithi139 bash[43301]: cluster 2022-04-28T20:48:22.480566+0000 mgr.y (mgr.24961) 1989 : cluster [DBG] pgmap v2347: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:25 smithi059 bash[61152]: cluster 2022-04-28T20:48:24.481579+0000 mgr.y (mgr.24961) 1990 : cluster [DBG] pgmap v2348: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:25 smithi059 bash[59745]: cluster 2022-04-28T20:48:24.481579+0000 mgr.y (mgr.24961) 1990 : cluster [DBG] pgmap v2348: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:25 smithi139 bash[43301]: cluster 2022-04-28T20:48:24.481579+0000 mgr.y (mgr.24961) 1990 : cluster [DBG] pgmap v2348: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:28.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:27 smithi059 bash[61152]: cluster 2022-04-28T20:48:26.482691+0000 mgr.y (mgr.24961) 1991 : cluster [DBG] pgmap v2349: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:27 smithi059 bash[59745]: cluster 2022-04-28T20:48:26.482691+0000 mgr.y (mgr.24961) 1991 : cluster [DBG] pgmap v2349: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:27 smithi139 bash[43301]: cluster 2022-04-28T20:48:26.482691+0000 mgr.y (mgr.24961) 1991 : cluster [DBG] pgmap v2349: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:30.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:29 smithi059 bash[61152]: cluster 2022-04-28T20:48:28.483326+0000 mgr.y (mgr.24961) 1992 : cluster [DBG] pgmap v2350: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:29 smithi059 bash[59745]: cluster 2022-04-28T20:48:28.483326+0000 mgr.y (mgr.24961) 1992 : cluster [DBG] pgmap v2350: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:29 smithi139 bash[43301]: cluster 2022-04-28T20:48:28.483326+0000 mgr.y (mgr.24961) 1992 : cluster [DBG] pgmap v2350: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:31 smithi059 bash[61152]: cluster 2022-04-28T20:48:30.484475+0000 mgr.y (mgr.24961) 1993 : cluster [DBG] pgmap v2351: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:31.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:31 smithi059 bash[59745]: cluster 2022-04-28T20:48:30.484475+0000 mgr.y (mgr.24961) 1993 : cluster [DBG] pgmap v2351: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:31 smithi139 bash[43301]: cluster 2022-04-28T20:48:30.484475+0000 mgr.y (mgr.24961) 1993 : cluster [DBG] pgmap v2351: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:33 smithi059 bash[61152]: cluster 2022-04-28T20:48:32.485465+0000 mgr.y (mgr.24961) 1994 : cluster [DBG] pgmap v2352: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:33.743Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:33 smithi059 bash[59745]: cluster 2022-04-28T20:48:32.485465+0000 mgr.y (mgr.24961) 1994 : cluster [DBG] pgmap v2352: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:33 smithi139 bash[43301]: cluster 2022-04-28T20:48:32.485465+0000 mgr.y (mgr.24961) 1994 : cluster [DBG] pgmap v2352: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:36.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:35 smithi059 bash[61152]: cluster 2022-04-28T20:48:34.486477+0000 mgr.y (mgr.24961) 1995 : cluster [DBG] pgmap v2353: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:35 smithi059 bash[59745]: cluster 2022-04-28T20:48:34.486477+0000 mgr.y (mgr.24961) 1995 : cluster [DBG] pgmap v2353: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:35 smithi139 bash[43301]: cluster 2022-04-28T20:48:34.486477+0000 mgr.y (mgr.24961) 1995 : cluster [DBG] pgmap v2353: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:37 smithi059 bash[59745]: cluster 2022-04-28T20:48:36.487581+0000 mgr.y (mgr.24961) 1996 : cluster [DBG] pgmap v2354: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:37 smithi059 bash[61152]: cluster 2022-04-28T20:48:36.487581+0000 mgr.y (mgr.24961) 1996 : cluster [DBG] pgmap v2354: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:37 smithi139 bash[43301]: cluster 2022-04-28T20:48:36.487581+0000 mgr.y (mgr.24961) 1996 : cluster [DBG] pgmap v2354: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:40.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:39 smithi059 bash[59745]: cluster 2022-04-28T20:48:38.488261+0000 mgr.y (mgr.24961) 1997 : cluster [DBG] pgmap v2355: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:39 smithi059 bash[61152]: cluster 2022-04-28T20:48:38.488261+0000 mgr.y (mgr.24961) 1997 : cluster [DBG] pgmap v2355: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:40.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:39 smithi139 bash[43301]: cluster 2022-04-28T20:48:38.488261+0000 mgr.y (mgr.24961) 1997 : cluster [DBG] pgmap v2355: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:42 smithi059 bash[61152]: cluster 2022-04-28T20:48:40.489417+0000 mgr.y (mgr.24961) 1998 : cluster [DBG] pgmap v2356: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:41.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:42 smithi059 bash[59745]: cluster 2022-04-28T20:48:40.489417+0000 mgr.y (mgr.24961) 1998 : cluster [DBG] pgmap v2356: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:41 smithi139 bash[43301]: cluster 2022-04-28T20:48:40.489417+0000 mgr.y (mgr.24961) 1998 : cluster [DBG] pgmap v2356: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:44.006 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:43.742Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:44.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:44 smithi059 bash[59745]: cluster 2022-04-28T20:48:42.490286+0000 mgr.y (mgr.24961) 1999 : cluster [DBG] pgmap v2357: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:44 smithi059 bash[61152]: cluster 2022-04-28T20:48:42.490286+0000 mgr.y (mgr.24961) 1999 : cluster [DBG] pgmap v2357: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:44 smithi139 bash[43301]: cluster 2022-04-28T20:48:42.490286+0000 mgr.y (mgr.24961) 1999 : cluster [DBG] pgmap v2357: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:46 smithi059 bash[59745]: cluster 2022-04-28T20:48:44.491346+0000 mgr.y (mgr.24961) 2000 : cluster [DBG] pgmap v2358: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:46 smithi059 bash[61152]: cluster 2022-04-28T20:48:44.491346+0000 mgr.y (mgr.24961) 2000 : cluster [DBG] pgmap v2358: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:46 smithi139 bash[43301]: cluster 2022-04-28T20:48:44.491346+0000 mgr.y (mgr.24961) 2000 : cluster [DBG] pgmap v2358: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:47 smithi059 bash[61152]: cluster 2022-04-28T20:48:46.492540+0000 mgr.y (mgr.24961) 2001 : cluster [DBG] pgmap v2359: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:47 smithi059 bash[59745]: cluster 2022-04-28T20:48:46.492540+0000 mgr.y (mgr.24961) 2001 : cluster [DBG] pgmap v2359: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:47.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:47 smithi139 bash[43301]: cluster 2022-04-28T20:48:46.492540+0000 mgr.y (mgr.24961) 2001 : cluster [DBG] pgmap v2359: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:49 smithi059 bash[61152]: cluster 2022-04-28T20:48:48.493194+0000 mgr.y (mgr.24961) 2002 : cluster [DBG] pgmap v2360: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:50.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:49 smithi059 bash[59745]: cluster 2022-04-28T20:48:48.493194+0000 mgr.y (mgr.24961) 2002 : cluster [DBG] pgmap v2360: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:49 smithi139 bash[43301]: cluster 2022-04-28T20:48:48.493194+0000 mgr.y (mgr.24961) 2002 : cluster [DBG] pgmap v2360: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:51 smithi059 bash[61152]: cluster 2022-04-28T20:48:50.494357+0000 mgr.y (mgr.24961) 2003 : cluster [DBG] pgmap v2361: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:51.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:51 smithi059 bash[59745]: cluster 2022-04-28T20:48:50.494357+0000 mgr.y (mgr.24961) 2003 : cluster [DBG] pgmap v2361: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:51 smithi139 bash[43301]: cluster 2022-04-28T20:48:50.494357+0000 mgr.y (mgr.24961) 2003 : cluster [DBG] pgmap v2361: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:53 smithi059 bash[61152]: cluster 2022-04-28T20:48:52.495278+0000 mgr.y (mgr.24961) 2004 : cluster [DBG] pgmap v2362: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:54.141 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:48:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:48:53.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:48:54.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:53 smithi059 bash[59745]: cluster 2022-04-28T20:48:52.495278+0000 mgr.y (mgr.24961) 2004 : cluster [DBG] pgmap v2362: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:53 smithi139 bash[43301]: cluster 2022-04-28T20:48:52.495278+0000 mgr.y (mgr.24961) 2004 : cluster [DBG] pgmap v2362: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:55 smithi059 bash[61152]: cluster 2022-04-28T20:48:54.496296+0000 mgr.y (mgr.24961) 2005 : cluster [DBG] pgmap v2363: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:55 smithi059 bash[59745]: cluster 2022-04-28T20:48:54.496296+0000 mgr.y (mgr.24961) 2005 : cluster [DBG] pgmap v2363: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:55 smithi139 bash[43301]: cluster 2022-04-28T20:48:54.496296+0000 mgr.y (mgr.24961) 2005 : cluster [DBG] pgmap v2363: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:57 smithi059 bash[59745]: cluster 2022-04-28T20:48:56.497387+0000 mgr.y (mgr.24961) 2006 : cluster [DBG] pgmap v2364: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:57 smithi059 bash[61152]: cluster 2022-04-28T20:48:56.497387+0000 mgr.y (mgr.24961) 2006 : cluster [DBG] pgmap v2364: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:48:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:57 smithi139 bash[43301]: cluster 2022-04-28T20:48:56.497387+0000 mgr.y (mgr.24961) 2006 : cluster [DBG] pgmap v2364: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[59745]: cluster 2022-04-28T20:48:58.498171+0000 mgr.y (mgr.24961) 2007 : cluster [DBG] pgmap v2365: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[59745]: audit 2022-04-28T20:48:59.522391+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:49:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[59745]: audit 2022-04-28T20:48:59.525263+0000 mon.a (mon.0) 2242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:49:00.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[59745]: audit 2022-04-28T20:48:59.527199+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:49:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[59745]: audit 2022-04-28T20:48:59.838099+0000 mon.a (mon.0) 2244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:00.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[59745]: audit 2022-04-28T20:48:59.843275+0000 mon.a (mon.0) 2245 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:00.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[61152]: cluster 2022-04-28T20:48:58.498171+0000 mgr.y (mgr.24961) 2007 : cluster [DBG] pgmap v2365: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:00.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[61152]: audit 2022-04-28T20:48:59.522391+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:49:00.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[61152]: audit 2022-04-28T20:48:59.525263+0000 mon.a (mon.0) 2242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:49:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[61152]: audit 2022-04-28T20:48:59.527199+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:49:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[61152]: audit 2022-04-28T20:48:59.838099+0000 mon.a (mon.0) 2244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:00.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:48:59 smithi059 bash[61152]: audit 2022-04-28T20:48:59.843275+0000 mon.a (mon.0) 2245 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:59 smithi139 bash[43301]: cluster 2022-04-28T20:48:58.498171+0000 mgr.y (mgr.24961) 2007 : cluster [DBG] pgmap v2365: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:59 smithi139 bash[43301]: audit 2022-04-28T20:48:59.522391+0000 mon.a (mon.0) 2241 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:49:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:59 smithi139 bash[43301]: audit 2022-04-28T20:48:59.525263+0000 mon.a (mon.0) 2242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:49:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:59 smithi139 bash[43301]: audit 2022-04-28T20:48:59.527199+0000 mon.a (mon.0) 2243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:49:00.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:59 smithi139 bash[43301]: audit 2022-04-28T20:48:59.838099+0000 mon.a (mon.0) 2244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:00.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:48:59 smithi139 bash[43301]: audit 2022-04-28T20:48:59.843275+0000 mon.a (mon.0) 2245 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:00 smithi059 bash[61152]: cephadm 2022-04-28T20:48:59.528948+0000 mgr.y (mgr.24961) 2008 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:49:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:00 smithi059 bash[61152]: audit 2022-04-28T20:49:00.135176+0000 mon.a (mon.0) 2246 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:00 smithi059 bash[59745]: cephadm 2022-04-28T20:48:59.528948+0000 mgr.y (mgr.24961) 2008 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:49:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:00 smithi059 bash[59745]: audit 2022-04-28T20:49:00.135176+0000 mon.a (mon.0) 2246 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:00 smithi139 bash[43301]: cephadm 2022-04-28T20:48:59.528948+0000 mgr.y (mgr.24961) 2008 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:49:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:00 smithi139 bash[43301]: audit 2022-04-28T20:49:00.135176+0000 mon.a (mon.0) 2246 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:49:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:01 smithi059 bash[61152]: cluster 2022-04-28T20:49:00.499429+0000 mgr.y (mgr.24961) 2009 : cluster [DBG] pgmap v2366: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:01.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:01 smithi059 bash[59745]: cluster 2022-04-28T20:49:00.499429+0000 mgr.y (mgr.24961) 2009 : cluster [DBG] pgmap v2366: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:01 smithi139 bash[43301]: cluster 2022-04-28T20:49:00.499429+0000 mgr.y (mgr.24961) 2009 : cluster [DBG] pgmap v2366: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:03 smithi059 bash[61152]: cluster 2022-04-28T20:49:02.500410+0000 mgr.y (mgr.24961) 2010 : cluster [DBG] pgmap v2367: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:03.744Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:03 smithi059 bash[59745]: cluster 2022-04-28T20:49:02.500410+0000 mgr.y (mgr.24961) 2010 : cluster [DBG] pgmap v2367: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:03 smithi139 bash[43301]: cluster 2022-04-28T20:49:02.500410+0000 mgr.y (mgr.24961) 2010 : cluster [DBG] pgmap v2367: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:05 smithi059 bash[61152]: cluster 2022-04-28T20:49:04.501403+0000 mgr.y (mgr.24961) 2011 : cluster [DBG] pgmap v2368: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:05 smithi059 bash[59745]: cluster 2022-04-28T20:49:04.501403+0000 mgr.y (mgr.24961) 2011 : cluster [DBG] pgmap v2368: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:05 smithi139 bash[43301]: cluster 2022-04-28T20:49:04.501403+0000 mgr.y (mgr.24961) 2011 : cluster [DBG] pgmap v2368: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:07 smithi059 bash[61152]: cluster 2022-04-28T20:49:06.502504+0000 mgr.y (mgr.24961) 2012 : cluster [DBG] pgmap v2369: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:07 smithi059 bash[59745]: cluster 2022-04-28T20:49:06.502504+0000 mgr.y (mgr.24961) 2012 : cluster [DBG] pgmap v2369: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:07 smithi139 bash[43301]: cluster 2022-04-28T20:49:06.502504+0000 mgr.y (mgr.24961) 2012 : cluster [DBG] pgmap v2369: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:10.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:09 smithi059 bash[61152]: cluster 2022-04-28T20:49:08.503210+0000 mgr.y (mgr.24961) 2013 : cluster [DBG] pgmap v2370: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:09 smithi059 bash[59745]: cluster 2022-04-28T20:49:08.503210+0000 mgr.y (mgr.24961) 2013 : cluster [DBG] pgmap v2370: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:09 smithi139 bash[43301]: cluster 2022-04-28T20:49:08.503210+0000 mgr.y (mgr.24961) 2013 : cluster [DBG] pgmap v2370: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:11 smithi059 bash[61152]: cluster 2022-04-28T20:49:10.504541+0000 mgr.y (mgr.24961) 2014 : cluster [DBG] pgmap v2371: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:11.851Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:11 smithi059 bash[59745]: cluster 2022-04-28T20:49:10.504541+0000 mgr.y (mgr.24961) 2014 : cluster [DBG] pgmap v2371: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:11 smithi139 bash[43301]: cluster 2022-04-28T20:49:10.504541+0000 mgr.y (mgr.24961) 2014 : cluster [DBG] pgmap v2371: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:13 smithi059 bash[61152]: cluster 2022-04-28T20:49:12.505581+0000 mgr.y (mgr.24961) 2015 : cluster [DBG] pgmap v2372: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:13.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:13 smithi059 bash[59745]: cluster 2022-04-28T20:49:12.505581+0000 mgr.y (mgr.24961) 2015 : cluster [DBG] pgmap v2372: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:13 smithi139 bash[43301]: cluster 2022-04-28T20:49:12.505581+0000 mgr.y (mgr.24961) 2015 : cluster [DBG] pgmap v2372: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:15 smithi059 bash[61152]: cluster 2022-04-28T20:49:14.506560+0000 mgr.y (mgr.24961) 2016 : cluster [DBG] pgmap v2373: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:15 smithi059 bash[59745]: cluster 2022-04-28T20:49:14.506560+0000 mgr.y (mgr.24961) 2016 : cluster [DBG] pgmap v2373: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:15 smithi139 bash[43301]: cluster 2022-04-28T20:49:14.506560+0000 mgr.y (mgr.24961) 2016 : cluster [DBG] pgmap v2373: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[59745]: cluster 2022-04-28T20:49:16.507779+0000 mgr.y (mgr.24961) 2017 : cluster [DBG] pgmap v2374: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[59745]: audit 2022-04-28T20:49:17.833017+0000 mon.a (mon.0) 2247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[59745]: audit 2022-04-28T20:49:17.833613+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[59745]: audit 2022-04-28T20:49:17.844208+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[59745]: audit 2022-04-28T20:49:17.845005+0000 mon.a (mon.0) 2250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2022-04-28T20:49:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[59745]: audit 2022-04-28T20:49:17.845622+0000 mon.a (mon.0) 2251 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[61152]: cluster 2022-04-28T20:49:16.507779+0000 mgr.y (mgr.24961) 2017 : cluster [DBG] pgmap v2374: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[61152]: audit 2022-04-28T20:49:17.833017+0000 mon.a (mon.0) 2247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[61152]: audit 2022-04-28T20:49:17.833613+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[61152]: audit 2022-04-28T20:49:17.844208+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:49:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[61152]: audit 2022-04-28T20:49:17.845005+0000 mon.a (mon.0) 2250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2022-04-28T20:49:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:17 smithi059 bash[61152]: audit 2022-04-28T20:49:17.845622+0000 mon.a (mon.0) 2251 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:49:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:17 smithi139 bash[43301]: cluster 2022-04-28T20:49:16.507779+0000 mgr.y (mgr.24961) 2017 : cluster [DBG] pgmap v2374: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:17 smithi139 bash[43301]: audit 2022-04-28T20:49:17.833017+0000 mon.a (mon.0) 2247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:17 smithi139 bash[43301]: audit 2022-04-28T20:49:17.833613+0000 mon.a (mon.0) 2248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:17 smithi139 bash[43301]: audit 2022-04-28T20:49:17.844208+0000 mon.a (mon.0) 2249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:49:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:17 smithi139 bash[43301]: audit 2022-04-28T20:49:17.845005+0000 mon.a (mon.0) 2250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2022-04-28T20:49:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:17 smithi139 bash[43301]: audit 2022-04-28T20:49:17.845622+0000 mon.a (mon.0) 2251 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:49:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:19 smithi059 bash[61152]: cluster 2022-04-28T20:49:18.508512+0000 mgr.y (mgr.24961) 2018 : cluster [DBG] pgmap v2375: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:19 smithi059 bash[59745]: cluster 2022-04-28T20:49:18.508512+0000 mgr.y (mgr.24961) 2018 : cluster [DBG] pgmap v2375: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:19 smithi139 bash[43301]: cluster 2022-04-28T20:49:18.508512+0000 mgr.y (mgr.24961) 2018 : cluster [DBG] pgmap v2375: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:21 smithi059 bash[61152]: cluster 2022-04-28T20:49:20.509666+0000 mgr.y (mgr.24961) 2019 : cluster [DBG] pgmap v2376: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:21.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:21 smithi059 bash[59745]: cluster 2022-04-28T20:49:20.509666+0000 mgr.y (mgr.24961) 2019 : cluster [DBG] pgmap v2376: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:21 smithi139 bash[43301]: cluster 2022-04-28T20:49:20.509666+0000 mgr.y (mgr.24961) 2019 : cluster [DBG] pgmap v2376: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:24.002 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:24 smithi059 bash[59745]: cluster 2022-04-28T20:49:22.510599+0000 mgr.y (mgr.24961) 2020 : cluster [DBG] pgmap v2377: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:24.003 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:23.745Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:24.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:24 smithi059 bash[61152]: cluster 2022-04-28T20:49:22.510599+0000 mgr.y (mgr.24961) 2020 : cluster [DBG] pgmap v2377: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:24 smithi139 bash[43301]: cluster 2022-04-28T20:49:22.510599+0000 mgr.y (mgr.24961) 2020 : cluster [DBG] pgmap v2377: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:26 smithi059 bash[61152]: cluster 2022-04-28T20:49:24.511614+0000 mgr.y (mgr.24961) 2021 : cluster [DBG] pgmap v2378: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:26 smithi059 bash[59745]: cluster 2022-04-28T20:49:24.511614+0000 mgr.y (mgr.24961) 2021 : cluster [DBG] pgmap v2378: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:26 smithi139 bash[43301]: cluster 2022-04-28T20:49:24.511614+0000 mgr.y (mgr.24961) 2021 : cluster [DBG] pgmap v2378: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:28 smithi059 bash[61152]: cluster 2022-04-28T20:49:26.512734+0000 mgr.y (mgr.24961) 2022 : cluster [DBG] pgmap v2379: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:28 smithi059 bash[59745]: cluster 2022-04-28T20:49:26.512734+0000 mgr.y (mgr.24961) 2022 : cluster [DBG] pgmap v2379: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:28 smithi139 bash[43301]: cluster 2022-04-28T20:49:26.512734+0000 mgr.y (mgr.24961) 2022 : cluster [DBG] pgmap v2379: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:30 smithi059 bash[61152]: cluster 2022-04-28T20:49:28.513428+0000 mgr.y (mgr.24961) 2023 : cluster [DBG] pgmap v2380: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:30 smithi059 bash[59745]: cluster 2022-04-28T20:49:28.513428+0000 mgr.y (mgr.24961) 2023 : cluster [DBG] pgmap v2380: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:30 smithi139 bash[43301]: cluster 2022-04-28T20:49:28.513428+0000 mgr.y (mgr.24961) 2023 : cluster [DBG] pgmap v2380: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:32 smithi059 bash[61152]: cluster 2022-04-28T20:49:30.514629+0000 mgr.y (mgr.24961) 2024 : cluster [DBG] pgmap v2381: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:32 smithi059 bash[59745]: cluster 2022-04-28T20:49:30.514629+0000 mgr.y (mgr.24961) 2024 : cluster [DBG] pgmap v2381: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:31.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:32 smithi139 bash[43301]: cluster 2022-04-28T20:49:30.514629+0000 mgr.y (mgr.24961) 2024 : cluster [DBG] pgmap v2381: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:33 smithi059 bash[61152]: cluster 2022-04-28T20:49:32.515528+0000 mgr.y (mgr.24961) 2025 : cluster [DBG] pgmap v2382: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:33 smithi059 bash[59745]: cluster 2022-04-28T20:49:32.515528+0000 mgr.y (mgr.24961) 2025 : cluster [DBG] pgmap v2382: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:33.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:33 smithi139 bash[43301]: cluster 2022-04-28T20:49:32.515528+0000 mgr.y (mgr.24961) 2025 : cluster [DBG] pgmap v2382: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:33.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:35 smithi059 bash[59745]: cluster 2022-04-28T20:49:34.516464+0000 mgr.y (mgr.24961) 2026 : cluster [DBG] pgmap v2383: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:35 smithi059 bash[61152]: cluster 2022-04-28T20:49:34.516464+0000 mgr.y (mgr.24961) 2026 : cluster [DBG] pgmap v2383: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:35 smithi139 bash[43301]: cluster 2022-04-28T20:49:34.516464+0000 mgr.y (mgr.24961) 2026 : cluster [DBG] pgmap v2383: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:37 smithi059 bash[61152]: cluster 2022-04-28T20:49:36.517570+0000 mgr.y (mgr.24961) 2027 : cluster [DBG] pgmap v2384: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:37 smithi059 bash[59745]: cluster 2022-04-28T20:49:36.517570+0000 mgr.y (mgr.24961) 2027 : cluster [DBG] pgmap v2384: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:37 smithi139 bash[43301]: cluster 2022-04-28T20:49:36.517570+0000 mgr.y (mgr.24961) 2027 : cluster [DBG] pgmap v2384: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:39 smithi059 bash[61152]: cluster 2022-04-28T20:49:38.518267+0000 mgr.y (mgr.24961) 2028 : cluster [DBG] pgmap v2385: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:39 smithi059 bash[59745]: cluster 2022-04-28T20:49:38.518267+0000 mgr.y (mgr.24961) 2028 : cluster [DBG] pgmap v2385: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:39 smithi139 bash[43301]: cluster 2022-04-28T20:49:38.518267+0000 mgr.y (mgr.24961) 2028 : cluster [DBG] pgmap v2385: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:41 smithi059 bash[61152]: cluster 2022-04-28T20:49:40.519387+0000 mgr.y (mgr.24961) 2029 : cluster [DBG] pgmap v2386: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:41.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:41 smithi059 bash[59745]: cluster 2022-04-28T20:49:40.519387+0000 mgr.y (mgr.24961) 2029 : cluster [DBG] pgmap v2386: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:41 smithi139 bash[43301]: cluster 2022-04-28T20:49:40.519387+0000 mgr.y (mgr.24961) 2029 : cluster [DBG] pgmap v2386: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:43 smithi059 bash[61152]: cluster 2022-04-28T20:49:42.520363+0000 mgr.y (mgr.24961) 2030 : cluster [DBG] pgmap v2387: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:43.747Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:43 smithi059 bash[59745]: cluster 2022-04-28T20:49:42.520363+0000 mgr.y (mgr.24961) 2030 : cluster [DBG] pgmap v2387: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:43 smithi139 bash[43301]: cluster 2022-04-28T20:49:42.520363+0000 mgr.y (mgr.24961) 2030 : cluster [DBG] pgmap v2387: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:45 smithi059 bash[61152]: cluster 2022-04-28T20:49:44.521367+0000 mgr.y (mgr.24961) 2031 : cluster [DBG] pgmap v2388: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:45 smithi059 bash[59745]: cluster 2022-04-28T20:49:44.521367+0000 mgr.y (mgr.24961) 2031 : cluster [DBG] pgmap v2388: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:45 smithi139 bash[43301]: cluster 2022-04-28T20:49:44.521367+0000 mgr.y (mgr.24961) 2031 : cluster [DBG] pgmap v2388: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:47 smithi059 bash[61152]: cluster 2022-04-28T20:49:46.522540+0000 mgr.y (mgr.24961) 2032 : cluster [DBG] pgmap v2389: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:47 smithi059 bash[59745]: cluster 2022-04-28T20:49:46.522540+0000 mgr.y (mgr.24961) 2032 : cluster [DBG] pgmap v2389: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:47 smithi139 bash[43301]: cluster 2022-04-28T20:49:46.522540+0000 mgr.y (mgr.24961) 2032 : cluster [DBG] pgmap v2389: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:49 smithi059 bash[61152]: cluster 2022-04-28T20:49:48.523271+0000 mgr.y (mgr.24961) 2033 : cluster [DBG] pgmap v2390: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:49 smithi059 bash[59745]: cluster 2022-04-28T20:49:48.523271+0000 mgr.y (mgr.24961) 2033 : cluster [DBG] pgmap v2390: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:49 smithi139 bash[43301]: cluster 2022-04-28T20:49:48.523271+0000 mgr.y (mgr.24961) 2033 : cluster [DBG] pgmap v2390: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:51 smithi059 bash[61152]: cluster 2022-04-28T20:49:50.524509+0000 mgr.y (mgr.24961) 2034 : cluster [DBG] pgmap v2391: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:51.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:51 smithi059 bash[59745]: cluster 2022-04-28T20:49:50.524509+0000 mgr.y (mgr.24961) 2034 : cluster [DBG] pgmap v2391: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:51 smithi139 bash[43301]: cluster 2022-04-28T20:49:50.524509+0000 mgr.y (mgr.24961) 2034 : cluster [DBG] pgmap v2391: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:53 smithi059 bash[61152]: cluster 2022-04-28T20:49:52.525504+0000 mgr.y (mgr.24961) 2035 : cluster [DBG] pgmap v2392: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:49:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:49:53.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:49:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:53 smithi059 bash[59745]: cluster 2022-04-28T20:49:52.525504+0000 mgr.y (mgr.24961) 2035 : cluster [DBG] pgmap v2392: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:53 smithi139 bash[43301]: cluster 2022-04-28T20:49:52.525504+0000 mgr.y (mgr.24961) 2035 : cluster [DBG] pgmap v2392: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:56.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:55 smithi059 bash[59745]: cluster 2022-04-28T20:49:54.526560+0000 mgr.y (mgr.24961) 2036 : cluster [DBG] pgmap v2393: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:55 smithi059 bash[61152]: cluster 2022-04-28T20:49:54.526560+0000 mgr.y (mgr.24961) 2036 : cluster [DBG] pgmap v2393: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:55 smithi139 bash[43301]: cluster 2022-04-28T20:49:54.526560+0000 mgr.y (mgr.24961) 2036 : cluster [DBG] pgmap v2393: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:49:57 smithi059 bash[59745]: cluster 2022-04-28T20:49:56.527632+0000 mgr.y (mgr.24961) 2037 : cluster [DBG] pgmap v2394: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:49:57 smithi059 bash[61152]: cluster 2022-04-28T20:49:56.527632+0000 mgr.y (mgr.24961) 2037 : cluster [DBG] pgmap v2394: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:49:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:49:57 smithi139 bash[43301]: cluster 2022-04-28T20:49:56.527632+0000 mgr.y (mgr.24961) 2037 : cluster [DBG] pgmap v2394: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:00 smithi059 bash[61152]: cluster 2022-04-28T20:49:58.528323+0000 mgr.y (mgr.24961) 2038 : cluster [DBG] pgmap v2395: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:00 smithi059 bash[59745]: cluster 2022-04-28T20:49:58.528323+0000 mgr.y (mgr.24961) 2038 : cluster [DBG] pgmap v2395: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:00 smithi139 bash[43301]: cluster 2022-04-28T20:49:58.528323+0000 mgr.y (mgr.24961) 2038 : cluster [DBG] pgmap v2395: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[61152]: cluster 2022-04-28T20:50:00.000138+0000 mon.a (mon.0) 2252 : cluster [INF] overall HEALTH_OK 2022-04-28T20:50:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[61152]: audit 2022-04-28T20:50:00.141382+0000 mon.a (mon.0) 2253 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:50:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[61152]: audit 2022-04-28T20:50:00.144029+0000 mon.a (mon.0) 2254 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:50:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[61152]: audit 2022-04-28T20:50:00.145776+0000 mon.a (mon.0) 2255 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:50:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[61152]: audit 2022-04-28T20:50:00.466138+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:50:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[59745]: cluster 2022-04-28T20:50:00.000138+0000 mon.a (mon.0) 2252 : cluster [INF] overall HEALTH_OK 2022-04-28T20:50:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[59745]: audit 2022-04-28T20:50:00.141382+0000 mon.a (mon.0) 2253 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:50:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[59745]: audit 2022-04-28T20:50:00.144029+0000 mon.a (mon.0) 2254 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:50:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[59745]: audit 2022-04-28T20:50:00.145776+0000 mon.a (mon.0) 2255 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:50:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[59745]: audit 2022-04-28T20:50:00.466138+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:50:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:01 smithi139 bash[43301]: cluster 2022-04-28T20:50:00.000138+0000 mon.a (mon.0) 2252 : cluster [INF] overall HEALTH_OK 2022-04-28T20:50:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:01 smithi139 bash[43301]: audit 2022-04-28T20:50:00.141382+0000 mon.a (mon.0) 2253 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:50:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:01 smithi139 bash[43301]: audit 2022-04-28T20:50:00.144029+0000 mon.a (mon.0) 2254 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:50:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:01 smithi139 bash[43301]: audit 2022-04-28T20:50:00.145776+0000 mon.a (mon.0) 2255 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:50:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:01 smithi139 bash[43301]: audit 2022-04-28T20:50:00.466138+0000 mon.a (mon.0) 2256 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:50:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:01.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:02 smithi059 bash[61152]: cephadm 2022-04-28T20:50:00.147198+0000 mgr.y (mgr.24961) 2039 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:50:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:02 smithi059 bash[61152]: cluster 2022-04-28T20:50:00.529520+0000 mgr.y (mgr.24961) 2040 : cluster [DBG] pgmap v2396: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:02 smithi059 bash[59745]: cephadm 2022-04-28T20:50:00.147198+0000 mgr.y (mgr.24961) 2039 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:50:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:02 smithi059 bash[59745]: cluster 2022-04-28T20:50:00.529520+0000 mgr.y (mgr.24961) 2040 : cluster [DBG] pgmap v2396: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:02 smithi139 bash[43301]: cephadm 2022-04-28T20:50:00.147198+0000 mgr.y (mgr.24961) 2039 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:50:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:02 smithi139 bash[43301]: cluster 2022-04-28T20:50:00.529520+0000 mgr.y (mgr.24961) 2040 : cluster [DBG] pgmap v2396: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:04.014 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:03.748Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:04.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:04 smithi059 bash[61152]: cluster 2022-04-28T20:50:02.530852+0000 mgr.y (mgr.24961) 2041 : cluster [DBG] pgmap v2397: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:04 smithi059 bash[59745]: cluster 2022-04-28T20:50:02.530852+0000 mgr.y (mgr.24961) 2041 : cluster [DBG] pgmap v2397: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:04 smithi139 bash[43301]: cluster 2022-04-28T20:50:02.530852+0000 mgr.y (mgr.24961) 2041 : cluster [DBG] pgmap v2397: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:06 smithi059 bash[61152]: cluster 2022-04-28T20:50:04.531958+0000 mgr.y (mgr.24961) 2042 : cluster [DBG] pgmap v2398: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:06 smithi059 bash[59745]: cluster 2022-04-28T20:50:04.531958+0000 mgr.y (mgr.24961) 2042 : cluster [DBG] pgmap v2398: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:06 smithi139 bash[43301]: cluster 2022-04-28T20:50:04.531958+0000 mgr.y (mgr.24961) 2042 : cluster [DBG] pgmap v2398: 105 pgs: 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[59745]: cluster 2022-04-28T20:50:06.029078+0000 mon.a (mon.0) 2257 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2022-04-28T20:50:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[59745]: audit 2022-04-28T20:50:06.032415+0000 mon.a (mon.0) 2258 : audit [INF] from='client.? 172.21.15.59:0/533921532' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78828-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:50:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[59745]: cluster 2022-04-28T20:50:06.532790+0000 mgr.y (mgr.24961) 2043 : cluster [DBG] pgmap v2400: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[59745]: audit 2022-04-28T20:50:07.033781+0000 mon.a (mon.0) 2259 : audit [INF] from='client.? 172.21.15.59:0/533921532' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78828-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:50:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[59745]: cluster 2022-04-28T20:50:07.034050+0000 mon.a (mon.0) 2260 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2022-04-28T20:50:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[61152]: cluster 2022-04-28T20:50:06.029078+0000 mon.a (mon.0) 2257 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2022-04-28T20:50:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[61152]: audit 2022-04-28T20:50:06.032415+0000 mon.a (mon.0) 2258 : audit [INF] from='client.? 172.21.15.59:0/533921532' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78828-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:50:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[61152]: cluster 2022-04-28T20:50:06.532790+0000 mgr.y (mgr.24961) 2043 : cluster [DBG] pgmap v2400: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[61152]: audit 2022-04-28T20:50:07.033781+0000 mon.a (mon.0) 2259 : audit [INF] from='client.? 172.21.15.59:0/533921532' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78828-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:50:07.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:07 smithi059 bash[61152]: cluster 2022-04-28T20:50:07.034050+0000 mon.a (mon.0) 2260 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2022-04-28T20:50:07.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:07 smithi139 bash[43301]: cluster 2022-04-28T20:50:06.029078+0000 mon.a (mon.0) 2257 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2022-04-28T20:50:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:07 smithi139 bash[43301]: audit 2022-04-28T20:50:06.032415+0000 mon.a (mon.0) 2258 : audit [INF] from='client.? 172.21.15.59:0/533921532' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78828-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:50:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:07 smithi139 bash[43301]: cluster 2022-04-28T20:50:06.532790+0000 mgr.y (mgr.24961) 2043 : cluster [DBG] pgmap v2400: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:07 smithi139 bash[43301]: audit 2022-04-28T20:50:07.033781+0000 mon.a (mon.0) 2259 : audit [INF] from='client.? 172.21.15.59:0/533921532' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-78828-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:50:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:07 smithi139 bash[43301]: cluster 2022-04-28T20:50:07.034050+0000 mon.a (mon.0) 2260 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2022-04-28T20:50:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:09 smithi059 bash[61152]: cluster 2022-04-28T20:50:08.051239+0000 mon.a (mon.0) 2261 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2022-04-28T20:50:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:09 smithi059 bash[61152]: cluster 2022-04-28T20:50:08.533540+0000 mgr.y (mgr.24961) 2044 : cluster [DBG] pgmap v2403: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:09 smithi059 bash[59745]: cluster 2022-04-28T20:50:08.051239+0000 mon.a (mon.0) 2261 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2022-04-28T20:50:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:09 smithi059 bash[59745]: cluster 2022-04-28T20:50:08.533540+0000 mgr.y (mgr.24961) 2044 : cluster [DBG] pgmap v2403: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:09 smithi139 bash[43301]: cluster 2022-04-28T20:50:08.051239+0000 mon.a (mon.0) 2261 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2022-04-28T20:50:09.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:09 smithi139 bash[43301]: cluster 2022-04-28T20:50:08.533540+0000 mgr.y (mgr.24961) 2044 : cluster [DBG] pgmap v2403: 137 pgs: 32 unknown, 105 active+clean; 24 KiB data, 725 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:11.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:11 smithi059 bash[61152]: cluster 2022-04-28T20:50:10.534840+0000 mgr.y (mgr.24961) 2045 : cluster [DBG] pgmap v2404: 137 pgs: 10 unknown, 127 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:11.856 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:11 smithi059 bash[59745]: cluster 2022-04-28T20:50:10.534840+0000 mgr.y (mgr.24961) 2045 : cluster [DBG] pgmap v2404: 137 pgs: 10 unknown, 127 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:11.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:11 smithi139 bash[43301]: cluster 2022-04-28T20:50:10.534840+0000 mgr.y (mgr.24961) 2045 : cluster [DBG] pgmap v2404: 137 pgs: 10 unknown, 127 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:11.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:13.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:13 smithi139 bash[43301]: cluster 2022-04-28T20:50:12.535904+0000 mgr.y (mgr.24961) 2046 : cluster [DBG] pgmap v2405: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:13 smithi059 bash[61152]: cluster 2022-04-28T20:50:12.535904+0000 mgr.y (mgr.24961) 2046 : cluster [DBG] pgmap v2405: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:13.749Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:13 smithi059 bash[59745]: cluster 2022-04-28T20:50:12.535904+0000 mgr.y (mgr.24961) 2046 : cluster [DBG] pgmap v2405: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:15 smithi059 bash[61152]: cluster 2022-04-28T20:50:14.537029+0000 mgr.y (mgr.24961) 2047 : cluster [DBG] pgmap v2406: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:15 smithi059 bash[59745]: cluster 2022-04-28T20:50:14.537029+0000 mgr.y (mgr.24961) 2047 : cluster [DBG] pgmap v2406: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:15 smithi139 bash[43301]: cluster 2022-04-28T20:50:14.537029+0000 mgr.y (mgr.24961) 2047 : cluster [DBG] pgmap v2406: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[59745]: cluster 2022-04-28T20:50:16.538242+0000 mgr.y (mgr.24961) 2048 : cluster [DBG] pgmap v2407: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[59745]: audit 2022-04-28T20:50:17.844529+0000 mon.a (mon.0) 2262 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[59745]: audit 2022-04-28T20:50:17.851145+0000 mon.a (mon.0) 2263 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[59745]: audit 2022-04-28T20:50:17.856293+0000 mon.a (mon.0) 2264 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[59745]: audit 2022-04-28T20:50:17.856789+0000 mon.a (mon.0) 2265 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:50:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[59745]: audit 2022-04-28T20:50:17.857257+0000 mon.a (mon.0) 2266 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:50:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[61152]: cluster 2022-04-28T20:50:16.538242+0000 mgr.y (mgr.24961) 2048 : cluster [DBG] pgmap v2407: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[61152]: audit 2022-04-28T20:50:17.844529+0000 mon.a (mon.0) 2262 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:50:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[61152]: audit 2022-04-28T20:50:17.851145+0000 mon.a (mon.0) 2263 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:50:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[61152]: audit 2022-04-28T20:50:17.856293+0000 mon.a (mon.0) 2264 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:50:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[61152]: audit 2022-04-28T20:50:17.856789+0000 mon.a (mon.0) 2265 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:50:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:17 smithi059 bash[61152]: audit 2022-04-28T20:50:17.857257+0000 mon.a (mon.0) 2266 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:50:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:17 smithi139 bash[43301]: cluster 2022-04-28T20:50:16.538242+0000 mgr.y (mgr.24961) 2048 : cluster [DBG] pgmap v2407: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:17 smithi139 bash[43301]: audit 2022-04-28T20:50:17.844529+0000 mon.a (mon.0) 2262 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:50:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:17 smithi139 bash[43301]: audit 2022-04-28T20:50:17.851145+0000 mon.a (mon.0) 2263 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:50:18.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:17 smithi139 bash[43301]: audit 2022-04-28T20:50:17.856293+0000 mon.a (mon.0) 2264 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T20:50:18.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:17 smithi139 bash[43301]: audit 2022-04-28T20:50:17.856789+0000 mon.a (mon.0) 2265 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:50:18.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:17 smithi139 bash[43301]: audit 2022-04-28T20:50:17.857257+0000 mon.a (mon.0) 2266 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T20:50:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:19 smithi059 bash[61152]: cluster 2022-04-28T20:50:18.539018+0000 mgr.y (mgr.24961) 2049 : cluster [DBG] pgmap v2408: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:19 smithi059 bash[59745]: cluster 2022-04-28T20:50:18.539018+0000 mgr.y (mgr.24961) 2049 : cluster [DBG] pgmap v2408: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:19 smithi139 bash[43301]: cluster 2022-04-28T20:50:18.539018+0000 mgr.y (mgr.24961) 2049 : cluster [DBG] pgmap v2408: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:21 smithi059 bash[61152]: cluster 2022-04-28T20:50:20.540414+0000 mgr.y (mgr.24961) 2050 : cluster [DBG] pgmap v2409: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:21.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:21 smithi059 bash[59745]: cluster 2022-04-28T20:50:20.540414+0000 mgr.y (mgr.24961) 2050 : cluster [DBG] pgmap v2409: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:21 smithi139 bash[43301]: cluster 2022-04-28T20:50:20.540414+0000 mgr.y (mgr.24961) 2050 : cluster [DBG] pgmap v2409: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:23 smithi059 bash[61152]: cluster 2022-04-28T20:50:22.541441+0000 mgr.y (mgr.24961) 2051 : cluster [DBG] pgmap v2410: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:23.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:23 smithi059 bash[59745]: cluster 2022-04-28T20:50:22.541441+0000 mgr.y (mgr.24961) 2051 : cluster [DBG] pgmap v2410: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:23 smithi139 bash[43301]: cluster 2022-04-28T20:50:22.541441+0000 mgr.y (mgr.24961) 2051 : cluster [DBG] pgmap v2410: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:25 smithi059 bash[61152]: cluster 2022-04-28T20:50:24.542580+0000 mgr.y (mgr.24961) 2052 : cluster [DBG] pgmap v2411: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:25 smithi059 bash[59745]: cluster 2022-04-28T20:50:24.542580+0000 mgr.y (mgr.24961) 2052 : cluster [DBG] pgmap v2411: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:25 smithi139 bash[43301]: cluster 2022-04-28T20:50:24.542580+0000 mgr.y (mgr.24961) 2052 : cluster [DBG] pgmap v2411: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:27 smithi059 bash[59745]: cluster 2022-04-28T20:50:26.543865+0000 mgr.y (mgr.24961) 2053 : cluster [DBG] pgmap v2412: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:27 smithi059 bash[61152]: cluster 2022-04-28T20:50:26.543865+0000 mgr.y (mgr.24961) 2053 : cluster [DBG] pgmap v2412: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:27 smithi139 bash[43301]: cluster 2022-04-28T20:50:26.543865+0000 mgr.y (mgr.24961) 2053 : cluster [DBG] pgmap v2412: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:29 smithi059 bash[61152]: cluster 2022-04-28T20:50:28.544579+0000 mgr.y (mgr.24961) 2054 : cluster [DBG] pgmap v2413: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:29 smithi059 bash[59745]: cluster 2022-04-28T20:50:28.544579+0000 mgr.y (mgr.24961) 2054 : cluster [DBG] pgmap v2413: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:29 smithi139 bash[43301]: cluster 2022-04-28T20:50:28.544579+0000 mgr.y (mgr.24961) 2054 : cluster [DBG] pgmap v2413: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:31 smithi059 bash[61152]: cluster 2022-04-28T20:50:30.545958+0000 mgr.y (mgr.24961) 2055 : cluster [DBG] pgmap v2414: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:31.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:31 smithi059 bash[59745]: cluster 2022-04-28T20:50:30.545958+0000 mgr.y (mgr.24961) 2055 : cluster [DBG] pgmap v2414: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:31 smithi139 bash[43301]: cluster 2022-04-28T20:50:30.545958+0000 mgr.y (mgr.24961) 2055 : cluster [DBG] pgmap v2414: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:33 smithi059 bash[61152]: cluster 2022-04-28T20:50:32.546967+0000 mgr.y (mgr.24961) 2056 : cluster [DBG] pgmap v2415: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:33.750Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:33 smithi059 bash[59745]: cluster 2022-04-28T20:50:32.546967+0000 mgr.y (mgr.24961) 2056 : cluster [DBG] pgmap v2415: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:33 smithi139 bash[43301]: cluster 2022-04-28T20:50:32.546967+0000 mgr.y (mgr.24961) 2056 : cluster [DBG] pgmap v2415: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:36 smithi059 bash[61152]: cluster 2022-04-28T20:50:34.548099+0000 mgr.y (mgr.24961) 2057 : cluster [DBG] pgmap v2416: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:36 smithi059 bash[59745]: cluster 2022-04-28T20:50:34.548099+0000 mgr.y (mgr.24961) 2057 : cluster [DBG] pgmap v2416: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:36 smithi139 bash[43301]: cluster 2022-04-28T20:50:34.548099+0000 mgr.y (mgr.24961) 2057 : cluster [DBG] pgmap v2416: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:38 smithi059 bash[61152]: cluster 2022-04-28T20:50:36.549335+0000 mgr.y (mgr.24961) 2058 : cluster [DBG] pgmap v2417: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:38 smithi059 bash[59745]: cluster 2022-04-28T20:50:36.549335+0000 mgr.y (mgr.24961) 2058 : cluster [DBG] pgmap v2417: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:38 smithi139 bash[43301]: cluster 2022-04-28T20:50:36.549335+0000 mgr.y (mgr.24961) 2058 : cluster [DBG] pgmap v2417: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:40 smithi059 bash[61152]: cluster 2022-04-28T20:50:38.550025+0000 mgr.y (mgr.24961) 2059 : cluster [DBG] pgmap v2418: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:40 smithi059 bash[59745]: cluster 2022-04-28T20:50:38.550025+0000 mgr.y (mgr.24961) 2059 : cluster [DBG] pgmap v2418: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:40 smithi139 bash[43301]: cluster 2022-04-28T20:50:38.550025+0000 mgr.y (mgr.24961) 2059 : cluster [DBG] pgmap v2418: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:42 smithi059 bash[59745]: cluster 2022-04-28T20:50:40.551400+0000 mgr.y (mgr.24961) 2060 : cluster [DBG] pgmap v2419: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:42 smithi059 bash[61152]: cluster 2022-04-28T20:50:40.551400+0000 mgr.y (mgr.24961) 2060 : cluster [DBG] pgmap v2419: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:41.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:42 smithi139 bash[43301]: cluster 2022-04-28T20:50:40.551400+0000 mgr.y (mgr.24961) 2060 : cluster [DBG] pgmap v2419: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:44.021 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:43.751Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:44 smithi059 bash[61152]: cluster 2022-04-28T20:50:42.552465+0000 mgr.y (mgr.24961) 2061 : cluster [DBG] pgmap v2420: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:44 smithi059 bash[59745]: cluster 2022-04-28T20:50:42.552465+0000 mgr.y (mgr.24961) 2061 : cluster [DBG] pgmap v2420: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:44 smithi139 bash[43301]: cluster 2022-04-28T20:50:42.552465+0000 mgr.y (mgr.24961) 2061 : cluster [DBG] pgmap v2420: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:46 smithi059 bash[61152]: cluster 2022-04-28T20:50:44.553587+0000 mgr.y (mgr.24961) 2062 : cluster [DBG] pgmap v2421: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:46 smithi059 bash[59745]: cluster 2022-04-28T20:50:44.553587+0000 mgr.y (mgr.24961) 2062 : cluster [DBG] pgmap v2421: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:46 smithi139 bash[43301]: cluster 2022-04-28T20:50:44.553587+0000 mgr.y (mgr.24961) 2062 : cluster [DBG] pgmap v2421: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:48 smithi059 bash[61152]: cluster 2022-04-28T20:50:46.554858+0000 mgr.y (mgr.24961) 2063 : cluster [DBG] pgmap v2422: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:48 smithi059 bash[59745]: cluster 2022-04-28T20:50:46.554858+0000 mgr.y (mgr.24961) 2063 : cluster [DBG] pgmap v2422: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:48 smithi139 bash[43301]: cluster 2022-04-28T20:50:46.554858+0000 mgr.y (mgr.24961) 2063 : cluster [DBG] pgmap v2422: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:50 smithi059 bash[59745]: cluster 2022-04-28T20:50:48.555591+0000 mgr.y (mgr.24961) 2064 : cluster [DBG] pgmap v2423: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:50 smithi059 bash[61152]: cluster 2022-04-28T20:50:48.555591+0000 mgr.y (mgr.24961) 2064 : cluster [DBG] pgmap v2423: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:50 smithi139 bash[43301]: cluster 2022-04-28T20:50:48.555591+0000 mgr.y (mgr.24961) 2064 : cluster [DBG] pgmap v2423: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:51.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:51 smithi139 bash[43301]: cluster 2022-04-28T20:50:50.557045+0000 mgr.y (mgr.24961) 2065 : cluster [DBG] pgmap v2424: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:51 smithi059 bash[61152]: cluster 2022-04-28T20:50:50.557045+0000 mgr.y (mgr.24961) 2065 : cluster [DBG] pgmap v2424: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:51 smithi059 bash[59745]: cluster 2022-04-28T20:50:50.557045+0000 mgr.y (mgr.24961) 2065 : cluster [DBG] pgmap v2424: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:51.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:53.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:53 smithi139 bash[43301]: cluster 2022-04-28T20:50:52.558111+0000 mgr.y (mgr.24961) 2066 : cluster [DBG] pgmap v2425: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:53 smithi059 bash[61152]: cluster 2022-04-28T20:50:52.558111+0000 mgr.y (mgr.24961) 2066 : cluster [DBG] pgmap v2425: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:50:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:50:53.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:50:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:53 smithi059 bash[59745]: cluster 2022-04-28T20:50:52.558111+0000 mgr.y (mgr.24961) 2066 : cluster [DBG] pgmap v2425: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:55 smithi059 bash[61152]: cluster 2022-04-28T20:50:54.559212+0000 mgr.y (mgr.24961) 2067 : cluster [DBG] pgmap v2426: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:55 smithi059 bash[59745]: cluster 2022-04-28T20:50:54.559212+0000 mgr.y (mgr.24961) 2067 : cluster [DBG] pgmap v2426: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:55 smithi139 bash[43301]: cluster 2022-04-28T20:50:54.559212+0000 mgr.y (mgr.24961) 2067 : cluster [DBG] pgmap v2426: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:57 smithi059 bash[61152]: cluster 2022-04-28T20:50:56.560504+0000 mgr.y (mgr.24961) 2068 : cluster [DBG] pgmap v2427: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:57 smithi059 bash[59745]: cluster 2022-04-28T20:50:56.560504+0000 mgr.y (mgr.24961) 2068 : cluster [DBG] pgmap v2427: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:50:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:57 smithi139 bash[43301]: cluster 2022-04-28T20:50:56.560504+0000 mgr.y (mgr.24961) 2068 : cluster [DBG] pgmap v2427: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:50:59 smithi059 bash[59745]: cluster 2022-04-28T20:50:58.561234+0000 mgr.y (mgr.24961) 2069 : cluster [DBG] pgmap v2428: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:50:59 smithi059 bash[61152]: cluster 2022-04-28T20:50:58.561234+0000 mgr.y (mgr.24961) 2069 : cluster [DBG] pgmap v2428: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:50:59 smithi139 bash[43301]: cluster 2022-04-28T20:50:58.561234+0000 mgr.y (mgr.24961) 2069 : cluster [DBG] pgmap v2428: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.473829+0000 mon.a (mon.0) 2267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:51:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.476768+0000 mon.a (mon.0) 2268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:51:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.478644+0000 mon.a (mon.0) 2269 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:51:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.771034+0000 mon.a (mon.0) 2270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.771951+0000 mon.a (mon.0) 2271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.772748+0000 mon.a (mon.0) 2272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.773492+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.774273+0000 mon.a (mon.0) 2274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.775029+0000 mon.a (mon.0) 2275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.775808+0000 mon.a (mon.0) 2276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.776561+0000 mon.a (mon.0) 2277 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.777364+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:51:01.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.778206+0000 mon.a (mon.0) 2279 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.808239+0000 mon.a (mon.0) 2280 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.809173+0000 mon.a (mon.0) 2281 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.810022+0000 mon.a (mon.0) 2282 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.473829+0000 mon.a (mon.0) 2267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:51:01.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.476768+0000 mon.a (mon.0) 2268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:51:01.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.478644+0000 mon.a (mon.0) 2269 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:51:01.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.771034+0000 mon.a (mon.0) 2270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.771951+0000 mon.a (mon.0) 2271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.772748+0000 mon.a (mon.0) 2272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.773492+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.774273+0000 mon.a (mon.0) 2274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.775029+0000 mon.a (mon.0) 2275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.775808+0000 mon.a (mon.0) 2276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.776561+0000 mon.a (mon.0) 2277 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.777364+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:51:01.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.778206+0000 mon.a (mon.0) 2279 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.808239+0000 mon.a (mon.0) 2280 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.809173+0000 mon.a (mon.0) 2281 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.810022+0000 mon.a (mon.0) 2282 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.810914+0000 mon.a (mon.0) 2283 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.811810+0000 mon.a (mon.0) 2284 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.812600+0000 mon.a (mon.0) 2285 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.813372+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.814129+0000 mon.a (mon.0) 2287 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.814892+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:51:01.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.815692+0000 mon.a (mon.0) 2289 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[59745]: audit 2022-04-28T20:51:00.823321+0000 mon.a (mon.0) 2290 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:51:01.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.810914+0000 mon.a (mon.0) 2283 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.811810+0000 mon.a (mon.0) 2284 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.812600+0000 mon.a (mon.0) 2285 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.813372+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.814129+0000 mon.a (mon.0) 2287 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.814892+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:51:01.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.815692+0000 mon.a (mon.0) 2289 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:00 smithi059 bash[61152]: audit 2022-04-28T20:51:00.823321+0000 mon.a (mon.0) 2290 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:51:01.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.473829+0000 mon.a (mon.0) 2267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:51:01.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.476768+0000 mon.a (mon.0) 2268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:51:01.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.478644+0000 mon.a (mon.0) 2269 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:51:01.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.771034+0000 mon.a (mon.0) 2270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.771951+0000 mon.a (mon.0) 2271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.772748+0000 mon.a (mon.0) 2272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.773492+0000 mon.a (mon.0) 2273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.774273+0000 mon.a (mon.0) 2274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.775029+0000 mon.a (mon.0) 2275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.775808+0000 mon.a (mon.0) 2276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.776561+0000 mon.a (mon.0) 2277 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.777364+0000 mon.a (mon.0) 2278 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:51:01.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.778206+0000 mon.a (mon.0) 2279 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.808239+0000 mon.a (mon.0) 2280 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.809173+0000 mon.a (mon.0) 2281 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.810022+0000 mon.a (mon.0) 2282 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.810914+0000 mon.a (mon.0) 2283 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.811810+0000 mon.a (mon.0) 2284 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.812600+0000 mon.a (mon.0) 2285 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.813372+0000 mon.a (mon.0) 2286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T20:51:01.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.814129+0000 mon.a (mon.0) 2287 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.814892+0000 mon.a (mon.0) 2288 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T20:51:01.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.815692+0000 mon.a (mon.0) 2289 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T20:51:01.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:00 smithi139 bash[43301]: audit 2022-04-28T20:51:00.823321+0000 mon.a (mon.0) 2290 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:51:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:01 smithi059 bash[59745]: cephadm 2022-04-28T20:51:00.480447+0000 mgr.y (mgr.24961) 2070 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:51:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:01 smithi059 bash[59745]: cluster 2022-04-28T20:51:00.562552+0000 mgr.y (mgr.24961) 2071 : cluster [DBG] pgmap v2429: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:01 smithi059 bash[61152]: cephadm 2022-04-28T20:51:00.480447+0000 mgr.y (mgr.24961) 2070 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:51:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:01 smithi059 bash[61152]: cluster 2022-04-28T20:51:00.562552+0000 mgr.y (mgr.24961) 2071 : cluster [DBG] pgmap v2429: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:01.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:01 smithi139 bash[43301]: cephadm 2022-04-28T20:51:00.480447+0000 mgr.y (mgr.24961) 2070 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:51:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:01 smithi139 bash[43301]: cluster 2022-04-28T20:51:00.562552+0000 mgr.y (mgr.24961) 2071 : cluster [DBG] pgmap v2429: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:03 smithi059 bash[59745]: cluster 2022-04-28T20:51:02.563599+0000 mgr.y (mgr.24961) 2072 : cluster [DBG] pgmap v2430: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:03 smithi059 bash[61152]: cluster 2022-04-28T20:51:02.563599+0000 mgr.y (mgr.24961) 2072 : cluster [DBG] pgmap v2430: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:03.752Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:03 smithi139 bash[43301]: cluster 2022-04-28T20:51:02.563599+0000 mgr.y (mgr.24961) 2072 : cluster [DBG] pgmap v2430: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:05 smithi059 bash[59745]: audit 2022-04-28T20:51:04.954162+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.59:0/2699668779' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:51:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:05 smithi059 bash[61152]: audit 2022-04-28T20:51:04.954162+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.59:0/2699668779' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:51:05.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:05 smithi139 bash[43301]: audit 2022-04-28T20:51:04.954162+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.59:0/2699668779' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T20:51:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:06 smithi059 bash[61152]: cluster 2022-04-28T20:51:04.564740+0000 mgr.y (mgr.24961) 2073 : cluster [DBG] pgmap v2431: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:06 smithi059 bash[59745]: cluster 2022-04-28T20:51:04.564740+0000 mgr.y (mgr.24961) 2073 : cluster [DBG] pgmap v2431: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:06 smithi139 bash[43301]: cluster 2022-04-28T20:51:04.564740+0000 mgr.y (mgr.24961) 2073 : cluster [DBG] pgmap v2431: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:08 smithi059 bash[61152]: cluster 2022-04-28T20:51:06.566070+0000 mgr.y (mgr.24961) 2074 : cluster [DBG] pgmap v2432: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:08 smithi059 bash[59745]: cluster 2022-04-28T20:51:06.566070+0000 mgr.y (mgr.24961) 2074 : cluster [DBG] pgmap v2432: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:08 smithi139 bash[43301]: cluster 2022-04-28T20:51:06.566070+0000 mgr.y (mgr.24961) 2074 : cluster [DBG] pgmap v2432: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:10 smithi059 bash[61152]: cluster 2022-04-28T20:51:08.566795+0000 mgr.y (mgr.24961) 2075 : cluster [DBG] pgmap v2433: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:10 smithi059 bash[59745]: cluster 2022-04-28T20:51:08.566795+0000 mgr.y (mgr.24961) 2075 : cluster [DBG] pgmap v2433: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:10 smithi139 bash[43301]: cluster 2022-04-28T20:51:08.566795+0000 mgr.y (mgr.24961) 2075 : cluster [DBG] pgmap v2433: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:12 smithi059 bash[59745]: cluster 2022-04-28T20:51:10.568163+0000 mgr.y (mgr.24961) 2076 : cluster [DBG] pgmap v2434: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:12 smithi059 bash[61152]: cluster 2022-04-28T20:51:10.568163+0000 mgr.y (mgr.24961) 2076 : cluster [DBG] pgmap v2434: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:11.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:12 smithi139 bash[43301]: cluster 2022-04-28T20:51:10.568163+0000 mgr.y (mgr.24961) 2076 : cluster [DBG] pgmap v2434: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:14.032 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:13.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:14.032 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:14 smithi059 bash[59745]: cluster 2022-04-28T20:51:12.569197+0000 mgr.y (mgr.24961) 2077 : cluster [DBG] pgmap v2435: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:14 smithi059 bash[61152]: cluster 2022-04-28T20:51:12.569197+0000 mgr.y (mgr.24961) 2077 : cluster [DBG] pgmap v2435: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:14 smithi139 bash[43301]: cluster 2022-04-28T20:51:12.569197+0000 mgr.y (mgr.24961) 2077 : cluster [DBG] pgmap v2435: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:16 smithi059 bash[59745]: cluster 2022-04-28T20:51:14.570317+0000 mgr.y (mgr.24961) 2078 : cluster [DBG] pgmap v2436: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:16 smithi059 bash[61152]: cluster 2022-04-28T20:51:14.570317+0000 mgr.y (mgr.24961) 2078 : cluster [DBG] pgmap v2436: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:16 smithi139 bash[43301]: cluster 2022-04-28T20:51:14.570317+0000 mgr.y (mgr.24961) 2078 : cluster [DBG] pgmap v2436: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[61152]: cluster 2022-04-28T20:51:16.571654+0000 mgr.y (mgr.24961) 2079 : cluster [DBG] pgmap v2437: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[61152]: audit 2022-04-28T20:51:17.866355+0000 mon.a (mon.0) 2291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:51:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[61152]: audit 2022-04-28T20:51:17.867046+0000 mon.a (mon.0) 2292 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[61152]: audit 2022-04-28T20:51:17.878244+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[61152]: audit 2022-04-28T20:51:17.878779+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T20:51:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[61152]: audit 2022-04-28T20:51:17.879310+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:51:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[59745]: cluster 2022-04-28T20:51:16.571654+0000 mgr.y (mgr.24961) 2079 : cluster [DBG] pgmap v2437: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[59745]: audit 2022-04-28T20:51:17.866355+0000 mon.a (mon.0) 2291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:51:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[59745]: audit 2022-04-28T20:51:17.867046+0000 mon.a (mon.0) 2292 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:51:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[59745]: audit 2022-04-28T20:51:17.878244+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:51:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[59745]: audit 2022-04-28T20:51:17.878779+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T20:51:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:18 smithi059 bash[59745]: audit 2022-04-28T20:51:17.879310+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:51:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:18 smithi139 bash[43301]: cluster 2022-04-28T20:51:16.571654+0000 mgr.y (mgr.24961) 2079 : cluster [DBG] pgmap v2437: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:18 smithi139 bash[43301]: audit 2022-04-28T20:51:17.866355+0000 mon.a (mon.0) 2291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:51:18.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:18 smithi139 bash[43301]: audit 2022-04-28T20:51:17.867046+0000 mon.a (mon.0) 2292 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:51:18.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:18 smithi139 bash[43301]: audit 2022-04-28T20:51:17.878244+0000 mon.a (mon.0) 2293 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:51:18.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:18 smithi139 bash[43301]: audit 2022-04-28T20:51:17.878779+0000 mon.a (mon.0) 2294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T20:51:18.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:18 smithi139 bash[43301]: audit 2022-04-28T20:51:17.879310+0000 mon.a (mon.0) 2295 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:51:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:20 smithi059 bash[61152]: cluster 2022-04-28T20:51:18.572390+0000 mgr.y (mgr.24961) 2080 : cluster [DBG] pgmap v2438: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:20 smithi059 bash[59745]: cluster 2022-04-28T20:51:18.572390+0000 mgr.y (mgr.24961) 2080 : cluster [DBG] pgmap v2438: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:20 smithi139 bash[43301]: cluster 2022-04-28T20:51:18.572390+0000 mgr.y (mgr.24961) 2080 : cluster [DBG] pgmap v2438: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:21 smithi059 bash[61152]: cluster 2022-04-28T20:51:20.573779+0000 mgr.y (mgr.24961) 2081 : cluster [DBG] pgmap v2439: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:21 smithi059 bash[59745]: cluster 2022-04-28T20:51:20.573779+0000 mgr.y (mgr.24961) 2081 : cluster [DBG] pgmap v2439: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:21 smithi139 bash[43301]: cluster 2022-04-28T20:51:20.573779+0000 mgr.y (mgr.24961) 2081 : cluster [DBG] pgmap v2439: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:21.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:23 smithi059 bash[61152]: cluster 2022-04-28T20:51:22.574806+0000 mgr.y (mgr.24961) 2082 : cluster [DBG] pgmap v2440: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:23.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:23 smithi059 bash[59745]: cluster 2022-04-28T20:51:22.574806+0000 mgr.y (mgr.24961) 2082 : cluster [DBG] pgmap v2440: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:23 smithi139 bash[43301]: cluster 2022-04-28T20:51:22.574806+0000 mgr.y (mgr.24961) 2082 : cluster [DBG] pgmap v2440: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:25 smithi059 bash[61152]: cluster 2022-04-28T20:51:24.575937+0000 mgr.y (mgr.24961) 2083 : cluster [DBG] pgmap v2441: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:25 smithi059 bash[59745]: cluster 2022-04-28T20:51:24.575937+0000 mgr.y (mgr.24961) 2083 : cluster [DBG] pgmap v2441: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:25 smithi139 bash[43301]: cluster 2022-04-28T20:51:24.575937+0000 mgr.y (mgr.24961) 2083 : cluster [DBG] pgmap v2441: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:27 smithi059 bash[61152]: cluster 2022-04-28T20:51:26.577225+0000 mgr.y (mgr.24961) 2084 : cluster [DBG] pgmap v2442: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:27 smithi059 bash[59745]: cluster 2022-04-28T20:51:26.577225+0000 mgr.y (mgr.24961) 2084 : cluster [DBG] pgmap v2442: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:27 smithi139 bash[43301]: cluster 2022-04-28T20:51:26.577225+0000 mgr.y (mgr.24961) 2084 : cluster [DBG] pgmap v2442: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:29 smithi059 bash[61152]: cluster 2022-04-28T20:51:28.577966+0000 mgr.y (mgr.24961) 2085 : cluster [DBG] pgmap v2443: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:29 smithi059 bash[59745]: cluster 2022-04-28T20:51:28.577966+0000 mgr.y (mgr.24961) 2085 : cluster [DBG] pgmap v2443: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:29 smithi139 bash[43301]: cluster 2022-04-28T20:51:28.577966+0000 mgr.y (mgr.24961) 2085 : cluster [DBG] pgmap v2443: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:31 smithi059 bash[61152]: cluster 2022-04-28T20:51:30.579299+0000 mgr.y (mgr.24961) 2086 : cluster [DBG] pgmap v2444: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:31.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:31 smithi059 bash[59745]: cluster 2022-04-28T20:51:30.579299+0000 mgr.y (mgr.24961) 2086 : cluster [DBG] pgmap v2444: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:31 smithi139 bash[43301]: cluster 2022-04-28T20:51:30.579299+0000 mgr.y (mgr.24961) 2086 : cluster [DBG] pgmap v2444: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:33 smithi059 bash[61152]: cluster 2022-04-28T20:51:32.580312+0000 mgr.y (mgr.24961) 2087 : cluster [DBG] pgmap v2445: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:33.753Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:33 smithi059 bash[59745]: cluster 2022-04-28T20:51:32.580312+0000 mgr.y (mgr.24961) 2087 : cluster [DBG] pgmap v2445: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:33 smithi139 bash[43301]: cluster 2022-04-28T20:51:32.580312+0000 mgr.y (mgr.24961) 2087 : cluster [DBG] pgmap v2445: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:36 smithi059 bash[61152]: cluster 2022-04-28T20:51:34.581437+0000 mgr.y (mgr.24961) 2088 : cluster [DBG] pgmap v2446: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:36.431 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:35 smithi059 bash[59745]: cluster 2022-04-28T20:51:34.581437+0000 mgr.y (mgr.24961) 2088 : cluster [DBG] pgmap v2446: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:36.432 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:35 smithi139 bash[43301]: cluster 2022-04-28T20:51:34.581437+0000 mgr.y (mgr.24961) 2088 : cluster [DBG] pgmap v2446: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:38 smithi059 bash[59745]: cluster 2022-04-28T20:51:36.582731+0000 mgr.y (mgr.24961) 2089 : cluster [DBG] pgmap v2447: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:38 smithi059 bash[61152]: cluster 2022-04-28T20:51:36.582731+0000 mgr.y (mgr.24961) 2089 : cluster [DBG] pgmap v2447: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:38 smithi139 bash[43301]: cluster 2022-04-28T20:51:36.582731+0000 mgr.y (mgr.24961) 2089 : cluster [DBG] pgmap v2447: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:40 smithi059 bash[61152]: cluster 2022-04-28T20:51:38.583466+0000 mgr.y (mgr.24961) 2090 : cluster [DBG] pgmap v2448: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:40 smithi059 bash[59745]: cluster 2022-04-28T20:51:38.583466+0000 mgr.y (mgr.24961) 2090 : cluster [DBG] pgmap v2448: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:40 smithi139 bash[43301]: cluster 2022-04-28T20:51:38.583466+0000 mgr.y (mgr.24961) 2090 : cluster [DBG] pgmap v2448: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:42 smithi059 bash[61152]: cluster 2022-04-28T20:51:40.584878+0000 mgr.y (mgr.24961) 2091 : cluster [DBG] pgmap v2449: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:41.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:42 smithi059 bash[59745]: cluster 2022-04-28T20:51:40.584878+0000 mgr.y (mgr.24961) 2091 : cluster [DBG] pgmap v2449: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:42 smithi139 bash[43301]: cluster 2022-04-28T20:51:40.584878+0000 mgr.y (mgr.24961) 2091 : cluster [DBG] pgmap v2449: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:51:44.026 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:43.754Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:44.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:44 smithi059 bash[59745]: cluster 2022-04-28T20:51:42.585924+0000 mgr.y (mgr.24961) 2092 : cluster [DBG] pgmap v2450: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:44 smithi059 bash[61152]: cluster 2022-04-28T20:51:42.585924+0000 mgr.y (mgr.24961) 2092 : cluster [DBG] pgmap v2450: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:44 smithi139 bash[43301]: cluster 2022-04-28T20:51:42.585924+0000 mgr.y (mgr.24961) 2092 : cluster [DBG] pgmap v2450: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:46 smithi059 bash[61152]: cluster 2022-04-28T20:51:44.587105+0000 mgr.y (mgr.24961) 2093 : cluster [DBG] pgmap v2451: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:51:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:46 smithi059 bash[59745]: cluster 2022-04-28T20:51:44.587105+0000 mgr.y (mgr.24961) 2093 : cluster [DBG] pgmap v2451: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:51:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:46 smithi139 bash[43301]: cluster 2022-04-28T20:51:44.587105+0000 mgr.y (mgr.24961) 2093 : cluster [DBG] pgmap v2451: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:51:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:48 smithi059 bash[61152]: cluster 2022-04-28T20:51:46.588548+0000 mgr.y (mgr.24961) 2094 : cluster [DBG] pgmap v2452: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:48 smithi059 bash[59745]: cluster 2022-04-28T20:51:46.588548+0000 mgr.y (mgr.24961) 2094 : cluster [DBG] pgmap v2452: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:48 smithi139 bash[43301]: cluster 2022-04-28T20:51:46.588548+0000 mgr.y (mgr.24961) 2094 : cluster [DBG] pgmap v2452: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:50 smithi059 bash[59745]: cluster 2022-04-28T20:51:48.589312+0000 mgr.y (mgr.24961) 2095 : cluster [DBG] pgmap v2453: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:50 smithi059 bash[61152]: cluster 2022-04-28T20:51:48.589312+0000 mgr.y (mgr.24961) 2095 : cluster [DBG] pgmap v2453: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:50 smithi139 bash[43301]: cluster 2022-04-28T20:51:48.589312+0000 mgr.y (mgr.24961) 2095 : cluster [DBG] pgmap v2453: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:52 smithi059 bash[61152]: cluster 2022-04-28T20:51:50.590751+0000 mgr.y (mgr.24961) 2096 : cluster [DBG] pgmap v2454: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:51:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:51.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:52 smithi059 bash[59745]: cluster 2022-04-28T20:51:50.590751+0000 mgr.y (mgr.24961) 2096 : cluster [DBG] pgmap v2454: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:51:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:52 smithi139 bash[43301]: cluster 2022-04-28T20:51:50.590751+0000 mgr.y (mgr.24961) 2096 : cluster [DBG] pgmap v2454: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:51:53.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:53 smithi139 bash[43301]: cluster 2022-04-28T20:51:52.591854+0000 mgr.y (mgr.24961) 2097 : cluster [DBG] pgmap v2455: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:53 smithi059 bash[61152]: cluster 2022-04-28T20:51:52.591854+0000 mgr.y (mgr.24961) 2097 : cluster [DBG] pgmap v2455: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:53 smithi059 bash[59745]: cluster 2022-04-28T20:51:52.591854+0000 mgr.y (mgr.24961) 2097 : cluster [DBG] pgmap v2455: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:51:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:51:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:51:53.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:51:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:55 smithi059 bash[59745]: cluster 2022-04-28T20:51:54.592959+0000 mgr.y (mgr.24961) 2098 : cluster [DBG] pgmap v2456: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:51:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:55 smithi059 bash[61152]: cluster 2022-04-28T20:51:54.592959+0000 mgr.y (mgr.24961) 2098 : cluster [DBG] pgmap v2456: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:51:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:55 smithi139 bash[43301]: cluster 2022-04-28T20:51:54.592959+0000 mgr.y (mgr.24961) 2098 : cluster [DBG] pgmap v2456: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:51:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:51:57 smithi059 bash[61152]: cluster 2022-04-28T20:51:56.594347+0000 mgr.y (mgr.24961) 2099 : cluster [DBG] pgmap v2457: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:51:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:51:57 smithi059 bash[59745]: cluster 2022-04-28T20:51:56.594347+0000 mgr.y (mgr.24961) 2099 : cluster [DBG] pgmap v2457: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:51:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:51:57 smithi139 bash[43301]: cluster 2022-04-28T20:51:56.594347+0000 mgr.y (mgr.24961) 2099 : cluster [DBG] pgmap v2457: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:00 smithi059 bash[61152]: cluster 2022-04-28T20:51:58.595222+0000 mgr.y (mgr.24961) 2100 : cluster [DBG] pgmap v2458: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:00 smithi059 bash[59745]: cluster 2022-04-28T20:51:58.595222+0000 mgr.y (mgr.24961) 2100 : cluster [DBG] pgmap v2458: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:00 smithi139 bash[43301]: cluster 2022-04-28T20:51:58.595222+0000 mgr.y (mgr.24961) 2100 : cluster [DBG] pgmap v2458: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[61152]: audit 2022-04-28T20:52:00.830066+0000 mon.a (mon.0) 2296 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:52:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[61152]: audit 2022-04-28T20:52:00.832959+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:52:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[61152]: audit 2022-04-28T20:52:00.834855+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:52:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[59745]: audit 2022-04-28T20:52:00.830066+0000 mon.a (mon.0) 2296 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:52:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[59745]: audit 2022-04-28T20:52:00.832959+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:52:01.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[59745]: audit 2022-04-28T20:52:00.834855+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:52:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:01 smithi139 bash[43301]: audit 2022-04-28T20:52:00.830066+0000 mon.a (mon.0) 2296 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:52:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:01 smithi139 bash[43301]: audit 2022-04-28T20:52:00.832959+0000 mon.a (mon.0) 2297 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:52:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:01 smithi139 bash[43301]: audit 2022-04-28T20:52:00.834855+0000 mon.a (mon.0) 2298 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:52:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:01.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:02 smithi059 bash[61152]: cluster 2022-04-28T20:52:00.596622+0000 mgr.y (mgr.24961) 2101 : cluster [DBG] pgmap v2459: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:02 smithi059 bash[61152]: cephadm 2022-04-28T20:52:00.836559+0000 mgr.y (mgr.24961) 2102 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:52:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:02 smithi059 bash[61152]: audit 2022-04-28T20:52:01.164437+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:52:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:02 smithi059 bash[59745]: cluster 2022-04-28T20:52:00.596622+0000 mgr.y (mgr.24961) 2101 : cluster [DBG] pgmap v2459: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:02 smithi059 bash[59745]: cephadm 2022-04-28T20:52:00.836559+0000 mgr.y (mgr.24961) 2102 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:52:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:02 smithi059 bash[59745]: audit 2022-04-28T20:52:01.164437+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:52:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:02 smithi139 bash[43301]: cluster 2022-04-28T20:52:00.596622+0000 mgr.y (mgr.24961) 2101 : cluster [DBG] pgmap v2459: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:02 smithi139 bash[43301]: cephadm 2022-04-28T20:52:00.836559+0000 mgr.y (mgr.24961) 2102 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:52:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:02 smithi139 bash[43301]: audit 2022-04-28T20:52:01.164437+0000 mon.a (mon.0) 2299 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:52:04.017 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:03.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:04 smithi059 bash[61152]: cluster 2022-04-28T20:52:02.597644+0000 mgr.y (mgr.24961) 2103 : cluster [DBG] pgmap v2460: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:04 smithi059 bash[59745]: cluster 2022-04-28T20:52:02.597644+0000 mgr.y (mgr.24961) 2103 : cluster [DBG] pgmap v2460: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:04 smithi139 bash[43301]: cluster 2022-04-28T20:52:02.597644+0000 mgr.y (mgr.24961) 2103 : cluster [DBG] pgmap v2460: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:06 smithi059 bash[61152]: cluster 2022-04-28T20:52:04.598732+0000 mgr.y (mgr.24961) 2104 : cluster [DBG] pgmap v2461: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:06 smithi059 bash[59745]: cluster 2022-04-28T20:52:04.598732+0000 mgr.y (mgr.24961) 2104 : cluster [DBG] pgmap v2461: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:06 smithi139 bash[43301]: cluster 2022-04-28T20:52:04.598732+0000 mgr.y (mgr.24961) 2104 : cluster [DBG] pgmap v2461: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:08 smithi059 bash[61152]: cluster 2022-04-28T20:52:06.600077+0000 mgr.y (mgr.24961) 2105 : cluster [DBG] pgmap v2462: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:08 smithi059 bash[59745]: cluster 2022-04-28T20:52:06.600077+0000 mgr.y (mgr.24961) 2105 : cluster [DBG] pgmap v2462: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:08 smithi139 bash[43301]: cluster 2022-04-28T20:52:06.600077+0000 mgr.y (mgr.24961) 2105 : cluster [DBG] pgmap v2462: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:10 smithi059 bash[61152]: cluster 2022-04-28T20:52:08.600827+0000 mgr.y (mgr.24961) 2106 : cluster [DBG] pgmap v2463: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:10 smithi059 bash[59745]: cluster 2022-04-28T20:52:08.600827+0000 mgr.y (mgr.24961) 2106 : cluster [DBG] pgmap v2463: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:10 smithi139 bash[43301]: cluster 2022-04-28T20:52:08.600827+0000 mgr.y (mgr.24961) 2106 : cluster [DBG] pgmap v2463: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:12 smithi059 bash[61152]: cluster 2022-04-28T20:52:10.602194+0000 mgr.y (mgr.24961) 2107 : cluster [DBG] pgmap v2464: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:11.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:12 smithi059 bash[59745]: cluster 2022-04-28T20:52:10.602194+0000 mgr.y (mgr.24961) 2107 : cluster [DBG] pgmap v2464: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:12 smithi139 bash[43301]: cluster 2022-04-28T20:52:10.602194+0000 mgr.y (mgr.24961) 2107 : cluster [DBG] pgmap v2464: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:14.043 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:13.756Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:14 smithi059 bash[59745]: cluster 2022-04-28T20:52:12.603203+0000 mgr.y (mgr.24961) 2108 : cluster [DBG] pgmap v2465: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:14 smithi059 bash[61152]: cluster 2022-04-28T20:52:12.603203+0000 mgr.y (mgr.24961) 2108 : cluster [DBG] pgmap v2465: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:14 smithi139 bash[43301]: cluster 2022-04-28T20:52:12.603203+0000 mgr.y (mgr.24961) 2108 : cluster [DBG] pgmap v2465: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:16 smithi059 bash[61152]: cluster 2022-04-28T20:52:14.604343+0000 mgr.y (mgr.24961) 2109 : cluster [DBG] pgmap v2466: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:16 smithi059 bash[59745]: cluster 2022-04-28T20:52:14.604343+0000 mgr.y (mgr.24961) 2109 : cluster [DBG] pgmap v2466: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:16 smithi139 bash[43301]: cluster 2022-04-28T20:52:14.604343+0000 mgr.y (mgr.24961) 2109 : cluster [DBG] pgmap v2466: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[59745]: cluster 2022-04-28T20:52:16.605649+0000 mgr.y (mgr.24961) 2110 : cluster [DBG] pgmap v2467: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[59745]: audit 2022-04-28T20:52:17.877600+0000 mon.a (mon.0) 2300 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:52:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[59745]: audit 2022-04-28T20:52:17.878017+0000 mon.a (mon.0) 2301 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:52:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[59745]: audit 2022-04-28T20:52:17.887900+0000 mon.a (mon.0) 2302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:52:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[59745]: audit 2022-04-28T20:52:17.888628+0000 mon.a (mon.0) 2303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:52:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[59745]: audit 2022-04-28T20:52:17.889049+0000 mon.a (mon.0) 2304 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:52:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[61152]: cluster 2022-04-28T20:52:16.605649+0000 mgr.y (mgr.24961) 2110 : cluster [DBG] pgmap v2467: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[61152]: audit 2022-04-28T20:52:17.877600+0000 mon.a (mon.0) 2300 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:52:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[61152]: audit 2022-04-28T20:52:17.878017+0000 mon.a (mon.0) 2301 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:52:18.406 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[61152]: audit 2022-04-28T20:52:17.887900+0000 mon.a (mon.0) 2302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:52:18.406 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[61152]: audit 2022-04-28T20:52:17.888628+0000 mon.a (mon.0) 2303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:52:18.406 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:18 smithi059 bash[61152]: audit 2022-04-28T20:52:17.889049+0000 mon.a (mon.0) 2304 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:52:18.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:18 smithi139 bash[43301]: cluster 2022-04-28T20:52:16.605649+0000 mgr.y (mgr.24961) 2110 : cluster [DBG] pgmap v2467: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:18.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:18 smithi139 bash[43301]: audit 2022-04-28T20:52:17.877600+0000 mon.a (mon.0) 2300 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:52:18.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:18 smithi139 bash[43301]: audit 2022-04-28T20:52:17.878017+0000 mon.a (mon.0) 2301 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:52:18.407 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:18 smithi139 bash[43301]: audit 2022-04-28T20:52:17.887900+0000 mon.a (mon.0) 2302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:52:18.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:18 smithi139 bash[43301]: audit 2022-04-28T20:52:17.888628+0000 mon.a (mon.0) 2303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T20:52:18.408 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:18 smithi139 bash[43301]: audit 2022-04-28T20:52:17.889049+0000 mon.a (mon.0) 2304 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T20:52:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:20 smithi059 bash[61152]: cluster 2022-04-28T20:52:18.606385+0000 mgr.y (mgr.24961) 2111 : cluster [DBG] pgmap v2468: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:20 smithi059 bash[59745]: cluster 2022-04-28T20:52:18.606385+0000 mgr.y (mgr.24961) 2111 : cluster [DBG] pgmap v2468: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:20 smithi139 bash[43301]: cluster 2022-04-28T20:52:18.606385+0000 mgr.y (mgr.24961) 2111 : cluster [DBG] pgmap v2468: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:21 smithi059 bash[61152]: cluster 2022-04-28T20:52:20.607796+0000 mgr.y (mgr.24961) 2112 : cluster [DBG] pgmap v2469: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:21 smithi059 bash[59745]: cluster 2022-04-28T20:52:20.607796+0000 mgr.y (mgr.24961) 2112 : cluster [DBG] pgmap v2469: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:21.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:21 smithi139 bash[43301]: cluster 2022-04-28T20:52:20.607796+0000 mgr.y (mgr.24961) 2112 : cluster [DBG] pgmap v2469: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:21.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:23 smithi059 bash[59745]: cluster 2022-04-28T20:52:22.608833+0000 mgr.y (mgr.24961) 2113 : cluster [DBG] pgmap v2470: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:23 smithi059 bash[61152]: cluster 2022-04-28T20:52:22.608833+0000 mgr.y (mgr.24961) 2113 : cluster [DBG] pgmap v2470: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:23.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:24.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:23 smithi139 bash[43301]: cluster 2022-04-28T20:52:22.608833+0000 mgr.y (mgr.24961) 2113 : cluster [DBG] pgmap v2470: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:26 smithi059 bash[59745]: cluster 2022-04-28T20:52:24.610027+0000 mgr.y (mgr.24961) 2114 : cluster [DBG] pgmap v2471: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:26 smithi059 bash[61152]: cluster 2022-04-28T20:52:24.610027+0000 mgr.y (mgr.24961) 2114 : cluster [DBG] pgmap v2471: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:25 smithi139 bash[43301]: cluster 2022-04-28T20:52:24.610027+0000 mgr.y (mgr.24961) 2114 : cluster [DBG] pgmap v2471: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:28 smithi059 bash[61152]: cluster 2022-04-28T20:52:26.611425+0000 mgr.y (mgr.24961) 2115 : cluster [DBG] pgmap v2472: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:28 smithi059 bash[59745]: cluster 2022-04-28T20:52:26.611425+0000 mgr.y (mgr.24961) 2115 : cluster [DBG] pgmap v2472: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:28 smithi139 bash[43301]: cluster 2022-04-28T20:52:26.611425+0000 mgr.y (mgr.24961) 2115 : cluster [DBG] pgmap v2472: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:30 smithi059 bash[61152]: cluster 2022-04-28T20:52:28.612224+0000 mgr.y (mgr.24961) 2116 : cluster [DBG] pgmap v2473: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:30 smithi059 bash[59745]: cluster 2022-04-28T20:52:28.612224+0000 mgr.y (mgr.24961) 2116 : cluster [DBG] pgmap v2473: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:30 smithi139 bash[43301]: cluster 2022-04-28T20:52:28.612224+0000 mgr.y (mgr.24961) 2116 : cluster [DBG] pgmap v2473: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:32 smithi059 bash[61152]: cluster 2022-04-28T20:52:30.613600+0000 mgr.y (mgr.24961) 2117 : cluster [DBG] pgmap v2474: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:31.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:32 smithi059 bash[59745]: cluster 2022-04-28T20:52:30.613600+0000 mgr.y (mgr.24961) 2117 : cluster [DBG] pgmap v2474: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:32 smithi139 bash[43301]: cluster 2022-04-28T20:52:30.613600+0000 mgr.y (mgr.24961) 2117 : cluster [DBG] pgmap v2474: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:34.036 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:33.758Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:34 smithi059 bash[61152]: cluster 2022-04-28T20:52:32.614743+0000 mgr.y (mgr.24961) 2118 : cluster [DBG] pgmap v2475: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:34 smithi059 bash[59745]: cluster 2022-04-28T20:52:32.614743+0000 mgr.y (mgr.24961) 2118 : cluster [DBG] pgmap v2475: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:34 smithi139 bash[43301]: cluster 2022-04-28T20:52:32.614743+0000 mgr.y (mgr.24961) 2118 : cluster [DBG] pgmap v2475: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:52:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:36 smithi059 bash[61152]: cluster 2022-04-28T20:52:34.615901+0000 mgr.y (mgr.24961) 2119 : cluster [DBG] pgmap v2476: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:36 smithi059 bash[59745]: cluster 2022-04-28T20:52:34.615901+0000 mgr.y (mgr.24961) 2119 : cluster [DBG] pgmap v2476: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:36 smithi139 bash[43301]: cluster 2022-04-28T20:52:34.615901+0000 mgr.y (mgr.24961) 2119 : cluster [DBG] pgmap v2476: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:38 smithi059 bash[61152]: cluster 2022-04-28T20:52:36.617208+0000 mgr.y (mgr.24961) 2120 : cluster [DBG] pgmap v2477: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:38 smithi059 bash[59745]: cluster 2022-04-28T20:52:36.617208+0000 mgr.y (mgr.24961) 2120 : cluster [DBG] pgmap v2477: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:38 smithi139 bash[43301]: cluster 2022-04-28T20:52:36.617208+0000 mgr.y (mgr.24961) 2120 : cluster [DBG] pgmap v2477: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:40 smithi059 bash[61152]: cluster 2022-04-28T20:52:38.617959+0000 mgr.y (mgr.24961) 2121 : cluster [DBG] pgmap v2478: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:40 smithi059 bash[59745]: cluster 2022-04-28T20:52:38.617959+0000 mgr.y (mgr.24961) 2121 : cluster [DBG] pgmap v2478: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:40 smithi139 bash[43301]: cluster 2022-04-28T20:52:38.617959+0000 mgr.y (mgr.24961) 2121 : cluster [DBG] pgmap v2478: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:42 smithi059 bash[59745]: cluster 2022-04-28T20:52:40.619354+0000 mgr.y (mgr.24961) 2122 : cluster [DBG] pgmap v2479: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:42 smithi059 bash[61152]: cluster 2022-04-28T20:52:40.619354+0000 mgr.y (mgr.24961) 2122 : cluster [DBG] pgmap v2479: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:41.862Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:42 smithi139 bash[43301]: cluster 2022-04-28T20:52:40.619354+0000 mgr.y (mgr.24961) 2122 : cluster [DBG] pgmap v2479: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:44.064 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:43.759Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:44.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:44 smithi059 bash[61152]: cluster 2022-04-28T20:52:42.620446+0000 mgr.y (mgr.24961) 2123 : cluster [DBG] pgmap v2480: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:44.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:44 smithi059 bash[59745]: cluster 2022-04-28T20:52:42.620446+0000 mgr.y (mgr.24961) 2123 : cluster [DBG] pgmap v2480: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:44 smithi139 bash[43301]: cluster 2022-04-28T20:52:42.620446+0000 mgr.y (mgr.24961) 2123 : cluster [DBG] pgmap v2480: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:45 smithi059 bash[61152]: cluster 2022-04-28T20:52:44.621817+0000 mgr.y (mgr.24961) 2124 : cluster [DBG] pgmap v2481: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:45.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:45 smithi059 bash[59745]: cluster 2022-04-28T20:52:44.621817+0000 mgr.y (mgr.24961) 2124 : cluster [DBG] pgmap v2481: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:45.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:45 smithi139 bash[43301]: cluster 2022-04-28T20:52:44.621817+0000 mgr.y (mgr.24961) 2124 : cluster [DBG] pgmap v2481: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:47 smithi059 bash[61152]: cluster 2022-04-28T20:52:46.623107+0000 mgr.y (mgr.24961) 2125 : cluster [DBG] pgmap v2482: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:47 smithi059 bash[59745]: cluster 2022-04-28T20:52:46.623107+0000 mgr.y (mgr.24961) 2125 : cluster [DBG] pgmap v2482: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:48.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:47 smithi139 bash[43301]: cluster 2022-04-28T20:52:46.623107+0000 mgr.y (mgr.24961) 2125 : cluster [DBG] pgmap v2482: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:50 smithi059 bash[61152]: cluster 2022-04-28T20:52:48.623860+0000 mgr.y (mgr.24961) 2126 : cluster [DBG] pgmap v2483: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:50 smithi059 bash[59745]: cluster 2022-04-28T20:52:48.623860+0000 mgr.y (mgr.24961) 2126 : cluster [DBG] pgmap v2483: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:50 smithi139 bash[43301]: cluster 2022-04-28T20:52:48.623860+0000 mgr.y (mgr.24961) 2126 : cluster [DBG] pgmap v2483: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:52 smithi059 bash[61152]: cluster 2022-04-28T20:52:50.625248+0000 mgr.y (mgr.24961) 2127 : cluster [DBG] pgmap v2484: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:51.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:52 smithi059 bash[59745]: cluster 2022-04-28T20:52:50.625248+0000 mgr.y (mgr.24961) 2127 : cluster [DBG] pgmap v2484: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:52 smithi139 bash[43301]: cluster 2022-04-28T20:52:50.625248+0000 mgr.y (mgr.24961) 2127 : cluster [DBG] pgmap v2484: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:54.018 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:52:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:52:53.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:52:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:54 smithi059 bash[61152]: cluster 2022-04-28T20:52:52.626258+0000 mgr.y (mgr.24961) 2128 : cluster [DBG] pgmap v2485: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:54 smithi059 bash[59745]: cluster 2022-04-28T20:52:52.626258+0000 mgr.y (mgr.24961) 2128 : cluster [DBG] pgmap v2485: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:54 smithi139 bash[43301]: cluster 2022-04-28T20:52:52.626258+0000 mgr.y (mgr.24961) 2128 : cluster [DBG] pgmap v2485: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:56 smithi059 bash[61152]: cluster 2022-04-28T20:52:54.627377+0000 mgr.y (mgr.24961) 2129 : cluster [DBG] pgmap v2486: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:56 smithi059 bash[59745]: cluster 2022-04-28T20:52:54.627377+0000 mgr.y (mgr.24961) 2129 : cluster [DBG] pgmap v2486: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:56 smithi139 bash[43301]: cluster 2022-04-28T20:52:54.627377+0000 mgr.y (mgr.24961) 2129 : cluster [DBG] pgmap v2486: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:52:58 smithi059 bash[61152]: cluster 2022-04-28T20:52:56.628622+0000 mgr.y (mgr.24961) 2130 : cluster [DBG] pgmap v2487: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:52:58 smithi059 bash[59745]: cluster 2022-04-28T20:52:56.628622+0000 mgr.y (mgr.24961) 2130 : cluster [DBG] pgmap v2487: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:52:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:52:58 smithi139 bash[43301]: cluster 2022-04-28T20:52:56.628622+0000 mgr.y (mgr.24961) 2130 : cluster [DBG] pgmap v2487: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:00 smithi059 bash[61152]: cluster 2022-04-28T20:52:58.629367+0000 mgr.y (mgr.24961) 2131 : cluster [DBG] pgmap v2488: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:00 smithi059 bash[59745]: cluster 2022-04-28T20:52:58.629367+0000 mgr.y (mgr.24961) 2131 : cluster [DBG] pgmap v2488: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:00 smithi139 bash[43301]: cluster 2022-04-28T20:52:58.629367+0000 mgr.y (mgr.24961) 2131 : cluster [DBG] pgmap v2488: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[61152]: cluster 2022-04-28T20:53:00.630740+0000 mgr.y (mgr.24961) 2132 : cluster [DBG] pgmap v2489: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[61152]: audit 2022-04-28T20:53:01.170614+0000 mon.a (mon.0) 2305 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:53:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[61152]: audit 2022-04-28T20:53:01.173356+0000 mon.a (mon.0) 2306 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[61152]: audit 2022-04-28T20:53:01.175256+0000 mon.a (mon.0) 2307 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:53:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[61152]: audit 2022-04-28T20:53:01.494595+0000 mon.a (mon.0) 2308 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:53:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[59745]: cluster 2022-04-28T20:53:00.630740+0000 mgr.y (mgr.24961) 2132 : cluster [DBG] pgmap v2489: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[59745]: audit 2022-04-28T20:53:01.170614+0000 mon.a (mon.0) 2305 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:53:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[59745]: audit 2022-04-28T20:53:01.173356+0000 mon.a (mon.0) 2306 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:53:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[59745]: audit 2022-04-28T20:53:01.175256+0000 mon.a (mon.0) 2307 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:53:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:02 smithi059 bash[59745]: audit 2022-04-28T20:53:01.494595+0000 mon.a (mon.0) 2308 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:53:02.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:01.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:02 smithi139 bash[43301]: cluster 2022-04-28T20:53:00.630740+0000 mgr.y (mgr.24961) 2132 : cluster [DBG] pgmap v2489: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:02 smithi139 bash[43301]: audit 2022-04-28T20:53:01.170614+0000 mon.a (mon.0) 2305 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:53:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:02 smithi139 bash[43301]: audit 2022-04-28T20:53:01.173356+0000 mon.a (mon.0) 2306 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:53:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:02 smithi139 bash[43301]: audit 2022-04-28T20:53:01.175256+0000 mon.a (mon.0) 2307 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:53:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:02 smithi139 bash[43301]: audit 2022-04-28T20:53:01.494595+0000 mon.a (mon.0) 2308 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:53:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:03 smithi059 bash[59745]: cephadm 2022-04-28T20:53:01.177008+0000 mgr.y (mgr.24961) 2133 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:53:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:03 smithi059 bash[61152]: cephadm 2022-04-28T20:53:01.177008+0000 mgr.y (mgr.24961) 2133 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:53:03.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:03 smithi139 bash[43301]: cephadm 2022-04-28T20:53:01.177008+0000 mgr.y (mgr.24961) 2133 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:53:04.056 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:03.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:04 smithi059 bash[61152]: cluster 2022-04-28T20:53:02.631841+0000 mgr.y (mgr.24961) 2134 : cluster [DBG] pgmap v2490: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:04 smithi059 bash[59745]: cluster 2022-04-28T20:53:02.631841+0000 mgr.y (mgr.24961) 2134 : cluster [DBG] pgmap v2490: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:04 smithi139 bash[43301]: cluster 2022-04-28T20:53:02.631841+0000 mgr.y (mgr.24961) 2134 : cluster [DBG] pgmap v2490: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:06 smithi059 bash[59745]: cluster 2022-04-28T20:53:04.632965+0000 mgr.y (mgr.24961) 2135 : cluster [DBG] pgmap v2491: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:06 smithi059 bash[61152]: cluster 2022-04-28T20:53:04.632965+0000 mgr.y (mgr.24961) 2135 : cluster [DBG] pgmap v2491: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:06 smithi139 bash[43301]: cluster 2022-04-28T20:53:04.632965+0000 mgr.y (mgr.24961) 2135 : cluster [DBG] pgmap v2491: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:08 smithi059 bash[61152]: cluster 2022-04-28T20:53:06.634304+0000 mgr.y (mgr.24961) 2136 : cluster [DBG] pgmap v2492: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:08 smithi059 bash[59745]: cluster 2022-04-28T20:53:06.634304+0000 mgr.y (mgr.24961) 2136 : cluster [DBG] pgmap v2492: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:08 smithi139 bash[43301]: cluster 2022-04-28T20:53:06.634304+0000 mgr.y (mgr.24961) 2136 : cluster [DBG] pgmap v2492: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:10 smithi059 bash[61152]: cluster 2022-04-28T20:53:08.635057+0000 mgr.y (mgr.24961) 2137 : cluster [DBG] pgmap v2493: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:10 smithi059 bash[59745]: cluster 2022-04-28T20:53:08.635057+0000 mgr.y (mgr.24961) 2137 : cluster [DBG] pgmap v2493: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:10 smithi139 bash[43301]: cluster 2022-04-28T20:53:08.635057+0000 mgr.y (mgr.24961) 2137 : cluster [DBG] pgmap v2493: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:12 smithi059 bash[61152]: cluster 2022-04-28T20:53:10.636610+0000 mgr.y (mgr.24961) 2138 : cluster [DBG] pgmap v2494: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:11.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:12 smithi059 bash[59745]: cluster 2022-04-28T20:53:10.636610+0000 mgr.y (mgr.24961) 2138 : cluster [DBG] pgmap v2494: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:12 smithi139 bash[43301]: cluster 2022-04-28T20:53:10.636610+0000 mgr.y (mgr.24961) 2138 : cluster [DBG] pgmap v2494: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:14.024 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:13.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:14 smithi059 bash[61152]: cluster 2022-04-28T20:53:12.637655+0000 mgr.y (mgr.24961) 2139 : cluster [DBG] pgmap v2495: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:14 smithi059 bash[59745]: cluster 2022-04-28T20:53:12.637655+0000 mgr.y (mgr.24961) 2139 : cluster [DBG] pgmap v2495: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:14 smithi139 bash[43301]: cluster 2022-04-28T20:53:12.637655+0000 mgr.y (mgr.24961) 2139 : cluster [DBG] pgmap v2495: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:16 smithi059 bash[59745]: cluster 2022-04-28T20:53:14.638757+0000 mgr.y (mgr.24961) 2140 : cluster [DBG] pgmap v2496: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:16 smithi059 bash[61152]: cluster 2022-04-28T20:53:14.638757+0000 mgr.y (mgr.24961) 2140 : cluster [DBG] pgmap v2496: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:16 smithi139 bash[43301]: cluster 2022-04-28T20:53:14.638757+0000 mgr.y (mgr.24961) 2140 : cluster [DBG] pgmap v2496: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[59745]: cluster 2022-04-28T20:53:16.640132+0000 mgr.y (mgr.24961) 2141 : cluster [DBG] pgmap v2497: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[59745]: audit 2022-04-28T20:53:17.890922+0000 mon.a (mon.0) 2309 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:53:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[59745]: audit 2022-04-28T20:53:17.891823+0000 mon.a (mon.0) 2310 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:53:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[59745]: audit 2022-04-28T20:53:17.901747+0000 mon.a (mon.0) 2311 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:53:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[59745]: audit 2022-04-28T20:53:17.902497+0000 mon.a (mon.0) 2312 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:53:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[59745]: audit 2022-04-28T20:53:17.902790+0000 mon.a (mon.0) 2313 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:53:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[61152]: cluster 2022-04-28T20:53:16.640132+0000 mgr.y (mgr.24961) 2141 : cluster [DBG] pgmap v2497: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[61152]: audit 2022-04-28T20:53:17.890922+0000 mon.a (mon.0) 2309 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:53:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[61152]: audit 2022-04-28T20:53:17.891823+0000 mon.a (mon.0) 2310 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:53:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[61152]: audit 2022-04-28T20:53:17.901747+0000 mon.a (mon.0) 2311 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:53:18.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[61152]: audit 2022-04-28T20:53:17.902497+0000 mon.a (mon.0) 2312 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:53:18.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:18 smithi059 bash[61152]: audit 2022-04-28T20:53:17.902790+0000 mon.a (mon.0) 2313 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:53:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:18 smithi139 bash[43301]: cluster 2022-04-28T20:53:16.640132+0000 mgr.y (mgr.24961) 2141 : cluster [DBG] pgmap v2497: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:18 smithi139 bash[43301]: audit 2022-04-28T20:53:17.890922+0000 mon.a (mon.0) 2309 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:53:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:18 smithi139 bash[43301]: audit 2022-04-28T20:53:17.891823+0000 mon.a (mon.0) 2310 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:53:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:18 smithi139 bash[43301]: audit 2022-04-28T20:53:17.901747+0000 mon.a (mon.0) 2311 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:53:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:18 smithi139 bash[43301]: audit 2022-04-28T20:53:17.902497+0000 mon.a (mon.0) 2312 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T20:53:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:18 smithi139 bash[43301]: audit 2022-04-28T20:53:17.902790+0000 mon.a (mon.0) 2313 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:53:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:20 smithi059 bash[61152]: cluster 2022-04-28T20:53:18.640945+0000 mgr.y (mgr.24961) 2142 : cluster [DBG] pgmap v2498: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:20 smithi059 bash[59745]: cluster 2022-04-28T20:53:18.640945+0000 mgr.y (mgr.24961) 2142 : cluster [DBG] pgmap v2498: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:20 smithi139 bash[43301]: cluster 2022-04-28T20:53:18.640945+0000 mgr.y (mgr.24961) 2142 : cluster [DBG] pgmap v2498: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:22 smithi059 bash[59745]: cluster 2022-04-28T20:53:20.642286+0000 mgr.y (mgr.24961) 2143 : cluster [DBG] pgmap v2499: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:22 smithi059 bash[61152]: cluster 2022-04-28T20:53:20.642286+0000 mgr.y (mgr.24961) 2143 : cluster [DBG] pgmap v2499: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:21.864Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:22 smithi139 bash[43301]: cluster 2022-04-28T20:53:20.642286+0000 mgr.y (mgr.24961) 2143 : cluster [DBG] pgmap v2499: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:24.050 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:23.762Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:24.051 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:24 smithi059 bash[59745]: cluster 2022-04-28T20:53:22.643394+0000 mgr.y (mgr.24961) 2144 : cluster [DBG] pgmap v2500: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:24 smithi059 bash[61152]: cluster 2022-04-28T20:53:22.643394+0000 mgr.y (mgr.24961) 2144 : cluster [DBG] pgmap v2500: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:24 smithi139 bash[43301]: cluster 2022-04-28T20:53:22.643394+0000 mgr.y (mgr.24961) 2144 : cluster [DBG] pgmap v2500: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:26 smithi059 bash[61152]: cluster 2022-04-28T20:53:24.644547+0000 mgr.y (mgr.24961) 2145 : cluster [DBG] pgmap v2501: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:26 smithi059 bash[59745]: cluster 2022-04-28T20:53:24.644547+0000 mgr.y (mgr.24961) 2145 : cluster [DBG] pgmap v2501: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:26 smithi139 bash[43301]: cluster 2022-04-28T20:53:24.644547+0000 mgr.y (mgr.24961) 2145 : cluster [DBG] pgmap v2501: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:28 smithi059 bash[61152]: cluster 2022-04-28T20:53:26.645818+0000 mgr.y (mgr.24961) 2146 : cluster [DBG] pgmap v2502: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:28 smithi059 bash[59745]: cluster 2022-04-28T20:53:26.645818+0000 mgr.y (mgr.24961) 2146 : cluster [DBG] pgmap v2502: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:28 smithi139 bash[43301]: cluster 2022-04-28T20:53:26.645818+0000 mgr.y (mgr.24961) 2146 : cluster [DBG] pgmap v2502: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:30 smithi059 bash[61152]: cluster 2022-04-28T20:53:28.646550+0000 mgr.y (mgr.24961) 2147 : cluster [DBG] pgmap v2503: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:30 smithi059 bash[59745]: cluster 2022-04-28T20:53:28.646550+0000 mgr.y (mgr.24961) 2147 : cluster [DBG] pgmap v2503: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:30 smithi139 bash[43301]: cluster 2022-04-28T20:53:28.646550+0000 mgr.y (mgr.24961) 2147 : cluster [DBG] pgmap v2503: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:32 smithi059 bash[61152]: cluster 2022-04-28T20:53:30.647928+0000 mgr.y (mgr.24961) 2148 : cluster [DBG] pgmap v2504: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:31.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:32 smithi059 bash[59745]: cluster 2022-04-28T20:53:30.647928+0000 mgr.y (mgr.24961) 2148 : cluster [DBG] pgmap v2504: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:32 smithi139 bash[43301]: cluster 2022-04-28T20:53:30.647928+0000 mgr.y (mgr.24961) 2148 : cluster [DBG] pgmap v2504: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:33 smithi059 bash[61152]: cluster 2022-04-28T20:53:32.649024+0000 mgr.y (mgr.24961) 2149 : cluster [DBG] pgmap v2505: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:33.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:33 smithi059 bash[59745]: cluster 2022-04-28T20:53:32.649024+0000 mgr.y (mgr.24961) 2149 : cluster [DBG] pgmap v2505: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:33 smithi139 bash[43301]: cluster 2022-04-28T20:53:32.649024+0000 mgr.y (mgr.24961) 2149 : cluster [DBG] pgmap v2505: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:33.761Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:36 smithi059 bash[59745]: cluster 2022-04-28T20:53:34.650258+0000 mgr.y (mgr.24961) 2150 : cluster [DBG] pgmap v2506: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:36 smithi059 bash[61152]: cluster 2022-04-28T20:53:34.650258+0000 mgr.y (mgr.24961) 2150 : cluster [DBG] pgmap v2506: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:36 smithi139 bash[43301]: cluster 2022-04-28T20:53:34.650258+0000 mgr.y (mgr.24961) 2150 : cluster [DBG] pgmap v2506: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:38 smithi059 bash[61152]: cluster 2022-04-28T20:53:36.651686+0000 mgr.y (mgr.24961) 2151 : cluster [DBG] pgmap v2507: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:38 smithi059 bash[59745]: cluster 2022-04-28T20:53:36.651686+0000 mgr.y (mgr.24961) 2151 : cluster [DBG] pgmap v2507: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:38 smithi139 bash[43301]: cluster 2022-04-28T20:53:36.651686+0000 mgr.y (mgr.24961) 2151 : cluster [DBG] pgmap v2507: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:40 smithi059 bash[61152]: cluster 2022-04-28T20:53:38.652442+0000 mgr.y (mgr.24961) 2152 : cluster [DBG] pgmap v2508: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:40 smithi059 bash[59745]: cluster 2022-04-28T20:53:38.652442+0000 mgr.y (mgr.24961) 2152 : cluster [DBG] pgmap v2508: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:40 smithi139 bash[43301]: cluster 2022-04-28T20:53:38.652442+0000 mgr.y (mgr.24961) 2152 : cluster [DBG] pgmap v2508: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:42 smithi059 bash[61152]: cluster 2022-04-28T20:53:40.653817+0000 mgr.y (mgr.24961) 2153 : cluster [DBG] pgmap v2509: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:41.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:42 smithi059 bash[59745]: cluster 2022-04-28T20:53:40.653817+0000 mgr.y (mgr.24961) 2153 : cluster [DBG] pgmap v2509: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:42 smithi139 bash[43301]: cluster 2022-04-28T20:53:40.653817+0000 mgr.y (mgr.24961) 2153 : cluster [DBG] pgmap v2509: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:44.045 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:43.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:44 smithi059 bash[61152]: cluster 2022-04-28T20:53:42.654848+0000 mgr.y (mgr.24961) 2154 : cluster [DBG] pgmap v2510: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:44 smithi059 bash[59745]: cluster 2022-04-28T20:53:42.654848+0000 mgr.y (mgr.24961) 2154 : cluster [DBG] pgmap v2510: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:44 smithi139 bash[43301]: cluster 2022-04-28T20:53:42.654848+0000 mgr.y (mgr.24961) 2154 : cluster [DBG] pgmap v2510: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:46 smithi059 bash[61152]: cluster 2022-04-28T20:53:44.655976+0000 mgr.y (mgr.24961) 2155 : cluster [DBG] pgmap v2511: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:46 smithi059 bash[59745]: cluster 2022-04-28T20:53:44.655976+0000 mgr.y (mgr.24961) 2155 : cluster [DBG] pgmap v2511: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:46 smithi139 bash[43301]: cluster 2022-04-28T20:53:44.655976+0000 mgr.y (mgr.24961) 2155 : cluster [DBG] pgmap v2511: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:48 smithi059 bash[61152]: cluster 2022-04-28T20:53:46.657250+0000 mgr.y (mgr.24961) 2156 : cluster [DBG] pgmap v2512: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:48 smithi059 bash[59745]: cluster 2022-04-28T20:53:46.657250+0000 mgr.y (mgr.24961) 2156 : cluster [DBG] pgmap v2512: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:48 smithi139 bash[43301]: cluster 2022-04-28T20:53:46.657250+0000 mgr.y (mgr.24961) 2156 : cluster [DBG] pgmap v2512: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:50 smithi059 bash[61152]: cluster 2022-04-28T20:53:48.657977+0000 mgr.y (mgr.24961) 2157 : cluster [DBG] pgmap v2513: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:50 smithi059 bash[59745]: cluster 2022-04-28T20:53:48.657977+0000 mgr.y (mgr.24961) 2157 : cluster [DBG] pgmap v2513: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:50 smithi139 bash[43301]: cluster 2022-04-28T20:53:48.657977+0000 mgr.y (mgr.24961) 2157 : cluster [DBG] pgmap v2513: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:52 smithi059 bash[61152]: cluster 2022-04-28T20:53:50.659511+0000 mgr.y (mgr.24961) 2158 : cluster [DBG] pgmap v2514: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:52.344 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:51.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:52.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:52 smithi059 bash[59745]: cluster 2022-04-28T20:53:50.659511+0000 mgr.y (mgr.24961) 2158 : cluster [DBG] pgmap v2514: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:52 smithi139 bash[43301]: cluster 2022-04-28T20:53:50.659511+0000 mgr.y (mgr.24961) 2158 : cluster [DBG] pgmap v2514: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:54.075 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:53:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:53:53.763Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:53:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:54 smithi059 bash[61152]: cluster 2022-04-28T20:53:52.660530+0000 mgr.y (mgr.24961) 2159 : cluster [DBG] pgmap v2515: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:54.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:54 smithi059 bash[59745]: cluster 2022-04-28T20:53:52.660530+0000 mgr.y (mgr.24961) 2159 : cluster [DBG] pgmap v2515: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:54 smithi139 bash[43301]: cluster 2022-04-28T20:53:52.660530+0000 mgr.y (mgr.24961) 2159 : cluster [DBG] pgmap v2515: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:55 smithi059 bash[61152]: cluster 2022-04-28T20:53:54.661638+0000 mgr.y (mgr.24961) 2160 : cluster [DBG] pgmap v2516: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:55 smithi059 bash[59745]: cluster 2022-04-28T20:53:54.661638+0000 mgr.y (mgr.24961) 2160 : cluster [DBG] pgmap v2516: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:55 smithi139 bash[43301]: cluster 2022-04-28T20:53:54.661638+0000 mgr.y (mgr.24961) 2160 : cluster [DBG] pgmap v2516: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:53:57 smithi059 bash[61152]: cluster 2022-04-28T20:53:56.662902+0000 mgr.y (mgr.24961) 2161 : cluster [DBG] pgmap v2517: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:53:57 smithi059 bash[59745]: cluster 2022-04-28T20:53:56.662902+0000 mgr.y (mgr.24961) 2161 : cluster [DBG] pgmap v2517: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:53:58.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:53:57 smithi139 bash[43301]: cluster 2022-04-28T20:53:56.662902+0000 mgr.y (mgr.24961) 2161 : cluster [DBG] pgmap v2517: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:00 smithi059 bash[61152]: cluster 2022-04-28T20:53:58.663633+0000 mgr.y (mgr.24961) 2162 : cluster [DBG] pgmap v2518: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:00 smithi059 bash[59745]: cluster 2022-04-28T20:53:58.663633+0000 mgr.y (mgr.24961) 2162 : cluster [DBG] pgmap v2518: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:00 smithi139 bash[43301]: cluster 2022-04-28T20:53:58.663633+0000 mgr.y (mgr.24961) 2162 : cluster [DBG] pgmap v2518: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[59745]: cluster 2022-04-28T20:54:00.664990+0000 mgr.y (mgr.24961) 2163 : cluster [DBG] pgmap v2519: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[59745]: audit 2022-04-28T20:54:01.502911+0000 mon.a (mon.0) 2314 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:54:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[59745]: audit 2022-04-28T20:54:01.506039+0000 mon.a (mon.0) 2315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:54:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[59745]: audit 2022-04-28T20:54:01.508076+0000 mon.a (mon.0) 2316 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:54:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[59745]: audit 2022-04-28T20:54:01.835169+0000 mon.a (mon.0) 2317 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:54:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[61152]: cluster 2022-04-28T20:54:00.664990+0000 mgr.y (mgr.24961) 2163 : cluster [DBG] pgmap v2519: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[61152]: audit 2022-04-28T20:54:01.502911+0000 mon.a (mon.0) 2314 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:54:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[61152]: audit 2022-04-28T20:54:01.506039+0000 mon.a (mon.0) 2315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:54:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[61152]: audit 2022-04-28T20:54:01.508076+0000 mon.a (mon.0) 2316 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:54:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:02 smithi059 bash[61152]: audit 2022-04-28T20:54:01.835169+0000 mon.a (mon.0) 2317 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:54:02.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:01.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:02 smithi139 bash[43301]: cluster 2022-04-28T20:54:00.664990+0000 mgr.y (mgr.24961) 2163 : cluster [DBG] pgmap v2519: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:02 smithi139 bash[43301]: audit 2022-04-28T20:54:01.502911+0000 mon.a (mon.0) 2314 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:02 smithi139 bash[43301]: audit 2022-04-28T20:54:01.506039+0000 mon.a (mon.0) 2315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:02 smithi139 bash[43301]: audit 2022-04-28T20:54:01.508076+0000 mon.a (mon.0) 2316 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:54:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:02 smithi139 bash[43301]: audit 2022-04-28T20:54:01.835169+0000 mon.a (mon.0) 2317 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:54:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:03 smithi059 bash[61152]: cephadm 2022-04-28T20:54:01.509954+0000 mgr.y (mgr.24961) 2164 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:54:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:03 smithi059 bash[59745]: cephadm 2022-04-28T20:54:01.509954+0000 mgr.y (mgr.24961) 2164 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:54:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:03 smithi139 bash[43301]: cephadm 2022-04-28T20:54:01.509954+0000 mgr.y (mgr.24961) 2164 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:54:04.048 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:03.764Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:04 smithi059 bash[61152]: cluster 2022-04-28T20:54:02.665944+0000 mgr.y (mgr.24961) 2165 : cluster [DBG] pgmap v2520: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:04 smithi059 bash[59745]: cluster 2022-04-28T20:54:02.665944+0000 mgr.y (mgr.24961) 2165 : cluster [DBG] pgmap v2520: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:04 smithi139 bash[43301]: cluster 2022-04-28T20:54:02.665944+0000 mgr.y (mgr.24961) 2165 : cluster [DBG] pgmap v2520: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:06 smithi059 bash[61152]: cluster 2022-04-28T20:54:04.667057+0000 mgr.y (mgr.24961) 2166 : cluster [DBG] pgmap v2521: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:06.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:06 smithi059 bash[59745]: cluster 2022-04-28T20:54:04.667057+0000 mgr.y (mgr.24961) 2166 : cluster [DBG] pgmap v2521: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:06 smithi139 bash[43301]: cluster 2022-04-28T20:54:04.667057+0000 mgr.y (mgr.24961) 2166 : cluster [DBG] pgmap v2521: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:08 smithi059 bash[59745]: cluster 2022-04-28T20:54:06.668343+0000 mgr.y (mgr.24961) 2167 : cluster [DBG] pgmap v2522: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:08 smithi059 bash[61152]: cluster 2022-04-28T20:54:06.668343+0000 mgr.y (mgr.24961) 2167 : cluster [DBG] pgmap v2522: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:08.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:08 smithi139 bash[43301]: cluster 2022-04-28T20:54:06.668343+0000 mgr.y (mgr.24961) 2167 : cluster [DBG] pgmap v2522: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:10 smithi059 bash[59745]: cluster 2022-04-28T20:54:08.669045+0000 mgr.y (mgr.24961) 2168 : cluster [DBG] pgmap v2523: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:10 smithi059 bash[61152]: cluster 2022-04-28T20:54:08.669045+0000 mgr.y (mgr.24961) 2168 : cluster [DBG] pgmap v2523: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:10.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:10 smithi139 bash[43301]: cluster 2022-04-28T20:54:08.669045+0000 mgr.y (mgr.24961) 2168 : cluster [DBG] pgmap v2523: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:12 smithi059 bash[61152]: cluster 2022-04-28T20:54:10.670464+0000 mgr.y (mgr.24961) 2169 : cluster [DBG] pgmap v2524: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:11.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:12.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:12 smithi059 bash[59745]: cluster 2022-04-28T20:54:10.670464+0000 mgr.y (mgr.24961) 2169 : cluster [DBG] pgmap v2524: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:12 smithi139 bash[43301]: cluster 2022-04-28T20:54:10.670464+0000 mgr.y (mgr.24961) 2169 : cluster [DBG] pgmap v2524: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:54:14.076 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:13.765Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:14 smithi059 bash[61152]: cluster 2022-04-28T20:54:12.671474+0000 mgr.y (mgr.24961) 2170 : cluster [DBG] pgmap v2525: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:14 smithi059 bash[59745]: cluster 2022-04-28T20:54:12.671474+0000 mgr.y (mgr.24961) 2170 : cluster [DBG] pgmap v2525: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:14 smithi139 bash[43301]: cluster 2022-04-28T20:54:12.671474+0000 mgr.y (mgr.24961) 2170 : cluster [DBG] pgmap v2525: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:16 smithi059 bash[61152]: cluster 2022-04-28T20:54:14.672621+0000 mgr.y (mgr.24961) 2171 : cluster [DBG] pgmap v2526: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:16 smithi059 bash[59745]: cluster 2022-04-28T20:54:14.672621+0000 mgr.y (mgr.24961) 2171 : cluster [DBG] pgmap v2526: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:16 smithi139 bash[43301]: cluster 2022-04-28T20:54:14.672621+0000 mgr.y (mgr.24961) 2171 : cluster [DBG] pgmap v2526: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:17 smithi059 bash[61152]: cluster 2022-04-28T20:54:16.673936+0000 mgr.y (mgr.24961) 2172 : cluster [DBG] pgmap v2527: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:17 smithi059 bash[59745]: cluster 2022-04-28T20:54:16.673936+0000 mgr.y (mgr.24961) 2172 : cluster [DBG] pgmap v2527: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:17 smithi139 bash[43301]: cluster 2022-04-28T20:54:16.673936+0000 mgr.y (mgr.24961) 2172 : cluster [DBG] pgmap v2527: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:18.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:18 smithi139 bash[43301]: audit 2022-04-28T20:54:17.905271+0000 mon.a (mon.0) 2318 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:54:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:18 smithi139 bash[43301]: audit 2022-04-28T20:54:17.905504+0000 mon.a (mon.0) 2319 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:54:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:18 smithi139 bash[43301]: audit 2022-04-28T20:54:17.915164+0000 mon.a (mon.0) 2320 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:54:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:18 smithi139 bash[43301]: audit 2022-04-28T20:54:17.916041+0000 mon.a (mon.0) 2321 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:54:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:18 smithi139 bash[43301]: audit 2022-04-28T20:54:17.916791+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:54:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[61152]: audit 2022-04-28T20:54:17.905271+0000 mon.a (mon.0) 2318 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:54:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[61152]: audit 2022-04-28T20:54:17.905504+0000 mon.a (mon.0) 2319 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:54:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[61152]: audit 2022-04-28T20:54:17.915164+0000 mon.a (mon.0) 2320 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:54:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[61152]: audit 2022-04-28T20:54:17.916041+0000 mon.a (mon.0) 2321 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:54:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[61152]: audit 2022-04-28T20:54:17.916791+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:54:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[59745]: audit 2022-04-28T20:54:17.905271+0000 mon.a (mon.0) 2318 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:54:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[59745]: audit 2022-04-28T20:54:17.905504+0000 mon.a (mon.0) 2319 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:54:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[59745]: audit 2022-04-28T20:54:17.915164+0000 mon.a (mon.0) 2320 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T20:54:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[59745]: audit 2022-04-28T20:54:17.916041+0000 mon.a (mon.0) 2321 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:54:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:18 smithi059 bash[59745]: audit 2022-04-28T20:54:17.916791+0000 mon.a (mon.0) 2322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T20:54:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:20 smithi059 bash[61152]: cluster 2022-04-28T20:54:18.674693+0000 mgr.y (mgr.24961) 2173 : cluster [DBG] pgmap v2528: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:20 smithi059 bash[59745]: cluster 2022-04-28T20:54:18.674693+0000 mgr.y (mgr.24961) 2173 : cluster [DBG] pgmap v2528: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:20 smithi139 bash[43301]: cluster 2022-04-28T20:54:18.674693+0000 mgr.y (mgr.24961) 2173 : cluster [DBG] pgmap v2528: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:22 smithi059 bash[59745]: cluster 2022-04-28T20:54:20.676190+0000 mgr.y (mgr.24961) 2174 : cluster [DBG] pgmap v2529: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:22 smithi059 bash[61152]: cluster 2022-04-28T20:54:20.676190+0000 mgr.y (mgr.24961) 2174 : cluster [DBG] pgmap v2529: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:21.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:22 smithi139 bash[43301]: cluster 2022-04-28T20:54:20.676190+0000 mgr.y (mgr.24961) 2174 : cluster [DBG] pgmap v2529: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:24.046 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:23.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:24 smithi059 bash[59745]: cluster 2022-04-28T20:54:22.677272+0000 mgr.y (mgr.24961) 2175 : cluster [DBG] pgmap v2530: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:24 smithi059 bash[61152]: cluster 2022-04-28T20:54:22.677272+0000 mgr.y (mgr.24961) 2175 : cluster [DBG] pgmap v2530: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:24 smithi139 bash[43301]: cluster 2022-04-28T20:54:22.677272+0000 mgr.y (mgr.24961) 2175 : cluster [DBG] pgmap v2530: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:26 smithi059 bash[59745]: cluster 2022-04-28T20:54:24.678423+0000 mgr.y (mgr.24961) 2176 : cluster [DBG] pgmap v2531: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:26.357 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:26 smithi059 bash[61152]: cluster 2022-04-28T20:54:24.678423+0000 mgr.y (mgr.24961) 2176 : cluster [DBG] pgmap v2531: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:26 smithi139 bash[43301]: cluster 2022-04-28T20:54:24.678423+0000 mgr.y (mgr.24961) 2176 : cluster [DBG] pgmap v2531: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:28 smithi059 bash[61152]: cluster 2022-04-28T20:54:26.679835+0000 mgr.y (mgr.24961) 2177 : cluster [DBG] pgmap v2532: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:28 smithi059 bash[59745]: cluster 2022-04-28T20:54:26.679835+0000 mgr.y (mgr.24961) 2177 : cluster [DBG] pgmap v2532: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:28 smithi139 bash[43301]: cluster 2022-04-28T20:54:26.679835+0000 mgr.y (mgr.24961) 2177 : cluster [DBG] pgmap v2532: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:54:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:30 smithi059 bash[59745]: cluster 2022-04-28T20:54:28.680618+0000 mgr.y (mgr.24961) 2178 : cluster [DBG] pgmap v2533: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:30.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:30 smithi059 bash[61152]: cluster 2022-04-28T20:54:28.680618+0000 mgr.y (mgr.24961) 2178 : cluster [DBG] pgmap v2533: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:30 smithi139 bash[43301]: cluster 2022-04-28T20:54:28.680618+0000 mgr.y (mgr.24961) 2178 : cluster [DBG] pgmap v2533: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:32 smithi059 bash[61152]: cluster 2022-04-28T20:54:30.682073+0000 mgr.y (mgr.24961) 2179 : cluster [DBG] pgmap v2534: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:31.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:32 smithi059 bash[59745]: cluster 2022-04-28T20:54:30.682073+0000 mgr.y (mgr.24961) 2179 : cluster [DBG] pgmap v2534: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:32 smithi139 bash[43301]: cluster 2022-04-28T20:54:30.682073+0000 mgr.y (mgr.24961) 2179 : cluster [DBG] pgmap v2534: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:34.073 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:33.767Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:34 smithi059 bash[59745]: cluster 2022-04-28T20:54:32.683142+0000 mgr.y (mgr.24961) 2180 : cluster [DBG] pgmap v2535: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:34 smithi059 bash[61152]: cluster 2022-04-28T20:54:32.683142+0000 mgr.y (mgr.24961) 2180 : cluster [DBG] pgmap v2535: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:34 smithi139 bash[43301]: cluster 2022-04-28T20:54:32.683142+0000 mgr.y (mgr.24961) 2180 : cluster [DBG] pgmap v2535: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:36 smithi059 bash[61152]: cluster 2022-04-28T20:54:34.684349+0000 mgr.y (mgr.24961) 2181 : cluster [DBG] pgmap v2536: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:36 smithi059 bash[59745]: cluster 2022-04-28T20:54:34.684349+0000 mgr.y (mgr.24961) 2181 : cluster [DBG] pgmap v2536: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:36.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:36 smithi139 bash[43301]: cluster 2022-04-28T20:54:34.684349+0000 mgr.y (mgr.24961) 2181 : cluster [DBG] pgmap v2536: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:38 smithi059 bash[61152]: cluster 2022-04-28T20:54:36.685593+0000 mgr.y (mgr.24961) 2182 : cluster [DBG] pgmap v2537: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:54:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:38 smithi059 bash[59745]: cluster 2022-04-28T20:54:36.685593+0000 mgr.y (mgr.24961) 2182 : cluster [DBG] pgmap v2537: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:54:38.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:38 smithi139 bash[43301]: cluster 2022-04-28T20:54:36.685593+0000 mgr.y (mgr.24961) 2182 : cluster [DBG] pgmap v2537: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:54:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:40 smithi059 bash[59745]: cluster 2022-04-28T20:54:38.686351+0000 mgr.y (mgr.24961) 2183 : cluster [DBG] pgmap v2538: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:40 smithi059 bash[61152]: cluster 2022-04-28T20:54:38.686351+0000 mgr.y (mgr.24961) 2183 : cluster [DBG] pgmap v2538: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:40 smithi139 bash[43301]: cluster 2022-04-28T20:54:38.686351+0000 mgr.y (mgr.24961) 2183 : cluster [DBG] pgmap v2538: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:42 smithi059 bash[61152]: cluster 2022-04-28T20:54:40.687896+0000 mgr.y (mgr.24961) 2184 : cluster [DBG] pgmap v2539: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:41.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:42.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:42 smithi059 bash[59745]: cluster 2022-04-28T20:54:40.687896+0000 mgr.y (mgr.24961) 2184 : cluster [DBG] pgmap v2539: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:42 smithi139 bash[43301]: cluster 2022-04-28T20:54:40.687896+0000 mgr.y (mgr.24961) 2184 : cluster [DBG] pgmap v2539: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:44.063 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:44 smithi059 bash[59745]: cluster 2022-04-28T20:54:42.688941+0000 mgr.y (mgr.24961) 2185 : cluster [DBG] pgmap v2540: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:44.064 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:43.766Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:44 smithi059 bash[61152]: cluster 2022-04-28T20:54:42.688941+0000 mgr.y (mgr.24961) 2185 : cluster [DBG] pgmap v2540: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:44.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:44 smithi139 bash[43301]: cluster 2022-04-28T20:54:42.688941+0000 mgr.y (mgr.24961) 2185 : cluster [DBG] pgmap v2540: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:46 smithi059 bash[61152]: cluster 2022-04-28T20:54:44.690084+0000 mgr.y (mgr.24961) 2186 : cluster [DBG] pgmap v2541: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:46 smithi059 bash[59745]: cluster 2022-04-28T20:54:44.690084+0000 mgr.y (mgr.24961) 2186 : cluster [DBG] pgmap v2541: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:46 smithi139 bash[43301]: cluster 2022-04-28T20:54:44.690084+0000 mgr.y (mgr.24961) 2186 : cluster [DBG] pgmap v2541: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:54:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:48 smithi059 bash[61152]: cluster 2022-04-28T20:54:46.691392+0000 mgr.y (mgr.24961) 2187 : cluster [DBG] pgmap v2542: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:48 smithi059 bash[59745]: cluster 2022-04-28T20:54:46.691392+0000 mgr.y (mgr.24961) 2187 : cluster [DBG] pgmap v2542: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:48 smithi139 bash[43301]: cluster 2022-04-28T20:54:46.691392+0000 mgr.y (mgr.24961) 2187 : cluster [DBG] pgmap v2542: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:50 smithi059 bash[59745]: cluster 2022-04-28T20:54:48.692161+0000 mgr.y (mgr.24961) 2188 : cluster [DBG] pgmap v2543: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:50 smithi059 bash[61152]: cluster 2022-04-28T20:54:48.692161+0000 mgr.y (mgr.24961) 2188 : cluster [DBG] pgmap v2543: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:50 smithi139 bash[43301]: cluster 2022-04-28T20:54:48.692161+0000 mgr.y (mgr.24961) 2188 : cluster [DBG] pgmap v2543: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:52 smithi059 bash[59745]: cluster 2022-04-28T20:54:50.693139+0000 mgr.y (mgr.24961) 2189 : cluster [DBG] pgmap v2544: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:52 smithi059 bash[61152]: cluster 2022-04-28T20:54:50.693139+0000 mgr.y (mgr.24961) 2189 : cluster [DBG] pgmap v2544: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:51.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:52.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:52 smithi139 bash[43301]: cluster 2022-04-28T20:54:50.693139+0000 mgr.y (mgr.24961) 2189 : cluster [DBG] pgmap v2544: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:54:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:53 smithi059 bash[61152]: cluster 2022-04-28T20:54:52.694070+0000 mgr.y (mgr.24961) 2190 : cluster [DBG] pgmap v2545: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:53.638 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:53 smithi059 bash[59745]: cluster 2022-04-28T20:54:52.694070+0000 mgr.y (mgr.24961) 2190 : cluster [DBG] pgmap v2545: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:53.639 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:53 smithi139 bash[43301]: cluster 2022-04-28T20:54:52.694070+0000 mgr.y (mgr.24961) 2190 : cluster [DBG] pgmap v2545: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:54:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:54:53.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:54:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:56 smithi059 bash[61152]: cluster 2022-04-28T20:54:54.695187+0000 mgr.y (mgr.24961) 2191 : cluster [DBG] pgmap v2546: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:56 smithi059 bash[59745]: cluster 2022-04-28T20:54:54.695187+0000 mgr.y (mgr.24961) 2191 : cluster [DBG] pgmap v2546: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:56 smithi139 bash[43301]: cluster 2022-04-28T20:54:54.695187+0000 mgr.y (mgr.24961) 2191 : cluster [DBG] pgmap v2546: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:54:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:54:58 smithi059 bash[61152]: cluster 2022-04-28T20:54:56.696539+0000 mgr.y (mgr.24961) 2192 : cluster [DBG] pgmap v2547: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:54:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:54:58 smithi059 bash[59745]: cluster 2022-04-28T20:54:56.696539+0000 mgr.y (mgr.24961) 2192 : cluster [DBG] pgmap v2547: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:54:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:54:58 smithi139 bash[43301]: cluster 2022-04-28T20:54:56.696539+0000 mgr.y (mgr.24961) 2192 : cluster [DBG] pgmap v2547: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:55:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:00 smithi059 bash[59745]: cluster 2022-04-28T20:54:58.697295+0000 mgr.y (mgr.24961) 2193 : cluster [DBG] pgmap v2548: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:55:00.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:00 smithi059 bash[61152]: cluster 2022-04-28T20:54:58.697295+0000 mgr.y (mgr.24961) 2193 : cluster [DBG] pgmap v2548: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:55:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:00 smithi139 bash[43301]: cluster 2022-04-28T20:54:58.697295+0000 mgr.y (mgr.24961) 2193 : cluster [DBG] pgmap v2548: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:55:02.069 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:02 smithi139 bash[43301]: cluster 2022-04-28T20:55:00.698672+0000 mgr.y (mgr.24961) 2194 : cluster [DBG] pgmap v2549: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:55:02.070 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:02 smithi139 bash[43301]: audit 2022-04-28T20:55:01.844127+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:55:02.070 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:02 smithi139 bash[43301]: audit 2022-04-28T20:55:01.847037+0000 mon.a (mon.0) 2324 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:55:02.070 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:02 smithi139 bash[43301]: audit 2022-04-28T20:55:01.848936+0000 mon.a (mon.0) 2325 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:55:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[59745]: cluster 2022-04-28T20:55:00.698672+0000 mgr.y (mgr.24961) 2194 : cluster [DBG] pgmap v2549: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:55:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[59745]: audit 2022-04-28T20:55:01.844127+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:55:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[59745]: audit 2022-04-28T20:55:01.847037+0000 mon.a (mon.0) 2324 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:55:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[59745]: audit 2022-04-28T20:55:01.848936+0000 mon.a (mon.0) 2325 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:55:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[61152]: cluster 2022-04-28T20:55:00.698672+0000 mgr.y (mgr.24961) 2194 : cluster [DBG] pgmap v2549: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:55:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[61152]: audit 2022-04-28T20:55:01.844127+0000 mon.a (mon.0) 2323 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:55:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[61152]: audit 2022-04-28T20:55:01.847037+0000 mon.a (mon.0) 2324 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:55:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:02 smithi059 bash[61152]: audit 2022-04-28T20:55:01.848936+0000 mon.a (mon.0) 2325 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:55:02.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:01.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:03 smithi059 bash[61152]: cephadm 2022-04-28T20:55:01.850649+0000 mgr.y (mgr.24961) 2195 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:55:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:03 smithi059 bash[61152]: audit 2022-04-28T20:55:02.171768+0000 mon.a (mon.0) 2326 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:55:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:03 smithi059 bash[59745]: cephadm 2022-04-28T20:55:01.850649+0000 mgr.y (mgr.24961) 2195 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:55:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:03 smithi059 bash[59745]: audit 2022-04-28T20:55:02.171768+0000 mon.a (mon.0) 2326 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:55:03.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:03 smithi139 bash[43301]: cephadm 2022-04-28T20:55:01.850649+0000 mgr.y (mgr.24961) 2195 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:55:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:03 smithi139 bash[43301]: audit 2022-04-28T20:55:02.171768+0000 mon.a (mon.0) 2326 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:55:04.073 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:03.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:04 smithi059 bash[61152]: cluster 2022-04-28T20:55:02.699727+0000 mgr.y (mgr.24961) 2196 : cluster [DBG] pgmap v2550: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:55:04.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:04 smithi059 bash[59745]: cluster 2022-04-28T20:55:02.699727+0000 mgr.y (mgr.24961) 2196 : cluster [DBG] pgmap v2550: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:55:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:04 smithi139 bash[43301]: cluster 2022-04-28T20:55:02.699727+0000 mgr.y (mgr.24961) 2196 : cluster [DBG] pgmap v2550: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:55:06.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:06 smithi059 bash[59745]: cluster 2022-04-28T20:55:04.700862+0000 mgr.y (mgr.24961) 2197 : cluster [DBG] pgmap v2551: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:55:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:06 smithi059 bash[61152]: cluster 2022-04-28T20:55:04.700862+0000 mgr.y (mgr.24961) 2197 : cluster [DBG] pgmap v2551: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:55:06.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:06 smithi139 bash[43301]: cluster 2022-04-28T20:55:04.700862+0000 mgr.y (mgr.24961) 2197 : cluster [DBG] pgmap v2551: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:55:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:08 smithi059 bash[61152]: cluster 2022-04-28T20:55:06.702244+0000 mgr.y (mgr.24961) 2198 : cluster [DBG] pgmap v2552: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:55:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:08 smithi059 bash[59745]: cluster 2022-04-28T20:55:06.702244+0000 mgr.y (mgr.24961) 2198 : cluster [DBG] pgmap v2552: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:55:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:08 smithi139 bash[43301]: cluster 2022-04-28T20:55:06.702244+0000 mgr.y (mgr.24961) 2198 : cluster [DBG] pgmap v2552: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:55:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:10 smithi139 bash[43301]: cluster 2022-04-28T20:55:08.703031+0000 mgr.y (mgr.24961) 2199 : cluster [DBG] pgmap v2553: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:55:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:10 smithi059 bash[61152]: cluster 2022-04-28T20:55:08.703031+0000 mgr.y (mgr.24961) 2199 : cluster [DBG] pgmap v2553: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:55:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:10 smithi059 bash[59745]: cluster 2022-04-28T20:55:08.703031+0000 mgr.y (mgr.24961) 2199 : cluster [DBG] pgmap v2553: 137 pgs: 137 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T20:55:11.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:11 smithi139 bash[43301]: cluster 2022-04-28T20:55:10.093350+0000 mon.a (mon.0) 2327 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2022-04-28T20:55:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:11 smithi139 bash[43301]: cluster 2022-04-28T20:55:10.703764+0000 mgr.y (mgr.24961) 2200 : cluster [DBG] pgmap v2555: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:11 smithi059 bash[61152]: cluster 2022-04-28T20:55:10.093350+0000 mon.a (mon.0) 2327 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2022-04-28T20:55:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:11 smithi059 bash[61152]: cluster 2022-04-28T20:55:10.703764+0000 mgr.y (mgr.24961) 2200 : cluster [DBG] pgmap v2555: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:11 smithi059 bash[59745]: cluster 2022-04-28T20:55:10.093350+0000 mon.a (mon.0) 2327 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2022-04-28T20:55:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:11 smithi059 bash[59745]: cluster 2022-04-28T20:55:10.703764+0000 mgr.y (mgr.24961) 2200 : cluster [DBG] pgmap v2555: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:11.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:13 smithi059 bash[61152]: cluster 2022-04-28T20:55:12.704700+0000 mgr.y (mgr.24961) 2201 : cluster [DBG] pgmap v2556: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:13.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:13 smithi059 bash[59745]: cluster 2022-04-28T20:55:12.704700+0000 mgr.y (mgr.24961) 2201 : cluster [DBG] pgmap v2556: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:14.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:13 smithi139 bash[43301]: cluster 2022-04-28T20:55:12.704700+0000 mgr.y (mgr.24961) 2201 : cluster [DBG] pgmap v2556: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:16 smithi059 bash[61152]: cluster 2022-04-28T20:55:14.705706+0000 mgr.y (mgr.24961) 2202 : cluster [DBG] pgmap v2557: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:16 smithi059 bash[59745]: cluster 2022-04-28T20:55:14.705706+0000 mgr.y (mgr.24961) 2202 : cluster [DBG] pgmap v2557: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:16.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:16 smithi139 bash[43301]: cluster 2022-04-28T20:55:14.705706+0000 mgr.y (mgr.24961) 2202 : cluster [DBG] pgmap v2557: 105 pgs: 105 active+clean; 24 KiB data, 726 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[61152]: cluster 2022-04-28T20:55:16.706858+0000 mgr.y (mgr.24961) 2203 : cluster [DBG] pgmap v2558: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[61152]: audit 2022-04-28T20:55:17.916759+0000 mon.a (mon.0) 2328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:55:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[61152]: audit 2022-04-28T20:55:17.917258+0000 mon.a (mon.0) 2329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:55:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[61152]: audit 2022-04-28T20:55:17.929048+0000 mon.a (mon.0) 2330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:55:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[61152]: audit 2022-04-28T20:55:17.929802+0000 mon.a (mon.0) 2331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:55:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[61152]: audit 2022-04-28T20:55:17.930259+0000 mon.a (mon.0) 2332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:55:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[59745]: cluster 2022-04-28T20:55:16.706858+0000 mgr.y (mgr.24961) 2203 : cluster [DBG] pgmap v2558: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[59745]: audit 2022-04-28T20:55:17.916759+0000 mon.a (mon.0) 2328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:55:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[59745]: audit 2022-04-28T20:55:17.917258+0000 mon.a (mon.0) 2329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:55:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[59745]: audit 2022-04-28T20:55:17.929048+0000 mon.a (mon.0) 2330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:55:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[59745]: audit 2022-04-28T20:55:17.929802+0000 mon.a (mon.0) 2331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:55:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:18 smithi059 bash[59745]: audit 2022-04-28T20:55:17.930259+0000 mon.a (mon.0) 2332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:18 smithi139 bash[43301]: cluster 2022-04-28T20:55:16.706858+0000 mgr.y (mgr.24961) 2203 : cluster [DBG] pgmap v2558: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:18 smithi139 bash[43301]: audit 2022-04-28T20:55:17.916759+0000 mon.a (mon.0) 2328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:18 smithi139 bash[43301]: audit 2022-04-28T20:55:17.917258+0000 mon.a (mon.0) 2329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:18 smithi139 bash[43301]: audit 2022-04-28T20:55:17.929048+0000 mon.a (mon.0) 2330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T20:55:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:18 smithi139 bash[43301]: audit 2022-04-28T20:55:17.929802+0000 mon.a (mon.0) 2331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T20:55:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:18 smithi139 bash[43301]: audit 2022-04-28T20:55:17.930259+0000 mon.a (mon.0) 2332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:55:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:20 smithi059 bash[59745]: cluster 2022-04-28T20:55:18.707646+0000 mgr.y (mgr.24961) 2204 : cluster [DBG] pgmap v2559: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:20.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:20 smithi059 bash[61152]: cluster 2022-04-28T20:55:18.707646+0000 mgr.y (mgr.24961) 2204 : cluster [DBG] pgmap v2559: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:20 smithi139 bash[43301]: cluster 2022-04-28T20:55:18.707646+0000 mgr.y (mgr.24961) 2204 : cluster [DBG] pgmap v2559: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:22 smithi059 bash[61152]: cluster 2022-04-28T20:55:20.708780+0000 mgr.y (mgr.24961) 2205 : cluster [DBG] pgmap v2560: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:22 smithi059 bash[59745]: cluster 2022-04-28T20:55:20.708780+0000 mgr.y (mgr.24961) 2205 : cluster [DBG] pgmap v2560: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:21.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:22 smithi139 bash[43301]: cluster 2022-04-28T20:55:20.708780+0000 mgr.y (mgr.24961) 2205 : cluster [DBG] pgmap v2560: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:24.079 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:23.770Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:24 smithi059 bash[61152]: cluster 2022-04-28T20:55:22.709718+0000 mgr.y (mgr.24961) 2206 : cluster [DBG] pgmap v2561: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:24 smithi059 bash[59745]: cluster 2022-04-28T20:55:22.709718+0000 mgr.y (mgr.24961) 2206 : cluster [DBG] pgmap v2561: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:24 smithi139 bash[43301]: cluster 2022-04-28T20:55:22.709718+0000 mgr.y (mgr.24961) 2206 : cluster [DBG] pgmap v2561: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:26 smithi059 bash[61152]: cluster 2022-04-28T20:55:24.710730+0000 mgr.y (mgr.24961) 2207 : cluster [DBG] pgmap v2562: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:26 smithi059 bash[59745]: cluster 2022-04-28T20:55:24.710730+0000 mgr.y (mgr.24961) 2207 : cluster [DBG] pgmap v2562: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:26.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:26 smithi139 bash[43301]: cluster 2022-04-28T20:55:24.710730+0000 mgr.y (mgr.24961) 2207 : cluster [DBG] pgmap v2562: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:28 smithi059 bash[61152]: cluster 2022-04-28T20:55:26.711886+0000 mgr.y (mgr.24961) 2208 : cluster [DBG] pgmap v2563: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:28 smithi059 bash[59745]: cluster 2022-04-28T20:55:26.711886+0000 mgr.y (mgr.24961) 2208 : cluster [DBG] pgmap v2563: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:28 smithi139 bash[43301]: cluster 2022-04-28T20:55:26.711886+0000 mgr.y (mgr.24961) 2208 : cluster [DBG] pgmap v2563: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:30 smithi139 bash[43301]: cluster 2022-04-28T20:55:28.712630+0000 mgr.y (mgr.24961) 2209 : cluster [DBG] pgmap v2564: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:30 smithi059 bash[61152]: cluster 2022-04-28T20:55:28.712630+0000 mgr.y (mgr.24961) 2209 : cluster [DBG] pgmap v2564: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:30.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:30 smithi059 bash[59745]: cluster 2022-04-28T20:55:28.712630+0000 mgr.y (mgr.24961) 2209 : cluster [DBG] pgmap v2564: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:31.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:31 smithi139 bash[43301]: cluster 2022-04-28T20:55:30.713832+0000 mgr.y (mgr.24961) 2210 : cluster [DBG] pgmap v2565: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:31 smithi059 bash[61152]: cluster 2022-04-28T20:55:30.713832+0000 mgr.y (mgr.24961) 2210 : cluster [DBG] pgmap v2565: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:31 smithi059 bash[59745]: cluster 2022-04-28T20:55:30.713832+0000 mgr.y (mgr.24961) 2210 : cluster [DBG] pgmap v2565: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:31.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:34.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:33 smithi059 bash[61152]: cluster 2022-04-28T20:55:32.715072+0000 mgr.y (mgr.24961) 2211 : cluster [DBG] pgmap v2566: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:34.096 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:33.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:34.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:33 smithi059 bash[59745]: cluster 2022-04-28T20:55:32.715072+0000 mgr.y (mgr.24961) 2211 : cluster [DBG] pgmap v2566: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:34.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:33 smithi139 bash[43301]: cluster 2022-04-28T20:55:32.715072+0000 mgr.y (mgr.24961) 2211 : cluster [DBG] pgmap v2566: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:36 smithi059 bash[61152]: cluster 2022-04-28T20:55:34.716196+0000 mgr.y (mgr.24961) 2212 : cluster [DBG] pgmap v2567: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:36 smithi059 bash[59745]: cluster 2022-04-28T20:55:34.716196+0000 mgr.y (mgr.24961) 2212 : cluster [DBG] pgmap v2567: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:36.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:36 smithi139 bash[43301]: cluster 2022-04-28T20:55:34.716196+0000 mgr.y (mgr.24961) 2212 : cluster [DBG] pgmap v2567: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:38 smithi059 bash[61152]: cluster 2022-04-28T20:55:36.717362+0000 mgr.y (mgr.24961) 2213 : cluster [DBG] pgmap v2568: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:38 smithi059 bash[59745]: cluster 2022-04-28T20:55:36.717362+0000 mgr.y (mgr.24961) 2213 : cluster [DBG] pgmap v2568: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:38 smithi139 bash[43301]: cluster 2022-04-28T20:55:36.717362+0000 mgr.y (mgr.24961) 2213 : cluster [DBG] pgmap v2568: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:40 smithi059 bash[61152]: cluster 2022-04-28T20:55:38.718079+0000 mgr.y (mgr.24961) 2214 : cluster [DBG] pgmap v2569: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:40 smithi059 bash[59745]: cluster 2022-04-28T20:55:38.718079+0000 mgr.y (mgr.24961) 2214 : cluster [DBG] pgmap v2569: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:40 smithi139 bash[43301]: cluster 2022-04-28T20:55:38.718079+0000 mgr.y (mgr.24961) 2214 : cluster [DBG] pgmap v2569: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:42 smithi059 bash[61152]: cluster 2022-04-28T20:55:40.719274+0000 mgr.y (mgr.24961) 2215 : cluster [DBG] pgmap v2570: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:41.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:42 smithi059 bash[59745]: cluster 2022-04-28T20:55:40.719274+0000 mgr.y (mgr.24961) 2215 : cluster [DBG] pgmap v2570: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:42 smithi139 bash[43301]: cluster 2022-04-28T20:55:40.719274+0000 mgr.y (mgr.24961) 2215 : cluster [DBG] pgmap v2570: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:43.772Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:44 smithi059 bash[59745]: cluster 2022-04-28T20:55:42.720184+0000 mgr.y (mgr.24961) 2216 : cluster [DBG] pgmap v2571: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:44 smithi059 bash[61152]: cluster 2022-04-28T20:55:42.720184+0000 mgr.y (mgr.24961) 2216 : cluster [DBG] pgmap v2571: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:44 smithi139 bash[43301]: cluster 2022-04-28T20:55:42.720184+0000 mgr.y (mgr.24961) 2216 : cluster [DBG] pgmap v2571: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:46 smithi139 bash[43301]: cluster 2022-04-28T20:55:44.721191+0000 mgr.y (mgr.24961) 2217 : cluster [DBG] pgmap v2572: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:46 smithi059 bash[59745]: cluster 2022-04-28T20:55:44.721191+0000 mgr.y (mgr.24961) 2217 : cluster [DBG] pgmap v2572: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:46 smithi059 bash[61152]: cluster 2022-04-28T20:55:44.721191+0000 mgr.y (mgr.24961) 2217 : cluster [DBG] pgmap v2572: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:47 smithi059 bash[59745]: cluster 2022-04-28T20:55:46.722356+0000 mgr.y (mgr.24961) 2218 : cluster [DBG] pgmap v2573: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:47 smithi059 bash[61152]: cluster 2022-04-28T20:55:46.722356+0000 mgr.y (mgr.24961) 2218 : cluster [DBG] pgmap v2573: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:47 smithi139 bash[43301]: cluster 2022-04-28T20:55:46.722356+0000 mgr.y (mgr.24961) 2218 : cluster [DBG] pgmap v2573: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:50 smithi059 bash[61152]: cluster 2022-04-28T20:55:48.723087+0000 mgr.y (mgr.24961) 2219 : cluster [DBG] pgmap v2574: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:50 smithi059 bash[59745]: cluster 2022-04-28T20:55:48.723087+0000 mgr.y (mgr.24961) 2219 : cluster [DBG] pgmap v2574: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:50 smithi139 bash[43301]: cluster 2022-04-28T20:55:48.723087+0000 mgr.y (mgr.24961) 2219 : cluster [DBG] pgmap v2574: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:52 smithi059 bash[61152]: cluster 2022-04-28T20:55:50.724278+0000 mgr.y (mgr.24961) 2220 : cluster [DBG] pgmap v2575: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:52 smithi059 bash[59745]: cluster 2022-04-28T20:55:50.724278+0000 mgr.y (mgr.24961) 2220 : cluster [DBG] pgmap v2575: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:51.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:52 smithi139 bash[43301]: cluster 2022-04-28T20:55:50.724278+0000 mgr.y (mgr.24961) 2220 : cluster [DBG] pgmap v2575: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:54.083 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:55:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:55:53.771Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:55:54.084 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:54 smithi059 bash[59745]: cluster 2022-04-28T20:55:52.725221+0000 mgr.y (mgr.24961) 2221 : cluster [DBG] pgmap v2576: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:54 smithi059 bash[61152]: cluster 2022-04-28T20:55:52.725221+0000 mgr.y (mgr.24961) 2221 : cluster [DBG] pgmap v2576: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:54.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:54 smithi139 bash[43301]: cluster 2022-04-28T20:55:52.725221+0000 mgr.y (mgr.24961) 2221 : cluster [DBG] pgmap v2576: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:56 smithi059 bash[61152]: cluster 2022-04-28T20:55:54.726222+0000 mgr.y (mgr.24961) 2222 : cluster [DBG] pgmap v2577: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:56 smithi059 bash[59745]: cluster 2022-04-28T20:55:54.726222+0000 mgr.y (mgr.24961) 2222 : cluster [DBG] pgmap v2577: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:56.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:56 smithi139 bash[43301]: cluster 2022-04-28T20:55:54.726222+0000 mgr.y (mgr.24961) 2222 : cluster [DBG] pgmap v2577: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:58.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:55:58 smithi139 bash[43301]: cluster 2022-04-28T20:55:56.727363+0000 mgr.y (mgr.24961) 2223 : cluster [DBG] pgmap v2578: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:55:58 smithi059 bash[61152]: cluster 2022-04-28T20:55:56.727363+0000 mgr.y (mgr.24961) 2223 : cluster [DBG] pgmap v2578: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:55:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:55:58 smithi059 bash[59745]: cluster 2022-04-28T20:55:56.727363+0000 mgr.y (mgr.24961) 2223 : cluster [DBG] pgmap v2578: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:00 smithi139 bash[43301]: cluster 2022-04-28T20:55:58.728113+0000 mgr.y (mgr.24961) 2224 : cluster [DBG] pgmap v2579: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:00 smithi059 bash[61152]: cluster 2022-04-28T20:55:58.728113+0000 mgr.y (mgr.24961) 2224 : cluster [DBG] pgmap v2579: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:00.601 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:00 smithi059 bash[59745]: cluster 2022-04-28T20:55:58.728113+0000 mgr.y (mgr.24961) 2224 : cluster [DBG] pgmap v2579: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:01 smithi059 bash[61152]: cluster 2022-04-28T20:56:00.729320+0000 mgr.y (mgr.24961) 2225 : cluster [DBG] pgmap v2580: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:01 smithi059 bash[59745]: cluster 2022-04-28T20:56:00.729320+0000 mgr.y (mgr.24961) 2225 : cluster [DBG] pgmap v2580: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:01 smithi139 bash[43301]: cluster 2022-04-28T20:56:00.729320+0000 mgr.y (mgr.24961) 2225 : cluster [DBG] pgmap v2580: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:02.237 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:01.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:02 smithi059 bash[59745]: audit 2022-04-28T20:56:02.181999+0000 mon.a (mon.0) 2333 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:56:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:02 smithi059 bash[59745]: audit 2022-04-28T20:56:02.184867+0000 mon.a (mon.0) 2334 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:56:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:02 smithi059 bash[59745]: audit 2022-04-28T20:56:02.186702+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:56:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:02 smithi059 bash[61152]: audit 2022-04-28T20:56:02.181999+0000 mon.a (mon.0) 2333 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:56:02.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:02 smithi059 bash[61152]: audit 2022-04-28T20:56:02.184867+0000 mon.a (mon.0) 2334 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:56:02.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:02 smithi059 bash[61152]: audit 2022-04-28T20:56:02.186702+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:56:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:02 smithi139 bash[43301]: audit 2022-04-28T20:56:02.181999+0000 mon.a (mon.0) 2333 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:56:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:02 smithi139 bash[43301]: audit 2022-04-28T20:56:02.184867+0000 mon.a (mon.0) 2334 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:56:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:02 smithi139 bash[43301]: audit 2022-04-28T20:56:02.186702+0000 mon.a (mon.0) 2335 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:56:03.774 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[61152]: cephadm 2022-04-28T20:56:02.188691+0000 mgr.y (mgr.24961) 2226 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:56:03.774 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[61152]: audit 2022-04-28T20:56:02.492292+0000 mon.a (mon.0) 2336 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:56:03.775 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[61152]: cluster 2022-04-28T20:56:02.730265+0000 mgr.y (mgr.24961) 2227 : cluster [DBG] pgmap v2581: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:03.775 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[59745]: cephadm 2022-04-28T20:56:02.188691+0000 mgr.y (mgr.24961) 2226 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:56:03.775 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[59745]: audit 2022-04-28T20:56:02.492292+0000 mon.a (mon.0) 2336 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:56:03.776 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[59745]: cluster 2022-04-28T20:56:02.730265+0000 mgr.y (mgr.24961) 2227 : cluster [DBG] pgmap v2581: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:03.776 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:03.773Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:03.868 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:03 smithi139 bash[43301]: cephadm 2022-04-28T20:56:02.188691+0000 mgr.y (mgr.24961) 2226 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:56:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:03 smithi139 bash[43301]: audit 2022-04-28T20:56:02.492292+0000 mon.a (mon.0) 2336 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:56:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:03 smithi139 bash[43301]: cluster 2022-04-28T20:56:02.730265+0000 mgr.y (mgr.24961) 2227 : cluster [DBG] pgmap v2581: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:06.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:06 smithi059 bash[61152]: cluster 2022-04-28T20:56:04.731270+0000 mgr.y (mgr.24961) 2228 : cluster [DBG] pgmap v2582: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:06.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:06 smithi059 bash[59745]: cluster 2022-04-28T20:56:04.731270+0000 mgr.y (mgr.24961) 2228 : cluster [DBG] pgmap v2582: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:06.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:06 smithi139 bash[43301]: cluster 2022-04-28T20:56:04.731270+0000 mgr.y (mgr.24961) 2228 : cluster [DBG] pgmap v2582: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:08 smithi059 bash[59745]: cluster 2022-04-28T20:56:06.732479+0000 mgr.y (mgr.24961) 2229 : cluster [DBG] pgmap v2583: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:08 smithi059 bash[61152]: cluster 2022-04-28T20:56:06.732479+0000 mgr.y (mgr.24961) 2229 : cluster [DBG] pgmap v2583: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:08 smithi139 bash[43301]: cluster 2022-04-28T20:56:06.732479+0000 mgr.y (mgr.24961) 2229 : cluster [DBG] pgmap v2583: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:10 smithi059 bash[59745]: cluster 2022-04-28T20:56:08.733179+0000 mgr.y (mgr.24961) 2230 : cluster [DBG] pgmap v2584: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:10 smithi059 bash[61152]: cluster 2022-04-28T20:56:08.733179+0000 mgr.y (mgr.24961) 2230 : cluster [DBG] pgmap v2584: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:10 smithi139 bash[43301]: cluster 2022-04-28T20:56:08.733179+0000 mgr.y (mgr.24961) 2230 : cluster [DBG] pgmap v2584: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:12 smithi059 bash[61152]: cluster 2022-04-28T20:56:10.734372+0000 mgr.y (mgr.24961) 2231 : cluster [DBG] pgmap v2585: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:11.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:12 smithi059 bash[59745]: cluster 2022-04-28T20:56:10.734372+0000 mgr.y (mgr.24961) 2231 : cluster [DBG] pgmap v2585: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:12 smithi139 bash[43301]: cluster 2022-04-28T20:56:10.734372+0000 mgr.y (mgr.24961) 2231 : cluster [DBG] pgmap v2585: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:13.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:14 smithi139 bash[43301]: cluster 2022-04-28T20:56:12.735303+0000 mgr.y (mgr.24961) 2232 : cluster [DBG] pgmap v2586: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:14 smithi059 bash[59745]: cluster 2022-04-28T20:56:12.735303+0000 mgr.y (mgr.24961) 2232 : cluster [DBG] pgmap v2586: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:14 smithi059 bash[61152]: cluster 2022-04-28T20:56:12.735303+0000 mgr.y (mgr.24961) 2232 : cluster [DBG] pgmap v2586: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:16 smithi139 bash[43301]: cluster 2022-04-28T20:56:14.736339+0000 mgr.y (mgr.24961) 2233 : cluster [DBG] pgmap v2587: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:16 smithi059 bash[61152]: cluster 2022-04-28T20:56:14.736339+0000 mgr.y (mgr.24961) 2233 : cluster [DBG] pgmap v2587: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:16 smithi059 bash[59745]: cluster 2022-04-28T20:56:14.736339+0000 mgr.y (mgr.24961) 2233 : cluster [DBG] pgmap v2587: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:17.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:17 smithi139 bash[43301]: cluster 2022-04-28T20:56:16.737469+0000 mgr.y (mgr.24961) 2234 : cluster [DBG] pgmap v2588: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:17 smithi059 bash[61152]: cluster 2022-04-28T20:56:16.737469+0000 mgr.y (mgr.24961) 2234 : cluster [DBG] pgmap v2588: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:17 smithi059 bash[59745]: cluster 2022-04-28T20:56:16.737469+0000 mgr.y (mgr.24961) 2234 : cluster [DBG] pgmap v2588: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[61152]: audit 2022-04-28T20:56:17.929969+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:56:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[61152]: audit 2022-04-28T20:56:17.930535+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:56:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[61152]: audit 2022-04-28T20:56:17.940627+0000 mon.a (mon.0) 2339 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:56:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[61152]: audit 2022-04-28T20:56:17.941369+0000 mon.a (mon.0) 2340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:56:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[61152]: audit 2022-04-28T20:56:17.941889+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:56:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[59745]: audit 2022-04-28T20:56:17.929969+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:56:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[59745]: audit 2022-04-28T20:56:17.930535+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:56:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[59745]: audit 2022-04-28T20:56:17.940627+0000 mon.a (mon.0) 2339 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:56:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[59745]: audit 2022-04-28T20:56:17.941369+0000 mon.a (mon.0) 2340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:56:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:18 smithi059 bash[59745]: audit 2022-04-28T20:56:17.941889+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:18 smithi139 bash[43301]: audit 2022-04-28T20:56:17.929969+0000 mon.a (mon.0) 2337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:18 smithi139 bash[43301]: audit 2022-04-28T20:56:17.930535+0000 mon.a (mon.0) 2338 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:18 smithi139 bash[43301]: audit 2022-04-28T20:56:17.940627+0000 mon.a (mon.0) 2339 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T20:56:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:18 smithi139 bash[43301]: audit 2022-04-28T20:56:17.941369+0000 mon.a (mon.0) 2340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T20:56:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:18 smithi139 bash[43301]: audit 2022-04-28T20:56:17.941889+0000 mon.a (mon.0) 2341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:56:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:20 smithi059 bash[61152]: cluster 2022-04-28T20:56:18.738162+0000 mgr.y (mgr.24961) 2235 : cluster [DBG] pgmap v2589: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:20 smithi059 bash[59745]: cluster 2022-04-28T20:56:18.738162+0000 mgr.y (mgr.24961) 2235 : cluster [DBG] pgmap v2589: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:20 smithi139 bash[43301]: cluster 2022-04-28T20:56:18.738162+0000 mgr.y (mgr.24961) 2235 : cluster [DBG] pgmap v2589: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:22 smithi059 bash[61152]: cluster 2022-04-28T20:56:20.739334+0000 mgr.y (mgr.24961) 2236 : cluster [DBG] pgmap v2590: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:21.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:22 smithi059 bash[59745]: cluster 2022-04-28T20:56:20.739334+0000 mgr.y (mgr.24961) 2236 : cluster [DBG] pgmap v2590: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:22 smithi139 bash[43301]: cluster 2022-04-28T20:56:20.739334+0000 mgr.y (mgr.24961) 2236 : cluster [DBG] pgmap v2590: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:24.085 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:23.774Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:24.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:24 smithi059 bash[59745]: cluster 2022-04-28T20:56:22.740249+0000 mgr.y (mgr.24961) 2237 : cluster [DBG] pgmap v2591: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:24 smithi059 bash[61152]: cluster 2022-04-28T20:56:22.740249+0000 mgr.y (mgr.24961) 2237 : cluster [DBG] pgmap v2591: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:24 smithi139 bash[43301]: cluster 2022-04-28T20:56:22.740249+0000 mgr.y (mgr.24961) 2237 : cluster [DBG] pgmap v2591: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:26 smithi059 bash[61152]: cluster 2022-04-28T20:56:24.741269+0000 mgr.y (mgr.24961) 2238 : cluster [DBG] pgmap v2592: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:26 smithi059 bash[59745]: cluster 2022-04-28T20:56:24.741269+0000 mgr.y (mgr.24961) 2238 : cluster [DBG] pgmap v2592: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:26.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:26 smithi139 bash[43301]: cluster 2022-04-28T20:56:24.741269+0000 mgr.y (mgr.24961) 2238 : cluster [DBG] pgmap v2592: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:28.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:28 smithi139 bash[43301]: cluster 2022-04-28T20:56:26.742391+0000 mgr.y (mgr.24961) 2239 : cluster [DBG] pgmap v2593: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:28 smithi059 bash[59745]: cluster 2022-04-28T20:56:26.742391+0000 mgr.y (mgr.24961) 2239 : cluster [DBG] pgmap v2593: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:28 smithi059 bash[61152]: cluster 2022-04-28T20:56:26.742391+0000 mgr.y (mgr.24961) 2239 : cluster [DBG] pgmap v2593: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:30 smithi139 bash[43301]: cluster 2022-04-28T20:56:28.743037+0000 mgr.y (mgr.24961) 2240 : cluster [DBG] pgmap v2594: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:30 smithi059 bash[59745]: cluster 2022-04-28T20:56:28.743037+0000 mgr.y (mgr.24961) 2240 : cluster [DBG] pgmap v2594: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:30 smithi059 bash[61152]: cluster 2022-04-28T20:56:28.743037+0000 mgr.y (mgr.24961) 2240 : cluster [DBG] pgmap v2594: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:32 smithi059 bash[61152]: cluster 2022-04-28T20:56:30.744187+0000 mgr.y (mgr.24961) 2241 : cluster [DBG] pgmap v2595: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:31.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:32 smithi059 bash[59745]: cluster 2022-04-28T20:56:30.744187+0000 mgr.y (mgr.24961) 2241 : cluster [DBG] pgmap v2595: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:32 smithi139 bash[43301]: cluster 2022-04-28T20:56:30.744187+0000 mgr.y (mgr.24961) 2241 : cluster [DBG] pgmap v2595: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[59745]: cluster 2022-04-28T20:56:32.745076+0000 mgr.y (mgr.24961) 2242 : cluster [DBG] pgmap v2596: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:33.638 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[59745]: cluster 2022-04-28T20:56:33.121996+0000 mon.a (mon.0) 2342 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2022-04-28T20:56:33.639 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[59745]: audit 2022-04-28T20:56:33.125122+0000 mon.a (mon.0) 2343 : audit [INF] from='client.? 172.21.15.59:0/2999613481' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-79405-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:56:33.639 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[61152]: cluster 2022-04-28T20:56:32.745076+0000 mgr.y (mgr.24961) 2242 : cluster [DBG] pgmap v2596: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:33.639 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[61152]: cluster 2022-04-28T20:56:33.121996+0000 mon.a (mon.0) 2342 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2022-04-28T20:56:33.640 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[61152]: audit 2022-04-28T20:56:33.125122+0000 mon.a (mon.0) 2343 : audit [INF] from='client.? 172.21.15.59:0/2999613481' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-79405-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:56:33.640 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:33 smithi139 bash[43301]: cluster 2022-04-28T20:56:32.745076+0000 mgr.y (mgr.24961) 2242 : cluster [DBG] pgmap v2596: 105 pgs: 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:33.640 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:33 smithi139 bash[43301]: cluster 2022-04-28T20:56:33.121996+0000 mon.a (mon.0) 2342 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2022-04-28T20:56:33.641 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:33 smithi139 bash[43301]: audit 2022-04-28T20:56:33.125122+0000 mon.a (mon.0) 2343 : audit [INF] from='client.? 172.21.15.59:0/2999613481' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-79405-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T20:56:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:33.775Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:35 smithi059 bash[61152]: audit 2022-04-28T20:56:34.129209+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? 172.21.15.59:0/2999613481' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-79405-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:56:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:35 smithi059 bash[61152]: cluster 2022-04-28T20:56:34.129729+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2022-04-28T20:56:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:35 smithi059 bash[61152]: cluster 2022-04-28T20:56:34.745852+0000 mgr.y (mgr.24961) 2243 : cluster [DBG] pgmap v2599: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:35 smithi059 bash[59745]: audit 2022-04-28T20:56:34.129209+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? 172.21.15.59:0/2999613481' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-79405-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:56:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:35 smithi059 bash[59745]: cluster 2022-04-28T20:56:34.129729+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2022-04-28T20:56:35.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:35 smithi059 bash[59745]: cluster 2022-04-28T20:56:34.745852+0000 mgr.y (mgr.24961) 2243 : cluster [DBG] pgmap v2599: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:35 smithi139 bash[43301]: audit 2022-04-28T20:56:34.129209+0000 mon.a (mon.0) 2344 : audit [INF] from='client.? 172.21.15.59:0/2999613481' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-79405-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T20:56:35.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:35 smithi139 bash[43301]: cluster 2022-04-28T20:56:34.129729+0000 mon.a (mon.0) 2345 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2022-04-28T20:56:35.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:35 smithi139 bash[43301]: cluster 2022-04-28T20:56:34.745852+0000 mgr.y (mgr.24961) 2243 : cluster [DBG] pgmap v2599: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:36 smithi059 bash[61152]: cluster 2022-04-28T20:56:35.128186+0000 mon.a (mon.0) 2346 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2022-04-28T20:56:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:36 smithi059 bash[59745]: cluster 2022-04-28T20:56:35.128186+0000 mon.a (mon.0) 2346 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2022-04-28T20:56:36.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:36 smithi139 bash[43301]: cluster 2022-04-28T20:56:35.128186+0000 mon.a (mon.0) 2346 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2022-04-28T20:56:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:37 smithi059 bash[61152]: cluster 2022-04-28T20:56:36.746860+0000 mgr.y (mgr.24961) 2244 : cluster [DBG] pgmap v2601: 137 pgs: 15 creating+peering, 122 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:37.607 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:37 smithi059 bash[59745]: cluster 2022-04-28T20:56:36.746860+0000 mgr.y (mgr.24961) 2244 : cluster [DBG] pgmap v2601: 137 pgs: 15 creating+peering, 122 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:37.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:37 smithi139 bash[43301]: cluster 2022-04-28T20:56:36.746860+0000 mgr.y (mgr.24961) 2244 : cluster [DBG] pgmap v2601: 137 pgs: 15 creating+peering, 122 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:40 smithi059 bash[61152]: cluster 2022-04-28T20:56:38.747738+0000 mgr.y (mgr.24961) 2245 : cluster [DBG] pgmap v2602: 137 pgs: 15 creating+peering, 122 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:40.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:40 smithi059 bash[59745]: cluster 2022-04-28T20:56:38.747738+0000 mgr.y (mgr.24961) 2245 : cluster [DBG] pgmap v2602: 137 pgs: 15 creating+peering, 122 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:40 smithi139 bash[43301]: cluster 2022-04-28T20:56:38.747738+0000 mgr.y (mgr.24961) 2245 : cluster [DBG] pgmap v2602: 137 pgs: 15 creating+peering, 122 active+clean; 24 KiB data, 727 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:42 smithi059 bash[59745]: cluster 2022-04-28T20:56:40.749290+0000 mgr.y (mgr.24961) 2246 : cluster [DBG] pgmap v2603: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:42 smithi059 bash[61152]: cluster 2022-04-28T20:56:40.749290+0000 mgr.y (mgr.24961) 2246 : cluster [DBG] pgmap v2603: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:41.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:42 smithi139 bash[43301]: cluster 2022-04-28T20:56:40.749290+0000 mgr.y (mgr.24961) 2246 : cluster [DBG] pgmap v2603: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:43.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:44 smithi139 bash[43301]: cluster 2022-04-28T20:56:42.750312+0000 mgr.y (mgr.24961) 2247 : cluster [DBG] pgmap v2604: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:44 smithi059 bash[59745]: cluster 2022-04-28T20:56:42.750312+0000 mgr.y (mgr.24961) 2247 : cluster [DBG] pgmap v2604: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:44.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:44 smithi059 bash[61152]: cluster 2022-04-28T20:56:42.750312+0000 mgr.y (mgr.24961) 2247 : cluster [DBG] pgmap v2604: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:46.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:46 smithi139 bash[43301]: cluster 2022-04-28T20:56:44.751471+0000 mgr.y (mgr.24961) 2248 : cluster [DBG] pgmap v2605: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:46.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:46 smithi059 bash[59745]: cluster 2022-04-28T20:56:44.751471+0000 mgr.y (mgr.24961) 2248 : cluster [DBG] pgmap v2605: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:46 smithi059 bash[61152]: cluster 2022-04-28T20:56:44.751471+0000 mgr.y (mgr.24961) 2248 : cluster [DBG] pgmap v2605: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:48 smithi139 bash[43301]: cluster 2022-04-28T20:56:46.752770+0000 mgr.y (mgr.24961) 2249 : cluster [DBG] pgmap v2606: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:48 smithi059 bash[61152]: cluster 2022-04-28T20:56:46.752770+0000 mgr.y (mgr.24961) 2249 : cluster [DBG] pgmap v2606: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:48 smithi059 bash[59745]: cluster 2022-04-28T20:56:46.752770+0000 mgr.y (mgr.24961) 2249 : cluster [DBG] pgmap v2606: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:50 smithi139 bash[43301]: cluster 2022-04-28T20:56:48.753503+0000 mgr.y (mgr.24961) 2250 : cluster [DBG] pgmap v2607: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:50 smithi059 bash[61152]: cluster 2022-04-28T20:56:48.753503+0000 mgr.y (mgr.24961) 2250 : cluster [DBG] pgmap v2607: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:50.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:50 smithi059 bash[59745]: cluster 2022-04-28T20:56:48.753503+0000 mgr.y (mgr.24961) 2250 : cluster [DBG] pgmap v2607: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:51 smithi059 bash[61152]: cluster 2022-04-28T20:56:50.754901+0000 mgr.y (mgr.24961) 2251 : cluster [DBG] pgmap v2608: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:51 smithi059 bash[59745]: cluster 2022-04-28T20:56:50.754901+0000 mgr.y (mgr.24961) 2251 : cluster [DBG] pgmap v2608: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:51 smithi139 bash[43301]: cluster 2022-04-28T20:56:50.754901+0000 mgr.y (mgr.24961) 2251 : cluster [DBG] pgmap v2608: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:51.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:53 smithi059 bash[61152]: cluster 2022-04-28T20:56:52.755938+0000 mgr.y (mgr.24961) 2252 : cluster [DBG] pgmap v2609: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:53.842 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:56:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:56:53.778Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:56:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:53 smithi059 bash[59745]: cluster 2022-04-28T20:56:52.755938+0000 mgr.y (mgr.24961) 2252 : cluster [DBG] pgmap v2609: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:54.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:53 smithi139 bash[43301]: cluster 2022-04-28T20:56:52.755938+0000 mgr.y (mgr.24961) 2252 : cluster [DBG] pgmap v2609: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:56.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:56 smithi059 bash[61152]: cluster 2022-04-28T20:56:54.757059+0000 mgr.y (mgr.24961) 2253 : cluster [DBG] pgmap v2610: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:56.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:56 smithi059 bash[59745]: cluster 2022-04-28T20:56:54.757059+0000 mgr.y (mgr.24961) 2253 : cluster [DBG] pgmap v2610: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:56.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:56 smithi139 bash[43301]: cluster 2022-04-28T20:56:54.757059+0000 mgr.y (mgr.24961) 2253 : cluster [DBG] pgmap v2610: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:56:58 smithi139 bash[43301]: cluster 2022-04-28T20:56:56.758317+0000 mgr.y (mgr.24961) 2254 : cluster [DBG] pgmap v2611: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:56:58 smithi059 bash[61152]: cluster 2022-04-28T20:56:56.758317+0000 mgr.y (mgr.24961) 2254 : cluster [DBG] pgmap v2611: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:56:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:56:58 smithi059 bash[59745]: cluster 2022-04-28T20:56:56.758317+0000 mgr.y (mgr.24961) 2254 : cluster [DBG] pgmap v2611: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:00 smithi139 bash[43301]: cluster 2022-04-28T20:56:58.759033+0000 mgr.y (mgr.24961) 2255 : cluster [DBG] pgmap v2612: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:00 smithi059 bash[61152]: cluster 2022-04-28T20:56:58.759033+0000 mgr.y (mgr.24961) 2255 : cluster [DBG] pgmap v2612: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:00 smithi059 bash[59745]: cluster 2022-04-28T20:56:58.759033+0000 mgr.y (mgr.24961) 2255 : cluster [DBG] pgmap v2612: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:02 smithi059 bash[61152]: cluster 2022-04-28T20:57:00.760538+0000 mgr.y (mgr.24961) 2256 : cluster [DBG] pgmap v2613: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:02 smithi059 bash[59745]: cluster 2022-04-28T20:57:00.760538+0000 mgr.y (mgr.24961) 2256 : cluster [DBG] pgmap v2613: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:01.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:02.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:02 smithi139 bash[43301]: cluster 2022-04-28T20:57:00.760538+0000 mgr.y (mgr.24961) 2256 : cluster [DBG] pgmap v2613: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:03 smithi139 bash[43301]: audit 2022-04-28T20:57:02.499794+0000 mon.a (mon.0) 2347 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:57:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:03 smithi139 bash[43301]: audit 2022-04-28T20:57:02.502627+0000 mon.a (mon.0) 2348 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:57:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:03 smithi139 bash[43301]: audit 2022-04-28T20:57:02.504487+0000 mon.a (mon.0) 2349 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:57:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:03 smithi139 bash[43301]: audit 2022-04-28T20:57:02.507732+0000 mon.a (mon.0) 2350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:57:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:03 smithi139 bash[43301]: audit 2022-04-28T20:57:02.508489+0000 mon.a (mon.0) 2351 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:57:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[59745]: audit 2022-04-28T20:57:02.499794+0000 mon.a (mon.0) 2347 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:57:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[59745]: audit 2022-04-28T20:57:02.502627+0000 mon.a (mon.0) 2348 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:57:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[59745]: audit 2022-04-28T20:57:02.504487+0000 mon.a (mon.0) 2349 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:57:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[59745]: audit 2022-04-28T20:57:02.507732+0000 mon.a (mon.0) 2350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:57:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[59745]: audit 2022-04-28T20:57:02.508489+0000 mon.a (mon.0) 2351 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:57:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[61152]: audit 2022-04-28T20:57:02.499794+0000 mon.a (mon.0) 2347 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:57:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[61152]: audit 2022-04-28T20:57:02.502627+0000 mon.a (mon.0) 2348 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:57:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[61152]: audit 2022-04-28T20:57:02.504487+0000 mon.a (mon.0) 2349 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:57:03.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[61152]: audit 2022-04-28T20:57:02.507732+0000 mon.a (mon.0) 2350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:57:03.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[61152]: audit 2022-04-28T20:57:02.508489+0000 mon.a (mon.0) 2351 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T20:57:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:03.776Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:04.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:04 smithi139 bash[43301]: cephadm 2022-04-28T20:57:02.506195+0000 mgr.y (mgr.24961) 2257 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:57:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:04 smithi139 bash[43301]: cluster 2022-04-28T20:57:02.761282+0000 mgr.y (mgr.24961) 2258 : cluster [DBG] pgmap v2614: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:04 smithi059 bash[59745]: cephadm 2022-04-28T20:57:02.506195+0000 mgr.y (mgr.24961) 2257 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:57:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:04 smithi059 bash[59745]: cluster 2022-04-28T20:57:02.761282+0000 mgr.y (mgr.24961) 2258 : cluster [DBG] pgmap v2614: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:04.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:04 smithi059 bash[61152]: cephadm 2022-04-28T20:57:02.506195+0000 mgr.y (mgr.24961) 2257 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:57:04.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:04 smithi059 bash[61152]: cluster 2022-04-28T20:57:02.761282+0000 mgr.y (mgr.24961) 2258 : cluster [DBG] pgmap v2614: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:05 smithi059 bash[59745]: cluster 2022-04-28T20:57:04.762359+0000 mgr.y (mgr.24961) 2259 : cluster [DBG] pgmap v2615: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:05 smithi059 bash[61152]: cluster 2022-04-28T20:57:04.762359+0000 mgr.y (mgr.24961) 2259 : cluster [DBG] pgmap v2615: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:05 smithi139 bash[43301]: cluster 2022-04-28T20:57:04.762359+0000 mgr.y (mgr.24961) 2259 : cluster [DBG] pgmap v2615: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[59745]: audit 2022-04-28T20:57:06.627122+0000 mon.a (mon.0) 2352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[59745]: cluster 2022-04-28T20:57:06.763613+0000 mgr.y (mgr.24961) 2260 : cluster [DBG] pgmap v2616: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[59745]: audit 2022-04-28T20:57:06.834613+0000 mon.a (mon.0) 2353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[59745]: audit 2022-04-28T20:57:07.128967+0000 mon.a (mon.0) 2354 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[61152]: audit 2022-04-28T20:57:06.627122+0000 mon.a (mon.0) 2352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[61152]: cluster 2022-04-28T20:57:06.763613+0000 mgr.y (mgr.24961) 2260 : cluster [DBG] pgmap v2616: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[61152]: audit 2022-04-28T20:57:06.834613+0000 mon.a (mon.0) 2353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:07 smithi059 bash[61152]: audit 2022-04-28T20:57:07.128967+0000 mon.a (mon.0) 2354 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:07 smithi139 bash[43301]: audit 2022-04-28T20:57:06.627122+0000 mon.a (mon.0) 2352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:07 smithi139 bash[43301]: cluster 2022-04-28T20:57:06.763613+0000 mgr.y (mgr.24961) 2260 : cluster [DBG] pgmap v2616: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:07 smithi139 bash[43301]: audit 2022-04-28T20:57:06.834613+0000 mon.a (mon.0) 2353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:07 smithi139 bash[43301]: audit 2022-04-28T20:57:07.128967+0000 mon.a (mon.0) 2354 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:57:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:10 smithi059 bash[61152]: cluster 2022-04-28T20:57:08.764340+0000 mgr.y (mgr.24961) 2261 : cluster [DBG] pgmap v2617: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:10 smithi059 bash[59745]: cluster 2022-04-28T20:57:08.764340+0000 mgr.y (mgr.24961) 2261 : cluster [DBG] pgmap v2617: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:10 smithi139 bash[43301]: cluster 2022-04-28T20:57:08.764340+0000 mgr.y (mgr.24961) 2261 : cluster [DBG] pgmap v2617: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:12 smithi059 bash[61152]: cluster 2022-04-28T20:57:10.765757+0000 mgr.y (mgr.24961) 2262 : cluster [DBG] pgmap v2618: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:12 smithi059 bash[59745]: cluster 2022-04-28T20:57:10.765757+0000 mgr.y (mgr.24961) 2262 : cluster [DBG] pgmap v2618: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:11.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:12.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:12 smithi139 bash[43301]: cluster 2022-04-28T20:57:10.765757+0000 mgr.y (mgr.24961) 2262 : cluster [DBG] pgmap v2618: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:13.779Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:14.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:14 smithi139 bash[43301]: cluster 2022-04-28T20:57:12.766737+0000 mgr.y (mgr.24961) 2263 : cluster [DBG] pgmap v2619: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:14 smithi059 bash[59745]: cluster 2022-04-28T20:57:12.766737+0000 mgr.y (mgr.24961) 2263 : cluster [DBG] pgmap v2619: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:14 smithi059 bash[61152]: cluster 2022-04-28T20:57:12.766737+0000 mgr.y (mgr.24961) 2263 : cluster [DBG] pgmap v2619: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:16.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:16 smithi139 bash[43301]: cluster 2022-04-28T20:57:14.767841+0000 mgr.y (mgr.24961) 2264 : cluster [DBG] pgmap v2620: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:16 smithi059 bash[59745]: cluster 2022-04-28T20:57:14.767841+0000 mgr.y (mgr.24961) 2264 : cluster [DBG] pgmap v2620: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:16 smithi059 bash[61152]: cluster 2022-04-28T20:57:14.767841+0000 mgr.y (mgr.24961) 2264 : cluster [DBG] pgmap v2620: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: cluster 2022-04-28T20:57:16.769104+0000 mgr.y (mgr.24961) 2265 : cluster [DBG] pgmap v2621: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: audit 2022-04-28T20:57:17.943849+0000 mon.a (mon.0) 2355 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: audit 2022-04-28T20:57:17.944390+0000 mon.a (mon.0) 2356 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: audit 2022-04-28T20:57:17.953078+0000 mon.a (mon.0) 2357 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:57:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: audit 2022-04-28T20:57:17.953689+0000 mon.a (mon.0) 2358 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:57:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: audit 2022-04-28T20:57:17.954308+0000 mon.a (mon.0) 2359 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:57:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:18 smithi139 bash[43301]: audit 2022-04-28T20:57:17.954883+0000 mon.a (mon.0) 2360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T20:57:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: cluster 2022-04-28T20:57:16.769104+0000 mgr.y (mgr.24961) 2265 : cluster [DBG] pgmap v2621: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: audit 2022-04-28T20:57:17.943849+0000 mon.a (mon.0) 2355 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:57:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: audit 2022-04-28T20:57:17.944390+0000 mon.a (mon.0) 2356 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:57:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: audit 2022-04-28T20:57:17.953078+0000 mon.a (mon.0) 2357 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:57:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: audit 2022-04-28T20:57:17.953689+0000 mon.a (mon.0) 2358 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:57:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: audit 2022-04-28T20:57:17.954308+0000 mon.a (mon.0) 2359 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:57:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[61152]: audit 2022-04-28T20:57:17.954883+0000 mon.a (mon.0) 2360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T20:57:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: cluster 2022-04-28T20:57:16.769104+0000 mgr.y (mgr.24961) 2265 : cluster [DBG] pgmap v2621: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: audit 2022-04-28T20:57:17.943849+0000 mon.a (mon.0) 2355 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:57:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: audit 2022-04-28T20:57:17.944390+0000 mon.a (mon.0) 2356 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:57:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: audit 2022-04-28T20:57:17.953078+0000 mon.a (mon.0) 2357 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:57:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: audit 2022-04-28T20:57:17.953689+0000 mon.a (mon.0) 2358 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:57:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: audit 2022-04-28T20:57:17.954308+0000 mon.a (mon.0) 2359 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T20:57:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:18 smithi059 bash[59745]: audit 2022-04-28T20:57:17.954883+0000 mon.a (mon.0) 2360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T20:57:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:20 smithi139 bash[43301]: cluster 2022-04-28T20:57:18.769832+0000 mgr.y (mgr.24961) 2266 : cluster [DBG] pgmap v2622: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:20 smithi059 bash[61152]: cluster 2022-04-28T20:57:18.769832+0000 mgr.y (mgr.24961) 2266 : cluster [DBG] pgmap v2622: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:20 smithi059 bash[59745]: cluster 2022-04-28T20:57:18.769832+0000 mgr.y (mgr.24961) 2266 : cluster [DBG] pgmap v2622: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:22 smithi059 bash[59745]: cluster 2022-04-28T20:57:20.771237+0000 mgr.y (mgr.24961) 2267 : cluster [DBG] pgmap v2623: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:22 smithi059 bash[61152]: cluster 2022-04-28T20:57:20.771237+0000 mgr.y (mgr.24961) 2267 : cluster [DBG] pgmap v2623: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:21.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:22 smithi139 bash[43301]: cluster 2022-04-28T20:57:20.771237+0000 mgr.y (mgr.24961) 2267 : cluster [DBG] pgmap v2623: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:23.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:24 smithi139 bash[43301]: cluster 2022-04-28T20:57:22.772254+0000 mgr.y (mgr.24961) 2268 : cluster [DBG] pgmap v2624: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:24 smithi059 bash[59745]: cluster 2022-04-28T20:57:22.772254+0000 mgr.y (mgr.24961) 2268 : cluster [DBG] pgmap v2624: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:24 smithi059 bash[61152]: cluster 2022-04-28T20:57:22.772254+0000 mgr.y (mgr.24961) 2268 : cluster [DBG] pgmap v2624: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:26 smithi139 bash[43301]: cluster 2022-04-28T20:57:24.773378+0000 mgr.y (mgr.24961) 2269 : cluster [DBG] pgmap v2625: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:26 smithi059 bash[61152]: cluster 2022-04-28T20:57:24.773378+0000 mgr.y (mgr.24961) 2269 : cluster [DBG] pgmap v2625: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:26.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:26 smithi059 bash[59745]: cluster 2022-04-28T20:57:24.773378+0000 mgr.y (mgr.24961) 2269 : cluster [DBG] pgmap v2625: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:28 smithi139 bash[43301]: cluster 2022-04-28T20:57:26.774659+0000 mgr.y (mgr.24961) 2270 : cluster [DBG] pgmap v2626: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:28 smithi059 bash[59745]: cluster 2022-04-28T20:57:26.774659+0000 mgr.y (mgr.24961) 2270 : cluster [DBG] pgmap v2626: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:28 smithi059 bash[61152]: cluster 2022-04-28T20:57:26.774659+0000 mgr.y (mgr.24961) 2270 : cluster [DBG] pgmap v2626: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:30 smithi139 bash[43301]: cluster 2022-04-28T20:57:28.775457+0000 mgr.y (mgr.24961) 2271 : cluster [DBG] pgmap v2627: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:30 smithi059 bash[61152]: cluster 2022-04-28T20:57:28.775457+0000 mgr.y (mgr.24961) 2271 : cluster [DBG] pgmap v2627: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:30 smithi059 bash[59745]: cluster 2022-04-28T20:57:28.775457+0000 mgr.y (mgr.24961) 2271 : cluster [DBG] pgmap v2627: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:32 smithi059 bash[61152]: cluster 2022-04-28T20:57:30.776852+0000 mgr.y (mgr.24961) 2272 : cluster [DBG] pgmap v2628: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:31.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:32 smithi059 bash[59745]: cluster 2022-04-28T20:57:30.776852+0000 mgr.y (mgr.24961) 2272 : cluster [DBG] pgmap v2628: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:32.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:32 smithi139 bash[43301]: cluster 2022-04-28T20:57:30.776852+0000 mgr.y (mgr.24961) 2272 : cluster [DBG] pgmap v2628: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:33.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:34.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:34 smithi059 bash[59745]: cluster 2022-04-28T20:57:32.777824+0000 mgr.y (mgr.24961) 2273 : cluster [DBG] pgmap v2629: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:34.632 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:34 smithi059 bash[61152]: cluster 2022-04-28T20:57:32.777824+0000 mgr.y (mgr.24961) 2273 : cluster [DBG] pgmap v2629: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:34.634 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:34 smithi139 bash[43301]: cluster 2022-04-28T20:57:32.777824+0000 mgr.y (mgr.24961) 2273 : cluster [DBG] pgmap v2629: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:35 smithi059 bash[59745]: cluster 2022-04-28T20:57:34.778972+0000 mgr.y (mgr.24961) 2274 : cluster [DBG] pgmap v2630: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:35 smithi059 bash[61152]: cluster 2022-04-28T20:57:34.778972+0000 mgr.y (mgr.24961) 2274 : cluster [DBG] pgmap v2630: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:35.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:35 smithi139 bash[43301]: cluster 2022-04-28T20:57:34.778972+0000 mgr.y (mgr.24961) 2274 : cluster [DBG] pgmap v2630: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:37 smithi059 bash[61152]: cluster 2022-04-28T20:57:36.780294+0000 mgr.y (mgr.24961) 2275 : cluster [DBG] pgmap v2631: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:37 smithi059 bash[59745]: cluster 2022-04-28T20:57:36.780294+0000 mgr.y (mgr.24961) 2275 : cluster [DBG] pgmap v2631: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:38.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:37 smithi139 bash[43301]: cluster 2022-04-28T20:57:36.780294+0000 mgr.y (mgr.24961) 2275 : cluster [DBG] pgmap v2631: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:40 smithi059 bash[61152]: cluster 2022-04-28T20:57:38.780993+0000 mgr.y (mgr.24961) 2276 : cluster [DBG] pgmap v2632: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:40 smithi059 bash[59745]: cluster 2022-04-28T20:57:38.780993+0000 mgr.y (mgr.24961) 2276 : cluster [DBG] pgmap v2632: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:40.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:40 smithi139 bash[43301]: cluster 2022-04-28T20:57:38.780993+0000 mgr.y (mgr.24961) 2276 : cluster [DBG] pgmap v2632: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:41 smithi059 bash[61152]: cluster 2022-04-28T20:57:40.782411+0000 mgr.y (mgr.24961) 2277 : cluster [DBG] pgmap v2633: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:41 smithi059 bash[59745]: cluster 2022-04-28T20:57:40.782411+0000 mgr.y (mgr.24961) 2277 : cluster [DBG] pgmap v2633: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:41 smithi139 bash[43301]: cluster 2022-04-28T20:57:40.782411+0000 mgr.y (mgr.24961) 2277 : cluster [DBG] pgmap v2633: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:41.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:43 smithi059 bash[61152]: cluster 2022-04-28T20:57:42.783381+0000 mgr.y (mgr.24961) 2278 : cluster [DBG] pgmap v2634: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:57:44.103 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:43.780Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:44.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:43 smithi059 bash[59745]: cluster 2022-04-28T20:57:42.783381+0000 mgr.y (mgr.24961) 2278 : cluster [DBG] pgmap v2634: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:57:44.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:43 smithi139 bash[43301]: cluster 2022-04-28T20:57:42.783381+0000 mgr.y (mgr.24961) 2278 : cluster [DBG] pgmap v2634: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:57:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:45 smithi059 bash[59745]: cluster 2022-04-28T20:57:44.784536+0000 mgr.y (mgr.24961) 2279 : cluster [DBG] pgmap v2635: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:45 smithi059 bash[61152]: cluster 2022-04-28T20:57:44.784536+0000 mgr.y (mgr.24961) 2279 : cluster [DBG] pgmap v2635: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:45 smithi139 bash[43301]: cluster 2022-04-28T20:57:44.784536+0000 mgr.y (mgr.24961) 2279 : cluster [DBG] pgmap v2635: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:47 smithi139 bash[43301]: cluster 2022-04-28T20:57:46.785768+0000 mgr.y (mgr.24961) 2280 : cluster [DBG] pgmap v2636: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:47 smithi059 bash[61152]: cluster 2022-04-28T20:57:46.785768+0000 mgr.y (mgr.24961) 2280 : cluster [DBG] pgmap v2636: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:47 smithi059 bash[59745]: cluster 2022-04-28T20:57:46.785768+0000 mgr.y (mgr.24961) 2280 : cluster [DBG] pgmap v2636: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:50 smithi059 bash[61152]: cluster 2022-04-28T20:57:48.786503+0000 mgr.y (mgr.24961) 2281 : cluster [DBG] pgmap v2637: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:57:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:50 smithi059 bash[59745]: cluster 2022-04-28T20:57:48.786503+0000 mgr.y (mgr.24961) 2281 : cluster [DBG] pgmap v2637: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:57:50.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:50 smithi139 bash[43301]: cluster 2022-04-28T20:57:48.786503+0000 mgr.y (mgr.24961) 2281 : cluster [DBG] pgmap v2637: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 596 B/s wr, 0 op/s 2022-04-28T20:57:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:51 smithi059 bash[61152]: cluster 2022-04-28T20:57:50.788063+0000 mgr.y (mgr.24961) 2282 : cluster [DBG] pgmap v2638: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:51 smithi059 bash[59745]: cluster 2022-04-28T20:57:50.788063+0000 mgr.y (mgr.24961) 2282 : cluster [DBG] pgmap v2638: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:51.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:51 smithi139 bash[43301]: cluster 2022-04-28T20:57:50.788063+0000 mgr.y (mgr.24961) 2282 : cluster [DBG] pgmap v2638: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 597 B/s wr, 0 op/s 2022-04-28T20:57:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:51.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:53 smithi059 bash[61152]: cluster 2022-04-28T20:57:52.789078+0000 mgr.y (mgr.24961) 2283 : cluster [DBG] pgmap v2639: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:57:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:57:53.781Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:57:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:53 smithi059 bash[59745]: cluster 2022-04-28T20:57:52.789078+0000 mgr.y (mgr.24961) 2283 : cluster [DBG] pgmap v2639: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:54.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:53 smithi139 bash[43301]: cluster 2022-04-28T20:57:52.789078+0000 mgr.y (mgr.24961) 2283 : cluster [DBG] pgmap v2639: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:56 smithi059 bash[61152]: cluster 2022-04-28T20:57:54.790189+0000 mgr.y (mgr.24961) 2284 : cluster [DBG] pgmap v2640: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:56 smithi059 bash[59745]: cluster 2022-04-28T20:57:54.790189+0000 mgr.y (mgr.24961) 2284 : cluster [DBG] pgmap v2640: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:56 smithi139 bash[43301]: cluster 2022-04-28T20:57:54.790189+0000 mgr.y (mgr.24961) 2284 : cluster [DBG] pgmap v2640: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:57:57 smithi059 bash[61152]: cluster 2022-04-28T20:57:56.791609+0000 mgr.y (mgr.24961) 2285 : cluster [DBG] pgmap v2641: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:57:57 smithi059 bash[59745]: cluster 2022-04-28T20:57:56.791609+0000 mgr.y (mgr.24961) 2285 : cluster [DBG] pgmap v2641: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:57:57.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:57:57 smithi139 bash[43301]: cluster 2022-04-28T20:57:56.791609+0000 mgr.y (mgr.24961) 2285 : cluster [DBG] pgmap v2641: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:00 smithi059 bash[61152]: cluster 2022-04-28T20:57:58.792361+0000 mgr.y (mgr.24961) 2286 : cluster [DBG] pgmap v2642: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:00 smithi059 bash[59745]: cluster 2022-04-28T20:57:58.792361+0000 mgr.y (mgr.24961) 2286 : cluster [DBG] pgmap v2642: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:00.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:00 smithi139 bash[43301]: cluster 2022-04-28T20:57:58.792361+0000 mgr.y (mgr.24961) 2286 : cluster [DBG] pgmap v2642: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:01 smithi059 bash[61152]: cluster 2022-04-28T20:58:00.793790+0000 mgr.y (mgr.24961) 2287 : cluster [DBG] pgmap v2643: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:01 smithi059 bash[59745]: cluster 2022-04-28T20:58:00.793790+0000 mgr.y (mgr.24961) 2287 : cluster [DBG] pgmap v2643: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:01 smithi139 bash[43301]: cluster 2022-04-28T20:58:00.793790+0000 mgr.y (mgr.24961) 2287 : cluster [DBG] pgmap v2643: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:01.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:03 smithi059 bash[61152]: cluster 2022-04-28T20:58:02.794758+0000 mgr.y (mgr.24961) 2288 : cluster [DBG] pgmap v2644: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:03.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:03 smithi059 bash[59745]: cluster 2022-04-28T20:58:02.794758+0000 mgr.y (mgr.24961) 2288 : cluster [DBG] pgmap v2644: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:03 smithi139 bash[43301]: cluster 2022-04-28T20:58:02.794758+0000 mgr.y (mgr.24961) 2288 : cluster [DBG] pgmap v2644: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:05 smithi059 bash[61152]: cluster 2022-04-28T20:58:04.795915+0000 mgr.y (mgr.24961) 2289 : cluster [DBG] pgmap v2645: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:05 smithi059 bash[59745]: cluster 2022-04-28T20:58:04.795915+0000 mgr.y (mgr.24961) 2289 : cluster [DBG] pgmap v2645: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:05 smithi139 bash[43301]: cluster 2022-04-28T20:58:04.795915+0000 mgr.y (mgr.24961) 2289 : cluster [DBG] pgmap v2645: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:07 smithi139 bash[43301]: cluster 2022-04-28T20:58:06.797169+0000 mgr.y (mgr.24961) 2290 : cluster [DBG] pgmap v2646: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:07 smithi139 bash[43301]: audit 2022-04-28T20:58:07.140424+0000 mon.a (mon.0) 2361 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:58:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:07 smithi139 bash[43301]: audit 2022-04-28T20:58:07.143065+0000 mon.a (mon.0) 2362 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:58:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:07 smithi139 bash[43301]: audit 2022-04-28T20:58:07.144957+0000 mon.a (mon.0) 2363 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:58:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:07 smithi139 bash[43301]: audit 2022-04-28T20:58:07.445357+0000 mon.a (mon.0) 2364 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:58:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[61152]: cluster 2022-04-28T20:58:06.797169+0000 mgr.y (mgr.24961) 2290 : cluster [DBG] pgmap v2646: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[61152]: audit 2022-04-28T20:58:07.140424+0000 mon.a (mon.0) 2361 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:58:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[61152]: audit 2022-04-28T20:58:07.143065+0000 mon.a (mon.0) 2362 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:58:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[61152]: audit 2022-04-28T20:58:07.144957+0000 mon.a (mon.0) 2363 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:58:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[61152]: audit 2022-04-28T20:58:07.445357+0000 mon.a (mon.0) 2364 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:58:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[59745]: cluster 2022-04-28T20:58:06.797169+0000 mgr.y (mgr.24961) 2290 : cluster [DBG] pgmap v2646: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[59745]: audit 2022-04-28T20:58:07.140424+0000 mon.a (mon.0) 2361 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:58:08.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[59745]: audit 2022-04-28T20:58:07.143065+0000 mon.a (mon.0) 2362 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:58:08.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[59745]: audit 2022-04-28T20:58:07.144957+0000 mon.a (mon.0) 2363 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:58:08.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:07 smithi059 bash[59745]: audit 2022-04-28T20:58:07.445357+0000 mon.a (mon.0) 2364 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:58:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:08 smithi139 bash[43301]: cephadm 2022-04-28T20:58:07.146579+0000 mgr.y (mgr.24961) 2291 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:58:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:08 smithi059 bash[59745]: cephadm 2022-04-28T20:58:07.146579+0000 mgr.y (mgr.24961) 2291 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:58:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:08 smithi059 bash[61152]: cephadm 2022-04-28T20:58:07.146579+0000 mgr.y (mgr.24961) 2291 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:58:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:10 smithi059 bash[61152]: cluster 2022-04-28T20:58:08.797885+0000 mgr.y (mgr.24961) 2292 : cluster [DBG] pgmap v2647: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:10 smithi059 bash[59745]: cluster 2022-04-28T20:58:08.797885+0000 mgr.y (mgr.24961) 2292 : cluster [DBG] pgmap v2647: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:10 smithi139 bash[43301]: cluster 2022-04-28T20:58:08.797885+0000 mgr.y (mgr.24961) 2292 : cluster [DBG] pgmap v2647: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:11 smithi059 bash[61152]: cluster 2022-04-28T20:58:10.799266+0000 mgr.y (mgr.24961) 2293 : cluster [DBG] pgmap v2648: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:11 smithi059 bash[59745]: cluster 2022-04-28T20:58:10.799266+0000 mgr.y (mgr.24961) 2293 : cluster [DBG] pgmap v2648: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:11 smithi139 bash[43301]: cluster 2022-04-28T20:58:10.799266+0000 mgr.y (mgr.24961) 2293 : cluster [DBG] pgmap v2648: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:11.886Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:13 smithi059 bash[61152]: cluster 2022-04-28T20:58:12.800251+0000 mgr.y (mgr.24961) 2294 : cluster [DBG] pgmap v2649: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:13.783Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:13 smithi059 bash[59745]: cluster 2022-04-28T20:58:12.800251+0000 mgr.y (mgr.24961) 2294 : cluster [DBG] pgmap v2649: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:14.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:13 smithi139 bash[43301]: cluster 2022-04-28T20:58:12.800251+0000 mgr.y (mgr.24961) 2294 : cluster [DBG] pgmap v2649: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:15 smithi059 bash[61152]: cluster 2022-04-28T20:58:14.801381+0000 mgr.y (mgr.24961) 2295 : cluster [DBG] pgmap v2650: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:15 smithi059 bash[59745]: cluster 2022-04-28T20:58:14.801381+0000 mgr.y (mgr.24961) 2295 : cluster [DBG] pgmap v2650: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:15 smithi139 bash[43301]: cluster 2022-04-28T20:58:14.801381+0000 mgr.y (mgr.24961) 2295 : cluster [DBG] pgmap v2650: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:18.118 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:17 smithi139 bash[43301]: cluster 2022-04-28T20:58:16.802624+0000 mgr.y (mgr.24961) 2296 : cluster [DBG] pgmap v2651: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:17 smithi059 bash[61152]: cluster 2022-04-28T20:58:16.802624+0000 mgr.y (mgr.24961) 2296 : cluster [DBG] pgmap v2651: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:17 smithi059 bash[59745]: cluster 2022-04-28T20:58:16.802624+0000 mgr.y (mgr.24961) 2296 : cluster [DBG] pgmap v2651: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[61152]: audit 2022-04-28T20:58:17.958609+0000 mon.a (mon.0) 2365 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:58:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[61152]: audit 2022-04-28T20:58:17.959074+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:58:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[61152]: audit 2022-04-28T20:58:17.967158+0000 mon.a (mon.0) 2367 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:58:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[61152]: audit 2022-04-28T20:58:17.967804+0000 mon.a (mon.0) 2368 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:58:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[61152]: audit 2022-04-28T20:58:17.968328+0000 mon.a (mon.0) 2369 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:58:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[61152]: audit 2022-04-28T20:58:17.968872+0000 mon.a (mon.0) 2370 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T20:58:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[59745]: audit 2022-04-28T20:58:17.958609+0000 mon.a (mon.0) 2365 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:58:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[59745]: audit 2022-04-28T20:58:17.959074+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:58:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[59745]: audit 2022-04-28T20:58:17.967158+0000 mon.a (mon.0) 2367 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:58:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[59745]: audit 2022-04-28T20:58:17.967804+0000 mon.a (mon.0) 2368 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:58:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[59745]: audit 2022-04-28T20:58:17.968328+0000 mon.a (mon.0) 2369 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:58:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:18 smithi059 bash[59745]: audit 2022-04-28T20:58:17.968872+0000 mon.a (mon.0) 2370 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T20:58:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:18 smithi139 bash[43301]: audit 2022-04-28T20:58:17.958609+0000 mon.a (mon.0) 2365 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:58:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:18 smithi139 bash[43301]: audit 2022-04-28T20:58:17.959074+0000 mon.a (mon.0) 2366 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:58:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:18 smithi139 bash[43301]: audit 2022-04-28T20:58:17.967158+0000 mon.a (mon.0) 2367 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T20:58:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:18 smithi139 bash[43301]: audit 2022-04-28T20:58:17.967804+0000 mon.a (mon.0) 2368 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T20:58:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:18 smithi139 bash[43301]: audit 2022-04-28T20:58:17.968328+0000 mon.a (mon.0) 2369 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T20:58:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:18 smithi139 bash[43301]: audit 2022-04-28T20:58:17.968872+0000 mon.a (mon.0) 2370 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T20:58:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:19 smithi059 bash[61152]: cluster 2022-04-28T20:58:18.803322+0000 mgr.y (mgr.24961) 2297 : cluster [DBG] pgmap v2652: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:19 smithi059 bash[59745]: cluster 2022-04-28T20:58:18.803322+0000 mgr.y (mgr.24961) 2297 : cluster [DBG] pgmap v2652: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:20.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:19 smithi139 bash[43301]: cluster 2022-04-28T20:58:18.803322+0000 mgr.y (mgr.24961) 2297 : cluster [DBG] pgmap v2652: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:21 smithi059 bash[61152]: cluster 2022-04-28T20:58:20.804714+0000 mgr.y (mgr.24961) 2298 : cluster [DBG] pgmap v2653: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:21.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:21 smithi059 bash[59745]: cluster 2022-04-28T20:58:20.804714+0000 mgr.y (mgr.24961) 2298 : cluster [DBG] pgmap v2653: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:22.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:21 smithi139 bash[43301]: cluster 2022-04-28T20:58:20.804714+0000 mgr.y (mgr.24961) 2298 : cluster [DBG] pgmap v2653: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:23 smithi059 bash[61152]: cluster 2022-04-28T20:58:22.805700+0000 mgr.y (mgr.24961) 2299 : cluster [DBG] pgmap v2654: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:23.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:23 smithi059 bash[59745]: cluster 2022-04-28T20:58:22.805700+0000 mgr.y (mgr.24961) 2299 : cluster [DBG] pgmap v2654: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:24.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:23 smithi139 bash[43301]: cluster 2022-04-28T20:58:22.805700+0000 mgr.y (mgr.24961) 2299 : cluster [DBG] pgmap v2654: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:25 smithi059 bash[61152]: cluster 2022-04-28T20:58:24.806839+0000 mgr.y (mgr.24961) 2300 : cluster [DBG] pgmap v2655: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:25 smithi059 bash[59745]: cluster 2022-04-28T20:58:24.806839+0000 mgr.y (mgr.24961) 2300 : cluster [DBG] pgmap v2655: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:25.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:25 smithi139 bash[43301]: cluster 2022-04-28T20:58:24.806839+0000 mgr.y (mgr.24961) 2300 : cluster [DBG] pgmap v2655: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:28.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:27 smithi139 bash[43301]: cluster 2022-04-28T20:58:26.808132+0000 mgr.y (mgr.24961) 2301 : cluster [DBG] pgmap v2656: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:27 smithi059 bash[59745]: cluster 2022-04-28T20:58:26.808132+0000 mgr.y (mgr.24961) 2301 : cluster [DBG] pgmap v2656: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:28.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:27 smithi059 bash[61152]: cluster 2022-04-28T20:58:26.808132+0000 mgr.y (mgr.24961) 2301 : cluster [DBG] pgmap v2656: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:29 smithi059 bash[61152]: cluster 2022-04-28T20:58:28.808809+0000 mgr.y (mgr.24961) 2302 : cluster [DBG] pgmap v2657: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:29 smithi059 bash[59745]: cluster 2022-04-28T20:58:28.808809+0000 mgr.y (mgr.24961) 2302 : cluster [DBG] pgmap v2657: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:29 smithi139 bash[43301]: cluster 2022-04-28T20:58:28.808809+0000 mgr.y (mgr.24961) 2302 : cluster [DBG] pgmap v2657: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:32.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:31 smithi059 bash[59745]: cluster 2022-04-28T20:58:30.810194+0000 mgr.y (mgr.24961) 2303 : cluster [DBG] pgmap v2658: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:31 smithi059 bash[61152]: cluster 2022-04-28T20:58:30.810194+0000 mgr.y (mgr.24961) 2303 : cluster [DBG] pgmap v2658: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:32.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:31.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:31 smithi139 bash[43301]: cluster 2022-04-28T20:58:30.810194+0000 mgr.y (mgr.24961) 2303 : cluster [DBG] pgmap v2658: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:33 smithi059 bash[61152]: cluster 2022-04-28T20:58:32.811181+0000 mgr.y (mgr.24961) 2304 : cluster [DBG] pgmap v2659: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:33.784Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:33 smithi059 bash[59745]: cluster 2022-04-28T20:58:32.811181+0000 mgr.y (mgr.24961) 2304 : cluster [DBG] pgmap v2659: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:34.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:33 smithi139 bash[43301]: cluster 2022-04-28T20:58:32.811181+0000 mgr.y (mgr.24961) 2304 : cluster [DBG] pgmap v2659: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:35 smithi059 bash[59745]: cluster 2022-04-28T20:58:34.812436+0000 mgr.y (mgr.24961) 2305 : cluster [DBG] pgmap v2660: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:35 smithi059 bash[61152]: cluster 2022-04-28T20:58:34.812436+0000 mgr.y (mgr.24961) 2305 : cluster [DBG] pgmap v2660: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:35 smithi139 bash[43301]: cluster 2022-04-28T20:58:34.812436+0000 mgr.y (mgr.24961) 2305 : cluster [DBG] pgmap v2660: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T20:58:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:37 smithi059 bash[59745]: cluster 2022-04-28T20:58:36.813776+0000 mgr.y (mgr.24961) 2306 : cluster [DBG] pgmap v2661: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:58:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:37 smithi059 bash[61152]: cluster 2022-04-28T20:58:36.813776+0000 mgr.y (mgr.24961) 2306 : cluster [DBG] pgmap v2661: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:58:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:37 smithi139 bash[43301]: cluster 2022-04-28T20:58:36.813776+0000 mgr.y (mgr.24961) 2306 : cluster [DBG] pgmap v2661: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:58:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:39 smithi059 bash[61152]: cluster 2022-04-28T20:58:38.814489+0000 mgr.y (mgr.24961) 2307 : cluster [DBG] pgmap v2662: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:58:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:39 smithi059 bash[59745]: cluster 2022-04-28T20:58:38.814489+0000 mgr.y (mgr.24961) 2307 : cluster [DBG] pgmap v2662: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:58:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:39 smithi139 bash[43301]: cluster 2022-04-28T20:58:38.814489+0000 mgr.y (mgr.24961) 2307 : cluster [DBG] pgmap v2662: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:58:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:41 smithi059 bash[61152]: cluster 2022-04-28T20:58:40.815905+0000 mgr.y (mgr.24961) 2308 : cluster [DBG] pgmap v2663: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:41.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:41 smithi059 bash[59745]: cluster 2022-04-28T20:58:40.815905+0000 mgr.y (mgr.24961) 2308 : cluster [DBG] pgmap v2663: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:41 smithi139 bash[43301]: cluster 2022-04-28T20:58:40.815905+0000 mgr.y (mgr.24961) 2308 : cluster [DBG] pgmap v2663: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:43 smithi059 bash[59745]: cluster 2022-04-28T20:58:42.816912+0000 mgr.y (mgr.24961) 2309 : cluster [DBG] pgmap v2664: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:43 smithi059 bash[61152]: cluster 2022-04-28T20:58:42.816912+0000 mgr.y (mgr.24961) 2309 : cluster [DBG] pgmap v2664: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:43.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:44.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:43 smithi139 bash[43301]: cluster 2022-04-28T20:58:42.816912+0000 mgr.y (mgr.24961) 2309 : cluster [DBG] pgmap v2664: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:45 smithi059 bash[61152]: cluster 2022-04-28T20:58:44.818077+0000 mgr.y (mgr.24961) 2310 : cluster [DBG] pgmap v2665: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:45 smithi059 bash[59745]: cluster 2022-04-28T20:58:44.818077+0000 mgr.y (mgr.24961) 2310 : cluster [DBG] pgmap v2665: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:45 smithi139 bash[43301]: cluster 2022-04-28T20:58:44.818077+0000 mgr.y (mgr.24961) 2310 : cluster [DBG] pgmap v2665: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T20:58:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:47 smithi059 bash[61152]: cluster 2022-04-28T20:58:46.819385+0000 mgr.y (mgr.24961) 2311 : cluster [DBG] pgmap v2666: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:47 smithi059 bash[59745]: cluster 2022-04-28T20:58:46.819385+0000 mgr.y (mgr.24961) 2311 : cluster [DBG] pgmap v2666: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:47 smithi139 bash[43301]: cluster 2022-04-28T20:58:46.819385+0000 mgr.y (mgr.24961) 2311 : cluster [DBG] pgmap v2666: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:49 smithi059 bash[61152]: cluster 2022-04-28T20:58:48.820138+0000 mgr.y (mgr.24961) 2312 : cluster [DBG] pgmap v2667: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:50.381 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:49 smithi059 bash[59745]: cluster 2022-04-28T20:58:48.820138+0000 mgr.y (mgr.24961) 2312 : cluster [DBG] pgmap v2667: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:50.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:49 smithi139 bash[43301]: cluster 2022-04-28T20:58:48.820138+0000 mgr.y (mgr.24961) 2312 : cluster [DBG] pgmap v2667: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:51 smithi059 bash[61152]: cluster 2022-04-28T20:58:50.821584+0000 mgr.y (mgr.24961) 2313 : cluster [DBG] pgmap v2668: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:51 smithi059 bash[59745]: cluster 2022-04-28T20:58:50.821584+0000 mgr.y (mgr.24961) 2313 : cluster [DBG] pgmap v2668: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:51.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:51 smithi139 bash[43301]: cluster 2022-04-28T20:58:50.821584+0000 mgr.y (mgr.24961) 2313 : cluster [DBG] pgmap v2668: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:53 smithi059 bash[61152]: cluster 2022-04-28T20:58:52.822251+0000 mgr.y (mgr.24961) 2314 : cluster [DBG] pgmap v2669: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:53 smithi059 bash[59745]: cluster 2022-04-28T20:58:52.822251+0000 mgr.y (mgr.24961) 2314 : cluster [DBG] pgmap v2669: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:58:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:58:53.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:58:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:53 smithi139 bash[43301]: cluster 2022-04-28T20:58:52.822251+0000 mgr.y (mgr.24961) 2314 : cluster [DBG] pgmap v2669: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:55 smithi059 bash[61152]: cluster 2022-04-28T20:58:54.823349+0000 mgr.y (mgr.24961) 2315 : cluster [DBG] pgmap v2670: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:55 smithi059 bash[59745]: cluster 2022-04-28T20:58:54.823349+0000 mgr.y (mgr.24961) 2315 : cluster [DBG] pgmap v2670: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:55.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:55 smithi139 bash[43301]: cluster 2022-04-28T20:58:54.823349+0000 mgr.y (mgr.24961) 2315 : cluster [DBG] pgmap v2670: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:57 smithi059 bash[59745]: cluster 2022-04-28T20:58:56.824632+0000 mgr.y (mgr.24961) 2316 : cluster [DBG] pgmap v2671: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:57 smithi059 bash[61152]: cluster 2022-04-28T20:58:56.824632+0000 mgr.y (mgr.24961) 2316 : cluster [DBG] pgmap v2671: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:58:58.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:57 smithi139 bash[43301]: cluster 2022-04-28T20:58:56.824632+0000 mgr.y (mgr.24961) 2316 : cluster [DBG] pgmap v2671: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:00.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:58:59 smithi059 bash[61152]: cluster 2022-04-28T20:58:58.825365+0000 mgr.y (mgr.24961) 2317 : cluster [DBG] pgmap v2672: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:00.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:58:59 smithi059 bash[59745]: cluster 2022-04-28T20:58:58.825365+0000 mgr.y (mgr.24961) 2317 : cluster [DBG] pgmap v2672: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:58:59 smithi139 bash[43301]: cluster 2022-04-28T20:58:58.825365+0000 mgr.y (mgr.24961) 2317 : cluster [DBG] pgmap v2672: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:01 smithi059 bash[59745]: cluster 2022-04-28T20:59:00.826781+0000 mgr.y (mgr.24961) 2318 : cluster [DBG] pgmap v2673: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2022-04-28T20:59:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:01 smithi059 bash[61152]: cluster 2022-04-28T20:59:00.826781+0000 mgr.y (mgr.24961) 2318 : cluster [DBG] pgmap v2673: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2022-04-28T20:59:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:01 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:01.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:01 smithi139 bash[43301]: cluster 2022-04-28T20:59:00.826781+0000 mgr.y (mgr.24961) 2318 : cluster [DBG] pgmap v2673: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2022-04-28T20:59:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:03 smithi059 bash[61152]: cluster 2022-04-28T20:59:02.827844+0000 mgr.y (mgr.24961) 2319 : cluster [DBG] pgmap v2674: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:03 smithi059 bash[59745]: cluster 2022-04-28T20:59:02.827844+0000 mgr.y (mgr.24961) 2319 : cluster [DBG] pgmap v2674: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:03 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:03.786Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:03 smithi139 bash[43301]: cluster 2022-04-28T20:59:02.827844+0000 mgr.y (mgr.24961) 2319 : cluster [DBG] pgmap v2674: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:05 smithi059 bash[59745]: cluster 2022-04-28T20:59:04.829085+0000 mgr.y (mgr.24961) 2320 : cluster [DBG] pgmap v2675: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:05 smithi059 bash[61152]: cluster 2022-04-28T20:59:04.829085+0000 mgr.y (mgr.24961) 2320 : cluster [DBG] pgmap v2675: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:05 smithi139 bash[43301]: cluster 2022-04-28T20:59:04.829085+0000 mgr.y (mgr.24961) 2320 : cluster [DBG] pgmap v2675: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[59745]: cluster 2022-04-28T20:59:06.830492+0000 mgr.y (mgr.24961) 2321 : cluster [DBG] pgmap v2676: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2022-04-28T20:59:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[59745]: audit 2022-04-28T20:59:07.452913+0000 mon.a (mon.0) 2371 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:59:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[59745]: audit 2022-04-28T20:59:07.455766+0000 mon.a (mon.0) 2372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:59:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[59745]: audit 2022-04-28T20:59:07.457609+0000 mon.a (mon.0) 2373 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:59:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[59745]: audit 2022-04-28T20:59:07.761827+0000 mon.a (mon.0) 2374 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[59745]: audit 2022-04-28T20:59:07.768396+0000 mon.a (mon.0) 2375 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[61152]: cluster 2022-04-28T20:59:06.830492+0000 mgr.y (mgr.24961) 2321 : cluster [DBG] pgmap v2676: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2022-04-28T20:59:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[61152]: audit 2022-04-28T20:59:07.452913+0000 mon.a (mon.0) 2371 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:59:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[61152]: audit 2022-04-28T20:59:07.455766+0000 mon.a (mon.0) 2372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:59:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[61152]: audit 2022-04-28T20:59:07.457609+0000 mon.a (mon.0) 2373 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:59:08.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[61152]: audit 2022-04-28T20:59:07.761827+0000 mon.a (mon.0) 2374 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:08.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:07 smithi059 bash[61152]: audit 2022-04-28T20:59:07.768396+0000 mon.a (mon.0) 2375 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:07 smithi139 bash[43301]: cluster 2022-04-28T20:59:06.830492+0000 mgr.y (mgr.24961) 2321 : cluster [DBG] pgmap v2676: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2022-04-28T20:59:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:07 smithi139 bash[43301]: audit 2022-04-28T20:59:07.452913+0000 mon.a (mon.0) 2371 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T20:59:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:07 smithi139 bash[43301]: audit 2022-04-28T20:59:07.455766+0000 mon.a (mon.0) 2372 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T20:59:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:07 smithi139 bash[43301]: audit 2022-04-28T20:59:07.457609+0000 mon.a (mon.0) 2373 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T20:59:08.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:07 smithi139 bash[43301]: audit 2022-04-28T20:59:07.761827+0000 mon.a (mon.0) 2374 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:08.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:07 smithi139 bash[43301]: audit 2022-04-28T20:59:07.768396+0000 mon.a (mon.0) 2375 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:09 smithi059 bash[61152]: cephadm 2022-04-28T20:59:07.459266+0000 mgr.y (mgr.24961) 2322 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:59:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:09 smithi059 bash[61152]: audit 2022-04-28T20:59:08.059332+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:09 smithi059 bash[59745]: cephadm 2022-04-28T20:59:07.459266+0000 mgr.y (mgr.24961) 2322 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:59:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:09 smithi059 bash[59745]: audit 2022-04-28T20:59:08.059332+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:09 smithi139 bash[43301]: cephadm 2022-04-28T20:59:07.459266+0000 mgr.y (mgr.24961) 2322 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T20:59:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:09 smithi139 bash[43301]: audit 2022-04-28T20:59:08.059332+0000 mon.a (mon.0) 2376 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T20:59:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:10 smithi059 bash[59745]: cluster 2022-04-28T20:59:08.831236+0000 mgr.y (mgr.24961) 2323 : cluster [DBG] pgmap v2677: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:10 smithi059 bash[61152]: cluster 2022-04-28T20:59:08.831236+0000 mgr.y (mgr.24961) 2323 : cluster [DBG] pgmap v2677: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:10 smithi139 bash[43301]: cluster 2022-04-28T20:59:08.831236+0000 mgr.y (mgr.24961) 2323 : cluster [DBG] pgmap v2677: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:12 smithi059 bash[59745]: cluster 2022-04-28T20:59:10.832634+0000 mgr.y (mgr.24961) 2324 : cluster [DBG] pgmap v2678: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:12.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:12 smithi059 bash[61152]: cluster 2022-04-28T20:59:10.832634+0000 mgr.y (mgr.24961) 2324 : cluster [DBG] pgmap v2678: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:12.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:11 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:11.890Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:12 smithi139 bash[43301]: cluster 2022-04-28T20:59:10.832634+0000 mgr.y (mgr.24961) 2324 : cluster [DBG] pgmap v2678: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:14.082 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:13 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:13.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:14 smithi059 bash[59745]: cluster 2022-04-28T20:59:12.833660+0000 mgr.y (mgr.24961) 2325 : cluster [DBG] pgmap v2679: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:14.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:14 smithi059 bash[61152]: cluster 2022-04-28T20:59:12.833660+0000 mgr.y (mgr.24961) 2325 : cluster [DBG] pgmap v2679: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:14 smithi139 bash[43301]: cluster 2022-04-28T20:59:12.833660+0000 mgr.y (mgr.24961) 2325 : cluster [DBG] pgmap v2679: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:15 smithi059 bash[61152]: cluster 2022-04-28T20:59:14.834842+0000 mgr.y (mgr.24961) 2326 : cluster [DBG] pgmap v2680: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:15 smithi059 bash[59745]: cluster 2022-04-28T20:59:14.834842+0000 mgr.y (mgr.24961) 2326 : cluster [DBG] pgmap v2680: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:15 smithi139 bash[43301]: cluster 2022-04-28T20:59:14.834842+0000 mgr.y (mgr.24961) 2326 : cluster [DBG] pgmap v2680: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:17 smithi059 bash[61152]: cluster 2022-04-28T20:59:16.836162+0000 mgr.y (mgr.24961) 2327 : cluster [DBG] pgmap v2681: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:17 smithi059 bash[59745]: cluster 2022-04-28T20:59:16.836162+0000 mgr.y (mgr.24961) 2327 : cluster [DBG] pgmap v2681: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:17 smithi139 bash[43301]: cluster 2022-04-28T20:59:16.836162+0000 mgr.y (mgr.24961) 2327 : cluster [DBG] pgmap v2681: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[59745]: audit 2022-04-28T20:59:17.974051+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:59:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[59745]: audit 2022-04-28T20:59:17.974512+0000 mon.a (mon.0) 2378 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:59:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[59745]: audit 2022-04-28T20:59:17.984120+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:59:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[59745]: audit 2022-04-28T20:59:17.984777+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:59:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[59745]: audit 2022-04-28T20:59:17.985336+0000 mon.a (mon.0) 2381 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:59:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[59745]: audit 2022-04-28T20:59:17.985880+0000 mon.a (mon.0) 2382 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T20:59:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[61152]: audit 2022-04-28T20:59:17.974051+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:59:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[61152]: audit 2022-04-28T20:59:17.974512+0000 mon.a (mon.0) 2378 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:59:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[61152]: audit 2022-04-28T20:59:17.984120+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:59:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[61152]: audit 2022-04-28T20:59:17.984777+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:59:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[61152]: audit 2022-04-28T20:59:17.985336+0000 mon.a (mon.0) 2381 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:59:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:18 smithi059 bash[61152]: audit 2022-04-28T20:59:17.985880+0000 mon.a (mon.0) 2382 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T20:59:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:18 smithi139 bash[43301]: audit 2022-04-28T20:59:17.974051+0000 mon.a (mon.0) 2377 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T20:59:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:18 smithi139 bash[43301]: audit 2022-04-28T20:59:17.974512+0000 mon.a (mon.0) 2378 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T20:59:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:18 smithi139 bash[43301]: audit 2022-04-28T20:59:17.984120+0000 mon.a (mon.0) 2379 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T20:59:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:18 smithi139 bash[43301]: audit 2022-04-28T20:59:17.984777+0000 mon.a (mon.0) 2380 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T20:59:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:18 smithi139 bash[43301]: audit 2022-04-28T20:59:17.985336+0000 mon.a (mon.0) 2381 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T20:59:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:18 smithi139 bash[43301]: audit 2022-04-28T20:59:17.985880+0000 mon.a (mon.0) 2382 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T20:59:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:19 smithi059 bash[61152]: cluster 2022-04-28T20:59:18.836894+0000 mgr.y (mgr.24961) 2328 : cluster [DBG] pgmap v2682: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:19 smithi059 bash[59745]: cluster 2022-04-28T20:59:18.836894+0000 mgr.y (mgr.24961) 2328 : cluster [DBG] pgmap v2682: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:19 smithi139 bash[43301]: cluster 2022-04-28T20:59:18.836894+0000 mgr.y (mgr.24961) 2328 : cluster [DBG] pgmap v2682: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:21 smithi059 bash[61152]: cluster 2022-04-28T20:59:20.838321+0000 mgr.y (mgr.24961) 2329 : cluster [DBG] pgmap v2683: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:21 smithi059 bash[59745]: cluster 2022-04-28T20:59:20.838321+0000 mgr.y (mgr.24961) 2329 : cluster [DBG] pgmap v2683: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:21 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:21.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:21 smithi139 bash[43301]: cluster 2022-04-28T20:59:20.838321+0000 mgr.y (mgr.24961) 2329 : cluster [DBG] pgmap v2683: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T20:59:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:23 smithi059 bash[59745]: cluster 2022-04-28T20:59:22.839329+0000 mgr.y (mgr.24961) 2330 : cluster [DBG] pgmap v2684: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:23 smithi059 bash[61152]: cluster 2022-04-28T20:59:22.839329+0000 mgr.y (mgr.24961) 2330 : cluster [DBG] pgmap v2684: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:23 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:23.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:23 smithi139 bash[43301]: cluster 2022-04-28T20:59:22.839329+0000 mgr.y (mgr.24961) 2330 : cluster [DBG] pgmap v2684: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:25 smithi059 bash[59745]: cluster 2022-04-28T20:59:24.840485+0000 mgr.y (mgr.24961) 2331 : cluster [DBG] pgmap v2685: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:25.655 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:25 smithi059 bash[61152]: cluster 2022-04-28T20:59:24.840485+0000 mgr.y (mgr.24961) 2331 : cluster [DBG] pgmap v2685: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:25.655 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:25 smithi139 bash[43301]: cluster 2022-04-28T20:59:24.840485+0000 mgr.y (mgr.24961) 2331 : cluster [DBG] pgmap v2685: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:27 smithi059 bash[59745]: cluster 2022-04-28T20:59:26.841753+0000 mgr.y (mgr.24961) 2332 : cluster [DBG] pgmap v2686: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2022-04-28T20:59:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:27 smithi059 bash[61152]: cluster 2022-04-28T20:59:26.841753+0000 mgr.y (mgr.24961) 2332 : cluster [DBG] pgmap v2686: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2022-04-28T20:59:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:27 smithi139 bash[43301]: cluster 2022-04-28T20:59:26.841753+0000 mgr.y (mgr.24961) 2332 : cluster [DBG] pgmap v2686: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2022-04-28T20:59:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:29 smithi059 bash[59745]: cluster 2022-04-28T20:59:28.842483+0000 mgr.y (mgr.24961) 2333 : cluster [DBG] pgmap v2687: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:29 smithi059 bash[61152]: cluster 2022-04-28T20:59:28.842483+0000 mgr.y (mgr.24961) 2333 : cluster [DBG] pgmap v2687: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:29 smithi139 bash[43301]: cluster 2022-04-28T20:59:28.842483+0000 mgr.y (mgr.24961) 2333 : cluster [DBG] pgmap v2687: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2022-04-28T20:59:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:31 smithi059 bash[61152]: cluster 2022-04-28T20:59:30.843931+0000 mgr.y (mgr.24961) 2334 : cluster [DBG] pgmap v2688: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2022-04-28T20:59:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:31 smithi059 bash[59745]: cluster 2022-04-28T20:59:30.843931+0000 mgr.y (mgr.24961) 2334 : cluster [DBG] pgmap v2688: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2022-04-28T20:59:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:31 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:31.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:31 smithi139 bash[43301]: cluster 2022-04-28T20:59:30.843931+0000 mgr.y (mgr.24961) 2334 : cluster [DBG] pgmap v2688: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2022-04-28T20:59:34.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:33 smithi059 bash[59745]: cluster 2022-04-28T20:59:32.844940+0000 mgr.y (mgr.24961) 2335 : cluster [DBG] pgmap v2689: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:34.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:33 smithi059 bash[61152]: cluster 2022-04-28T20:59:32.844940+0000 mgr.y (mgr.24961) 2335 : cluster [DBG] pgmap v2689: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:33 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:33.787Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:33 smithi139 bash[43301]: cluster 2022-04-28T20:59:32.844940+0000 mgr.y (mgr.24961) 2335 : cluster [DBG] pgmap v2689: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:35 smithi059 bash[61152]: cluster 2022-04-28T20:59:34.846126+0000 mgr.y (mgr.24961) 2336 : cluster [DBG] pgmap v2690: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:35 smithi059 bash[59745]: cluster 2022-04-28T20:59:34.846126+0000 mgr.y (mgr.24961) 2336 : cluster [DBG] pgmap v2690: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:35 smithi139 bash[43301]: cluster 2022-04-28T20:59:34.846126+0000 mgr.y (mgr.24961) 2336 : cluster [DBG] pgmap v2690: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:37 smithi059 bash[59745]: cluster 2022-04-28T20:59:36.847419+0000 mgr.y (mgr.24961) 2337 : cluster [DBG] pgmap v2691: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2022-04-28T20:59:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:37 smithi059 bash[61152]: cluster 2022-04-28T20:59:36.847419+0000 mgr.y (mgr.24961) 2337 : cluster [DBG] pgmap v2691: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2022-04-28T20:59:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:37 smithi139 bash[43301]: cluster 2022-04-28T20:59:36.847419+0000 mgr.y (mgr.24961) 2337 : cluster [DBG] pgmap v2691: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 0 op/s 2022-04-28T20:59:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:39 smithi059 bash[59745]: cluster 2022-04-28T20:59:38.848187+0000 mgr.y (mgr.24961) 2338 : cluster [DBG] pgmap v2692: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:39 smithi059 bash[61152]: cluster 2022-04-28T20:59:38.848187+0000 mgr.y (mgr.24961) 2338 : cluster [DBG] pgmap v2692: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:40.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:39 smithi139 bash[43301]: cluster 2022-04-28T20:59:38.848187+0000 mgr.y (mgr.24961) 2338 : cluster [DBG] pgmap v2692: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:41 smithi059 bash[61152]: cluster 2022-04-28T20:59:40.849688+0000 mgr.y (mgr.24961) 2339 : cluster [DBG] pgmap v2693: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2022-04-28T20:59:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:41 smithi059 bash[59745]: cluster 2022-04-28T20:59:40.849688+0000 mgr.y (mgr.24961) 2339 : cluster [DBG] pgmap v2693: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2022-04-28T20:59:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:41 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:41.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:42.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:41 smithi139 bash[43301]: cluster 2022-04-28T20:59:40.849688+0000 mgr.y (mgr.24961) 2339 : cluster [DBG] pgmap v2693: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 0 op/s 2022-04-28T20:59:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:43 smithi059 bash[61152]: cluster 2022-04-28T20:59:42.850812+0000 mgr.y (mgr.24961) 2340 : cluster [DBG] pgmap v2694: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:43 smithi059 bash[59745]: cluster 2022-04-28T20:59:42.850812+0000 mgr.y (mgr.24961) 2340 : cluster [DBG] pgmap v2694: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:43 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:43.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:43 smithi139 bash[43301]: cluster 2022-04-28T20:59:42.850812+0000 mgr.y (mgr.24961) 2340 : cluster [DBG] pgmap v2694: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:45 smithi059 bash[61152]: cluster 2022-04-28T20:59:44.852023+0000 mgr.y (mgr.24961) 2341 : cluster [DBG] pgmap v2695: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:45 smithi059 bash[59745]: cluster 2022-04-28T20:59:44.852023+0000 mgr.y (mgr.24961) 2341 : cluster [DBG] pgmap v2695: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:45 smithi139 bash[43301]: cluster 2022-04-28T20:59:44.852023+0000 mgr.y (mgr.24961) 2341 : cluster [DBG] pgmap v2695: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2022-04-28T20:59:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:47 smithi059 bash[59745]: cluster 2022-04-28T20:59:46.853323+0000 mgr.y (mgr.24961) 2342 : cluster [DBG] pgmap v2696: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2022-04-28T20:59:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:47 smithi059 bash[61152]: cluster 2022-04-28T20:59:46.853323+0000 mgr.y (mgr.24961) 2342 : cluster [DBG] pgmap v2696: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2022-04-28T20:59:48.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:47 smithi139 bash[43301]: cluster 2022-04-28T20:59:46.853323+0000 mgr.y (mgr.24961) 2342 : cluster [DBG] pgmap v2696: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 596 B/s rd, 0 op/s 2022-04-28T20:59:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:49 smithi059 bash[59745]: cluster 2022-04-28T20:59:48.854069+0000 mgr.y (mgr.24961) 2343 : cluster [DBG] pgmap v2697: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:50.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:49 smithi059 bash[61152]: cluster 2022-04-28T20:59:48.854069+0000 mgr.y (mgr.24961) 2343 : cluster [DBG] pgmap v2697: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:49 smithi139 bash[43301]: cluster 2022-04-28T20:59:48.854069+0000 mgr.y (mgr.24961) 2343 : cluster [DBG] pgmap v2697: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:51 smithi059 bash[61152]: cluster 2022-04-28T20:59:50.855509+0000 mgr.y (mgr.24961) 2344 : cluster [DBG] pgmap v2698: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:51 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:51.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:51 smithi059 bash[59745]: cluster 2022-04-28T20:59:50.855509+0000 mgr.y (mgr.24961) 2344 : cluster [DBG] pgmap v2698: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:51 smithi139 bash[43301]: cluster 2022-04-28T20:59:50.855509+0000 mgr.y (mgr.24961) 2344 : cluster [DBG] pgmap v2698: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T20:59:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:53 smithi059 bash[61152]: cluster 2022-04-28T20:59:52.856482+0000 mgr.y (mgr.24961) 2345 : cluster [DBG] pgmap v2699: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 20:59:53 smithi059 bash[72035]: level=error ts=2022-04-28T20:59:53.789Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T20:59:54.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:53 smithi059 bash[59745]: cluster 2022-04-28T20:59:52.856482+0000 mgr.y (mgr.24961) 2345 : cluster [DBG] pgmap v2699: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:53 smithi139 bash[43301]: cluster 2022-04-28T20:59:52.856482+0000 mgr.y (mgr.24961) 2345 : cluster [DBG] pgmap v2699: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:55 smithi059 bash[59745]: cluster 2022-04-28T20:59:54.857634+0000 mgr.y (mgr.24961) 2346 : cluster [DBG] pgmap v2700: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:55 smithi059 bash[61152]: cluster 2022-04-28T20:59:54.857634+0000 mgr.y (mgr.24961) 2346 : cluster [DBG] pgmap v2700: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:55 smithi139 bash[43301]: cluster 2022-04-28T20:59:54.857634+0000 mgr.y (mgr.24961) 2346 : cluster [DBG] pgmap v2700: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:57 smithi059 bash[61152]: cluster 2022-04-28T20:59:56.858903+0000 mgr.y (mgr.24961) 2347 : cluster [DBG] pgmap v2701: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:57 smithi059 bash[59745]: cluster 2022-04-28T20:59:56.858903+0000 mgr.y (mgr.24961) 2347 : cluster [DBG] pgmap v2701: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T20:59:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:57 smithi139 bash[43301]: cluster 2022-04-28T20:59:56.858903+0000 mgr.y (mgr.24961) 2347 : cluster [DBG] pgmap v2701: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 20:59:59 smithi059 bash[61152]: cluster 2022-04-28T20:59:58.859771+0000 mgr.y (mgr.24961) 2348 : cluster [DBG] pgmap v2702: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 20:59:59 smithi059 bash[59745]: cluster 2022-04-28T20:59:58.859771+0000 mgr.y (mgr.24961) 2348 : cluster [DBG] pgmap v2702: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:00.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 20:59:59 smithi139 bash[43301]: cluster 2022-04-28T20:59:58.859771+0000 mgr.y (mgr.24961) 2348 : cluster [DBG] pgmap v2702: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:00 smithi059 bash[59745]: cluster 2022-04-28T21:00:00.000134+0000 mon.a (mon.0) 2383 : cluster [INF] overall HEALTH_OK 2022-04-28T21:00:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:00 smithi059 bash[61152]: cluster 2022-04-28T21:00:00.000134+0000 mon.a (mon.0) 2383 : cluster [INF] overall HEALTH_OK 2022-04-28T21:00:01.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:00 smithi139 bash[43301]: cluster 2022-04-28T21:00:00.000134+0000 mon.a (mon.0) 2383 : cluster [INF] overall HEALTH_OK 2022-04-28T21:00:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:01 smithi059 bash[61152]: cluster 2022-04-28T21:00:00.861222+0000 mgr.y (mgr.24961) 2349 : cluster [DBG] pgmap v2703: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:01.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:01 smithi059 bash[59745]: cluster 2022-04-28T21:00:00.861222+0000 mgr.y (mgr.24961) 2349 : cluster [DBG] pgmap v2703: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:02.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:01 smithi139 bash[43301]: cluster 2022-04-28T21:00:00.861222+0000 mgr.y (mgr.24961) 2349 : cluster [DBG] pgmap v2703: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:03 smithi059 bash[59745]: cluster 2022-04-28T21:00:02.862202+0000 mgr.y (mgr.24961) 2350 : cluster [DBG] pgmap v2704: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:03 smithi059 bash[61152]: cluster 2022-04-28T21:00:02.862202+0000 mgr.y (mgr.24961) 2350 : cluster [DBG] pgmap v2704: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:03.790Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:03 smithi139 bash[43301]: cluster 2022-04-28T21:00:02.862202+0000 mgr.y (mgr.24961) 2350 : cluster [DBG] pgmap v2704: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:05 smithi059 bash[61152]: cluster 2022-04-28T21:00:04.863360+0000 mgr.y (mgr.24961) 2351 : cluster [DBG] pgmap v2705: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:05 smithi059 bash[59745]: cluster 2022-04-28T21:00:04.863360+0000 mgr.y (mgr.24961) 2351 : cluster [DBG] pgmap v2705: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:05 smithi139 bash[43301]: cluster 2022-04-28T21:00:04.863360+0000 mgr.y (mgr.24961) 2351 : cluster [DBG] pgmap v2705: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:07 smithi059 bash[61152]: cluster 2022-04-28T21:00:06.864620+0000 mgr.y (mgr.24961) 2352 : cluster [DBG] pgmap v2706: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:07 smithi059 bash[59745]: cluster 2022-04-28T21:00:06.864620+0000 mgr.y (mgr.24961) 2352 : cluster [DBG] pgmap v2706: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:07 smithi139 bash[43301]: cluster 2022-04-28T21:00:06.864620+0000 mgr.y (mgr.24961) 2352 : cluster [DBG] pgmap v2706: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[59745]: audit 2022-04-28T21:00:08.066017+0000 mon.a (mon.0) 2384 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:00:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[59745]: audit 2022-04-28T21:00:08.068870+0000 mon.a (mon.0) 2385 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:00:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[59745]: audit 2022-04-28T21:00:08.070675+0000 mon.a (mon.0) 2386 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:00:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[59745]: cephadm 2022-04-28T21:00:08.072280+0000 mgr.y (mgr.24961) 2353 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:00:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[59745]: audit 2022-04-28T21:00:08.371655+0000 mon.a (mon.0) 2387 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:00:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[61152]: audit 2022-04-28T21:00:08.066017+0000 mon.a (mon.0) 2384 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:00:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[61152]: audit 2022-04-28T21:00:08.068870+0000 mon.a (mon.0) 2385 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:00:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[61152]: audit 2022-04-28T21:00:08.070675+0000 mon.a (mon.0) 2386 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:00:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[61152]: cephadm 2022-04-28T21:00:08.072280+0000 mgr.y (mgr.24961) 2353 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:00:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:08 smithi059 bash[61152]: audit 2022-04-28T21:00:08.371655+0000 mon.a (mon.0) 2387 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:00:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:08 smithi139 bash[43301]: audit 2022-04-28T21:00:08.066017+0000 mon.a (mon.0) 2384 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:00:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:08 smithi139 bash[43301]: audit 2022-04-28T21:00:08.068870+0000 mon.a (mon.0) 2385 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:00:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:08 smithi139 bash[43301]: audit 2022-04-28T21:00:08.070675+0000 mon.a (mon.0) 2386 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:00:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:08 smithi139 bash[43301]: cephadm 2022-04-28T21:00:08.072280+0000 mgr.y (mgr.24961) 2353 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:00:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:08 smithi139 bash[43301]: audit 2022-04-28T21:00:08.371655+0000 mon.a (mon.0) 2387 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:00:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:09 smithi059 bash[61152]: cluster 2022-04-28T21:00:08.865362+0000 mgr.y (mgr.24961) 2354 : cluster [DBG] pgmap v2707: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:10.401 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:09 smithi059 bash[59745]: cluster 2022-04-28T21:00:08.865362+0000 mgr.y (mgr.24961) 2354 : cluster [DBG] pgmap v2707: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:10.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:09 smithi139 bash[43301]: cluster 2022-04-28T21:00:08.865362+0000 mgr.y (mgr.24961) 2354 : cluster [DBG] pgmap v2707: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:11 smithi059 bash[59745]: cluster 2022-04-28T21:00:10.866792+0000 mgr.y (mgr.24961) 2355 : cluster [DBG] pgmap v2708: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T21:00:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:11 smithi059 bash[61152]: cluster 2022-04-28T21:00:10.866792+0000 mgr.y (mgr.24961) 2355 : cluster [DBG] pgmap v2708: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T21:00:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:11.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:11 smithi139 bash[43301]: cluster 2022-04-28T21:00:10.866792+0000 mgr.y (mgr.24961) 2355 : cluster [DBG] pgmap v2708: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2022-04-28T21:00:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:13 smithi059 bash[59745]: cluster 2022-04-28T21:00:12.867805+0000 mgr.y (mgr.24961) 2356 : cluster [DBG] pgmap v2709: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:13 smithi059 bash[61152]: cluster 2022-04-28T21:00:12.867805+0000 mgr.y (mgr.24961) 2356 : cluster [DBG] pgmap v2709: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:13.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:13 smithi139 bash[43301]: cluster 2022-04-28T21:00:12.867805+0000 mgr.y (mgr.24961) 2356 : cluster [DBG] pgmap v2709: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:15 smithi059 bash[61152]: cluster 2022-04-28T21:00:14.868950+0000 mgr.y (mgr.24961) 2357 : cluster [DBG] pgmap v2710: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:15 smithi059 bash[59745]: cluster 2022-04-28T21:00:14.868950+0000 mgr.y (mgr.24961) 2357 : cluster [DBG] pgmap v2710: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:15 smithi139 bash[43301]: cluster 2022-04-28T21:00:14.868950+0000 mgr.y (mgr.24961) 2357 : cluster [DBG] pgmap v2710: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:17 smithi059 bash[59745]: cluster 2022-04-28T21:00:16.870242+0000 mgr.y (mgr.24961) 2358 : cluster [DBG] pgmap v2711: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:17 smithi059 bash[61152]: cluster 2022-04-28T21:00:16.870242+0000 mgr.y (mgr.24961) 2358 : cluster [DBG] pgmap v2711: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:17 smithi139 bash[43301]: cluster 2022-04-28T21:00:16.870242+0000 mgr.y (mgr.24961) 2358 : cluster [DBG] pgmap v2711: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[59745]: audit 2022-04-28T21:00:17.987356+0000 mon.a (mon.0) 2388 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:00:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[59745]: audit 2022-04-28T21:00:17.988307+0000 mon.a (mon.0) 2389 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:00:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[59745]: audit 2022-04-28T21:00:17.999226+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:00:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[59745]: audit 2022-04-28T21:00:17.999758+0000 mon.a (mon.0) 2391 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2022-04-28T21:00:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[59745]: audit 2022-04-28T21:00:18.000298+0000 mon.a (mon.0) 2392 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:00:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[59745]: audit 2022-04-28T21:00:18.000828+0000 mon.a (mon.0) 2393 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:00:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[61152]: audit 2022-04-28T21:00:17.987356+0000 mon.a (mon.0) 2388 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:00:19.355 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[61152]: audit 2022-04-28T21:00:17.988307+0000 mon.a (mon.0) 2389 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:00:19.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[61152]: audit 2022-04-28T21:00:17.999226+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:00:19.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[61152]: audit 2022-04-28T21:00:17.999758+0000 mon.a (mon.0) 2391 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2022-04-28T21:00:19.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[61152]: audit 2022-04-28T21:00:18.000298+0000 mon.a (mon.0) 2392 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:00:19.356 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:18 smithi059 bash[61152]: audit 2022-04-28T21:00:18.000828+0000 mon.a (mon.0) 2393 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:00:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:18 smithi139 bash[43301]: audit 2022-04-28T21:00:17.987356+0000 mon.a (mon.0) 2388 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:00:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:18 smithi139 bash[43301]: audit 2022-04-28T21:00:17.988307+0000 mon.a (mon.0) 2389 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:00:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:18 smithi139 bash[43301]: audit 2022-04-28T21:00:17.999226+0000 mon.a (mon.0) 2390 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:00:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:18 smithi139 bash[43301]: audit 2022-04-28T21:00:17.999758+0000 mon.a (mon.0) 2391 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2022-04-28T21:00:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:18 smithi139 bash[43301]: audit 2022-04-28T21:00:18.000298+0000 mon.a (mon.0) 2392 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:00:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:18 smithi139 bash[43301]: audit 2022-04-28T21:00:18.000828+0000 mon.a (mon.0) 2393 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:00:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:19 smithi059 bash[61152]: cluster 2022-04-28T21:00:18.870963+0000 mgr.y (mgr.24961) 2359 : cluster [DBG] pgmap v2712: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:19 smithi059 bash[59745]: cluster 2022-04-28T21:00:18.870963+0000 mgr.y (mgr.24961) 2359 : cluster [DBG] pgmap v2712: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:19 smithi139 bash[43301]: cluster 2022-04-28T21:00:18.870963+0000 mgr.y (mgr.24961) 2359 : cluster [DBG] pgmap v2712: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:21 smithi059 bash[59745]: cluster 2022-04-28T21:00:20.872429+0000 mgr.y (mgr.24961) 2360 : cluster [DBG] pgmap v2713: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:21 smithi059 bash[61152]: cluster 2022-04-28T21:00:20.872429+0000 mgr.y (mgr.24961) 2360 : cluster [DBG] pgmap v2713: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:22.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:21.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:21 smithi139 bash[43301]: cluster 2022-04-28T21:00:20.872429+0000 mgr.y (mgr.24961) 2360 : cluster [DBG] pgmap v2713: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:23 smithi059 bash[59745]: cluster 2022-04-28T21:00:22.873428+0000 mgr.y (mgr.24961) 2361 : cluster [DBG] pgmap v2714: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:23 smithi059 bash[61152]: cluster 2022-04-28T21:00:22.873428+0000 mgr.y (mgr.24961) 2361 : cluster [DBG] pgmap v2714: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:23.791Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:23 smithi139 bash[43301]: cluster 2022-04-28T21:00:22.873428+0000 mgr.y (mgr.24961) 2361 : cluster [DBG] pgmap v2714: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:25 smithi059 bash[61152]: cluster 2022-04-28T21:00:24.874574+0000 mgr.y (mgr.24961) 2362 : cluster [DBG] pgmap v2715: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:25 smithi059 bash[59745]: cluster 2022-04-28T21:00:24.874574+0000 mgr.y (mgr.24961) 2362 : cluster [DBG] pgmap v2715: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:25.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:25 smithi139 bash[43301]: cluster 2022-04-28T21:00:24.874574+0000 mgr.y (mgr.24961) 2362 : cluster [DBG] pgmap v2715: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:27 smithi059 bash[59745]: cluster 2022-04-28T21:00:26.875872+0000 mgr.y (mgr.24961) 2363 : cluster [DBG] pgmap v2716: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:28.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:27 smithi059 bash[61152]: cluster 2022-04-28T21:00:26.875872+0000 mgr.y (mgr.24961) 2363 : cluster [DBG] pgmap v2716: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:27 smithi139 bash[43301]: cluster 2022-04-28T21:00:26.875872+0000 mgr.y (mgr.24961) 2363 : cluster [DBG] pgmap v2716: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:29 smithi059 bash[61152]: cluster 2022-04-28T21:00:28.876583+0000 mgr.y (mgr.24961) 2364 : cluster [DBG] pgmap v2717: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:29 smithi059 bash[59745]: cluster 2022-04-28T21:00:28.876583+0000 mgr.y (mgr.24961) 2364 : cluster [DBG] pgmap v2717: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:30.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:29 smithi139 bash[43301]: cluster 2022-04-28T21:00:28.876583+0000 mgr.y (mgr.24961) 2364 : cluster [DBG] pgmap v2717: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:31 smithi059 bash[61152]: cluster 2022-04-28T21:00:30.877997+0000 mgr.y (mgr.24961) 2365 : cluster [DBG] pgmap v2718: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:31.896Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:31 smithi059 bash[59745]: cluster 2022-04-28T21:00:30.877997+0000 mgr.y (mgr.24961) 2365 : cluster [DBG] pgmap v2718: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:31 smithi139 bash[43301]: cluster 2022-04-28T21:00:30.877997+0000 mgr.y (mgr.24961) 2365 : cluster [DBG] pgmap v2718: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2022-04-28T21:00:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:33 smithi059 bash[61152]: cluster 2022-04-28T21:00:32.878972+0000 mgr.y (mgr.24961) 2366 : cluster [DBG] pgmap v2719: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:33.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:33 smithi059 bash[59745]: cluster 2022-04-28T21:00:32.878972+0000 mgr.y (mgr.24961) 2366 : cluster [DBG] pgmap v2719: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:33 smithi139 bash[43301]: cluster 2022-04-28T21:00:32.878972+0000 mgr.y (mgr.24961) 2366 : cluster [DBG] pgmap v2719: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:35 smithi059 bash[61152]: cluster 2022-04-28T21:00:34.880157+0000 mgr.y (mgr.24961) 2367 : cluster [DBG] pgmap v2720: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:35 smithi059 bash[59745]: cluster 2022-04-28T21:00:34.880157+0000 mgr.y (mgr.24961) 2367 : cluster [DBG] pgmap v2720: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:35 smithi139 bash[43301]: cluster 2022-04-28T21:00:34.880157+0000 mgr.y (mgr.24961) 2367 : cluster [DBG] pgmap v2720: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:37 smithi059 bash[61152]: cluster 2022-04-28T21:00:36.881415+0000 mgr.y (mgr.24961) 2368 : cluster [DBG] pgmap v2721: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:37 smithi059 bash[59745]: cluster 2022-04-28T21:00:36.881415+0000 mgr.y (mgr.24961) 2368 : cluster [DBG] pgmap v2721: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:37 smithi139 bash[43301]: cluster 2022-04-28T21:00:36.881415+0000 mgr.y (mgr.24961) 2368 : cluster [DBG] pgmap v2721: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:00:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:39 smithi059 bash[61152]: cluster 2022-04-28T21:00:38.882161+0000 mgr.y (mgr.24961) 2369 : cluster [DBG] pgmap v2722: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:39 smithi059 bash[59745]: cluster 2022-04-28T21:00:38.882161+0000 mgr.y (mgr.24961) 2369 : cluster [DBG] pgmap v2722: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:39 smithi139 bash[43301]: cluster 2022-04-28T21:00:38.882161+0000 mgr.y (mgr.24961) 2369 : cluster [DBG] pgmap v2722: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:41 smithi059 bash[61152]: cluster 2022-04-28T21:00:40.883594+0000 mgr.y (mgr.24961) 2370 : cluster [DBG] pgmap v2723: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:41.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:42.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:41 smithi059 bash[59745]: cluster 2022-04-28T21:00:40.883594+0000 mgr.y (mgr.24961) 2370 : cluster [DBG] pgmap v2723: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:41 smithi139 bash[43301]: cluster 2022-04-28T21:00:40.883594+0000 mgr.y (mgr.24961) 2370 : cluster [DBG] pgmap v2723: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:00:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:43 smithi059 bash[61152]: cluster 2022-04-28T21:00:42.884658+0000 mgr.y (mgr.24961) 2371 : cluster [DBG] pgmap v2724: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:43.792Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:43 smithi059 bash[59745]: cluster 2022-04-28T21:00:42.884658+0000 mgr.y (mgr.24961) 2371 : cluster [DBG] pgmap v2724: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:43 smithi139 bash[43301]: cluster 2022-04-28T21:00:42.884658+0000 mgr.y (mgr.24961) 2371 : cluster [DBG] pgmap v2724: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:45 smithi059 bash[61152]: cluster 2022-04-28T21:00:44.885824+0000 mgr.y (mgr.24961) 2372 : cluster [DBG] pgmap v2725: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:45 smithi059 bash[59745]: cluster 2022-04-28T21:00:44.885824+0000 mgr.y (mgr.24961) 2372 : cluster [DBG] pgmap v2725: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:45 smithi139 bash[43301]: cluster 2022-04-28T21:00:44.885824+0000 mgr.y (mgr.24961) 2372 : cluster [DBG] pgmap v2725: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:47 smithi059 bash[61152]: cluster 2022-04-28T21:00:46.887072+0000 mgr.y (mgr.24961) 2373 : cluster [DBG] pgmap v2726: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:47 smithi059 bash[59745]: cluster 2022-04-28T21:00:46.887072+0000 mgr.y (mgr.24961) 2373 : cluster [DBG] pgmap v2726: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:47 smithi139 bash[43301]: cluster 2022-04-28T21:00:46.887072+0000 mgr.y (mgr.24961) 2373 : cluster [DBG] pgmap v2726: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:49 smithi059 bash[61152]: cluster 2022-04-28T21:00:48.887808+0000 mgr.y (mgr.24961) 2374 : cluster [DBG] pgmap v2727: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:49 smithi059 bash[59745]: cluster 2022-04-28T21:00:48.887808+0000 mgr.y (mgr.24961) 2374 : cluster [DBG] pgmap v2727: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:49 smithi139 bash[43301]: cluster 2022-04-28T21:00:48.887808+0000 mgr.y (mgr.24961) 2374 : cluster [DBG] pgmap v2727: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:51 smithi059 bash[61152]: cluster 2022-04-28T21:00:50.889204+0000 mgr.y (mgr.24961) 2375 : cluster [DBG] pgmap v2728: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:51.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:51 smithi059 bash[59745]: cluster 2022-04-28T21:00:50.889204+0000 mgr.y (mgr.24961) 2375 : cluster [DBG] pgmap v2728: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:51 smithi139 bash[43301]: cluster 2022-04-28T21:00:50.889204+0000 mgr.y (mgr.24961) 2375 : cluster [DBG] pgmap v2728: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:53 smithi059 bash[61152]: cluster 2022-04-28T21:00:52.890174+0000 mgr.y (mgr.24961) 2376 : cluster [DBG] pgmap v2729: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:54.115 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:00:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:00:53.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:00:54.116 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:53 smithi059 bash[59745]: cluster 2022-04-28T21:00:52.890174+0000 mgr.y (mgr.24961) 2376 : cluster [DBG] pgmap v2729: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:53 smithi139 bash[43301]: cluster 2022-04-28T21:00:52.890174+0000 mgr.y (mgr.24961) 2376 : cluster [DBG] pgmap v2729: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:55 smithi059 bash[61152]: cluster 2022-04-28T21:00:54.891280+0000 mgr.y (mgr.24961) 2377 : cluster [DBG] pgmap v2730: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:55 smithi059 bash[59745]: cluster 2022-04-28T21:00:54.891280+0000 mgr.y (mgr.24961) 2377 : cluster [DBG] pgmap v2730: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:55 smithi139 bash[43301]: cluster 2022-04-28T21:00:54.891280+0000 mgr.y (mgr.24961) 2377 : cluster [DBG] pgmap v2730: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:57 smithi059 bash[61152]: cluster 2022-04-28T21:00:56.892516+0000 mgr.y (mgr.24961) 2378 : cluster [DBG] pgmap v2731: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:57 smithi059 bash[59745]: cluster 2022-04-28T21:00:56.892516+0000 mgr.y (mgr.24961) 2378 : cluster [DBG] pgmap v2731: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:00:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:57 smithi139 bash[43301]: cluster 2022-04-28T21:00:56.892516+0000 mgr.y (mgr.24961) 2378 : cluster [DBG] pgmap v2731: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:00:59 smithi059 bash[61152]: cluster 2022-04-28T21:00:58.893252+0000 mgr.y (mgr.24961) 2379 : cluster [DBG] pgmap v2732: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:00:59 smithi059 bash[59745]: cluster 2022-04-28T21:00:58.893252+0000 mgr.y (mgr.24961) 2379 : cluster [DBG] pgmap v2732: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:00:59 smithi139 bash[43301]: cluster 2022-04-28T21:00:58.893252+0000 mgr.y (mgr.24961) 2379 : cluster [DBG] pgmap v2732: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:01 smithi059 bash[61152]: cluster 2022-04-28T21:01:00.894619+0000 mgr.y (mgr.24961) 2380 : cluster [DBG] pgmap v2733: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:01.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:01 smithi059 bash[59745]: cluster 2022-04-28T21:01:00.894619+0000 mgr.y (mgr.24961) 2380 : cluster [DBG] pgmap v2733: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:01 smithi139 bash[43301]: cluster 2022-04-28T21:01:00.894619+0000 mgr.y (mgr.24961) 2380 : cluster [DBG] pgmap v2733: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:03 smithi059 bash[61152]: cluster 2022-04-28T21:01:02.895714+0000 mgr.y (mgr.24961) 2381 : cluster [DBG] pgmap v2734: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:03.793Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:03 smithi059 bash[59745]: cluster 2022-04-28T21:01:02.895714+0000 mgr.y (mgr.24961) 2381 : cluster [DBG] pgmap v2734: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:03 smithi139 bash[43301]: cluster 2022-04-28T21:01:02.895714+0000 mgr.y (mgr.24961) 2381 : cluster [DBG] pgmap v2734: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:05 smithi059 bash[61152]: cluster 2022-04-28T21:01:04.896920+0000 mgr.y (mgr.24961) 2382 : cluster [DBG] pgmap v2735: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:05 smithi059 bash[59745]: cluster 2022-04-28T21:01:04.896920+0000 mgr.y (mgr.24961) 2382 : cluster [DBG] pgmap v2735: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:05 smithi139 bash[43301]: cluster 2022-04-28T21:01:04.896920+0000 mgr.y (mgr.24961) 2382 : cluster [DBG] pgmap v2735: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:08.347 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:07 smithi059 bash[61152]: cluster 2022-04-28T21:01:06.898196+0000 mgr.y (mgr.24961) 2383 : cluster [DBG] pgmap v2736: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:08.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:07 smithi059 bash[59745]: cluster 2022-04-28T21:01:06.898196+0000 mgr.y (mgr.24961) 2383 : cluster [DBG] pgmap v2736: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:07 smithi139 bash[43301]: cluster 2022-04-28T21:01:06.898196+0000 mgr.y (mgr.24961) 2383 : cluster [DBG] pgmap v2736: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.379125+0000 mon.a (mon.0) 2394 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:01:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.382033+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:01:09.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.383950+0000 mon.a (mon.0) 2396 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:01:09.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.674844+0000 mon.a (mon.0) 2397 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.676820+0000 mon.a (mon.0) 2398 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.677352+0000 mon.a (mon.0) 2399 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.677895+0000 mon.a (mon.0) 2400 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.678429+0000 mon.a (mon.0) 2401 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.678975+0000 mon.a (mon.0) 2402 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.679567+0000 mon.a (mon.0) 2403 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.680093+0000 mon.a (mon.0) 2404 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.680623+0000 mon.a (mon.0) 2405 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.681160+0000 mon.a (mon.0) 2406 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.681691+0000 mon.a (mon.0) 2407 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.682215+0000 mon.a (mon.0) 2408 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.682737+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.379125+0000 mon.a (mon.0) 2394 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:01:09.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.382033+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:01:09.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.383950+0000 mon.a (mon.0) 2396 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:01:09.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.674844+0000 mon.a (mon.0) 2397 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.676820+0000 mon.a (mon.0) 2398 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.677352+0000 mon.a (mon.0) 2399 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.677895+0000 mon.a (mon.0) 2400 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.678429+0000 mon.a (mon.0) 2401 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.349 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.678975+0000 mon.a (mon.0) 2402 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.679567+0000 mon.a (mon.0) 2403 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.680093+0000 mon.a (mon.0) 2404 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.680623+0000 mon.a (mon.0) 2405 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.681160+0000 mon.a (mon.0) 2406 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.681691+0000 mon.a (mon.0) 2407 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.682215+0000 mon.a (mon.0) 2408 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.682737+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.683254+0000 mon.a (mon.0) 2410 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.351 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.683846+0000 mon.a (mon.0) 2411 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.684166+0000 mon.a (mon.0) 2412 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.684422+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:01:09.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.684633+0000 mon.a (mon.0) 2414 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:01:09.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.684928+0000 mon.a (mon.0) 2415 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.685144+0000 mon.a (mon.0) 2416 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[59745]: audit 2022-04-28T21:01:08.692471+0000 mon.a (mon.0) 2417 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:01:09.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.683254+0000 mon.a (mon.0) 2410 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.683846+0000 mon.a (mon.0) 2411 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.684166+0000 mon.a (mon.0) 2412 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.353 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.684422+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:01:09.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.684633+0000 mon.a (mon.0) 2414 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:01:09.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.684928+0000 mon.a (mon.0) 2415 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.685144+0000 mon.a (mon.0) 2416 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.354 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:08 smithi059 bash[61152]: audit 2022-04-28T21:01:08.692471+0000 mon.a (mon.0) 2417 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:01:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.379125+0000 mon.a (mon.0) 2394 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:01:09.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.382033+0000 mon.a (mon.0) 2395 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:01:09.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.383950+0000 mon.a (mon.0) 2396 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:01:09.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.674844+0000 mon.a (mon.0) 2397 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.676820+0000 mon.a (mon.0) 2398 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.677352+0000 mon.a (mon.0) 2399 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.677895+0000 mon.a (mon.0) 2400 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.678429+0000 mon.a (mon.0) 2401 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.678975+0000 mon.a (mon.0) 2402 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.679567+0000 mon.a (mon.0) 2403 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.680093+0000 mon.a (mon.0) 2404 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.680623+0000 mon.a (mon.0) 2405 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.681160+0000 mon.a (mon.0) 2406 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.681691+0000 mon.a (mon.0) 2407 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.682215+0000 mon.a (mon.0) 2408 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.682737+0000 mon.a (mon.0) 2409 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.683254+0000 mon.a (mon.0) 2410 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:01:09.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.683846+0000 mon.a (mon.0) 2411 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.684166+0000 mon.a (mon.0) 2412 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.684422+0000 mon.a (mon.0) 2413 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:01:09.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.684633+0000 mon.a (mon.0) 2414 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:01:09.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.684928+0000 mon.a (mon.0) 2415 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.685144+0000 mon.a (mon.0) 2416 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:01:09.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:08 smithi139 bash[43301]: audit 2022-04-28T21:01:08.692471+0000 mon.a (mon.0) 2417 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:01:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:09 smithi059 bash[61152]: cephadm 2022-04-28T21:01:08.385712+0000 mgr.y (mgr.24961) 2384 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:01:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:09 smithi059 bash[61152]: cluster 2022-04-28T21:01:08.898992+0000 mgr.y (mgr.24961) 2385 : cluster [DBG] pgmap v2737: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:09 smithi059 bash[59745]: cephadm 2022-04-28T21:01:08.385712+0000 mgr.y (mgr.24961) 2384 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:01:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:09 smithi059 bash[59745]: cluster 2022-04-28T21:01:08.898992+0000 mgr.y (mgr.24961) 2385 : cluster [DBG] pgmap v2737: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:09 smithi139 bash[43301]: cephadm 2022-04-28T21:01:08.385712+0000 mgr.y (mgr.24961) 2384 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:01:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:09 smithi139 bash[43301]: cluster 2022-04-28T21:01:08.898992+0000 mgr.y (mgr.24961) 2385 : cluster [DBG] pgmap v2737: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:11 smithi059 bash[61152]: cluster 2022-04-28T21:01:10.900428+0000 mgr.y (mgr.24961) 2386 : cluster [DBG] pgmap v2738: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:11.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:11 smithi059 bash[59745]: cluster 2022-04-28T21:01:10.900428+0000 mgr.y (mgr.24961) 2386 : cluster [DBG] pgmap v2738: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:11 smithi139 bash[43301]: cluster 2022-04-28T21:01:10.900428+0000 mgr.y (mgr.24961) 2386 : cluster [DBG] pgmap v2738: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:13 smithi059 bash[59745]: cluster 2022-04-28T21:01:12.901517+0000 mgr.y (mgr.24961) 2387 : cluster [DBG] pgmap v2739: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:13 smithi059 bash[61152]: cluster 2022-04-28T21:01:12.901517+0000 mgr.y (mgr.24961) 2387 : cluster [DBG] pgmap v2739: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:13.794Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:13 smithi139 bash[43301]: cluster 2022-04-28T21:01:12.901517+0000 mgr.y (mgr.24961) 2387 : cluster [DBG] pgmap v2739: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:15 smithi059 bash[59745]: cluster 2022-04-28T21:01:14.902766+0000 mgr.y (mgr.24961) 2388 : cluster [DBG] pgmap v2740: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:15 smithi059 bash[61152]: cluster 2022-04-28T21:01:14.902766+0000 mgr.y (mgr.24961) 2388 : cluster [DBG] pgmap v2740: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:15 smithi139 bash[43301]: cluster 2022-04-28T21:01:14.902766+0000 mgr.y (mgr.24961) 2388 : cluster [DBG] pgmap v2740: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:17 smithi059 bash[61152]: cluster 2022-04-28T21:01:16.904066+0000 mgr.y (mgr.24961) 2389 : cluster [DBG] pgmap v2741: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:17 smithi059 bash[59745]: cluster 2022-04-28T21:01:16.904066+0000 mgr.y (mgr.24961) 2389 : cluster [DBG] pgmap v2741: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:17 smithi139 bash[43301]: cluster 2022-04-28T21:01:16.904066+0000 mgr.y (mgr.24961) 2389 : cluster [DBG] pgmap v2741: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[61152]: audit 2022-04-28T21:01:18.001689+0000 mon.a (mon.0) 2418 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:01:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[61152]: audit 2022-04-28T21:01:18.002251+0000 mon.a (mon.0) 2419 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:01:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[61152]: audit 2022-04-28T21:01:18.009222+0000 mon.a (mon.0) 2420 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[61152]: audit 2022-04-28T21:01:18.009968+0000 mon.a (mon.0) 2421 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[61152]: audit 2022-04-28T21:01:18.010508+0000 mon.a (mon.0) 2422 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[61152]: audit 2022-04-28T21:01:18.011030+0000 mon.a (mon.0) 2423 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:01:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[59745]: audit 2022-04-28T21:01:18.001689+0000 mon.a (mon.0) 2418 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:01:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[59745]: audit 2022-04-28T21:01:18.002251+0000 mon.a (mon.0) 2419 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:01:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[59745]: audit 2022-04-28T21:01:18.009222+0000 mon.a (mon.0) 2420 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[59745]: audit 2022-04-28T21:01:18.009968+0000 mon.a (mon.0) 2421 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[59745]: audit 2022-04-28T21:01:18.010508+0000 mon.a (mon.0) 2422 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:18 smithi059 bash[59745]: audit 2022-04-28T21:01:18.011030+0000 mon.a (mon.0) 2423 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:01:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:18 smithi139 bash[43301]: audit 2022-04-28T21:01:18.001689+0000 mon.a (mon.0) 2418 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:01:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:18 smithi139 bash[43301]: audit 2022-04-28T21:01:18.002251+0000 mon.a (mon.0) 2419 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:01:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:18 smithi139 bash[43301]: audit 2022-04-28T21:01:18.009222+0000 mon.a (mon.0) 2420 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:18 smithi139 bash[43301]: audit 2022-04-28T21:01:18.009968+0000 mon.a (mon.0) 2421 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:18 smithi139 bash[43301]: audit 2022-04-28T21:01:18.010508+0000 mon.a (mon.0) 2422 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:01:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:18 smithi139 bash[43301]: audit 2022-04-28T21:01:18.011030+0000 mon.a (mon.0) 2423 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:01:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:19 smithi059 bash[61152]: cluster 2022-04-28T21:01:18.904773+0000 mgr.y (mgr.24961) 2390 : cluster [DBG] pgmap v2742: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:19 smithi059 bash[59745]: cluster 2022-04-28T21:01:18.904773+0000 mgr.y (mgr.24961) 2390 : cluster [DBG] pgmap v2742: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:19 smithi139 bash[43301]: cluster 2022-04-28T21:01:18.904773+0000 mgr.y (mgr.24961) 2390 : cluster [DBG] pgmap v2742: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:21 smithi059 bash[61152]: cluster 2022-04-28T21:01:20.906273+0000 mgr.y (mgr.24961) 2391 : cluster [DBG] pgmap v2743: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:22.352 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:21.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:22.352 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:21 smithi059 bash[59745]: cluster 2022-04-28T21:01:20.906273+0000 mgr.y (mgr.24961) 2391 : cluster [DBG] pgmap v2743: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:21 smithi139 bash[43301]: cluster 2022-04-28T21:01:20.906273+0000 mgr.y (mgr.24961) 2391 : cluster [DBG] pgmap v2743: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:23 smithi059 bash[61152]: cluster 2022-04-28T21:01:22.907286+0000 mgr.y (mgr.24961) 2392 : cluster [DBG] pgmap v2744: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:23.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:23 smithi059 bash[59745]: cluster 2022-04-28T21:01:22.907286+0000 mgr.y (mgr.24961) 2392 : cluster [DBG] pgmap v2744: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:23 smithi139 bash[43301]: cluster 2022-04-28T21:01:22.907286+0000 mgr.y (mgr.24961) 2392 : cluster [DBG] pgmap v2744: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:25 smithi059 bash[61152]: cluster 2022-04-28T21:01:24.908492+0000 mgr.y (mgr.24961) 2393 : cluster [DBG] pgmap v2745: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:25 smithi059 bash[59745]: cluster 2022-04-28T21:01:24.908492+0000 mgr.y (mgr.24961) 2393 : cluster [DBG] pgmap v2745: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:25 smithi139 bash[43301]: cluster 2022-04-28T21:01:24.908492+0000 mgr.y (mgr.24961) 2393 : cluster [DBG] pgmap v2745: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:27 smithi059 bash[61152]: cluster 2022-04-28T21:01:26.909794+0000 mgr.y (mgr.24961) 2394 : cluster [DBG] pgmap v2746: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:27 smithi059 bash[59745]: cluster 2022-04-28T21:01:26.909794+0000 mgr.y (mgr.24961) 2394 : cluster [DBG] pgmap v2746: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:27 smithi139 bash[43301]: cluster 2022-04-28T21:01:26.909794+0000 mgr.y (mgr.24961) 2394 : cluster [DBG] pgmap v2746: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:29 smithi059 bash[61152]: cluster 2022-04-28T21:01:28.910516+0000 mgr.y (mgr.24961) 2395 : cluster [DBG] pgmap v2747: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:29 smithi059 bash[59745]: cluster 2022-04-28T21:01:28.910516+0000 mgr.y (mgr.24961) 2395 : cluster [DBG] pgmap v2747: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:29 smithi139 bash[43301]: cluster 2022-04-28T21:01:28.910516+0000 mgr.y (mgr.24961) 2395 : cluster [DBG] pgmap v2747: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:31 smithi059 bash[61152]: cluster 2022-04-28T21:01:30.911948+0000 mgr.y (mgr.24961) 2396 : cluster [DBG] pgmap v2748: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:31.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:31 smithi059 bash[59745]: cluster 2022-04-28T21:01:30.911948+0000 mgr.y (mgr.24961) 2396 : cluster [DBG] pgmap v2748: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:31 smithi139 bash[43301]: cluster 2022-04-28T21:01:30.911948+0000 mgr.y (mgr.24961) 2396 : cluster [DBG] pgmap v2748: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:33 smithi059 bash[61152]: cluster 2022-04-28T21:01:32.912931+0000 mgr.y (mgr.24961) 2397 : cluster [DBG] pgmap v2749: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:33.795Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:33 smithi059 bash[59745]: cluster 2022-04-28T21:01:32.912931+0000 mgr.y (mgr.24961) 2397 : cluster [DBG] pgmap v2749: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:33 smithi139 bash[43301]: cluster 2022-04-28T21:01:32.912931+0000 mgr.y (mgr.24961) 2397 : cluster [DBG] pgmap v2749: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:35 smithi059 bash[61152]: cluster 2022-04-28T21:01:34.914020+0000 mgr.y (mgr.24961) 2398 : cluster [DBG] pgmap v2750: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:35.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:35 smithi059 bash[59745]: cluster 2022-04-28T21:01:34.914020+0000 mgr.y (mgr.24961) 2398 : cluster [DBG] pgmap v2750: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:35 smithi139 bash[43301]: cluster 2022-04-28T21:01:34.914020+0000 mgr.y (mgr.24961) 2398 : cluster [DBG] pgmap v2750: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:01:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:37 smithi059 bash[59745]: cluster 2022-04-28T21:01:36.915283+0000 mgr.y (mgr.24961) 2399 : cluster [DBG] pgmap v2751: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:01:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:37 smithi059 bash[61152]: cluster 2022-04-28T21:01:36.915283+0000 mgr.y (mgr.24961) 2399 : cluster [DBG] pgmap v2751: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:01:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:37 smithi139 bash[43301]: cluster 2022-04-28T21:01:36.915283+0000 mgr.y (mgr.24961) 2399 : cluster [DBG] pgmap v2751: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:01:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:39 smithi059 bash[61152]: cluster 2022-04-28T21:01:38.916047+0000 mgr.y (mgr.24961) 2400 : cluster [DBG] pgmap v2752: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:01:40.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:39 smithi059 bash[59745]: cluster 2022-04-28T21:01:38.916047+0000 mgr.y (mgr.24961) 2400 : cluster [DBG] pgmap v2752: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:01:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:39 smithi139 bash[43301]: cluster 2022-04-28T21:01:38.916047+0000 mgr.y (mgr.24961) 2400 : cluster [DBG] pgmap v2752: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:01:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:41 smithi059 bash[61152]: cluster 2022-04-28T21:01:40.917462+0000 mgr.y (mgr.24961) 2401 : cluster [DBG] pgmap v2753: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:41 smithi059 bash[59745]: cluster 2022-04-28T21:01:40.917462+0000 mgr.y (mgr.24961) 2401 : cluster [DBG] pgmap v2753: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:41.900Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:41 smithi139 bash[43301]: cluster 2022-04-28T21:01:40.917462+0000 mgr.y (mgr.24961) 2401 : cluster [DBG] pgmap v2753: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:44.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:43 smithi059 bash[59745]: cluster 2022-04-28T21:01:42.918432+0000 mgr.y (mgr.24961) 2402 : cluster [DBG] pgmap v2754: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:44.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:43 smithi059 bash[61152]: cluster 2022-04-28T21:01:42.918432+0000 mgr.y (mgr.24961) 2402 : cluster [DBG] pgmap v2754: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:43.796Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:43 smithi139 bash[43301]: cluster 2022-04-28T21:01:42.918432+0000 mgr.y (mgr.24961) 2402 : cluster [DBG] pgmap v2754: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:45 smithi059 bash[59745]: cluster 2022-04-28T21:01:44.919596+0000 mgr.y (mgr.24961) 2403 : cluster [DBG] pgmap v2755: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:45 smithi059 bash[61152]: cluster 2022-04-28T21:01:44.919596+0000 mgr.y (mgr.24961) 2403 : cluster [DBG] pgmap v2755: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:45 smithi139 bash[43301]: cluster 2022-04-28T21:01:44.919596+0000 mgr.y (mgr.24961) 2403 : cluster [DBG] pgmap v2755: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:47 smithi059 bash[61152]: cluster 2022-04-28T21:01:46.920869+0000 mgr.y (mgr.24961) 2404 : cluster [DBG] pgmap v2756: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2022-04-28T21:01:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:47 smithi059 bash[59745]: cluster 2022-04-28T21:01:46.920869+0000 mgr.y (mgr.24961) 2404 : cluster [DBG] pgmap v2756: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2022-04-28T21:01:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:47 smithi139 bash[43301]: cluster 2022-04-28T21:01:46.920869+0000 mgr.y (mgr.24961) 2404 : cluster [DBG] pgmap v2756: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 170 B/s wr, 0 op/s 2022-04-28T21:01:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:49 smithi059 bash[61152]: cluster 2022-04-28T21:01:48.921597+0000 mgr.y (mgr.24961) 2405 : cluster [DBG] pgmap v2757: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:49 smithi059 bash[59745]: cluster 2022-04-28T21:01:48.921597+0000 mgr.y (mgr.24961) 2405 : cluster [DBG] pgmap v2757: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:49 smithi139 bash[43301]: cluster 2022-04-28T21:01:48.921597+0000 mgr.y (mgr.24961) 2405 : cluster [DBG] pgmap v2757: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:52 smithi059 bash[61152]: cluster 2022-04-28T21:01:50.923102+0000 mgr.y (mgr.24961) 2406 : cluster [DBG] pgmap v2758: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:51.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:52 smithi059 bash[59745]: cluster 2022-04-28T21:01:50.923102+0000 mgr.y (mgr.24961) 2406 : cluster [DBG] pgmap v2758: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:51 smithi139 bash[43301]: cluster 2022-04-28T21:01:50.923102+0000 mgr.y (mgr.24961) 2406 : cluster [DBG] pgmap v2758: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:54.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:53 smithi059 bash[61152]: cluster 2022-04-28T21:01:52.924124+0000 mgr.y (mgr.24961) 2407 : cluster [DBG] pgmap v2759: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:01:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:01:53.797Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:01:54.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:53 smithi059 bash[59745]: cluster 2022-04-28T21:01:52.924124+0000 mgr.y (mgr.24961) 2407 : cluster [DBG] pgmap v2759: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:53 smithi139 bash[43301]: cluster 2022-04-28T21:01:52.924124+0000 mgr.y (mgr.24961) 2407 : cluster [DBG] pgmap v2759: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:55 smithi059 bash[61152]: cluster 2022-04-28T21:01:54.925260+0000 mgr.y (mgr.24961) 2408 : cluster [DBG] pgmap v2760: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:55 smithi059 bash[59745]: cluster 2022-04-28T21:01:54.925260+0000 mgr.y (mgr.24961) 2408 : cluster [DBG] pgmap v2760: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:55 smithi139 bash[43301]: cluster 2022-04-28T21:01:54.925260+0000 mgr.y (mgr.24961) 2408 : cluster [DBG] pgmap v2760: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:58.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:57 smithi059 bash[59745]: cluster 2022-04-28T21:01:56.926539+0000 mgr.y (mgr.24961) 2409 : cluster [DBG] pgmap v2761: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:58.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:57 smithi059 bash[61152]: cluster 2022-04-28T21:01:56.926539+0000 mgr.y (mgr.24961) 2409 : cluster [DBG] pgmap v2761: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:01:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:57 smithi139 bash[43301]: cluster 2022-04-28T21:01:56.926539+0000 mgr.y (mgr.24961) 2409 : cluster [DBG] pgmap v2761: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:01:59 smithi059 bash[61152]: cluster 2022-04-28T21:01:58.927185+0000 mgr.y (mgr.24961) 2410 : cluster [DBG] pgmap v2762: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:01:59 smithi059 bash[59745]: cluster 2022-04-28T21:01:58.927185+0000 mgr.y (mgr.24961) 2410 : cluster [DBG] pgmap v2762: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:01:59 smithi139 bash[43301]: cluster 2022-04-28T21:01:58.927185+0000 mgr.y (mgr.24961) 2410 : cluster [DBG] pgmap v2762: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:01 smithi059 bash[61152]: cluster 2022-04-28T21:02:00.928600+0000 mgr.y (mgr.24961) 2411 : cluster [DBG] pgmap v2763: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:01.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:01 smithi059 bash[59745]: cluster 2022-04-28T21:02:00.928600+0000 mgr.y (mgr.24961) 2411 : cluster [DBG] pgmap v2763: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:01 smithi139 bash[43301]: cluster 2022-04-28T21:02:00.928600+0000 mgr.y (mgr.24961) 2411 : cluster [DBG] pgmap v2763: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:04 smithi059 bash[61152]: cluster 2022-04-28T21:02:02.929582+0000 mgr.y (mgr.24961) 2412 : cluster [DBG] pgmap v2764: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:03.798Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:04 smithi059 bash[59745]: cluster 2022-04-28T21:02:02.929582+0000 mgr.y (mgr.24961) 2412 : cluster [DBG] pgmap v2764: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:03 smithi139 bash[43301]: cluster 2022-04-28T21:02:02.929582+0000 mgr.y (mgr.24961) 2412 : cluster [DBG] pgmap v2764: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:05 smithi059 bash[61152]: cluster 2022-04-28T21:02:04.930700+0000 mgr.y (mgr.24961) 2413 : cluster [DBG] pgmap v2765: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:05 smithi059 bash[59745]: cluster 2022-04-28T21:02:04.930700+0000 mgr.y (mgr.24961) 2413 : cluster [DBG] pgmap v2765: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:05.618 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:05 smithi139 bash[43301]: cluster 2022-04-28T21:02:04.930700+0000 mgr.y (mgr.24961) 2413 : cluster [DBG] pgmap v2765: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:08.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:07 smithi059 bash[59745]: cluster 2022-04-28T21:02:06.931957+0000 mgr.y (mgr.24961) 2414 : cluster [DBG] pgmap v2766: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:07 smithi059 bash[61152]: cluster 2022-04-28T21:02:06.931957+0000 mgr.y (mgr.24961) 2414 : cluster [DBG] pgmap v2766: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:08.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:07 smithi139 bash[43301]: cluster 2022-04-28T21:02:06.931957+0000 mgr.y (mgr.24961) 2414 : cluster [DBG] pgmap v2766: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:08 smithi059 bash[59745]: audit 2022-04-28T21:02:08.699408+0000 mon.a (mon.0) 2424 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:02:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:08 smithi059 bash[59745]: audit 2022-04-28T21:02:08.702214+0000 mon.a (mon.0) 2425 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:02:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:08 smithi059 bash[59745]: audit 2022-04-28T21:02:08.704101+0000 mon.a (mon.0) 2426 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:02:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:08 smithi059 bash[61152]: audit 2022-04-28T21:02:08.699408+0000 mon.a (mon.0) 2424 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:02:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:08 smithi059 bash[61152]: audit 2022-04-28T21:02:08.702214+0000 mon.a (mon.0) 2425 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:02:09.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:08 smithi059 bash[61152]: audit 2022-04-28T21:02:08.704101+0000 mon.a (mon.0) 2426 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:02:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:08 smithi139 bash[43301]: audit 2022-04-28T21:02:08.699408+0000 mon.a (mon.0) 2424 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:02:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:08 smithi139 bash[43301]: audit 2022-04-28T21:02:08.702214+0000 mon.a (mon.0) 2425 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:02:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:08 smithi139 bash[43301]: audit 2022-04-28T21:02:08.704101+0000 mon.a (mon.0) 2426 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:02:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:10 smithi059 bash[59745]: cephadm 2022-04-28T21:02:08.705870+0000 mgr.y (mgr.24961) 2415 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:02:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:10 smithi059 bash[59745]: cluster 2022-04-28T21:02:08.932643+0000 mgr.y (mgr.24961) 2416 : cluster [DBG] pgmap v2767: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:10 smithi059 bash[59745]: audit 2022-04-28T21:02:09.005095+0000 mon.a (mon.0) 2427 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:02:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:10 smithi059 bash[61152]: cephadm 2022-04-28T21:02:08.705870+0000 mgr.y (mgr.24961) 2415 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:02:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:10 smithi059 bash[61152]: cluster 2022-04-28T21:02:08.932643+0000 mgr.y (mgr.24961) 2416 : cluster [DBG] pgmap v2767: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:10 smithi059 bash[61152]: audit 2022-04-28T21:02:09.005095+0000 mon.a (mon.0) 2427 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:02:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:10 smithi139 bash[43301]: cephadm 2022-04-28T21:02:08.705870+0000 mgr.y (mgr.24961) 2415 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:02:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:10 smithi139 bash[43301]: cluster 2022-04-28T21:02:08.932643+0000 mgr.y (mgr.24961) 2416 : cluster [DBG] pgmap v2767: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:10 smithi139 bash[43301]: audit 2022-04-28T21:02:09.005095+0000 mon.a (mon.0) 2427 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:02:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:12 smithi059 bash[59745]: cluster 2022-04-28T21:02:10.934038+0000 mgr.y (mgr.24961) 2417 : cluster [DBG] pgmap v2768: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:12.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:12 smithi059 bash[61152]: cluster 2022-04-28T21:02:10.934038+0000 mgr.y (mgr.24961) 2417 : cluster [DBG] pgmap v2768: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:11.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:12 smithi139 bash[43301]: cluster 2022-04-28T21:02:10.934038+0000 mgr.y (mgr.24961) 2417 : cluster [DBG] pgmap v2768: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:14 smithi059 bash[59745]: cluster 2022-04-28T21:02:12.935011+0000 mgr.y (mgr.24961) 2418 : cluster [DBG] pgmap v2769: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:14 smithi059 bash[61152]: cluster 2022-04-28T21:02:12.935011+0000 mgr.y (mgr.24961) 2418 : cluster [DBG] pgmap v2769: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:13.799Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:14 smithi139 bash[43301]: cluster 2022-04-28T21:02:12.935011+0000 mgr.y (mgr.24961) 2418 : cluster [DBG] pgmap v2769: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:15 smithi059 bash[59745]: cluster 2022-04-28T21:02:14.936128+0000 mgr.y (mgr.24961) 2419 : cluster [DBG] pgmap v2770: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:15 smithi059 bash[61152]: cluster 2022-04-28T21:02:14.936128+0000 mgr.y (mgr.24961) 2419 : cluster [DBG] pgmap v2770: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:15 smithi139 bash[43301]: cluster 2022-04-28T21:02:14.936128+0000 mgr.y (mgr.24961) 2419 : cluster [DBG] pgmap v2770: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:17 smithi059 bash[59745]: cluster 2022-04-28T21:02:16.937369+0000 mgr.y (mgr.24961) 2420 : cluster [DBG] pgmap v2771: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:17 smithi059 bash[61152]: cluster 2022-04-28T21:02:16.937369+0000 mgr.y (mgr.24961) 2420 : cluster [DBG] pgmap v2771: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:17 smithi139 bash[43301]: cluster 2022-04-28T21:02:16.937369+0000 mgr.y (mgr.24961) 2420 : cluster [DBG] pgmap v2771: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[59745]: audit 2022-04-28T21:02:18.013959+0000 mon.a (mon.0) 2428 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:02:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[59745]: audit 2022-04-28T21:02:18.014667+0000 mon.a (mon.0) 2429 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:02:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[59745]: audit 2022-04-28T21:02:18.025518+0000 mon.a (mon.0) 2430 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:02:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[59745]: audit 2022-04-28T21:02:18.026140+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:02:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[59745]: audit 2022-04-28T21:02:18.026798+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:02:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[59745]: audit 2022-04-28T21:02:18.027435+0000 mon.a (mon.0) 2433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:02:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[61152]: audit 2022-04-28T21:02:18.013959+0000 mon.a (mon.0) 2428 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:02:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[61152]: audit 2022-04-28T21:02:18.014667+0000 mon.a (mon.0) 2429 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:02:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[61152]: audit 2022-04-28T21:02:18.025518+0000 mon.a (mon.0) 2430 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:02:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[61152]: audit 2022-04-28T21:02:18.026140+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:02:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[61152]: audit 2022-04-28T21:02:18.026798+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:02:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:19 smithi059 bash[61152]: audit 2022-04-28T21:02:18.027435+0000 mon.a (mon.0) 2433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:02:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:19 smithi139 bash[43301]: audit 2022-04-28T21:02:18.013959+0000 mon.a (mon.0) 2428 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:02:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:19 smithi139 bash[43301]: audit 2022-04-28T21:02:18.014667+0000 mon.a (mon.0) 2429 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:02:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:19 smithi139 bash[43301]: audit 2022-04-28T21:02:18.025518+0000 mon.a (mon.0) 2430 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:02:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:19 smithi139 bash[43301]: audit 2022-04-28T21:02:18.026140+0000 mon.a (mon.0) 2431 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:02:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:19 smithi139 bash[43301]: audit 2022-04-28T21:02:18.026798+0000 mon.a (mon.0) 2432 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:02:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:19 smithi139 bash[43301]: audit 2022-04-28T21:02:18.027435+0000 mon.a (mon.0) 2433 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:02:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:20 smithi059 bash[61152]: cluster 2022-04-28T21:02:18.938054+0000 mgr.y (mgr.24961) 2421 : cluster [DBG] pgmap v2772: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:20 smithi059 bash[59745]: cluster 2022-04-28T21:02:18.938054+0000 mgr.y (mgr.24961) 2421 : cluster [DBG] pgmap v2772: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:20 smithi139 bash[43301]: cluster 2022-04-28T21:02:18.938054+0000 mgr.y (mgr.24961) 2421 : cluster [DBG] pgmap v2772: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:22 smithi059 bash[61152]: cluster 2022-04-28T21:02:20.939448+0000 mgr.y (mgr.24961) 2422 : cluster [DBG] pgmap v2773: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:22 smithi059 bash[59745]: cluster 2022-04-28T21:02:20.939448+0000 mgr.y (mgr.24961) 2422 : cluster [DBG] pgmap v2773: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:21.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:22 smithi139 bash[43301]: cluster 2022-04-28T21:02:20.939448+0000 mgr.y (mgr.24961) 2422 : cluster [DBG] pgmap v2773: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:24 smithi059 bash[61152]: cluster 2022-04-28T21:02:22.940445+0000 mgr.y (mgr.24961) 2423 : cluster [DBG] pgmap v2774: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:24 smithi059 bash[59745]: cluster 2022-04-28T21:02:22.940445+0000 mgr.y (mgr.24961) 2423 : cluster [DBG] pgmap v2774: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:23.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:24 smithi139 bash[43301]: cluster 2022-04-28T21:02:22.940445+0000 mgr.y (mgr.24961) 2423 : cluster [DBG] pgmap v2774: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:25 smithi059 bash[61152]: cluster 2022-04-28T21:02:24.941614+0000 mgr.y (mgr.24961) 2424 : cluster [DBG] pgmap v2775: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:25 smithi059 bash[59745]: cluster 2022-04-28T21:02:24.941614+0000 mgr.y (mgr.24961) 2424 : cluster [DBG] pgmap v2775: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:25 smithi139 bash[43301]: cluster 2022-04-28T21:02:24.941614+0000 mgr.y (mgr.24961) 2424 : cluster [DBG] pgmap v2775: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:02:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:28 smithi059 bash[61152]: cluster 2022-04-28T21:02:26.942882+0000 mgr.y (mgr.24961) 2425 : cluster [DBG] pgmap v2776: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:28 smithi059 bash[59745]: cluster 2022-04-28T21:02:26.942882+0000 mgr.y (mgr.24961) 2425 : cluster [DBG] pgmap v2776: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:27 smithi139 bash[43301]: cluster 2022-04-28T21:02:26.942882+0000 mgr.y (mgr.24961) 2425 : cluster [DBG] pgmap v2776: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:30 smithi059 bash[61152]: cluster 2022-04-28T21:02:28.943640+0000 mgr.y (mgr.24961) 2426 : cluster [DBG] pgmap v2777: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:30 smithi059 bash[59745]: cluster 2022-04-28T21:02:28.943640+0000 mgr.y (mgr.24961) 2426 : cluster [DBG] pgmap v2777: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:30 smithi139 bash[43301]: cluster 2022-04-28T21:02:28.943640+0000 mgr.y (mgr.24961) 2426 : cluster [DBG] pgmap v2777: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:32 smithi059 bash[61152]: cluster 2022-04-28T21:02:30.945030+0000 mgr.y (mgr.24961) 2427 : cluster [DBG] pgmap v2778: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:31.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:32 smithi059 bash[59745]: cluster 2022-04-28T21:02:30.945030+0000 mgr.y (mgr.24961) 2427 : cluster [DBG] pgmap v2778: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:32 smithi139 bash[43301]: cluster 2022-04-28T21:02:30.945030+0000 mgr.y (mgr.24961) 2427 : cluster [DBG] pgmap v2778: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:02:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:34 smithi059 bash[61152]: cluster 2022-04-28T21:02:32.946007+0000 mgr.y (mgr.24961) 2428 : cluster [DBG] pgmap v2779: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:33.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:34 smithi059 bash[59745]: cluster 2022-04-28T21:02:32.946007+0000 mgr.y (mgr.24961) 2428 : cluster [DBG] pgmap v2779: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:34 smithi139 bash[43301]: cluster 2022-04-28T21:02:32.946007+0000 mgr.y (mgr.24961) 2428 : cluster [DBG] pgmap v2779: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:35 smithi059 bash[61152]: cluster 2022-04-28T21:02:34.947173+0000 mgr.y (mgr.24961) 2429 : cluster [DBG] pgmap v2780: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:35 smithi059 bash[59745]: cluster 2022-04-28T21:02:34.947173+0000 mgr.y (mgr.24961) 2429 : cluster [DBG] pgmap v2780: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:35 smithi139 bash[43301]: cluster 2022-04-28T21:02:34.947173+0000 mgr.y (mgr.24961) 2429 : cluster [DBG] pgmap v2780: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:38 smithi059 bash[61152]: cluster 2022-04-28T21:02:36.948552+0000 mgr.y (mgr.24961) 2430 : cluster [DBG] pgmap v2781: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:38 smithi059 bash[59745]: cluster 2022-04-28T21:02:36.948552+0000 mgr.y (mgr.24961) 2430 : cluster [DBG] pgmap v2781: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:38 smithi139 bash[43301]: cluster 2022-04-28T21:02:36.948552+0000 mgr.y (mgr.24961) 2430 : cluster [DBG] pgmap v2781: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:40 smithi059 bash[61152]: cluster 2022-04-28T21:02:38.949398+0000 mgr.y (mgr.24961) 2431 : cluster [DBG] pgmap v2782: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:40 smithi059 bash[59745]: cluster 2022-04-28T21:02:38.949398+0000 mgr.y (mgr.24961) 2431 : cluster [DBG] pgmap v2782: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:40 smithi139 bash[43301]: cluster 2022-04-28T21:02:38.949398+0000 mgr.y (mgr.24961) 2431 : cluster [DBG] pgmap v2782: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:42 smithi059 bash[61152]: cluster 2022-04-28T21:02:40.950820+0000 mgr.y (mgr.24961) 2432 : cluster [DBG] pgmap v2783: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:41.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:42 smithi059 bash[59745]: cluster 2022-04-28T21:02:40.950820+0000 mgr.y (mgr.24961) 2432 : cluster [DBG] pgmap v2783: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:42 smithi139 bash[43301]: cluster 2022-04-28T21:02:40.950820+0000 mgr.y (mgr.24961) 2432 : cluster [DBG] pgmap v2783: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:44 smithi059 bash[61152]: cluster 2022-04-28T21:02:42.951820+0000 mgr.y (mgr.24961) 2433 : cluster [DBG] pgmap v2784: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:43.800Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:44 smithi059 bash[59745]: cluster 2022-04-28T21:02:42.951820+0000 mgr.y (mgr.24961) 2433 : cluster [DBG] pgmap v2784: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:44 smithi139 bash[43301]: cluster 2022-04-28T21:02:42.951820+0000 mgr.y (mgr.24961) 2433 : cluster [DBG] pgmap v2784: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:45 smithi059 bash[61152]: cluster 2022-04-28T21:02:44.952949+0000 mgr.y (mgr.24961) 2434 : cluster [DBG] pgmap v2785: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:45 smithi059 bash[59745]: cluster 2022-04-28T21:02:44.952949+0000 mgr.y (mgr.24961) 2434 : cluster [DBG] pgmap v2785: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:45 smithi139 bash[43301]: cluster 2022-04-28T21:02:44.952949+0000 mgr.y (mgr.24961) 2434 : cluster [DBG] pgmap v2785: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:48 smithi059 bash[59745]: cluster 2022-04-28T21:02:46.954167+0000 mgr.y (mgr.24961) 2435 : cluster [DBG] pgmap v2786: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:48.463 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:48 smithi059 bash[61152]: cluster 2022-04-28T21:02:46.954167+0000 mgr.y (mgr.24961) 2435 : cluster [DBG] pgmap v2786: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:48.464 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:48 smithi139 bash[43301]: cluster 2022-04-28T21:02:46.954167+0000 mgr.y (mgr.24961) 2435 : cluster [DBG] pgmap v2786: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:50 smithi059 bash[61152]: cluster 2022-04-28T21:02:48.954907+0000 mgr.y (mgr.24961) 2436 : cluster [DBG] pgmap v2787: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:50 smithi059 bash[59745]: cluster 2022-04-28T21:02:48.954907+0000 mgr.y (mgr.24961) 2436 : cluster [DBG] pgmap v2787: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:50.368 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:50 smithi139 bash[43301]: cluster 2022-04-28T21:02:48.954907+0000 mgr.y (mgr.24961) 2436 : cluster [DBG] pgmap v2787: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:52 smithi059 bash[59745]: cluster 2022-04-28T21:02:50.956392+0000 mgr.y (mgr.24961) 2437 : cluster [DBG] pgmap v2788: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:52 smithi059 bash[61152]: cluster 2022-04-28T21:02:50.956392+0000 mgr.y (mgr.24961) 2437 : cluster [DBG] pgmap v2788: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:51.905Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:52 smithi139 bash[43301]: cluster 2022-04-28T21:02:50.956392+0000 mgr.y (mgr.24961) 2437 : cluster [DBG] pgmap v2788: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:54 smithi059 bash[59745]: cluster 2022-04-28T21:02:52.957474+0000 mgr.y (mgr.24961) 2438 : cluster [DBG] pgmap v2789: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:54 smithi059 bash[61152]: cluster 2022-04-28T21:02:52.957474+0000 mgr.y (mgr.24961) 2438 : cluster [DBG] pgmap v2789: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:02:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:02:53.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:02:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:54 smithi139 bash[43301]: cluster 2022-04-28T21:02:52.957474+0000 mgr.y (mgr.24961) 2438 : cluster [DBG] pgmap v2789: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:55 smithi059 bash[61152]: cluster 2022-04-28T21:02:54.958757+0000 mgr.y (mgr.24961) 2439 : cluster [DBG] pgmap v2790: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:55 smithi059 bash[59745]: cluster 2022-04-28T21:02:54.958757+0000 mgr.y (mgr.24961) 2439 : cluster [DBG] pgmap v2790: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:55 smithi139 bash[43301]: cluster 2022-04-28T21:02:54.958757+0000 mgr.y (mgr.24961) 2439 : cluster [DBG] pgmap v2790: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:02:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:02:58 smithi059 bash[61152]: cluster 2022-04-28T21:02:56.960188+0000 mgr.y (mgr.24961) 2440 : cluster [DBG] pgmap v2791: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:02:58 smithi059 bash[59745]: cluster 2022-04-28T21:02:56.960188+0000 mgr.y (mgr.24961) 2440 : cluster [DBG] pgmap v2791: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:02:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:02:58 smithi139 bash[43301]: cluster 2022-04-28T21:02:56.960188+0000 mgr.y (mgr.24961) 2440 : cluster [DBG] pgmap v2791: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:00.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:00 smithi059 bash[61152]: cluster 2022-04-28T21:02:58.960934+0000 mgr.y (mgr.24961) 2441 : cluster [DBG] pgmap v2792: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:00.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:00 smithi059 bash[59745]: cluster 2022-04-28T21:02:58.960934+0000 mgr.y (mgr.24961) 2441 : cluster [DBG] pgmap v2792: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:00 smithi139 bash[43301]: cluster 2022-04-28T21:02:58.960934+0000 mgr.y (mgr.24961) 2441 : cluster [DBG] pgmap v2792: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:02 smithi059 bash[59745]: cluster 2022-04-28T21:03:00.962380+0000 mgr.y (mgr.24961) 2442 : cluster [DBG] pgmap v2793: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:02 smithi059 bash[61152]: cluster 2022-04-28T21:03:00.962380+0000 mgr.y (mgr.24961) 2442 : cluster [DBG] pgmap v2793: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:01.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:02 smithi139 bash[43301]: cluster 2022-04-28T21:03:00.962380+0000 mgr.y (mgr.24961) 2442 : cluster [DBG] pgmap v2793: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:04.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:04 smithi059 bash[59745]: cluster 2022-04-28T21:03:02.963510+0000 mgr.y (mgr.24961) 2443 : cluster [DBG] pgmap v2794: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:04.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:04 smithi059 bash[61152]: cluster 2022-04-28T21:03:02.963510+0000 mgr.y (mgr.24961) 2443 : cluster [DBG] pgmap v2794: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:03.802Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:04 smithi139 bash[43301]: cluster 2022-04-28T21:03:02.963510+0000 mgr.y (mgr.24961) 2443 : cluster [DBG] pgmap v2794: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:05 smithi059 bash[61152]: cluster 2022-04-28T21:03:04.964782+0000 mgr.y (mgr.24961) 2444 : cluster [DBG] pgmap v2795: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:05 smithi059 bash[59745]: cluster 2022-04-28T21:03:04.964782+0000 mgr.y (mgr.24961) 2444 : cluster [DBG] pgmap v2795: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:05 smithi139 bash[43301]: cluster 2022-04-28T21:03:04.964782+0000 mgr.y (mgr.24961) 2444 : cluster [DBG] pgmap v2795: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:08 smithi059 bash[61152]: cluster 2022-04-28T21:03:06.966086+0000 mgr.y (mgr.24961) 2445 : cluster [DBG] pgmap v2796: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:08 smithi059 bash[59745]: cluster 2022-04-28T21:03:06.966086+0000 mgr.y (mgr.24961) 2445 : cluster [DBG] pgmap v2796: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:08 smithi139 bash[43301]: cluster 2022-04-28T21:03:06.966086+0000 mgr.y (mgr.24961) 2445 : cluster [DBG] pgmap v2796: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:03:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:09 smithi059 bash[61152]: audit 2022-04-28T21:03:09.010371+0000 mon.a (mon.0) 2434 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:03:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:09 smithi059 bash[61152]: audit 2022-04-28T21:03:09.013254+0000 mon.a (mon.0) 2435 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:03:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:09 smithi059 bash[61152]: audit 2022-04-28T21:03:09.015183+0000 mon.a (mon.0) 2436 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:03:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:09 smithi059 bash[59745]: audit 2022-04-28T21:03:09.010371+0000 mon.a (mon.0) 2434 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:03:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:09 smithi059 bash[59745]: audit 2022-04-28T21:03:09.013254+0000 mon.a (mon.0) 2435 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:03:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:09 smithi059 bash[59745]: audit 2022-04-28T21:03:09.015183+0000 mon.a (mon.0) 2436 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:03:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:09 smithi139 bash[43301]: audit 2022-04-28T21:03:09.010371+0000 mon.a (mon.0) 2434 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:03:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:09 smithi139 bash[43301]: audit 2022-04-28T21:03:09.013254+0000 mon.a (mon.0) 2435 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:03:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:09 smithi139 bash[43301]: audit 2022-04-28T21:03:09.015183+0000 mon.a (mon.0) 2436 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:03:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:10 smithi059 bash[61152]: cluster 2022-04-28T21:03:08.966826+0000 mgr.y (mgr.24961) 2446 : cluster [DBG] pgmap v2797: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:10 smithi059 bash[61152]: cephadm 2022-04-28T21:03:09.016936+0000 mgr.y (mgr.24961) 2447 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:03:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:10 smithi059 bash[61152]: audit 2022-04-28T21:03:09.337730+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:03:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:10 smithi059 bash[59745]: cluster 2022-04-28T21:03:08.966826+0000 mgr.y (mgr.24961) 2446 : cluster [DBG] pgmap v2797: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:10 smithi059 bash[59745]: cephadm 2022-04-28T21:03:09.016936+0000 mgr.y (mgr.24961) 2447 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:03:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:10 smithi059 bash[59745]: audit 2022-04-28T21:03:09.337730+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:03:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:10 smithi139 bash[43301]: cluster 2022-04-28T21:03:08.966826+0000 mgr.y (mgr.24961) 2446 : cluster [DBG] pgmap v2797: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:10 smithi139 bash[43301]: cephadm 2022-04-28T21:03:09.016936+0000 mgr.y (mgr.24961) 2447 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:03:10.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:10 smithi139 bash[43301]: audit 2022-04-28T21:03:09.337730+0000 mon.a (mon.0) 2437 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:03:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:12 smithi059 bash[61152]: cluster 2022-04-28T21:03:10.968241+0000 mgr.y (mgr.24961) 2448 : cluster [DBG] pgmap v2798: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:11.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:12 smithi059 bash[59745]: cluster 2022-04-28T21:03:10.968241+0000 mgr.y (mgr.24961) 2448 : cluster [DBG] pgmap v2798: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:12 smithi139 bash[43301]: cluster 2022-04-28T21:03:10.968241+0000 mgr.y (mgr.24961) 2448 : cluster [DBG] pgmap v2798: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:14 smithi059 bash[61152]: cluster 2022-04-28T21:03:12.969309+0000 mgr.y (mgr.24961) 2449 : cluster [DBG] pgmap v2799: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:13.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:14 smithi059 bash[59745]: cluster 2022-04-28T21:03:12.969309+0000 mgr.y (mgr.24961) 2449 : cluster [DBG] pgmap v2799: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:14 smithi139 bash[43301]: cluster 2022-04-28T21:03:12.969309+0000 mgr.y (mgr.24961) 2449 : cluster [DBG] pgmap v2799: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:15 smithi059 bash[61152]: cluster 2022-04-28T21:03:14.970578+0000 mgr.y (mgr.24961) 2450 : cluster [DBG] pgmap v2800: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:15 smithi059 bash[59745]: cluster 2022-04-28T21:03:14.970578+0000 mgr.y (mgr.24961) 2450 : cluster [DBG] pgmap v2800: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:15 smithi139 bash[43301]: cluster 2022-04-28T21:03:14.970578+0000 mgr.y (mgr.24961) 2450 : cluster [DBG] pgmap v2800: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:18 smithi059 bash[61152]: cluster 2022-04-28T21:03:16.971916+0000 mgr.y (mgr.24961) 2451 : cluster [DBG] pgmap v2801: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:18.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:18 smithi059 bash[59745]: cluster 2022-04-28T21:03:16.971916+0000 mgr.y (mgr.24961) 2451 : cluster [DBG] pgmap v2801: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:18 smithi139 bash[43301]: cluster 2022-04-28T21:03:16.971916+0000 mgr.y (mgr.24961) 2451 : cluster [DBG] pgmap v2801: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[61152]: audit 2022-04-28T21:03:18.027594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:03:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[61152]: audit 2022-04-28T21:03:18.027805+0000 mon.a (mon.0) 2439 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:03:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[61152]: audit 2022-04-28T21:03:18.037403+0000 mon.a (mon.0) 2440 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:03:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[61152]: audit 2022-04-28T21:03:18.037881+0000 mon.a (mon.0) 2441 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:03:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[61152]: audit 2022-04-28T21:03:18.038107+0000 mon.a (mon.0) 2442 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:03:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[61152]: audit 2022-04-28T21:03:18.038341+0000 mon.a (mon.0) 2443 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:03:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[59745]: audit 2022-04-28T21:03:18.027594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:03:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[59745]: audit 2022-04-28T21:03:18.027805+0000 mon.a (mon.0) 2439 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:03:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[59745]: audit 2022-04-28T21:03:18.037403+0000 mon.a (mon.0) 2440 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:03:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[59745]: audit 2022-04-28T21:03:18.037881+0000 mon.a (mon.0) 2441 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:03:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[59745]: audit 2022-04-28T21:03:18.038107+0000 mon.a (mon.0) 2442 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:03:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:19 smithi059 bash[59745]: audit 2022-04-28T21:03:18.038341+0000 mon.a (mon.0) 2443 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:03:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:19 smithi139 bash[43301]: audit 2022-04-28T21:03:18.027594+0000 mon.a (mon.0) 2438 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:03:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:19 smithi139 bash[43301]: audit 2022-04-28T21:03:18.027805+0000 mon.a (mon.0) 2439 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:03:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:19 smithi139 bash[43301]: audit 2022-04-28T21:03:18.037403+0000 mon.a (mon.0) 2440 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:03:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:19 smithi139 bash[43301]: audit 2022-04-28T21:03:18.037881+0000 mon.a (mon.0) 2441 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:03:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:19 smithi139 bash[43301]: audit 2022-04-28T21:03:18.038107+0000 mon.a (mon.0) 2442 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:03:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:19 smithi139 bash[43301]: audit 2022-04-28T21:03:18.038341+0000 mon.a (mon.0) 2443 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:03:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:20 smithi059 bash[61152]: cluster 2022-04-28T21:03:18.972648+0000 mgr.y (mgr.24961) 2452 : cluster [DBG] pgmap v2802: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:20 smithi059 bash[59745]: cluster 2022-04-28T21:03:18.972648+0000 mgr.y (mgr.24961) 2452 : cluster [DBG] pgmap v2802: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:20 smithi139 bash[43301]: cluster 2022-04-28T21:03:18.972648+0000 mgr.y (mgr.24961) 2452 : cluster [DBG] pgmap v2802: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:22 smithi059 bash[61152]: cluster 2022-04-28T21:03:20.974061+0000 mgr.y (mgr.24961) 2453 : cluster [DBG] pgmap v2803: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:21.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:22 smithi059 bash[59745]: cluster 2022-04-28T21:03:20.974061+0000 mgr.y (mgr.24961) 2453 : cluster [DBG] pgmap v2803: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:22 smithi139 bash[43301]: cluster 2022-04-28T21:03:20.974061+0000 mgr.y (mgr.24961) 2453 : cluster [DBG] pgmap v2803: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:24.056 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:23.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:24.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:24 smithi059 bash[59745]: cluster 2022-04-28T21:03:22.975044+0000 mgr.y (mgr.24961) 2454 : cluster [DBG] pgmap v2804: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:24 smithi059 bash[61152]: cluster 2022-04-28T21:03:22.975044+0000 mgr.y (mgr.24961) 2454 : cluster [DBG] pgmap v2804: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:24 smithi139 bash[43301]: cluster 2022-04-28T21:03:22.975044+0000 mgr.y (mgr.24961) 2454 : cluster [DBG] pgmap v2804: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:25 smithi059 bash[61152]: cluster 2022-04-28T21:03:24.976210+0000 mgr.y (mgr.24961) 2455 : cluster [DBG] pgmap v2805: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:25 smithi059 bash[59745]: cluster 2022-04-28T21:03:24.976210+0000 mgr.y (mgr.24961) 2455 : cluster [DBG] pgmap v2805: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:25 smithi139 bash[43301]: cluster 2022-04-28T21:03:24.976210+0000 mgr.y (mgr.24961) 2455 : cluster [DBG] pgmap v2805: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:28 smithi059 bash[61152]: cluster 2022-04-28T21:03:26.977543+0000 mgr.y (mgr.24961) 2456 : cluster [DBG] pgmap v2806: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:28 smithi059 bash[59745]: cluster 2022-04-28T21:03:26.977543+0000 mgr.y (mgr.24961) 2456 : cluster [DBG] pgmap v2806: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:28 smithi139 bash[43301]: cluster 2022-04-28T21:03:26.977543+0000 mgr.y (mgr.24961) 2456 : cluster [DBG] pgmap v2806: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:30 smithi059 bash[61152]: cluster 2022-04-28T21:03:28.978301+0000 mgr.y (mgr.24961) 2457 : cluster [DBG] pgmap v2807: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:30 smithi059 bash[59745]: cluster 2022-04-28T21:03:28.978301+0000 mgr.y (mgr.24961) 2457 : cluster [DBG] pgmap v2807: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:30 smithi139 bash[43301]: cluster 2022-04-28T21:03:28.978301+0000 mgr.y (mgr.24961) 2457 : cluster [DBG] pgmap v2807: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:32 smithi059 bash[61152]: cluster 2022-04-28T21:03:30.979754+0000 mgr.y (mgr.24961) 2458 : cluster [DBG] pgmap v2808: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:31.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:32 smithi059 bash[59745]: cluster 2022-04-28T21:03:30.979754+0000 mgr.y (mgr.24961) 2458 : cluster [DBG] pgmap v2808: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:32 smithi139 bash[43301]: cluster 2022-04-28T21:03:30.979754+0000 mgr.y (mgr.24961) 2458 : cluster [DBG] pgmap v2808: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:34 smithi059 bash[61152]: cluster 2022-04-28T21:03:32.980724+0000 mgr.y (mgr.24961) 2459 : cluster [DBG] pgmap v2809: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:33.803Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:34 smithi059 bash[59745]: cluster 2022-04-28T21:03:32.980724+0000 mgr.y (mgr.24961) 2459 : cluster [DBG] pgmap v2809: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:34 smithi139 bash[43301]: cluster 2022-04-28T21:03:32.980724+0000 mgr.y (mgr.24961) 2459 : cluster [DBG] pgmap v2809: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:35 smithi059 bash[61152]: cluster 2022-04-28T21:03:34.981879+0000 mgr.y (mgr.24961) 2460 : cluster [DBG] pgmap v2810: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:35 smithi059 bash[59745]: cluster 2022-04-28T21:03:34.981879+0000 mgr.y (mgr.24961) 2460 : cluster [DBG] pgmap v2810: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:35 smithi139 bash[43301]: cluster 2022-04-28T21:03:34.981879+0000 mgr.y (mgr.24961) 2460 : cluster [DBG] pgmap v2810: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:38 smithi059 bash[61152]: cluster 2022-04-28T21:03:36.983138+0000 mgr.y (mgr.24961) 2461 : cluster [DBG] pgmap v2811: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:38 smithi059 bash[59745]: cluster 2022-04-28T21:03:36.983138+0000 mgr.y (mgr.24961) 2461 : cluster [DBG] pgmap v2811: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:38 smithi139 bash[43301]: cluster 2022-04-28T21:03:36.983138+0000 mgr.y (mgr.24961) 2461 : cluster [DBG] pgmap v2811: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:40 smithi059 bash[61152]: cluster 2022-04-28T21:03:38.983838+0000 mgr.y (mgr.24961) 2462 : cluster [DBG] pgmap v2812: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:40 smithi059 bash[59745]: cluster 2022-04-28T21:03:38.983838+0000 mgr.y (mgr.24961) 2462 : cluster [DBG] pgmap v2812: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:40 smithi139 bash[43301]: cluster 2022-04-28T21:03:38.983838+0000 mgr.y (mgr.24961) 2462 : cluster [DBG] pgmap v2812: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:42 smithi059 bash[61152]: cluster 2022-04-28T21:03:40.985230+0000 mgr.y (mgr.24961) 2463 : cluster [DBG] pgmap v2813: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:41.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:42 smithi059 bash[59745]: cluster 2022-04-28T21:03:40.985230+0000 mgr.y (mgr.24961) 2463 : cluster [DBG] pgmap v2813: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:42 smithi139 bash[43301]: cluster 2022-04-28T21:03:40.985230+0000 mgr.y (mgr.24961) 2463 : cluster [DBG] pgmap v2813: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:44.059 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:43.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:44.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:44 smithi059 bash[59745]: cluster 2022-04-28T21:03:42.986228+0000 mgr.y (mgr.24961) 2464 : cluster [DBG] pgmap v2814: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:44 smithi059 bash[61152]: cluster 2022-04-28T21:03:42.986228+0000 mgr.y (mgr.24961) 2464 : cluster [DBG] pgmap v2814: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:44.479 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:44 smithi139 bash[43301]: cluster 2022-04-28T21:03:42.986228+0000 mgr.y (mgr.24961) 2464 : cluster [DBG] pgmap v2814: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:45 smithi059 bash[61152]: cluster 2022-04-28T21:03:44.987420+0000 mgr.y (mgr.24961) 2465 : cluster [DBG] pgmap v2815: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:45 smithi059 bash[59745]: cluster 2022-04-28T21:03:44.987420+0000 mgr.y (mgr.24961) 2465 : cluster [DBG] pgmap v2815: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:45 smithi139 bash[43301]: cluster 2022-04-28T21:03:44.987420+0000 mgr.y (mgr.24961) 2465 : cluster [DBG] pgmap v2815: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:48 smithi059 bash[61152]: cluster 2022-04-28T21:03:46.988646+0000 mgr.y (mgr.24961) 2466 : cluster [DBG] pgmap v2816: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:48.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:48 smithi059 bash[59745]: cluster 2022-04-28T21:03:46.988646+0000 mgr.y (mgr.24961) 2466 : cluster [DBG] pgmap v2816: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:48 smithi139 bash[43301]: cluster 2022-04-28T21:03:46.988646+0000 mgr.y (mgr.24961) 2466 : cluster [DBG] pgmap v2816: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:50 smithi059 bash[61152]: cluster 2022-04-28T21:03:48.989367+0000 mgr.y (mgr.24961) 2467 : cluster [DBG] pgmap v2817: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:50 smithi059 bash[59745]: cluster 2022-04-28T21:03:48.989367+0000 mgr.y (mgr.24961) 2467 : cluster [DBG] pgmap v2817: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:50 smithi139 bash[43301]: cluster 2022-04-28T21:03:48.989367+0000 mgr.y (mgr.24961) 2467 : cluster [DBG] pgmap v2817: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:52 smithi059 bash[61152]: cluster 2022-04-28T21:03:50.990743+0000 mgr.y (mgr.24961) 2468 : cluster [DBG] pgmap v2818: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:52.355 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:51.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:52.360 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:52 smithi059 bash[59745]: cluster 2022-04-28T21:03:50.990743+0000 mgr.y (mgr.24961) 2468 : cluster [DBG] pgmap v2818: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:52 smithi139 bash[43301]: cluster 2022-04-28T21:03:50.990743+0000 mgr.y (mgr.24961) 2468 : cluster [DBG] pgmap v2818: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:54.063 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:03:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:03:53.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:03:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:54 smithi059 bash[61152]: cluster 2022-04-28T21:03:52.991793+0000 mgr.y (mgr.24961) 2469 : cluster [DBG] pgmap v2819: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:54.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:54 smithi059 bash[59745]: cluster 2022-04-28T21:03:52.991793+0000 mgr.y (mgr.24961) 2469 : cluster [DBG] pgmap v2819: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:54 smithi139 bash[43301]: cluster 2022-04-28T21:03:52.991793+0000 mgr.y (mgr.24961) 2469 : cluster [DBG] pgmap v2819: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:55 smithi059 bash[61152]: cluster 2022-04-28T21:03:54.993093+0000 mgr.y (mgr.24961) 2470 : cluster [DBG] pgmap v2820: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:55 smithi059 bash[59745]: cluster 2022-04-28T21:03:54.993093+0000 mgr.y (mgr.24961) 2470 : cluster [DBG] pgmap v2820: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:55 smithi139 bash[43301]: cluster 2022-04-28T21:03:54.993093+0000 mgr.y (mgr.24961) 2470 : cluster [DBG] pgmap v2820: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:03:58 smithi059 bash[61152]: cluster 2022-04-28T21:03:56.994303+0000 mgr.y (mgr.24961) 2471 : cluster [DBG] pgmap v2821: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:03:58 smithi059 bash[59745]: cluster 2022-04-28T21:03:56.994303+0000 mgr.y (mgr.24961) 2471 : cluster [DBG] pgmap v2821: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:03:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:03:58 smithi139 bash[43301]: cluster 2022-04-28T21:03:56.994303+0000 mgr.y (mgr.24961) 2471 : cluster [DBG] pgmap v2821: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:00.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:00 smithi059 bash[59745]: cluster 2022-04-28T21:03:58.995141+0000 mgr.y (mgr.24961) 2472 : cluster [DBG] pgmap v2822: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:00.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:00 smithi059 bash[61152]: cluster 2022-04-28T21:03:58.995141+0000 mgr.y (mgr.24961) 2472 : cluster [DBG] pgmap v2822: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:00 smithi139 bash[43301]: cluster 2022-04-28T21:03:58.995141+0000 mgr.y (mgr.24961) 2472 : cluster [DBG] pgmap v2822: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:02 smithi059 bash[61152]: cluster 2022-04-28T21:04:00.996688+0000 mgr.y (mgr.24961) 2473 : cluster [DBG] pgmap v2823: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:01.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:02 smithi059 bash[59745]: cluster 2022-04-28T21:04:00.996688+0000 mgr.y (mgr.24961) 2473 : cluster [DBG] pgmap v2823: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:02 smithi139 bash[43301]: cluster 2022-04-28T21:04:00.996688+0000 mgr.y (mgr.24961) 2473 : cluster [DBG] pgmap v2823: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:04.070 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:03.804Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:04.071 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:04 smithi059 bash[59745]: cluster 2022-04-28T21:04:02.997717+0000 mgr.y (mgr.24961) 2474 : cluster [DBG] pgmap v2824: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:04 smithi059 bash[61152]: cluster 2022-04-28T21:04:02.997717+0000 mgr.y (mgr.24961) 2474 : cluster [DBG] pgmap v2824: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:04 smithi139 bash[43301]: cluster 2022-04-28T21:04:02.997717+0000 mgr.y (mgr.24961) 2474 : cluster [DBG] pgmap v2824: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:05 smithi059 bash[61152]: cluster 2022-04-28T21:04:04.998992+0000 mgr.y (mgr.24961) 2475 : cluster [DBG] pgmap v2825: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:05 smithi059 bash[59745]: cluster 2022-04-28T21:04:04.998992+0000 mgr.y (mgr.24961) 2475 : cluster [DBG] pgmap v2825: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:05 smithi139 bash[43301]: cluster 2022-04-28T21:04:04.998992+0000 mgr.y (mgr.24961) 2475 : cluster [DBG] pgmap v2825: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:08.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:08 smithi059 bash[61152]: cluster 2022-04-28T21:04:07.000357+0000 mgr.y (mgr.24961) 2476 : cluster [DBG] pgmap v2826: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:08 smithi059 bash[59745]: cluster 2022-04-28T21:04:07.000357+0000 mgr.y (mgr.24961) 2476 : cluster [DBG] pgmap v2826: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:08 smithi139 bash[43301]: cluster 2022-04-28T21:04:07.000357+0000 mgr.y (mgr.24961) 2476 : cluster [DBG] pgmap v2826: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[59745]: cluster 2022-04-28T21:04:09.001200+0000 mgr.y (mgr.24961) 2477 : cluster [DBG] pgmap v2827: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[59745]: audit 2022-04-28T21:04:09.348358+0000 mon.a (mon.0) 2444 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:04:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[59745]: audit 2022-04-28T21:04:09.351340+0000 mon.a (mon.0) 2445 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:04:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[59745]: audit 2022-04-28T21:04:09.353181+0000 mon.a (mon.0) 2446 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:04:10.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[59745]: audit 2022-04-28T21:04:09.678160+0000 mon.a (mon.0) 2447 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:04:10.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[61152]: cluster 2022-04-28T21:04:09.001200+0000 mgr.y (mgr.24961) 2477 : cluster [DBG] pgmap v2827: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:10.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[61152]: audit 2022-04-28T21:04:09.348358+0000 mon.a (mon.0) 2444 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:04:10.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[61152]: audit 2022-04-28T21:04:09.351340+0000 mon.a (mon.0) 2445 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:04:10.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[61152]: audit 2022-04-28T21:04:09.353181+0000 mon.a (mon.0) 2446 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:04:10.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:10 smithi059 bash[61152]: audit 2022-04-28T21:04:09.678160+0000 mon.a (mon.0) 2447 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:04:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:10 smithi139 bash[43301]: cluster 2022-04-28T21:04:09.001200+0000 mgr.y (mgr.24961) 2477 : cluster [DBG] pgmap v2827: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:10 smithi139 bash[43301]: audit 2022-04-28T21:04:09.348358+0000 mon.a (mon.0) 2444 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:04:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:10 smithi139 bash[43301]: audit 2022-04-28T21:04:09.351340+0000 mon.a (mon.0) 2445 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:04:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:10 smithi139 bash[43301]: audit 2022-04-28T21:04:09.353181+0000 mon.a (mon.0) 2446 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:04:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:10 smithi139 bash[43301]: audit 2022-04-28T21:04:09.678160+0000 mon.a (mon.0) 2447 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:04:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:11 smithi059 bash[61152]: cephadm 2022-04-28T21:04:09.354732+0000 mgr.y (mgr.24961) 2478 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:04:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:11 smithi059 bash[59745]: cephadm 2022-04-28T21:04:09.354732+0000 mgr.y (mgr.24961) 2478 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:04:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:11 smithi139 bash[43301]: cephadm 2022-04-28T21:04:09.354732+0000 mgr.y (mgr.24961) 2478 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:04:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:12 smithi059 bash[61152]: cluster 2022-04-28T21:04:11.002626+0000 mgr.y (mgr.24961) 2479 : cluster [DBG] pgmap v2828: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:11.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:12 smithi059 bash[59745]: cluster 2022-04-28T21:04:11.002626+0000 mgr.y (mgr.24961) 2479 : cluster [DBG] pgmap v2828: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:12 smithi139 bash[43301]: cluster 2022-04-28T21:04:11.002626+0000 mgr.y (mgr.24961) 2479 : cluster [DBG] pgmap v2828: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:14.086 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:13.805Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:14.086 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:14 smithi059 bash[59745]: cluster 2022-04-28T21:04:13.003647+0000 mgr.y (mgr.24961) 2480 : cluster [DBG] pgmap v2829: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:14 smithi059 bash[61152]: cluster 2022-04-28T21:04:13.003647+0000 mgr.y (mgr.24961) 2480 : cluster [DBG] pgmap v2829: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:14.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:14 smithi139 bash[43301]: cluster 2022-04-28T21:04:13.003647+0000 mgr.y (mgr.24961) 2480 : cluster [DBG] pgmap v2829: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:15 smithi059 bash[59745]: cluster 2022-04-28T21:04:15.004899+0000 mgr.y (mgr.24961) 2481 : cluster [DBG] pgmap v2830: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:15 smithi059 bash[61152]: cluster 2022-04-28T21:04:15.004899+0000 mgr.y (mgr.24961) 2481 : cluster [DBG] pgmap v2830: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:15 smithi139 bash[43301]: cluster 2022-04-28T21:04:15.004899+0000 mgr.y (mgr.24961) 2481 : cluster [DBG] pgmap v2830: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: cluster 2022-04-28T21:04:17.006100+0000 mgr.y (mgr.24961) 2482 : cluster [DBG] pgmap v2831: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: audit 2022-04-28T21:04:18.042251+0000 mon.a (mon.0) 2448 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:04:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: audit 2022-04-28T21:04:18.042870+0000 mon.a (mon.0) 2449 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:04:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: audit 2022-04-28T21:04:18.051642+0000 mon.a (mon.0) 2450 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:04:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: audit 2022-04-28T21:04:18.052615+0000 mon.a (mon.0) 2451 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T21:04:18.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: audit 2022-04-28T21:04:18.053284+0000 mon.a (mon.0) 2452 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:04:18.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[61152]: audit 2022-04-28T21:04:18.053662+0000 mon.a (mon.0) 2453 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:04:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: cluster 2022-04-28T21:04:17.006100+0000 mgr.y (mgr.24961) 2482 : cluster [DBG] pgmap v2831: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: audit 2022-04-28T21:04:18.042251+0000 mon.a (mon.0) 2448 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:04:18.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: audit 2022-04-28T21:04:18.042870+0000 mon.a (mon.0) 2449 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:04:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: audit 2022-04-28T21:04:18.051642+0000 mon.a (mon.0) 2450 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:04:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: audit 2022-04-28T21:04:18.052615+0000 mon.a (mon.0) 2451 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T21:04:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: audit 2022-04-28T21:04:18.053284+0000 mon.a (mon.0) 2452 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:04:18.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:18 smithi059 bash[59745]: audit 2022-04-28T21:04:18.053662+0000 mon.a (mon.0) 2453 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:04:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: cluster 2022-04-28T21:04:17.006100+0000 mgr.y (mgr.24961) 2482 : cluster [DBG] pgmap v2831: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: audit 2022-04-28T21:04:18.042251+0000 mon.a (mon.0) 2448 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:04:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: audit 2022-04-28T21:04:18.042870+0000 mon.a (mon.0) 2449 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:04:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: audit 2022-04-28T21:04:18.051642+0000 mon.a (mon.0) 2450 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:04:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: audit 2022-04-28T21:04:18.052615+0000 mon.a (mon.0) 2451 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 5]}]: dispatch 2022-04-28T21:04:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: audit 2022-04-28T21:04:18.053284+0000 mon.a (mon.0) 2452 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:04:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:18 smithi139 bash[43301]: audit 2022-04-28T21:04:18.053662+0000 mon.a (mon.0) 2453 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:04:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:20 smithi059 bash[61152]: cluster 2022-04-28T21:04:19.006843+0000 mgr.y (mgr.24961) 2483 : cluster [DBG] pgmap v2832: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:20 smithi059 bash[59745]: cluster 2022-04-28T21:04:19.006843+0000 mgr.y (mgr.24961) 2483 : cluster [DBG] pgmap v2832: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:20 smithi139 bash[43301]: cluster 2022-04-28T21:04:19.006843+0000 mgr.y (mgr.24961) 2483 : cluster [DBG] pgmap v2832: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:22 smithi059 bash[61152]: cluster 2022-04-28T21:04:21.008314+0000 mgr.y (mgr.24961) 2484 : cluster [DBG] pgmap v2833: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:22.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:21.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:22 smithi059 bash[59745]: cluster 2022-04-28T21:04:21.008314+0000 mgr.y (mgr.24961) 2484 : cluster [DBG] pgmap v2833: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:22 smithi139 bash[43301]: cluster 2022-04-28T21:04:21.008314+0000 mgr.y (mgr.24961) 2484 : cluster [DBG] pgmap v2833: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:24.081 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:23.807Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:24.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:24 smithi059 bash[61152]: cluster 2022-04-28T21:04:23.009293+0000 mgr.y (mgr.24961) 2485 : cluster [DBG] pgmap v2834: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:24 smithi059 bash[59745]: cluster 2022-04-28T21:04:23.009293+0000 mgr.y (mgr.24961) 2485 : cluster [DBG] pgmap v2834: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:24 smithi139 bash[43301]: cluster 2022-04-28T21:04:23.009293+0000 mgr.y (mgr.24961) 2485 : cluster [DBG] pgmap v2834: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:25 smithi059 bash[61152]: cluster 2022-04-28T21:04:25.010474+0000 mgr.y (mgr.24961) 2486 : cluster [DBG] pgmap v2835: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:25 smithi059 bash[59745]: cluster 2022-04-28T21:04:25.010474+0000 mgr.y (mgr.24961) 2486 : cluster [DBG] pgmap v2835: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:25 smithi139 bash[43301]: cluster 2022-04-28T21:04:25.010474+0000 mgr.y (mgr.24961) 2486 : cluster [DBG] pgmap v2835: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:28 smithi059 bash[61152]: cluster 2022-04-28T21:04:27.011697+0000 mgr.y (mgr.24961) 2487 : cluster [DBG] pgmap v2836: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:28 smithi059 bash[59745]: cluster 2022-04-28T21:04:27.011697+0000 mgr.y (mgr.24961) 2487 : cluster [DBG] pgmap v2836: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:28 smithi139 bash[43301]: cluster 2022-04-28T21:04:27.011697+0000 mgr.y (mgr.24961) 2487 : cluster [DBG] pgmap v2836: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:30 smithi059 bash[61152]: cluster 2022-04-28T21:04:29.012414+0000 mgr.y (mgr.24961) 2488 : cluster [DBG] pgmap v2837: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:30 smithi059 bash[59745]: cluster 2022-04-28T21:04:29.012414+0000 mgr.y (mgr.24961) 2488 : cluster [DBG] pgmap v2837: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:30 smithi139 bash[43301]: cluster 2022-04-28T21:04:29.012414+0000 mgr.y (mgr.24961) 2488 : cluster [DBG] pgmap v2837: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:32 smithi059 bash[61152]: cluster 2022-04-28T21:04:31.013825+0000 mgr.y (mgr.24961) 2489 : cluster [DBG] pgmap v2838: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:31.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:32 smithi059 bash[59745]: cluster 2022-04-28T21:04:31.013825+0000 mgr.y (mgr.24961) 2489 : cluster [DBG] pgmap v2838: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:32 smithi139 bash[43301]: cluster 2022-04-28T21:04:31.013825+0000 mgr.y (mgr.24961) 2489 : cluster [DBG] pgmap v2838: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:34.086 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:33.806Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:34.087 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:34 smithi059 bash[59745]: cluster 2022-04-28T21:04:33.014820+0000 mgr.y (mgr.24961) 2490 : cluster [DBG] pgmap v2839: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:34 smithi059 bash[61152]: cluster 2022-04-28T21:04:33.014820+0000 mgr.y (mgr.24961) 2490 : cluster [DBG] pgmap v2839: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:34 smithi139 bash[43301]: cluster 2022-04-28T21:04:33.014820+0000 mgr.y (mgr.24961) 2490 : cluster [DBG] pgmap v2839: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:35 smithi059 bash[61152]: cluster 2022-04-28T21:04:35.016044+0000 mgr.y (mgr.24961) 2491 : cluster [DBG] pgmap v2840: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:35 smithi059 bash[59745]: cluster 2022-04-28T21:04:35.016044+0000 mgr.y (mgr.24961) 2491 : cluster [DBG] pgmap v2840: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:35 smithi139 bash[43301]: cluster 2022-04-28T21:04:35.016044+0000 mgr.y (mgr.24961) 2491 : cluster [DBG] pgmap v2840: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:38 smithi059 bash[61152]: cluster 2022-04-28T21:04:37.017238+0000 mgr.y (mgr.24961) 2492 : cluster [DBG] pgmap v2841: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:38 smithi059 bash[59745]: cluster 2022-04-28T21:04:37.017238+0000 mgr.y (mgr.24961) 2492 : cluster [DBG] pgmap v2841: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:38 smithi139 bash[43301]: cluster 2022-04-28T21:04:37.017238+0000 mgr.y (mgr.24961) 2492 : cluster [DBG] pgmap v2841: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:40 smithi059 bash[61152]: cluster 2022-04-28T21:04:39.018089+0000 mgr.y (mgr.24961) 2493 : cluster [DBG] pgmap v2842: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:40.359 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:40 smithi059 bash[59745]: cluster 2022-04-28T21:04:39.018089+0000 mgr.y (mgr.24961) 2493 : cluster [DBG] pgmap v2842: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:40 smithi139 bash[43301]: cluster 2022-04-28T21:04:39.018089+0000 mgr.y (mgr.24961) 2493 : cluster [DBG] pgmap v2842: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:42 smithi059 bash[61152]: cluster 2022-04-28T21:04:41.019391+0000 mgr.y (mgr.24961) 2494 : cluster [DBG] pgmap v2843: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:41.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:42 smithi059 bash[59745]: cluster 2022-04-28T21:04:41.019391+0000 mgr.y (mgr.24961) 2494 : cluster [DBG] pgmap v2843: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:42 smithi139 bash[43301]: cluster 2022-04-28T21:04:41.019391+0000 mgr.y (mgr.24961) 2494 : cluster [DBG] pgmap v2843: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:43.809Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:44 smithi059 bash[61152]: cluster 2022-04-28T21:04:43.020414+0000 mgr.y (mgr.24961) 2495 : cluster [DBG] pgmap v2844: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:44 smithi059 bash[59745]: cluster 2022-04-28T21:04:43.020414+0000 mgr.y (mgr.24961) 2495 : cluster [DBG] pgmap v2844: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:44 smithi139 bash[43301]: cluster 2022-04-28T21:04:43.020414+0000 mgr.y (mgr.24961) 2495 : cluster [DBG] pgmap v2844: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:45 smithi059 bash[61152]: cluster 2022-04-28T21:04:45.021628+0000 mgr.y (mgr.24961) 2496 : cluster [DBG] pgmap v2845: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:45 smithi059 bash[59745]: cluster 2022-04-28T21:04:45.021628+0000 mgr.y (mgr.24961) 2496 : cluster [DBG] pgmap v2845: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:45 smithi139 bash[43301]: cluster 2022-04-28T21:04:45.021628+0000 mgr.y (mgr.24961) 2496 : cluster [DBG] pgmap v2845: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:48.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:48 smithi059 bash[61152]: cluster 2022-04-28T21:04:47.022829+0000 mgr.y (mgr.24961) 2497 : cluster [DBG] pgmap v2846: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:48.356 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:48 smithi059 bash[59745]: cluster 2022-04-28T21:04:47.022829+0000 mgr.y (mgr.24961) 2497 : cluster [DBG] pgmap v2846: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:48 smithi139 bash[43301]: cluster 2022-04-28T21:04:47.022829+0000 mgr.y (mgr.24961) 2497 : cluster [DBG] pgmap v2846: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:50 smithi059 bash[61152]: cluster 2022-04-28T21:04:49.023721+0000 mgr.y (mgr.24961) 2498 : cluster [DBG] pgmap v2847: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:50 smithi059 bash[59745]: cluster 2022-04-28T21:04:49.023721+0000 mgr.y (mgr.24961) 2498 : cluster [DBG] pgmap v2847: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:50.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:50 smithi139 bash[43301]: cluster 2022-04-28T21:04:49.023721+0000 mgr.y (mgr.24961) 2498 : cluster [DBG] pgmap v2847: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:52 smithi059 bash[59745]: cluster 2022-04-28T21:04:51.025005+0000 mgr.y (mgr.24961) 2499 : cluster [DBG] pgmap v2848: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:52 smithi059 bash[61152]: cluster 2022-04-28T21:04:51.025005+0000 mgr.y (mgr.24961) 2499 : cluster [DBG] pgmap v2848: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:51.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:52 smithi139 bash[43301]: cluster 2022-04-28T21:04:51.025005+0000 mgr.y (mgr.24961) 2499 : cluster [DBG] pgmap v2848: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:04:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:04:53.812Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:04:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:54 smithi139 bash[43301]: cluster 2022-04-28T21:04:53.026002+0000 mgr.y (mgr.24961) 2500 : cluster [DBG] pgmap v2849: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:54 smithi059 bash[59745]: cluster 2022-04-28T21:04:53.026002+0000 mgr.y (mgr.24961) 2500 : cluster [DBG] pgmap v2849: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:54 smithi059 bash[61152]: cluster 2022-04-28T21:04:53.026002+0000 mgr.y (mgr.24961) 2500 : cluster [DBG] pgmap v2849: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:55 smithi059 bash[59745]: cluster 2022-04-28T21:04:55.027264+0000 mgr.y (mgr.24961) 2501 : cluster [DBG] pgmap v2850: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:55.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:55 smithi059 bash[61152]: cluster 2022-04-28T21:04:55.027264+0000 mgr.y (mgr.24961) 2501 : cluster [DBG] pgmap v2850: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:55 smithi139 bash[43301]: cluster 2022-04-28T21:04:55.027264+0000 mgr.y (mgr.24961) 2501 : cluster [DBG] pgmap v2850: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:04:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:04:58 smithi059 bash[61152]: cluster 2022-04-28T21:04:57.028556+0000 mgr.y (mgr.24961) 2502 : cluster [DBG] pgmap v2851: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:04:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:04:58 smithi059 bash[59745]: cluster 2022-04-28T21:04:57.028556+0000 mgr.y (mgr.24961) 2502 : cluster [DBG] pgmap v2851: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:04:58.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:04:58 smithi139 bash[43301]: cluster 2022-04-28T21:04:57.028556+0000 mgr.y (mgr.24961) 2502 : cluster [DBG] pgmap v2851: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:00.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:00 smithi139 bash[43301]: cluster 2022-04-28T21:04:59.029555+0000 mgr.y (mgr.24961) 2503 : cluster [DBG] pgmap v2852: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:00 smithi059 bash[61152]: cluster 2022-04-28T21:04:59.029555+0000 mgr.y (mgr.24961) 2503 : cluster [DBG] pgmap v2852: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:00 smithi059 bash[59745]: cluster 2022-04-28T21:04:59.029555+0000 mgr.y (mgr.24961) 2503 : cluster [DBG] pgmap v2852: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:02 smithi059 bash[61152]: cluster 2022-04-28T21:05:01.030985+0000 mgr.y (mgr.24961) 2504 : cluster [DBG] pgmap v2853: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:01.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:02 smithi059 bash[59745]: cluster 2022-04-28T21:05:01.030985+0000 mgr.y (mgr.24961) 2504 : cluster [DBG] pgmap v2853: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:02.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:02 smithi139 bash[43301]: cluster 2022-04-28T21:05:01.030985+0000 mgr.y (mgr.24961) 2504 : cluster [DBG] pgmap v2853: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:03.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:04.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:04 smithi139 bash[43301]: cluster 2022-04-28T21:05:03.031923+0000 mgr.y (mgr.24961) 2505 : cluster [DBG] pgmap v2854: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:04 smithi059 bash[61152]: cluster 2022-04-28T21:05:03.031923+0000 mgr.y (mgr.24961) 2505 : cluster [DBG] pgmap v2854: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:04.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:04 smithi059 bash[59745]: cluster 2022-04-28T21:05:03.031923+0000 mgr.y (mgr.24961) 2505 : cluster [DBG] pgmap v2854: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:05 smithi059 bash[61152]: cluster 2022-04-28T21:05:05.033128+0000 mgr.y (mgr.24961) 2506 : cluster [DBG] pgmap v2855: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:05 smithi059 bash[59745]: cluster 2022-04-28T21:05:05.033128+0000 mgr.y (mgr.24961) 2506 : cluster [DBG] pgmap v2855: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:05 smithi139 bash[43301]: cluster 2022-04-28T21:05:05.033128+0000 mgr.y (mgr.24961) 2506 : cluster [DBG] pgmap v2855: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:08.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:08 smithi139 bash[43301]: cluster 2022-04-28T21:05:07.034341+0000 mgr.y (mgr.24961) 2507 : cluster [DBG] pgmap v2856: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:08 smithi059 bash[61152]: cluster 2022-04-28T21:05:07.034341+0000 mgr.y (mgr.24961) 2507 : cluster [DBG] pgmap v2856: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:08 smithi059 bash[59745]: cluster 2022-04-28T21:05:07.034341+0000 mgr.y (mgr.24961) 2507 : cluster [DBG] pgmap v2856: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:10 smithi139 bash[43301]: cluster 2022-04-28T21:05:09.035252+0000 mgr.y (mgr.24961) 2508 : cluster [DBG] pgmap v2857: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:10 smithi139 bash[43301]: audit 2022-04-28T21:05:09.684938+0000 mon.a (mon.0) 2454 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:05:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:10 smithi139 bash[43301]: audit 2022-04-28T21:05:09.687746+0000 mon.a (mon.0) 2455 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:05:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:10 smithi139 bash[43301]: audit 2022-04-28T21:05:09.689566+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:05:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:10 smithi139 bash[43301]: audit 2022-04-28T21:05:10.006666+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:05:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[59745]: cluster 2022-04-28T21:05:09.035252+0000 mgr.y (mgr.24961) 2508 : cluster [DBG] pgmap v2857: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[59745]: audit 2022-04-28T21:05:09.684938+0000 mon.a (mon.0) 2454 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:05:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[59745]: audit 2022-04-28T21:05:09.687746+0000 mon.a (mon.0) 2455 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:05:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[59745]: audit 2022-04-28T21:05:09.689566+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:05:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[59745]: audit 2022-04-28T21:05:10.006666+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:05:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[61152]: cluster 2022-04-28T21:05:09.035252+0000 mgr.y (mgr.24961) 2508 : cluster [DBG] pgmap v2857: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[61152]: audit 2022-04-28T21:05:09.684938+0000 mon.a (mon.0) 2454 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:05:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[61152]: audit 2022-04-28T21:05:09.687746+0000 mon.a (mon.0) 2455 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:05:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[61152]: audit 2022-04-28T21:05:09.689566+0000 mon.a (mon.0) 2456 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:05:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:10 smithi059 bash[61152]: audit 2022-04-28T21:05:10.006666+0000 mon.a (mon.0) 2457 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:05:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:11 smithi139 bash[43301]: cephadm 2022-04-28T21:05:09.691167+0000 mgr.y (mgr.24961) 2509 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:05:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:11 smithi059 bash[61152]: cephadm 2022-04-28T21:05:09.691167+0000 mgr.y (mgr.24961) 2509 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:05:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:11 smithi059 bash[59745]: cephadm 2022-04-28T21:05:09.691167+0000 mgr.y (mgr.24961) 2509 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:05:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:12 smithi059 bash[61152]: cluster 2022-04-28T21:05:11.036755+0000 mgr.y (mgr.24961) 2510 : cluster [DBG] pgmap v2858: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:11.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:12 smithi059 bash[59745]: cluster 2022-04-28T21:05:11.036755+0000 mgr.y (mgr.24961) 2510 : cluster [DBG] pgmap v2858: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:12.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:12 smithi139 bash[43301]: cluster 2022-04-28T21:05:11.036755+0000 mgr.y (mgr.24961) 2510 : cluster [DBG] pgmap v2858: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 0 B/s rd, 0 op/s 2022-04-28T21:05:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:13.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:14 smithi059 bash[61152]: cluster 2022-04-28T21:05:13.037646+0000 mgr.y (mgr.24961) 2511 : cluster [DBG] pgmap v2859: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:14 smithi059 bash[59745]: cluster 2022-04-28T21:05:13.037646+0000 mgr.y (mgr.24961) 2511 : cluster [DBG] pgmap v2859: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:14.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:14 smithi139 bash[43301]: cluster 2022-04-28T21:05:13.037646+0000 mgr.y (mgr.24961) 2511 : cluster [DBG] pgmap v2859: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:15 smithi059 bash[61152]: cluster 2022-04-28T21:05:15.038813+0000 mgr.y (mgr.24961) 2512 : cluster [DBG] pgmap v2860: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:15 smithi059 bash[59745]: cluster 2022-04-28T21:05:15.038813+0000 mgr.y (mgr.24961) 2512 : cluster [DBG] pgmap v2860: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:15 smithi139 bash[43301]: cluster 2022-04-28T21:05:15.038813+0000 mgr.y (mgr.24961) 2512 : cluster [DBG] pgmap v2860: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: cluster 2022-04-28T21:05:17.040122+0000 mgr.y (mgr.24961) 2513 : cluster [DBG] pgmap v2861: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:18.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: audit 2022-04-28T21:05:18.054679+0000 mon.a (mon.0) 2458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:05:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: audit 2022-04-28T21:05:18.055556+0000 mon.a (mon.0) 2459 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:05:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: audit 2022-04-28T21:05:18.065379+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:05:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: audit 2022-04-28T21:05:18.066390+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:05:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: audit 2022-04-28T21:05:18.066749+0000 mon.a (mon.0) 2462 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T21:05:18.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:18 smithi139 bash[43301]: audit 2022-04-28T21:05:18.067082+0000 mon.a (mon.0) 2463 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:05:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: cluster 2022-04-28T21:05:17.040122+0000 mgr.y (mgr.24961) 2513 : cluster [DBG] pgmap v2861: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: audit 2022-04-28T21:05:18.054679+0000 mon.a (mon.0) 2458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: audit 2022-04-28T21:05:18.055556+0000 mon.a (mon.0) 2459 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: audit 2022-04-28T21:05:18.065379+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:05:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: audit 2022-04-28T21:05:18.066390+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:05:18.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: audit 2022-04-28T21:05:18.066749+0000 mon.a (mon.0) 2462 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T21:05:18.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[61152]: audit 2022-04-28T21:05:18.067082+0000 mon.a (mon.0) 2463 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:05:18.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: cluster 2022-04-28T21:05:17.040122+0000 mgr.y (mgr.24961) 2513 : cluster [DBG] pgmap v2861: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:18.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: audit 2022-04-28T21:05:18.054679+0000 mon.a (mon.0) 2458 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:05:18.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: audit 2022-04-28T21:05:18.055556+0000 mon.a (mon.0) 2459 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:05:18.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: audit 2022-04-28T21:05:18.065379+0000 mon.a (mon.0) 2460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:05:18.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: audit 2022-04-28T21:05:18.066390+0000 mon.a (mon.0) 2461 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:05:18.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: audit 2022-04-28T21:05:18.066749+0000 mon.a (mon.0) 2462 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T21:05:18.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:18 smithi059 bash[59745]: audit 2022-04-28T21:05:18.067082+0000 mon.a (mon.0) 2463 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:05:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:20 smithi139 bash[43301]: cluster 2022-04-28T21:05:19.041039+0000 mgr.y (mgr.24961) 2514 : cluster [DBG] pgmap v2862: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:20 smithi059 bash[61152]: cluster 2022-04-28T21:05:19.041039+0000 mgr.y (mgr.24961) 2514 : cluster [DBG] pgmap v2862: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:20 smithi059 bash[59745]: cluster 2022-04-28T21:05:19.041039+0000 mgr.y (mgr.24961) 2514 : cluster [DBG] pgmap v2862: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:22 smithi059 bash[61152]: cluster 2022-04-28T21:05:21.042430+0000 mgr.y (mgr.24961) 2515 : cluster [DBG] pgmap v2863: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:22.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:21.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:22 smithi059 bash[59745]: cluster 2022-04-28T21:05:21.042430+0000 mgr.y (mgr.24961) 2515 : cluster [DBG] pgmap v2863: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:22.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:22 smithi139 bash[43301]: cluster 2022-04-28T21:05:21.042430+0000 mgr.y (mgr.24961) 2515 : cluster [DBG] pgmap v2863: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:23.810Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:24 smithi139 bash[43301]: cluster 2022-04-28T21:05:23.043269+0000 mgr.y (mgr.24961) 2516 : cluster [DBG] pgmap v2864: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:24 smithi059 bash[61152]: cluster 2022-04-28T21:05:23.043269+0000 mgr.y (mgr.24961) 2516 : cluster [DBG] pgmap v2864: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:24.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:24 smithi059 bash[59745]: cluster 2022-04-28T21:05:23.043269+0000 mgr.y (mgr.24961) 2516 : cluster [DBG] pgmap v2864: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:25 smithi059 bash[61152]: cluster 2022-04-28T21:05:25.044455+0000 mgr.y (mgr.24961) 2517 : cluster [DBG] pgmap v2865: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:25 smithi059 bash[59745]: cluster 2022-04-28T21:05:25.044455+0000 mgr.y (mgr.24961) 2517 : cluster [DBG] pgmap v2865: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:25 smithi139 bash[43301]: cluster 2022-04-28T21:05:25.044455+0000 mgr.y (mgr.24961) 2517 : cluster [DBG] pgmap v2865: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:28 smithi139 bash[43301]: cluster 2022-04-28T21:05:27.045702+0000 mgr.y (mgr.24961) 2518 : cluster [DBG] pgmap v2866: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:28 smithi059 bash[61152]: cluster 2022-04-28T21:05:27.045702+0000 mgr.y (mgr.24961) 2518 : cluster [DBG] pgmap v2866: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:28 smithi059 bash[59745]: cluster 2022-04-28T21:05:27.045702+0000 mgr.y (mgr.24961) 2518 : cluster [DBG] pgmap v2866: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:30.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:30 smithi139 bash[43301]: cluster 2022-04-28T21:05:29.046658+0000 mgr.y (mgr.24961) 2519 : cluster [DBG] pgmap v2867: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:30 smithi059 bash[61152]: cluster 2022-04-28T21:05:29.046658+0000 mgr.y (mgr.24961) 2519 : cluster [DBG] pgmap v2867: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:30 smithi059 bash[59745]: cluster 2022-04-28T21:05:29.046658+0000 mgr.y (mgr.24961) 2519 : cluster [DBG] pgmap v2867: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:32 smithi059 bash[61152]: cluster 2022-04-28T21:05:31.048122+0000 mgr.y (mgr.24961) 2520 : cluster [DBG] pgmap v2868: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:31.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:32 smithi059 bash[59745]: cluster 2022-04-28T21:05:31.048122+0000 mgr.y (mgr.24961) 2520 : cluster [DBG] pgmap v2868: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:32.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:32 smithi139 bash[43301]: cluster 2022-04-28T21:05:31.048122+0000 mgr.y (mgr.24961) 2520 : cluster [DBG] pgmap v2868: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:33.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:34.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:34 smithi059 bash[59745]: cluster 2022-04-28T21:05:33.048984+0000 mgr.y (mgr.24961) 2521 : cluster [DBG] pgmap v2869: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:34.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:34 smithi059 bash[61152]: cluster 2022-04-28T21:05:33.048984+0000 mgr.y (mgr.24961) 2521 : cluster [DBG] pgmap v2869: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:34 smithi139 bash[43301]: cluster 2022-04-28T21:05:33.048984+0000 mgr.y (mgr.24961) 2521 : cluster [DBG] pgmap v2869: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:35 smithi059 bash[61152]: cluster 2022-04-28T21:05:35.050145+0000 mgr.y (mgr.24961) 2522 : cluster [DBG] pgmap v2870: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:35 smithi059 bash[59745]: cluster 2022-04-28T21:05:35.050145+0000 mgr.y (mgr.24961) 2522 : cluster [DBG] pgmap v2870: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:35 smithi139 bash[43301]: cluster 2022-04-28T21:05:35.050145+0000 mgr.y (mgr.24961) 2522 : cluster [DBG] pgmap v2870: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:38 smithi139 bash[43301]: cluster 2022-04-28T21:05:37.051423+0000 mgr.y (mgr.24961) 2523 : cluster [DBG] pgmap v2871: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:38 smithi059 bash[61152]: cluster 2022-04-28T21:05:37.051423+0000 mgr.y (mgr.24961) 2523 : cluster [DBG] pgmap v2871: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:38 smithi059 bash[59745]: cluster 2022-04-28T21:05:37.051423+0000 mgr.y (mgr.24961) 2523 : cluster [DBG] pgmap v2871: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:40 smithi139 bash[43301]: cluster 2022-04-28T21:05:39.052328+0000 mgr.y (mgr.24961) 2524 : cluster [DBG] pgmap v2872: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:40 smithi059 bash[61152]: cluster 2022-04-28T21:05:39.052328+0000 mgr.y (mgr.24961) 2524 : cluster [DBG] pgmap v2872: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:40 smithi059 bash[59745]: cluster 2022-04-28T21:05:39.052328+0000 mgr.y (mgr.24961) 2524 : cluster [DBG] pgmap v2872: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:42 smithi059 bash[61152]: cluster 2022-04-28T21:05:41.053706+0000 mgr.y (mgr.24961) 2525 : cluster [DBG] pgmap v2873: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:41.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:42 smithi059 bash[59745]: cluster 2022-04-28T21:05:41.053706+0000 mgr.y (mgr.24961) 2525 : cluster [DBG] pgmap v2873: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:42 smithi139 bash[43301]: cluster 2022-04-28T21:05:41.053706+0000 mgr.y (mgr.24961) 2525 : cluster [DBG] pgmap v2873: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:43.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:44 smithi059 bash[61152]: cluster 2022-04-28T21:05:43.054577+0000 mgr.y (mgr.24961) 2526 : cluster [DBG] pgmap v2874: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:44 smithi059 bash[59745]: cluster 2022-04-28T21:05:43.054577+0000 mgr.y (mgr.24961) 2526 : cluster [DBG] pgmap v2874: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:44 smithi139 bash[43301]: cluster 2022-04-28T21:05:43.054577+0000 mgr.y (mgr.24961) 2526 : cluster [DBG] pgmap v2874: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:45 smithi059 bash[61152]: cluster 2022-04-28T21:05:45.055805+0000 mgr.y (mgr.24961) 2527 : cluster [DBG] pgmap v2875: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:45 smithi059 bash[59745]: cluster 2022-04-28T21:05:45.055805+0000 mgr.y (mgr.24961) 2527 : cluster [DBG] pgmap v2875: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:45 smithi139 bash[43301]: cluster 2022-04-28T21:05:45.055805+0000 mgr.y (mgr.24961) 2527 : cluster [DBG] pgmap v2875: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:48.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:48 smithi139 bash[43301]: cluster 2022-04-28T21:05:47.057026+0000 mgr.y (mgr.24961) 2528 : cluster [DBG] pgmap v2876: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:48 smithi059 bash[61152]: cluster 2022-04-28T21:05:47.057026+0000 mgr.y (mgr.24961) 2528 : cluster [DBG] pgmap v2876: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:48 smithi059 bash[59745]: cluster 2022-04-28T21:05:47.057026+0000 mgr.y (mgr.24961) 2528 : cluster [DBG] pgmap v2876: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:50 smithi059 bash[61152]: cluster 2022-04-28T21:05:49.057873+0000 mgr.y (mgr.24961) 2529 : cluster [DBG] pgmap v2877: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:50 smithi059 bash[59745]: cluster 2022-04-28T21:05:49.057873+0000 mgr.y (mgr.24961) 2529 : cluster [DBG] pgmap v2877: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:50 smithi139 bash[43301]: cluster 2022-04-28T21:05:49.057873+0000 mgr.y (mgr.24961) 2529 : cluster [DBG] pgmap v2877: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:52 smithi059 bash[61152]: cluster 2022-04-28T21:05:51.059264+0000 mgr.y (mgr.24961) 2530 : cluster [DBG] pgmap v2878: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:51.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:52 smithi059 bash[59745]: cluster 2022-04-28T21:05:51.059264+0000 mgr.y (mgr.24961) 2530 : cluster [DBG] pgmap v2878: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:52 smithi139 bash[43301]: cluster 2022-04-28T21:05:51.059264+0000 mgr.y (mgr.24961) 2530 : cluster [DBG] pgmap v2878: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:54.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:05:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:05:53.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:05:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:54 smithi059 bash[61152]: cluster 2022-04-28T21:05:53.060126+0000 mgr.y (mgr.24961) 2531 : cluster [DBG] pgmap v2879: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:54 smithi059 bash[59745]: cluster 2022-04-28T21:05:53.060126+0000 mgr.y (mgr.24961) 2531 : cluster [DBG] pgmap v2879: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:54 smithi139 bash[43301]: cluster 2022-04-28T21:05:53.060126+0000 mgr.y (mgr.24961) 2531 : cluster [DBG] pgmap v2879: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:55 smithi059 bash[61152]: cluster 2022-04-28T21:05:55.061261+0000 mgr.y (mgr.24961) 2532 : cluster [DBG] pgmap v2880: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:55 smithi059 bash[59745]: cluster 2022-04-28T21:05:55.061261+0000 mgr.y (mgr.24961) 2532 : cluster [DBG] pgmap v2880: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:55 smithi139 bash[43301]: cluster 2022-04-28T21:05:55.061261+0000 mgr.y (mgr.24961) 2532 : cluster [DBG] pgmap v2880: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:05:58 smithi059 bash[61152]: cluster 2022-04-28T21:05:57.062578+0000 mgr.y (mgr.24961) 2533 : cluster [DBG] pgmap v2881: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:05:58 smithi059 bash[59745]: cluster 2022-04-28T21:05:57.062578+0000 mgr.y (mgr.24961) 2533 : cluster [DBG] pgmap v2881: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:05:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:05:58 smithi139 bash[43301]: cluster 2022-04-28T21:05:57.062578+0000 mgr.y (mgr.24961) 2533 : cluster [DBG] pgmap v2881: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:00 smithi059 bash[61152]: cluster 2022-04-28T21:05:59.063716+0000 mgr.y (mgr.24961) 2534 : cluster [DBG] pgmap v2882: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:00 smithi059 bash[59745]: cluster 2022-04-28T21:05:59.063716+0000 mgr.y (mgr.24961) 2534 : cluster [DBG] pgmap v2882: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:00 smithi139 bash[43301]: cluster 2022-04-28T21:05:59.063716+0000 mgr.y (mgr.24961) 2534 : cluster [DBG] pgmap v2882: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:02 smithi059 bash[61152]: cluster 2022-04-28T21:06:01.065015+0000 mgr.y (mgr.24961) 2535 : cluster [DBG] pgmap v2883: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:01.915Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:02 smithi059 bash[59745]: cluster 2022-04-28T21:06:01.065015+0000 mgr.y (mgr.24961) 2535 : cluster [DBG] pgmap v2883: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:02 smithi139 bash[43301]: cluster 2022-04-28T21:06:01.065015+0000 mgr.y (mgr.24961) 2535 : cluster [DBG] pgmap v2883: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:03.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:04.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:04 smithi059 bash[61152]: cluster 2022-04-28T21:06:03.065871+0000 mgr.y (mgr.24961) 2536 : cluster [DBG] pgmap v2884: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:04.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:04 smithi059 bash[59745]: cluster 2022-04-28T21:06:03.065871+0000 mgr.y (mgr.24961) 2536 : cluster [DBG] pgmap v2884: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:04 smithi139 bash[43301]: cluster 2022-04-28T21:06:03.065871+0000 mgr.y (mgr.24961) 2536 : cluster [DBG] pgmap v2884: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:05 smithi059 bash[59745]: cluster 2022-04-28T21:06:05.067129+0000 mgr.y (mgr.24961) 2537 : cluster [DBG] pgmap v2885: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:05 smithi059 bash[61152]: cluster 2022-04-28T21:06:05.067129+0000 mgr.y (mgr.24961) 2537 : cluster [DBG] pgmap v2885: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:05 smithi139 bash[43301]: cluster 2022-04-28T21:06:05.067129+0000 mgr.y (mgr.24961) 2537 : cluster [DBG] pgmap v2885: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:08 smithi059 bash[61152]: cluster 2022-04-28T21:06:07.068267+0000 mgr.y (mgr.24961) 2538 : cluster [DBG] pgmap v2886: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:08 smithi059 bash[59745]: cluster 2022-04-28T21:06:07.068267+0000 mgr.y (mgr.24961) 2538 : cluster [DBG] pgmap v2886: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:08 smithi139 bash[43301]: cluster 2022-04-28T21:06:07.068267+0000 mgr.y (mgr.24961) 2538 : cluster [DBG] pgmap v2886: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:10.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[59745]: cluster 2022-04-28T21:06:09.069297+0000 mgr.y (mgr.24961) 2539 : cluster [DBG] pgmap v2887: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:10.652 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[59745]: audit 2022-04-28T21:06:10.013354+0000 mon.a (mon.0) 2464 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:06:10.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[59745]: audit 2022-04-28T21:06:10.016206+0000 mon.a (mon.0) 2465 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:06:10.653 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[59745]: audit 2022-04-28T21:06:10.018026+0000 mon.a (mon.0) 2466 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:06:10.654 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[61152]: cluster 2022-04-28T21:06:09.069297+0000 mgr.y (mgr.24961) 2539 : cluster [DBG] pgmap v2887: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:10.655 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[61152]: audit 2022-04-28T21:06:10.013354+0000 mon.a (mon.0) 2464 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:06:10.655 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[61152]: audit 2022-04-28T21:06:10.016206+0000 mon.a (mon.0) 2465 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:06:10.656 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:10 smithi059 bash[61152]: audit 2022-04-28T21:06:10.018026+0000 mon.a (mon.0) 2466 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:06:10.657 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:10 smithi139 bash[43301]: cluster 2022-04-28T21:06:09.069297+0000 mgr.y (mgr.24961) 2539 : cluster [DBG] pgmap v2887: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:10.657 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:10 smithi139 bash[43301]: audit 2022-04-28T21:06:10.013354+0000 mon.a (mon.0) 2464 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:06:10.657 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:10 smithi139 bash[43301]: audit 2022-04-28T21:06:10.016206+0000 mon.a (mon.0) 2465 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:06:10.658 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:10 smithi139 bash[43301]: audit 2022-04-28T21:06:10.018026+0000 mon.a (mon.0) 2466 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:06:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:11 smithi059 bash[61152]: cephadm 2022-04-28T21:06:10.019688+0000 mgr.y (mgr.24961) 2540 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:06:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:11 smithi059 bash[61152]: audit 2022-04-28T21:06:10.352556+0000 mon.a (mon.0) 2467 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:06:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:11 smithi059 bash[59745]: cephadm 2022-04-28T21:06:10.019688+0000 mgr.y (mgr.24961) 2540 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:06:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:11 smithi059 bash[59745]: audit 2022-04-28T21:06:10.352556+0000 mon.a (mon.0) 2467 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:06:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:11 smithi139 bash[43301]: cephadm 2022-04-28T21:06:10.019688+0000 mgr.y (mgr.24961) 2540 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:06:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:11 smithi139 bash[43301]: audit 2022-04-28T21:06:10.352556+0000 mon.a (mon.0) 2467 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:06:12.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:12 smithi059 bash[61152]: cluster 2022-04-28T21:06:11.070599+0000 mgr.y (mgr.24961) 2541 : cluster [DBG] pgmap v2888: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:11.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:12.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:12 smithi059 bash[59745]: cluster 2022-04-28T21:06:11.070599+0000 mgr.y (mgr.24961) 2541 : cluster [DBG] pgmap v2888: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:12 smithi139 bash[43301]: cluster 2022-04-28T21:06:11.070599+0000 mgr.y (mgr.24961) 2541 : cluster [DBG] pgmap v2888: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:13.813Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:14 smithi059 bash[61152]: cluster 2022-04-28T21:06:13.071327+0000 mgr.y (mgr.24961) 2542 : cluster [DBG] pgmap v2889: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:14 smithi059 bash[59745]: cluster 2022-04-28T21:06:13.071327+0000 mgr.y (mgr.24961) 2542 : cluster [DBG] pgmap v2889: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:14.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:14 smithi139 bash[43301]: cluster 2022-04-28T21:06:13.071327+0000 mgr.y (mgr.24961) 2542 : cluster [DBG] pgmap v2889: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:15 smithi059 bash[61152]: cluster 2022-04-28T21:06:15.072598+0000 mgr.y (mgr.24961) 2543 : cluster [DBG] pgmap v2890: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:15 smithi059 bash[59745]: cluster 2022-04-28T21:06:15.072598+0000 mgr.y (mgr.24961) 2543 : cluster [DBG] pgmap v2890: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:15 smithi139 bash[43301]: cluster 2022-04-28T21:06:15.072598+0000 mgr.y (mgr.24961) 2543 : cluster [DBG] pgmap v2890: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: cluster 2022-04-28T21:06:17.073704+0000 mgr.y (mgr.24961) 2544 : cluster [DBG] pgmap v2891: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: audit 2022-04-28T21:06:18.069544+0000 mon.a (mon.0) 2468 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:06:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: audit 2022-04-28T21:06:18.070097+0000 mon.a (mon.0) 2469 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:06:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: audit 2022-04-28T21:06:18.082308+0000 mon.a (mon.0) 2470 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:06:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: audit 2022-04-28T21:06:18.082947+0000 mon.a (mon.0) 2471 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:06:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: audit 2022-04-28T21:06:18.083753+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:06:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[61152]: audit 2022-04-28T21:06:18.084492+0000 mon.a (mon.0) 2473 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:06:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: cluster 2022-04-28T21:06:17.073704+0000 mgr.y (mgr.24961) 2544 : cluster [DBG] pgmap v2891: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: audit 2022-04-28T21:06:18.069544+0000 mon.a (mon.0) 2468 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:06:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: audit 2022-04-28T21:06:18.070097+0000 mon.a (mon.0) 2469 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:06:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: audit 2022-04-28T21:06:18.082308+0000 mon.a (mon.0) 2470 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:06:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: audit 2022-04-28T21:06:18.082947+0000 mon.a (mon.0) 2471 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:06:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: audit 2022-04-28T21:06:18.083753+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:06:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:18 smithi059 bash[59745]: audit 2022-04-28T21:06:18.084492+0000 mon.a (mon.0) 2473 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:06:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: cluster 2022-04-28T21:06:17.073704+0000 mgr.y (mgr.24961) 2544 : cluster [DBG] pgmap v2891: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:18.659 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: audit 2022-04-28T21:06:18.069544+0000 mon.a (mon.0) 2468 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:06:18.659 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: audit 2022-04-28T21:06:18.070097+0000 mon.a (mon.0) 2469 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:06:18.660 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: audit 2022-04-28T21:06:18.082308+0000 mon.a (mon.0) 2470 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:06:18.660 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: audit 2022-04-28T21:06:18.082947+0000 mon.a (mon.0) 2471 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:06:18.660 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: audit 2022-04-28T21:06:18.083753+0000 mon.a (mon.0) 2472 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:06:18.660 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:18 smithi139 bash[43301]: audit 2022-04-28T21:06:18.084492+0000 mon.a (mon.0) 2473 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:06:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:20 smithi059 bash[59745]: cluster 2022-04-28T21:06:19.074732+0000 mgr.y (mgr.24961) 2545 : cluster [DBG] pgmap v2892: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:20 smithi059 bash[61152]: cluster 2022-04-28T21:06:19.074732+0000 mgr.y (mgr.24961) 2545 : cluster [DBG] pgmap v2892: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:20 smithi139 bash[43301]: cluster 2022-04-28T21:06:19.074732+0000 mgr.y (mgr.24961) 2545 : cluster [DBG] pgmap v2892: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:22.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:22 smithi059 bash[59745]: cluster 2022-04-28T21:06:21.076108+0000 mgr.y (mgr.24961) 2546 : cluster [DBG] pgmap v2893: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:06:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:22 smithi059 bash[61152]: cluster 2022-04-28T21:06:21.076108+0000 mgr.y (mgr.24961) 2546 : cluster [DBG] pgmap v2893: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:06:22.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:21.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:22 smithi139 bash[43301]: cluster 2022-04-28T21:06:21.076108+0000 mgr.y (mgr.24961) 2546 : cluster [DBG] pgmap v2893: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:06:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:23.815Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:24 smithi059 bash[59745]: cluster 2022-04-28T21:06:23.076837+0000 mgr.y (mgr.24961) 2547 : cluster [DBG] pgmap v2894: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:06:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:24 smithi059 bash[61152]: cluster 2022-04-28T21:06:23.076837+0000 mgr.y (mgr.24961) 2547 : cluster [DBG] pgmap v2894: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:06:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:24 smithi139 bash[43301]: cluster 2022-04-28T21:06:23.076837+0000 mgr.y (mgr.24961) 2547 : cluster [DBG] pgmap v2894: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2022-04-28T21:06:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:25 smithi059 bash[61152]: cluster 2022-04-28T21:06:25.078105+0000 mgr.y (mgr.24961) 2548 : cluster [DBG] pgmap v2895: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:25 smithi059 bash[59745]: cluster 2022-04-28T21:06:25.078105+0000 mgr.y (mgr.24961) 2548 : cluster [DBG] pgmap v2895: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:25.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:25 smithi139 bash[43301]: cluster 2022-04-28T21:06:25.078105+0000 mgr.y (mgr.24961) 2548 : cluster [DBG] pgmap v2895: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:28 smithi059 bash[61152]: cluster 2022-04-28T21:06:27.079218+0000 mgr.y (mgr.24961) 2549 : cluster [DBG] pgmap v2896: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:28 smithi059 bash[59745]: cluster 2022-04-28T21:06:27.079218+0000 mgr.y (mgr.24961) 2549 : cluster [DBG] pgmap v2896: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:28 smithi139 bash[43301]: cluster 2022-04-28T21:06:27.079218+0000 mgr.y (mgr.24961) 2549 : cluster [DBG] pgmap v2896: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:30 smithi059 bash[61152]: cluster 2022-04-28T21:06:29.080259+0000 mgr.y (mgr.24961) 2550 : cluster [DBG] pgmap v2897: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:30 smithi059 bash[59745]: cluster 2022-04-28T21:06:29.080259+0000 mgr.y (mgr.24961) 2550 : cluster [DBG] pgmap v2897: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:30 smithi139 bash[43301]: cluster 2022-04-28T21:06:29.080259+0000 mgr.y (mgr.24961) 2550 : cluster [DBG] pgmap v2897: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:32 smithi059 bash[61152]: cluster 2022-04-28T21:06:31.081607+0000 mgr.y (mgr.24961) 2551 : cluster [DBG] pgmap v2898: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:32.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:31.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:32 smithi059 bash[59745]: cluster 2022-04-28T21:06:31.081607+0000 mgr.y (mgr.24961) 2551 : cluster [DBG] pgmap v2898: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:32 smithi139 bash[43301]: cluster 2022-04-28T21:06:31.081607+0000 mgr.y (mgr.24961) 2551 : cluster [DBG] pgmap v2898: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:06:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:33.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:34.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:34 smithi059 bash[59745]: cluster 2022-04-28T21:06:33.082458+0000 mgr.y (mgr.24961) 2552 : cluster [DBG] pgmap v2899: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:06:34.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:34 smithi059 bash[61152]: cluster 2022-04-28T21:06:33.082458+0000 mgr.y (mgr.24961) 2552 : cluster [DBG] pgmap v2899: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:06:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:34 smithi139 bash[43301]: cluster 2022-04-28T21:06:33.082458+0000 mgr.y (mgr.24961) 2552 : cluster [DBG] pgmap v2899: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:06:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:35 smithi059 bash[61152]: cluster 2022-04-28T21:06:35.083859+0000 mgr.y (mgr.24961) 2553 : cluster [DBG] pgmap v2900: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:06:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:35 smithi059 bash[59745]: cluster 2022-04-28T21:06:35.083859+0000 mgr.y (mgr.24961) 2553 : cluster [DBG] pgmap v2900: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:06:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:35 smithi139 bash[43301]: cluster 2022-04-28T21:06:35.083859+0000 mgr.y (mgr.24961) 2553 : cluster [DBG] pgmap v2900: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2022-04-28T21:06:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:38 smithi059 bash[61152]: cluster 2022-04-28T21:06:37.085024+0000 mgr.y (mgr.24961) 2554 : cluster [DBG] pgmap v2901: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:38.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:38 smithi059 bash[59745]: cluster 2022-04-28T21:06:37.085024+0000 mgr.y (mgr.24961) 2554 : cluster [DBG] pgmap v2901: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:38 smithi139 bash[43301]: cluster 2022-04-28T21:06:37.085024+0000 mgr.y (mgr.24961) 2554 : cluster [DBG] pgmap v2901: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:40 smithi059 bash[59745]: cluster 2022-04-28T21:06:39.086043+0000 mgr.y (mgr.24961) 2555 : cluster [DBG] pgmap v2902: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:40 smithi059 bash[61152]: cluster 2022-04-28T21:06:39.086043+0000 mgr.y (mgr.24961) 2555 : cluster [DBG] pgmap v2902: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:40 smithi139 bash[43301]: cluster 2022-04-28T21:06:39.086043+0000 mgr.y (mgr.24961) 2555 : cluster [DBG] pgmap v2902: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:42 smithi059 bash[61152]: cluster 2022-04-28T21:06:41.087397+0000 mgr.y (mgr.24961) 2556 : cluster [DBG] pgmap v2903: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:41.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:42 smithi059 bash[59745]: cluster 2022-04-28T21:06:41.087397+0000 mgr.y (mgr.24961) 2556 : cluster [DBG] pgmap v2903: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:42 smithi139 bash[43301]: cluster 2022-04-28T21:06:41.087397+0000 mgr.y (mgr.24961) 2556 : cluster [DBG] pgmap v2903: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:06:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:43.816Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:44.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:44 smithi059 bash[61152]: cluster 2022-04-28T21:06:43.088167+0000 mgr.y (mgr.24961) 2557 : cluster [DBG] pgmap v2904: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:44 smithi059 bash[59745]: cluster 2022-04-28T21:06:43.088167+0000 mgr.y (mgr.24961) 2557 : cluster [DBG] pgmap v2904: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:44 smithi139 bash[43301]: cluster 2022-04-28T21:06:43.088167+0000 mgr.y (mgr.24961) 2557 : cluster [DBG] pgmap v2904: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:45 smithi059 bash[61152]: cluster 2022-04-28T21:06:45.089516+0000 mgr.y (mgr.24961) 2558 : cluster [DBG] pgmap v2905: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:45.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:45 smithi059 bash[59745]: cluster 2022-04-28T21:06:45.089516+0000 mgr.y (mgr.24961) 2558 : cluster [DBG] pgmap v2905: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:45 smithi139 bash[43301]: cluster 2022-04-28T21:06:45.089516+0000 mgr.y (mgr.24961) 2558 : cluster [DBG] pgmap v2905: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:46 smithi059 bash[61152]: audit 2022-04-28T21:06:46.106923+0000 mon.c (mon.1) 32 : audit [DBG] from='client.? 172.21.15.59:0/1296063907' entity='client.admin' cmd=[{"prefix": "osd map", "pool": "test-rados-api-smithi059-79405-1", "object": "image1"}]: dispatch 2022-04-28T21:06:46.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:46 smithi059 bash[59745]: audit 2022-04-28T21:06:46.106923+0000 mon.c (mon.1) 32 : audit [DBG] from='client.? 172.21.15.59:0/1296063907' entity='client.admin' cmd=[{"prefix": "osd map", "pool": "test-rados-api-smithi059-79405-1", "object": "image1"}]: dispatch 2022-04-28T21:06:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:46 smithi139 bash[43301]: audit 2022-04-28T21:06:46.106923+0000 mon.c (mon.1) 32 : audit [DBG] from='client.? 172.21.15.59:0/1296063907' entity='client.admin' cmd=[{"prefix": "osd map", "pool": "test-rados-api-smithi059-79405-1", "object": "image1"}]: dispatch 2022-04-28T21:06:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:47 smithi059 bash[61152]: cluster 2022-04-28T21:06:47.090678+0000 mgr.y (mgr.24961) 2559 : cluster [DBG] pgmap v2906: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:47 smithi059 bash[59745]: cluster 2022-04-28T21:06:47.090678+0000 mgr.y (mgr.24961) 2559 : cluster [DBG] pgmap v2906: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:47 smithi139 bash[43301]: cluster 2022-04-28T21:06:47.090678+0000 mgr.y (mgr.24961) 2559 : cluster [DBG] pgmap v2906: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:50 smithi059 bash[61152]: cluster 2022-04-28T21:06:49.091744+0000 mgr.y (mgr.24961) 2560 : cluster [DBG] pgmap v2907: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:50 smithi059 bash[59745]: cluster 2022-04-28T21:06:49.091744+0000 mgr.y (mgr.24961) 2560 : cluster [DBG] pgmap v2907: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:50 smithi139 bash[43301]: cluster 2022-04-28T21:06:49.091744+0000 mgr.y (mgr.24961) 2560 : cluster [DBG] pgmap v2907: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:52 smithi059 bash[61152]: cluster 2022-04-28T21:06:51.093069+0000 mgr.y (mgr.24961) 2561 : cluster [DBG] pgmap v2908: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:52 smithi059 bash[59745]: cluster 2022-04-28T21:06:51.093069+0000 mgr.y (mgr.24961) 2561 : cluster [DBG] pgmap v2908: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:51.924Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:52 smithi139 bash[43301]: cluster 2022-04-28T21:06:51.093069+0000 mgr.y (mgr.24961) 2561 : cluster [DBG] pgmap v2908: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:06:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:06:53.817Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:06:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:54 smithi059 bash[59745]: cluster 2022-04-28T21:06:53.093756+0000 mgr.y (mgr.24961) 2562 : cluster [DBG] pgmap v2909: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:54 smithi059 bash[61152]: cluster 2022-04-28T21:06:53.093756+0000 mgr.y (mgr.24961) 2562 : cluster [DBG] pgmap v2909: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:54 smithi139 bash[43301]: cluster 2022-04-28T21:06:53.093756+0000 mgr.y (mgr.24961) 2562 : cluster [DBG] pgmap v2909: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:55 smithi059 bash[59745]: cluster 2022-04-28T21:06:55.094938+0000 mgr.y (mgr.24961) 2563 : cluster [DBG] pgmap v2910: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:55.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:55 smithi059 bash[61152]: cluster 2022-04-28T21:06:55.094938+0000 mgr.y (mgr.24961) 2563 : cluster [DBG] pgmap v2910: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:55 smithi139 bash[43301]: cluster 2022-04-28T21:06:55.094938+0000 mgr.y (mgr.24961) 2563 : cluster [DBG] pgmap v2910: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:06:58 smithi059 bash[59745]: cluster 2022-04-28T21:06:57.096159+0000 mgr.y (mgr.24961) 2564 : cluster [DBG] pgmap v2911: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:06:58 smithi059 bash[61152]: cluster 2022-04-28T21:06:57.096159+0000 mgr.y (mgr.24961) 2564 : cluster [DBG] pgmap v2911: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:06:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:06:58 smithi139 bash[43301]: cluster 2022-04-28T21:06:57.096159+0000 mgr.y (mgr.24961) 2564 : cluster [DBG] pgmap v2911: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:00.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:00 smithi059 bash[61152]: cluster 2022-04-28T21:06:59.097275+0000 mgr.y (mgr.24961) 2565 : cluster [DBG] pgmap v2912: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:00 smithi059 bash[59745]: cluster 2022-04-28T21:06:59.097275+0000 mgr.y (mgr.24961) 2565 : cluster [DBG] pgmap v2912: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:00 smithi139 bash[43301]: cluster 2022-04-28T21:06:59.097275+0000 mgr.y (mgr.24961) 2565 : cluster [DBG] pgmap v2912: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:02 smithi059 bash[61152]: cluster 2022-04-28T21:07:01.098734+0000 mgr.y (mgr.24961) 2566 : cluster [DBG] pgmap v2913: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:01.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:02.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:02 smithi059 bash[59745]: cluster 2022-04-28T21:07:01.098734+0000 mgr.y (mgr.24961) 2566 : cluster [DBG] pgmap v2913: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:02 smithi139 bash[43301]: cluster 2022-04-28T21:07:01.098734+0000 mgr.y (mgr.24961) 2566 : cluster [DBG] pgmap v2913: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:03.818Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:04.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:04 smithi059 bash[61152]: cluster 2022-04-28T21:07:03.099433+0000 mgr.y (mgr.24961) 2567 : cluster [DBG] pgmap v2914: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:04.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:04 smithi059 bash[59745]: cluster 2022-04-28T21:07:03.099433+0000 mgr.y (mgr.24961) 2567 : cluster [DBG] pgmap v2914: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:04 smithi139 bash[43301]: cluster 2022-04-28T21:07:03.099433+0000 mgr.y (mgr.24961) 2567 : cluster [DBG] pgmap v2914: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:05 smithi059 bash[61152]: cluster 2022-04-28T21:07:05.100659+0000 mgr.y (mgr.24961) 2568 : cluster [DBG] pgmap v2915: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:05 smithi059 bash[59745]: cluster 2022-04-28T21:07:05.100659+0000 mgr.y (mgr.24961) 2568 : cluster [DBG] pgmap v2915: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:05 smithi139 bash[43301]: cluster 2022-04-28T21:07:05.100659+0000 mgr.y (mgr.24961) 2568 : cluster [DBG] pgmap v2915: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:08 smithi059 bash[61152]: cluster 2022-04-28T21:07:07.101830+0000 mgr.y (mgr.24961) 2569 : cluster [DBG] pgmap v2916: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:08 smithi059 bash[59745]: cluster 2022-04-28T21:07:07.101830+0000 mgr.y (mgr.24961) 2569 : cluster [DBG] pgmap v2916: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:08 smithi139 bash[43301]: cluster 2022-04-28T21:07:07.101830+0000 mgr.y (mgr.24961) 2569 : cluster [DBG] pgmap v2916: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:10 smithi059 bash[61152]: cluster 2022-04-28T21:07:09.102861+0000 mgr.y (mgr.24961) 2570 : cluster [DBG] pgmap v2917: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:10 smithi059 bash[59745]: cluster 2022-04-28T21:07:09.102861+0000 mgr.y (mgr.24961) 2570 : cluster [DBG] pgmap v2917: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:10 smithi139 bash[43301]: cluster 2022-04-28T21:07:09.102861+0000 mgr.y (mgr.24961) 2570 : cluster [DBG] pgmap v2917: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[61152]: audit 2022-04-28T21:07:10.360046+0000 mon.a (mon.0) 2474 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:07:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[61152]: audit 2022-04-28T21:07:10.363091+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:07:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[61152]: audit 2022-04-28T21:07:10.365135+0000 mon.a (mon.0) 2476 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:07:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[61152]: audit 2022-04-28T21:07:10.368487+0000 mon.a (mon.0) 2477 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:07:11.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[61152]: audit 2022-04-28T21:07:10.369026+0000 mon.a (mon.0) 2478 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:07:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[59745]: audit 2022-04-28T21:07:10.360046+0000 mon.a (mon.0) 2474 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:07:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[59745]: audit 2022-04-28T21:07:10.363091+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:07:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[59745]: audit 2022-04-28T21:07:10.365135+0000 mon.a (mon.0) 2476 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:07:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[59745]: audit 2022-04-28T21:07:10.368487+0000 mon.a (mon.0) 2477 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:07:11.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[59745]: audit 2022-04-28T21:07:10.369026+0000 mon.a (mon.0) 2478 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:07:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:11 smithi139 bash[43301]: audit 2022-04-28T21:07:10.360046+0000 mon.a (mon.0) 2474 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:07:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:11 smithi139 bash[43301]: audit 2022-04-28T21:07:10.363091+0000 mon.a (mon.0) 2475 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:07:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:11 smithi139 bash[43301]: audit 2022-04-28T21:07:10.365135+0000 mon.a (mon.0) 2476 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:07:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:11 smithi139 bash[43301]: audit 2022-04-28T21:07:10.368487+0000 mon.a (mon.0) 2477 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:07:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:11 smithi139 bash[43301]: audit 2022-04-28T21:07:10.369026+0000 mon.a (mon.0) 2478 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:07:12.184 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:11.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:12.184 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:12 smithi059 bash[59745]: cephadm 2022-04-28T21:07:10.366973+0000 mgr.y (mgr.24961) 2571 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:07:12.185 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:12 smithi059 bash[59745]: cluster 2022-04-28T21:07:11.104256+0000 mgr.y (mgr.24961) 2572 : cluster [DBG] pgmap v2918: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:12 smithi059 bash[61152]: cephadm 2022-04-28T21:07:10.366973+0000 mgr.y (mgr.24961) 2571 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:07:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:12 smithi059 bash[61152]: cluster 2022-04-28T21:07:11.104256+0000 mgr.y (mgr.24961) 2572 : cluster [DBG] pgmap v2918: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:12 smithi139 bash[43301]: cephadm 2022-04-28T21:07:10.366973+0000 mgr.y (mgr.24961) 2571 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:07:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:12 smithi139 bash[43301]: cluster 2022-04-28T21:07:11.104256+0000 mgr.y (mgr.24961) 2572 : cluster [DBG] pgmap v2918: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:13.814Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:14 smithi059 bash[61152]: cluster 2022-04-28T21:07:13.104969+0000 mgr.y (mgr.24961) 2573 : cluster [DBG] pgmap v2919: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:14 smithi059 bash[59745]: cluster 2022-04-28T21:07:13.104969+0000 mgr.y (mgr.24961) 2573 : cluster [DBG] pgmap v2919: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:14.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:14 smithi139 bash[43301]: cluster 2022-04-28T21:07:13.104969+0000 mgr.y (mgr.24961) 2573 : cluster [DBG] pgmap v2919: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[61152]: audit 2022-04-28T21:07:14.513009+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[61152]: audit 2022-04-28T21:07:14.662450+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[61152]: audit 2022-04-28T21:07:14.956090+0000 mon.a (mon.0) 2481 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[61152]: cluster 2022-04-28T21:07:15.106298+0000 mgr.y (mgr.24961) 2574 : cluster [DBG] pgmap v2920: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[59745]: audit 2022-04-28T21:07:14.513009+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[59745]: audit 2022-04-28T21:07:14.662450+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[59745]: audit 2022-04-28T21:07:14.956090+0000 mon.a (mon.0) 2481 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:15 smithi059 bash[59745]: cluster 2022-04-28T21:07:15.106298+0000 mgr.y (mgr.24961) 2574 : cluster [DBG] pgmap v2920: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:15 smithi139 bash[43301]: audit 2022-04-28T21:07:14.513009+0000 mon.a (mon.0) 2479 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:15 smithi139 bash[43301]: audit 2022-04-28T21:07:14.662450+0000 mon.a (mon.0) 2480 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:15 smithi139 bash[43301]: audit 2022-04-28T21:07:14.956090+0000 mon.a (mon.0) 2481 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:07:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:15 smithi139 bash[43301]: cluster 2022-04-28T21:07:15.106298+0000 mgr.y (mgr.24961) 2574 : cluster [DBG] pgmap v2920: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: cluster 2022-04-28T21:07:17.107284+0000 mgr.y (mgr.24961) 2575 : cluster [DBG] pgmap v2921: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: audit 2022-04-28T21:07:18.082624+0000 mon.a (mon.0) 2482 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:07:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: audit 2022-04-28T21:07:18.083090+0000 mon.a (mon.0) 2483 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:07:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: audit 2022-04-28T21:07:18.095745+0000 mon.a (mon.0) 2484 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:07:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: audit 2022-04-28T21:07:18.096393+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:07:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: audit 2022-04-28T21:07:18.096929+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:07:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[61152]: audit 2022-04-28T21:07:18.097456+0000 mon.a (mon.0) 2487 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:07:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: cluster 2022-04-28T21:07:17.107284+0000 mgr.y (mgr.24961) 2575 : cluster [DBG] pgmap v2921: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: audit 2022-04-28T21:07:18.082624+0000 mon.a (mon.0) 2482 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:07:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: audit 2022-04-28T21:07:18.083090+0000 mon.a (mon.0) 2483 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:07:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: audit 2022-04-28T21:07:18.095745+0000 mon.a (mon.0) 2484 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:07:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: audit 2022-04-28T21:07:18.096393+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:07:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: audit 2022-04-28T21:07:18.096929+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:07:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:18 smithi059 bash[59745]: audit 2022-04-28T21:07:18.097456+0000 mon.a (mon.0) 2487 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: cluster 2022-04-28T21:07:17.107284+0000 mgr.y (mgr.24961) 2575 : cluster [DBG] pgmap v2921: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: audit 2022-04-28T21:07:18.082624+0000 mon.a (mon.0) 2482 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: audit 2022-04-28T21:07:18.083090+0000 mon.a (mon.0) 2483 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:07:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: audit 2022-04-28T21:07:18.095745+0000 mon.a (mon.0) 2484 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:07:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: audit 2022-04-28T21:07:18.096393+0000 mon.a (mon.0) 2485 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:07:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: audit 2022-04-28T21:07:18.096929+0000 mon.a (mon.0) 2486 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:07:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:18 smithi139 bash[43301]: audit 2022-04-28T21:07:18.097456+0000 mon.a (mon.0) 2487 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:07:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:20 smithi059 bash[61152]: cluster 2022-04-28T21:07:19.108311+0000 mgr.y (mgr.24961) 2576 : cluster [DBG] pgmap v2922: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:20 smithi059 bash[59745]: cluster 2022-04-28T21:07:19.108311+0000 mgr.y (mgr.24961) 2576 : cluster [DBG] pgmap v2922: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:20 smithi139 bash[43301]: cluster 2022-04-28T21:07:19.108311+0000 mgr.y (mgr.24961) 2576 : cluster [DBG] pgmap v2922: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:22.177 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:21.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:22.177 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:22 smithi059 bash[59745]: cluster 2022-04-28T21:07:21.109631+0000 mgr.y (mgr.24961) 2577 : cluster [DBG] pgmap v2923: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:22 smithi059 bash[61152]: cluster 2022-04-28T21:07:21.109631+0000 mgr.y (mgr.24961) 2577 : cluster [DBG] pgmap v2923: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:22 smithi139 bash[43301]: cluster 2022-04-28T21:07:21.109631+0000 mgr.y (mgr.24961) 2577 : cluster [DBG] pgmap v2923: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:24.094 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:23.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:24 smithi059 bash[61152]: cluster 2022-04-28T21:07:23.110349+0000 mgr.y (mgr.24961) 2578 : cluster [DBG] pgmap v2924: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:24 smithi059 bash[59745]: cluster 2022-04-28T21:07:23.110349+0000 mgr.y (mgr.24961) 2578 : cluster [DBG] pgmap v2924: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:24 smithi139 bash[43301]: cluster 2022-04-28T21:07:23.110349+0000 mgr.y (mgr.24961) 2578 : cluster [DBG] pgmap v2924: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:25 smithi059 bash[61152]: cluster 2022-04-28T21:07:25.111758+0000 mgr.y (mgr.24961) 2579 : cluster [DBG] pgmap v2925: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:25 smithi059 bash[59745]: cluster 2022-04-28T21:07:25.111758+0000 mgr.y (mgr.24961) 2579 : cluster [DBG] pgmap v2925: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:25 smithi139 bash[43301]: cluster 2022-04-28T21:07:25.111758+0000 mgr.y (mgr.24961) 2579 : cluster [DBG] pgmap v2925: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:28 smithi059 bash[61152]: cluster 2022-04-28T21:07:27.112761+0000 mgr.y (mgr.24961) 2580 : cluster [DBG] pgmap v2926: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:28 smithi059 bash[59745]: cluster 2022-04-28T21:07:27.112761+0000 mgr.y (mgr.24961) 2580 : cluster [DBG] pgmap v2926: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:28 smithi139 bash[43301]: cluster 2022-04-28T21:07:27.112761+0000 mgr.y (mgr.24961) 2580 : cluster [DBG] pgmap v2926: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:30 smithi059 bash[61152]: cluster 2022-04-28T21:07:29.113776+0000 mgr.y (mgr.24961) 2581 : cluster [DBG] pgmap v2927: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:30 smithi059 bash[59745]: cluster 2022-04-28T21:07:29.113776+0000 mgr.y (mgr.24961) 2581 : cluster [DBG] pgmap v2927: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:30 smithi139 bash[43301]: cluster 2022-04-28T21:07:29.113776+0000 mgr.y (mgr.24961) 2581 : cluster [DBG] pgmap v2927: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:32.183 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:31.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:32.183 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:32 smithi059 bash[59745]: cluster 2022-04-28T21:07:31.115213+0000 mgr.y (mgr.24961) 2582 : cluster [DBG] pgmap v2928: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:32 smithi059 bash[61152]: cluster 2022-04-28T21:07:31.115213+0000 mgr.y (mgr.24961) 2582 : cluster [DBG] pgmap v2928: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:32 smithi139 bash[43301]: cluster 2022-04-28T21:07:31.115213+0000 mgr.y (mgr.24961) 2582 : cluster [DBG] pgmap v2928: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:33.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:34.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:34 smithi059 bash[61152]: cluster 2022-04-28T21:07:33.115978+0000 mgr.y (mgr.24961) 2583 : cluster [DBG] pgmap v2929: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:34.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:34 smithi059 bash[59745]: cluster 2022-04-28T21:07:33.115978+0000 mgr.y (mgr.24961) 2583 : cluster [DBG] pgmap v2929: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:34 smithi139 bash[43301]: cluster 2022-04-28T21:07:33.115978+0000 mgr.y (mgr.24961) 2583 : cluster [DBG] pgmap v2929: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:35 smithi059 bash[61152]: cluster 2022-04-28T21:07:35.117339+0000 mgr.y (mgr.24961) 2584 : cluster [DBG] pgmap v2930: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:35 smithi059 bash[59745]: cluster 2022-04-28T21:07:35.117339+0000 mgr.y (mgr.24961) 2584 : cluster [DBG] pgmap v2930: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:35 smithi139 bash[43301]: cluster 2022-04-28T21:07:35.117339+0000 mgr.y (mgr.24961) 2584 : cluster [DBG] pgmap v2930: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:38 smithi059 bash[61152]: cluster 2022-04-28T21:07:37.118449+0000 mgr.y (mgr.24961) 2585 : cluster [DBG] pgmap v2931: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:38 smithi059 bash[59745]: cluster 2022-04-28T21:07:37.118449+0000 mgr.y (mgr.24961) 2585 : cluster [DBG] pgmap v2931: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:38 smithi139 bash[43301]: cluster 2022-04-28T21:07:37.118449+0000 mgr.y (mgr.24961) 2585 : cluster [DBG] pgmap v2931: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:40 smithi059 bash[61152]: cluster 2022-04-28T21:07:39.119530+0000 mgr.y (mgr.24961) 2586 : cluster [DBG] pgmap v2932: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:40 smithi059 bash[59745]: cluster 2022-04-28T21:07:39.119530+0000 mgr.y (mgr.24961) 2586 : cluster [DBG] pgmap v2932: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:40 smithi139 bash[43301]: cluster 2022-04-28T21:07:39.119530+0000 mgr.y (mgr.24961) 2586 : cluster [DBG] pgmap v2932: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:42.188 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:41.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:42.188 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:42 smithi059 bash[59745]: cluster 2022-04-28T21:07:41.120838+0000 mgr.y (mgr.24961) 2587 : cluster [DBG] pgmap v2933: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:42 smithi059 bash[61152]: cluster 2022-04-28T21:07:41.120838+0000 mgr.y (mgr.24961) 2587 : cluster [DBG] pgmap v2933: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:42 smithi139 bash[43301]: cluster 2022-04-28T21:07:41.120838+0000 mgr.y (mgr.24961) 2587 : cluster [DBG] pgmap v2933: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:43.819Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:44 smithi059 bash[61152]: cluster 2022-04-28T21:07:43.121547+0000 mgr.y (mgr.24961) 2588 : cluster [DBG] pgmap v2934: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:44 smithi059 bash[59745]: cluster 2022-04-28T21:07:43.121547+0000 mgr.y (mgr.24961) 2588 : cluster [DBG] pgmap v2934: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:44 smithi139 bash[43301]: cluster 2022-04-28T21:07:43.121547+0000 mgr.y (mgr.24961) 2588 : cluster [DBG] pgmap v2934: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:45 smithi139 bash[43301]: cluster 2022-04-28T21:07:45.122945+0000 mgr.y (mgr.24961) 2589 : cluster [DBG] pgmap v2935: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:45 smithi059 bash[61152]: cluster 2022-04-28T21:07:45.122945+0000 mgr.y (mgr.24961) 2589 : cluster [DBG] pgmap v2935: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:45 smithi059 bash[59745]: cluster 2022-04-28T21:07:45.122945+0000 mgr.y (mgr.24961) 2589 : cluster [DBG] pgmap v2935: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:48 smithi059 bash[61152]: cluster 2022-04-28T21:07:47.123944+0000 mgr.y (mgr.24961) 2590 : cluster [DBG] pgmap v2936: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:48.606 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:48 smithi059 bash[59745]: cluster 2022-04-28T21:07:47.123944+0000 mgr.y (mgr.24961) 2590 : cluster [DBG] pgmap v2936: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:48 smithi139 bash[43301]: cluster 2022-04-28T21:07:47.123944+0000 mgr.y (mgr.24961) 2590 : cluster [DBG] pgmap v2936: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:50 smithi059 bash[61152]: cluster 2022-04-28T21:07:49.124957+0000 mgr.y (mgr.24961) 2591 : cluster [DBG] pgmap v2937: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:50 smithi059 bash[59745]: cluster 2022-04-28T21:07:49.124957+0000 mgr.y (mgr.24961) 2591 : cluster [DBG] pgmap v2937: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:50 smithi139 bash[43301]: cluster 2022-04-28T21:07:49.124957+0000 mgr.y (mgr.24961) 2591 : cluster [DBG] pgmap v2937: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:52.198 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:51.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:52.199 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:52 smithi059 bash[59745]: cluster 2022-04-28T21:07:51.126275+0000 mgr.y (mgr.24961) 2592 : cluster [DBG] pgmap v2938: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:52.199 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:52 smithi059 bash[61152]: cluster 2022-04-28T21:07:51.126275+0000 mgr.y (mgr.24961) 2592 : cluster [DBG] pgmap v2938: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:52 smithi139 bash[43301]: cluster 2022-04-28T21:07:51.126275+0000 mgr.y (mgr.24961) 2592 : cluster [DBG] pgmap v2938: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:07:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:07:53.820Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:07:54.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:54 smithi059 bash[61152]: cluster 2022-04-28T21:07:53.126942+0000 mgr.y (mgr.24961) 2593 : cluster [DBG] pgmap v2939: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:54 smithi059 bash[59745]: cluster 2022-04-28T21:07:53.126942+0000 mgr.y (mgr.24961) 2593 : cluster [DBG] pgmap v2939: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:54 smithi139 bash[43301]: cluster 2022-04-28T21:07:53.126942+0000 mgr.y (mgr.24961) 2593 : cluster [DBG] pgmap v2939: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:55 smithi059 bash[61152]: cluster 2022-04-28T21:07:55.128441+0000 mgr.y (mgr.24961) 2594 : cluster [DBG] pgmap v2940: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:55.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:55 smithi059 bash[59745]: cluster 2022-04-28T21:07:55.128441+0000 mgr.y (mgr.24961) 2594 : cluster [DBG] pgmap v2940: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:55 smithi139 bash[43301]: cluster 2022-04-28T21:07:55.128441+0000 mgr.y (mgr.24961) 2594 : cluster [DBG] pgmap v2940: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:07:58 smithi059 bash[61152]: cluster 2022-04-28T21:07:57.129452+0000 mgr.y (mgr.24961) 2595 : cluster [DBG] pgmap v2941: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:07:58 smithi059 bash[59745]: cluster 2022-04-28T21:07:57.129452+0000 mgr.y (mgr.24961) 2595 : cluster [DBG] pgmap v2941: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:07:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:07:58 smithi139 bash[43301]: cluster 2022-04-28T21:07:57.129452+0000 mgr.y (mgr.24961) 2595 : cluster [DBG] pgmap v2941: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:00 smithi059 bash[59745]: cluster 2022-04-28T21:07:59.130538+0000 mgr.y (mgr.24961) 2596 : cluster [DBG] pgmap v2942: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:00 smithi059 bash[61152]: cluster 2022-04-28T21:07:59.130538+0000 mgr.y (mgr.24961) 2596 : cluster [DBG] pgmap v2942: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:00 smithi139 bash[43301]: cluster 2022-04-28T21:07:59.130538+0000 mgr.y (mgr.24961) 2596 : cluster [DBG] pgmap v2942: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:02.199 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:01.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:02 smithi059 bash[61152]: cluster 2022-04-28T21:08:01.132021+0000 mgr.y (mgr.24961) 2597 : cluster [DBG] pgmap v2943: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:02 smithi059 bash[59745]: cluster 2022-04-28T21:08:01.132021+0000 mgr.y (mgr.24961) 2597 : cluster [DBG] pgmap v2943: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:02 smithi139 bash[43301]: cluster 2022-04-28T21:08:01.132021+0000 mgr.y (mgr.24961) 2597 : cluster [DBG] pgmap v2943: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:03.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:04 smithi059 bash[59745]: cluster 2022-04-28T21:08:03.132758+0000 mgr.y (mgr.24961) 2598 : cluster [DBG] pgmap v2944: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:04.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:04 smithi059 bash[61152]: cluster 2022-04-28T21:08:03.132758+0000 mgr.y (mgr.24961) 2598 : cluster [DBG] pgmap v2944: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:04 smithi139 bash[43301]: cluster 2022-04-28T21:08:03.132758+0000 mgr.y (mgr.24961) 2598 : cluster [DBG] pgmap v2944: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:05 smithi139 bash[43301]: cluster 2022-04-28T21:08:05.134158+0000 mgr.y (mgr.24961) 2599 : cluster [DBG] pgmap v2945: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:05 smithi059 bash[59745]: cluster 2022-04-28T21:08:05.134158+0000 mgr.y (mgr.24961) 2599 : cluster [DBG] pgmap v2945: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:05 smithi059 bash[61152]: cluster 2022-04-28T21:08:05.134158+0000 mgr.y (mgr.24961) 2599 : cluster [DBG] pgmap v2945: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:08 smithi059 bash[61152]: cluster 2022-04-28T21:08:07.135272+0000 mgr.y (mgr.24961) 2600 : cluster [DBG] pgmap v2946: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:08 smithi059 bash[59745]: cluster 2022-04-28T21:08:07.135272+0000 mgr.y (mgr.24961) 2600 : cluster [DBG] pgmap v2946: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:08 smithi139 bash[43301]: cluster 2022-04-28T21:08:07.135272+0000 mgr.y (mgr.24961) 2600 : cluster [DBG] pgmap v2946: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:10 smithi059 bash[61152]: cluster 2022-04-28T21:08:09.136324+0000 mgr.y (mgr.24961) 2601 : cluster [DBG] pgmap v2947: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:10 smithi059 bash[59745]: cluster 2022-04-28T21:08:09.136324+0000 mgr.y (mgr.24961) 2601 : cluster [DBG] pgmap v2947: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:10 smithi139 bash[43301]: cluster 2022-04-28T21:08:09.136324+0000 mgr.y (mgr.24961) 2601 : cluster [DBG] pgmap v2947: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:12.203 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:11.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:12 smithi059 bash[61152]: cluster 2022-04-28T21:08:11.137781+0000 mgr.y (mgr.24961) 2602 : cluster [DBG] pgmap v2948: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:12 smithi059 bash[59745]: cluster 2022-04-28T21:08:11.137781+0000 mgr.y (mgr.24961) 2602 : cluster [DBG] pgmap v2948: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:12 smithi139 bash[43301]: cluster 2022-04-28T21:08:11.137781+0000 mgr.y (mgr.24961) 2602 : cluster [DBG] pgmap v2948: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:13.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:14 smithi059 bash[61152]: cluster 2022-04-28T21:08:13.138533+0000 mgr.y (mgr.24961) 2603 : cluster [DBG] pgmap v2949: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:14 smithi059 bash[59745]: cluster 2022-04-28T21:08:13.138533+0000 mgr.y (mgr.24961) 2603 : cluster [DBG] pgmap v2949: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:14.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:14 smithi139 bash[43301]: cluster 2022-04-28T21:08:13.138533+0000 mgr.y (mgr.24961) 2603 : cluster [DBG] pgmap v2949: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:15 smithi139 bash[43301]: audit 2022-04-28T21:08:14.965248+0000 mon.a (mon.0) 2488 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:08:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:15 smithi139 bash[43301]: audit 2022-04-28T21:08:14.968247+0000 mon.a (mon.0) 2489 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:08:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:15 smithi139 bash[43301]: audit 2022-04-28T21:08:14.970211+0000 mon.a (mon.0) 2490 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:08:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:15 smithi139 bash[43301]: cephadm 2022-04-28T21:08:14.971985+0000 mgr.y (mgr.24961) 2604 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:08:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:15 smithi139 bash[43301]: cluster 2022-04-28T21:08:15.139926+0000 mgr.y (mgr.24961) 2605 : cluster [DBG] pgmap v2950: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:15 smithi139 bash[43301]: audit 2022-04-28T21:08:15.290741+0000 mon.a (mon.0) 2491 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:08:15.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[59745]: audit 2022-04-28T21:08:14.965248+0000 mon.a (mon.0) 2488 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:08:15.932 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[59745]: audit 2022-04-28T21:08:14.968247+0000 mon.a (mon.0) 2489 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:08:15.932 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[59745]: audit 2022-04-28T21:08:14.970211+0000 mon.a (mon.0) 2490 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:08:15.932 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[59745]: cephadm 2022-04-28T21:08:14.971985+0000 mgr.y (mgr.24961) 2604 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:08:15.933 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[59745]: cluster 2022-04-28T21:08:15.139926+0000 mgr.y (mgr.24961) 2605 : cluster [DBG] pgmap v2950: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:15.933 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[59745]: audit 2022-04-28T21:08:15.290741+0000 mon.a (mon.0) 2491 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:08:15.933 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[61152]: audit 2022-04-28T21:08:14.965248+0000 mon.a (mon.0) 2488 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:08:15.933 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[61152]: audit 2022-04-28T21:08:14.968247+0000 mon.a (mon.0) 2489 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:08:15.933 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[61152]: audit 2022-04-28T21:08:14.970211+0000 mon.a (mon.0) 2490 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:08:15.934 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[61152]: cephadm 2022-04-28T21:08:14.971985+0000 mgr.y (mgr.24961) 2604 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:08:15.934 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[61152]: cluster 2022-04-28T21:08:15.139926+0000 mgr.y (mgr.24961) 2605 : cluster [DBG] pgmap v2950: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:15.934 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:15 smithi059 bash[61152]: audit 2022-04-28T21:08:15.290741+0000 mon.a (mon.0) 2491 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:08:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: cluster 2022-04-28T21:08:17.141089+0000 mgr.y (mgr.24961) 2606 : cluster [DBG] pgmap v2951: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: audit 2022-04-28T21:08:18.096669+0000 mon.a (mon.0) 2492 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:08:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: audit 2022-04-28T21:08:18.097407+0000 mon.a (mon.0) 2493 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:08:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: audit 2022-04-28T21:08:18.108850+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:08:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: audit 2022-04-28T21:08:18.109591+0000 mon.a (mon.0) 2495 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:08:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: audit 2022-04-28T21:08:18.110756+0000 mon.a (mon.0) 2496 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:08:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[61152]: audit 2022-04-28T21:08:18.111207+0000 mon.a (mon.0) 2497 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:08:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: cluster 2022-04-28T21:08:17.141089+0000 mgr.y (mgr.24961) 2606 : cluster [DBG] pgmap v2951: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: audit 2022-04-28T21:08:18.096669+0000 mon.a (mon.0) 2492 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:08:18.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: audit 2022-04-28T21:08:18.097407+0000 mon.a (mon.0) 2493 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:08:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: audit 2022-04-28T21:08:18.108850+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:08:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: audit 2022-04-28T21:08:18.109591+0000 mon.a (mon.0) 2495 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:08:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: audit 2022-04-28T21:08:18.110756+0000 mon.a (mon.0) 2496 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:08:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:18 smithi059 bash[59745]: audit 2022-04-28T21:08:18.111207+0000 mon.a (mon.0) 2497 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:08:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: cluster 2022-04-28T21:08:17.141089+0000 mgr.y (mgr.24961) 2606 : cluster [DBG] pgmap v2951: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: audit 2022-04-28T21:08:18.096669+0000 mon.a (mon.0) 2492 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:08:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: audit 2022-04-28T21:08:18.097407+0000 mon.a (mon.0) 2493 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:08:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: audit 2022-04-28T21:08:18.108850+0000 mon.a (mon.0) 2494 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:08:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: audit 2022-04-28T21:08:18.109591+0000 mon.a (mon.0) 2495 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:08:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: audit 2022-04-28T21:08:18.110756+0000 mon.a (mon.0) 2496 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:08:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:18 smithi139 bash[43301]: audit 2022-04-28T21:08:18.111207+0000 mon.a (mon.0) 2497 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:08:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:20 smithi059 bash[61152]: cluster 2022-04-28T21:08:19.142141+0000 mgr.y (mgr.24961) 2607 : cluster [DBG] pgmap v2952: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:20.687 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:20 smithi059 bash[59745]: cluster 2022-04-28T21:08:19.142141+0000 mgr.y (mgr.24961) 2607 : cluster [DBG] pgmap v2952: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:20.688 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:20 smithi139 bash[43301]: cluster 2022-04-28T21:08:19.142141+0000 mgr.y (mgr.24961) 2607 : cluster [DBG] pgmap v2952: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:22.210 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:21.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:22.211 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:22 smithi059 bash[59745]: cluster 2022-04-28T21:08:21.143525+0000 mgr.y (mgr.24961) 2608 : cluster [DBG] pgmap v2953: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:22 smithi059 bash[61152]: cluster 2022-04-28T21:08:21.143525+0000 mgr.y (mgr.24961) 2608 : cluster [DBG] pgmap v2953: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:22 smithi139 bash[43301]: cluster 2022-04-28T21:08:21.143525+0000 mgr.y (mgr.24961) 2608 : cluster [DBG] pgmap v2953: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:23.821Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:24 smithi059 bash[61152]: cluster 2022-04-28T21:08:23.144274+0000 mgr.y (mgr.24961) 2609 : cluster [DBG] pgmap v2954: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:24 smithi059 bash[59745]: cluster 2022-04-28T21:08:23.144274+0000 mgr.y (mgr.24961) 2609 : cluster [DBG] pgmap v2954: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:24 smithi139 bash[43301]: cluster 2022-04-28T21:08:23.144274+0000 mgr.y (mgr.24961) 2609 : cluster [DBG] pgmap v2954: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:25 smithi139 bash[43301]: cluster 2022-04-28T21:08:25.145655+0000 mgr.y (mgr.24961) 2610 : cluster [DBG] pgmap v2955: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:25 smithi059 bash[61152]: cluster 2022-04-28T21:08:25.145655+0000 mgr.y (mgr.24961) 2610 : cluster [DBG] pgmap v2955: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:25.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:25 smithi059 bash[59745]: cluster 2022-04-28T21:08:25.145655+0000 mgr.y (mgr.24961) 2610 : cluster [DBG] pgmap v2955: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:28 smithi059 bash[61152]: cluster 2022-04-28T21:08:27.146861+0000 mgr.y (mgr.24961) 2611 : cluster [DBG] pgmap v2956: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:28 smithi059 bash[59745]: cluster 2022-04-28T21:08:27.146861+0000 mgr.y (mgr.24961) 2611 : cluster [DBG] pgmap v2956: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:28 smithi139 bash[43301]: cluster 2022-04-28T21:08:27.146861+0000 mgr.y (mgr.24961) 2611 : cluster [DBG] pgmap v2956: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:30 smithi059 bash[61152]: cluster 2022-04-28T21:08:29.148022+0000 mgr.y (mgr.24961) 2612 : cluster [DBG] pgmap v2957: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:30 smithi059 bash[59745]: cluster 2022-04-28T21:08:29.148022+0000 mgr.y (mgr.24961) 2612 : cluster [DBG] pgmap v2957: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:30 smithi139 bash[43301]: cluster 2022-04-28T21:08:29.148022+0000 mgr.y (mgr.24961) 2612 : cluster [DBG] pgmap v2957: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:32.214 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:31.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:32 smithi059 bash[61152]: cluster 2022-04-28T21:08:31.149363+0000 mgr.y (mgr.24961) 2613 : cluster [DBG] pgmap v2958: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:32.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:32 smithi059 bash[59745]: cluster 2022-04-28T21:08:31.149363+0000 mgr.y (mgr.24961) 2613 : cluster [DBG] pgmap v2958: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:32 smithi139 bash[43301]: cluster 2022-04-28T21:08:31.149363+0000 mgr.y (mgr.24961) 2613 : cluster [DBG] pgmap v2958: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:33.822Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:34.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:34 smithi059 bash[61152]: cluster 2022-04-28T21:08:33.150154+0000 mgr.y (mgr.24961) 2614 : cluster [DBG] pgmap v2959: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:34.786 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:34 smithi059 bash[59745]: cluster 2022-04-28T21:08:33.150154+0000 mgr.y (mgr.24961) 2614 : cluster [DBG] pgmap v2959: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:34.787 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:34 smithi139 bash[43301]: cluster 2022-04-28T21:08:33.150154+0000 mgr.y (mgr.24961) 2614 : cluster [DBG] pgmap v2959: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:35 smithi139 bash[43301]: cluster 2022-04-28T21:08:35.151598+0000 mgr.y (mgr.24961) 2615 : cluster [DBG] pgmap v2960: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:35.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:35 smithi059 bash[61152]: cluster 2022-04-28T21:08:35.151598+0000 mgr.y (mgr.24961) 2615 : cluster [DBG] pgmap v2960: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:35 smithi059 bash[59745]: cluster 2022-04-28T21:08:35.151598+0000 mgr.y (mgr.24961) 2615 : cluster [DBG] pgmap v2960: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:38 smithi059 bash[61152]: cluster 2022-04-28T21:08:37.152566+0000 mgr.y (mgr.24961) 2616 : cluster [DBG] pgmap v2961: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:38 smithi059 bash[59745]: cluster 2022-04-28T21:08:37.152566+0000 mgr.y (mgr.24961) 2616 : cluster [DBG] pgmap v2961: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:38 smithi139 bash[43301]: cluster 2022-04-28T21:08:37.152566+0000 mgr.y (mgr.24961) 2616 : cluster [DBG] pgmap v2961: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:40 smithi059 bash[61152]: cluster 2022-04-28T21:08:39.153597+0000 mgr.y (mgr.24961) 2617 : cluster [DBG] pgmap v2962: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:40 smithi059 bash[59745]: cluster 2022-04-28T21:08:39.153597+0000 mgr.y (mgr.24961) 2617 : cluster [DBG] pgmap v2962: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:40 smithi139 bash[43301]: cluster 2022-04-28T21:08:39.153597+0000 mgr.y (mgr.24961) 2617 : cluster [DBG] pgmap v2962: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:42.220 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:41.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:42 smithi059 bash[61152]: cluster 2022-04-28T21:08:41.154913+0000 mgr.y (mgr.24961) 2618 : cluster [DBG] pgmap v2963: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:42 smithi059 bash[59745]: cluster 2022-04-28T21:08:41.154913+0000 mgr.y (mgr.24961) 2618 : cluster [DBG] pgmap v2963: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:42 smithi139 bash[43301]: cluster 2022-04-28T21:08:41.154913+0000 mgr.y (mgr.24961) 2618 : cluster [DBG] pgmap v2963: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:43.824Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:44 smithi059 bash[61152]: cluster 2022-04-28T21:08:43.155674+0000 mgr.y (mgr.24961) 2619 : cluster [DBG] pgmap v2964: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:44 smithi059 bash[59745]: cluster 2022-04-28T21:08:43.155674+0000 mgr.y (mgr.24961) 2619 : cluster [DBG] pgmap v2964: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:44 smithi139 bash[43301]: cluster 2022-04-28T21:08:43.155674+0000 mgr.y (mgr.24961) 2619 : cluster [DBG] pgmap v2964: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:45 smithi139 bash[43301]: cluster 2022-04-28T21:08:45.157048+0000 mgr.y (mgr.24961) 2620 : cluster [DBG] pgmap v2965: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:45 smithi059 bash[61152]: cluster 2022-04-28T21:08:45.157048+0000 mgr.y (mgr.24961) 2620 : cluster [DBG] pgmap v2965: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:45 smithi059 bash[59745]: cluster 2022-04-28T21:08:45.157048+0000 mgr.y (mgr.24961) 2620 : cluster [DBG] pgmap v2965: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:48 smithi059 bash[61152]: cluster 2022-04-28T21:08:47.157993+0000 mgr.y (mgr.24961) 2621 : cluster [DBG] pgmap v2966: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:48 smithi059 bash[59745]: cluster 2022-04-28T21:08:47.157993+0000 mgr.y (mgr.24961) 2621 : cluster [DBG] pgmap v2966: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:48 smithi139 bash[43301]: cluster 2022-04-28T21:08:47.157993+0000 mgr.y (mgr.24961) 2621 : cluster [DBG] pgmap v2966: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:50 smithi059 bash[61152]: cluster 2022-04-28T21:08:49.159113+0000 mgr.y (mgr.24961) 2622 : cluster [DBG] pgmap v2967: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:50.674 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:50 smithi059 bash[59745]: cluster 2022-04-28T21:08:49.159113+0000 mgr.y (mgr.24961) 2622 : cluster [DBG] pgmap v2967: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:50.675 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:50 smithi139 bash[43301]: cluster 2022-04-28T21:08:49.159113+0000 mgr.y (mgr.24961) 2622 : cluster [DBG] pgmap v2967: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:52.225 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:51.931Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:52.226 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:52 smithi059 bash[59745]: cluster 2022-04-28T21:08:51.160606+0000 mgr.y (mgr.24961) 2623 : cluster [DBG] pgmap v2968: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:52 smithi059 bash[61152]: cluster 2022-04-28T21:08:51.160606+0000 mgr.y (mgr.24961) 2623 : cluster [DBG] pgmap v2968: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:52.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:52 smithi139 bash[43301]: cluster 2022-04-28T21:08:51.160606+0000 mgr.y (mgr.24961) 2623 : cluster [DBG] pgmap v2968: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:08:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:08:53.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:08:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:54 smithi059 bash[61152]: cluster 2022-04-28T21:08:53.161436+0000 mgr.y (mgr.24961) 2624 : cluster [DBG] pgmap v2969: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:54 smithi059 bash[59745]: cluster 2022-04-28T21:08:53.161436+0000 mgr.y (mgr.24961) 2624 : cluster [DBG] pgmap v2969: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:54 smithi139 bash[43301]: cluster 2022-04-28T21:08:53.161436+0000 mgr.y (mgr.24961) 2624 : cluster [DBG] pgmap v2969: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:55 smithi139 bash[43301]: cluster 2022-04-28T21:08:55.162929+0000 mgr.y (mgr.24961) 2625 : cluster [DBG] pgmap v2970: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:55 smithi059 bash[61152]: cluster 2022-04-28T21:08:55.162929+0000 mgr.y (mgr.24961) 2625 : cluster [DBG] pgmap v2970: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:55 smithi059 bash[59745]: cluster 2022-04-28T21:08:55.162929+0000 mgr.y (mgr.24961) 2625 : cluster [DBG] pgmap v2970: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:08:58 smithi059 bash[61152]: cluster 2022-04-28T21:08:57.163904+0000 mgr.y (mgr.24961) 2626 : cluster [DBG] pgmap v2971: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:08:58 smithi059 bash[59745]: cluster 2022-04-28T21:08:57.163904+0000 mgr.y (mgr.24961) 2626 : cluster [DBG] pgmap v2971: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:08:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:08:58 smithi139 bash[43301]: cluster 2022-04-28T21:08:57.163904+0000 mgr.y (mgr.24961) 2626 : cluster [DBG] pgmap v2971: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:00 smithi059 bash[61152]: cluster 2022-04-28T21:08:59.164906+0000 mgr.y (mgr.24961) 2627 : cluster [DBG] pgmap v2972: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:00 smithi059 bash[59745]: cluster 2022-04-28T21:08:59.164906+0000 mgr.y (mgr.24961) 2627 : cluster [DBG] pgmap v2972: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:00 smithi139 bash[43301]: cluster 2022-04-28T21:08:59.164906+0000 mgr.y (mgr.24961) 2627 : cluster [DBG] pgmap v2972: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:02.232 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:01.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:02 smithi059 bash[61152]: cluster 2022-04-28T21:09:01.166250+0000 mgr.y (mgr.24961) 2628 : cluster [DBG] pgmap v2973: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:02 smithi059 bash[59745]: cluster 2022-04-28T21:09:01.166250+0000 mgr.y (mgr.24961) 2628 : cluster [DBG] pgmap v2973: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:02 smithi139 bash[43301]: cluster 2022-04-28T21:09:01.166250+0000 mgr.y (mgr.24961) 2628 : cluster [DBG] pgmap v2973: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:03.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:04 smithi059 bash[61152]: cluster 2022-04-28T21:09:03.167052+0000 mgr.y (mgr.24961) 2629 : cluster [DBG] pgmap v2974: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:04.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:04 smithi059 bash[59745]: cluster 2022-04-28T21:09:03.167052+0000 mgr.y (mgr.24961) 2629 : cluster [DBG] pgmap v2974: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:04 smithi139 bash[43301]: cluster 2022-04-28T21:09:03.167052+0000 mgr.y (mgr.24961) 2629 : cluster [DBG] pgmap v2974: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:05 smithi059 bash[61152]: cluster 2022-04-28T21:09:05.168575+0000 mgr.y (mgr.24961) 2630 : cluster [DBG] pgmap v2975: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:05 smithi059 bash[59745]: cluster 2022-04-28T21:09:05.168575+0000 mgr.y (mgr.24961) 2630 : cluster [DBG] pgmap v2975: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:05 smithi139 bash[43301]: cluster 2022-04-28T21:09:05.168575+0000 mgr.y (mgr.24961) 2630 : cluster [DBG] pgmap v2975: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:08 smithi059 bash[61152]: cluster 2022-04-28T21:09:07.169599+0000 mgr.y (mgr.24961) 2631 : cluster [DBG] pgmap v2976: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:08 smithi059 bash[59745]: cluster 2022-04-28T21:09:07.169599+0000 mgr.y (mgr.24961) 2631 : cluster [DBG] pgmap v2976: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:08 smithi139 bash[43301]: cluster 2022-04-28T21:09:07.169599+0000 mgr.y (mgr.24961) 2631 : cluster [DBG] pgmap v2976: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:10 smithi059 bash[61152]: cluster 2022-04-28T21:09:09.170618+0000 mgr.y (mgr.24961) 2632 : cluster [DBG] pgmap v2977: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:10 smithi059 bash[59745]: cluster 2022-04-28T21:09:09.170618+0000 mgr.y (mgr.24961) 2632 : cluster [DBG] pgmap v2977: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:10 smithi139 bash[43301]: cluster 2022-04-28T21:09:09.170618+0000 mgr.y (mgr.24961) 2632 : cluster [DBG] pgmap v2977: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:12.237 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:11.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:12.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:12 smithi059 bash[61152]: cluster 2022-04-28T21:09:11.171984+0000 mgr.y (mgr.24961) 2633 : cluster [DBG] pgmap v2978: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:12.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:12 smithi059 bash[59745]: cluster 2022-04-28T21:09:11.171984+0000 mgr.y (mgr.24961) 2633 : cluster [DBG] pgmap v2978: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:12 smithi139 bash[43301]: cluster 2022-04-28T21:09:11.171984+0000 mgr.y (mgr.24961) 2633 : cluster [DBG] pgmap v2978: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:13.825Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:14.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:14 smithi059 bash[59745]: cluster 2022-04-28T21:09:13.172791+0000 mgr.y (mgr.24961) 2634 : cluster [DBG] pgmap v2979: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:14.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:14 smithi059 bash[61152]: cluster 2022-04-28T21:09:13.172791+0000 mgr.y (mgr.24961) 2634 : cluster [DBG] pgmap v2979: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:14.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:14 smithi139 bash[43301]: cluster 2022-04-28T21:09:13.172791+0000 mgr.y (mgr.24961) 2634 : cluster [DBG] pgmap v2979: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:15.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[59745]: cluster 2022-04-28T21:09:15.174187+0000 mgr.y (mgr.24961) 2635 : cluster [DBG] pgmap v2980: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[59745]: audit 2022-04-28T21:09:15.299618+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:09:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[59745]: audit 2022-04-28T21:09:15.304121+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:09:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[59745]: audit 2022-04-28T21:09:15.305699+0000 mon.a (mon.0) 2500 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:09:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[61152]: cluster 2022-04-28T21:09:15.174187+0000 mgr.y (mgr.24961) 2635 : cluster [DBG] pgmap v2980: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:15.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[61152]: audit 2022-04-28T21:09:15.299618+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:09:15.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[61152]: audit 2022-04-28T21:09:15.304121+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:09:15.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:15 smithi059 bash[61152]: audit 2022-04-28T21:09:15.305699+0000 mon.a (mon.0) 2500 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:09:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:15 smithi139 bash[43301]: cluster 2022-04-28T21:09:15.174187+0000 mgr.y (mgr.24961) 2635 : cluster [DBG] pgmap v2980: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:15 smithi139 bash[43301]: audit 2022-04-28T21:09:15.299618+0000 mon.a (mon.0) 2498 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:09:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:15 smithi139 bash[43301]: audit 2022-04-28T21:09:15.304121+0000 mon.a (mon.0) 2499 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:09:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:15 smithi139 bash[43301]: audit 2022-04-28T21:09:15.305699+0000 mon.a (mon.0) 2500 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:09:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:16 smithi139 bash[43301]: cephadm 2022-04-28T21:09:15.307136+0000 mgr.y (mgr.24961) 2636 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:09:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:16 smithi139 bash[43301]: audit 2022-04-28T21:09:15.598177+0000 mon.a (mon.0) 2501 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:16 smithi139 bash[43301]: audit 2022-04-28T21:09:15.607776+0000 mon.a (mon.0) 2502 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:16 smithi139 bash[43301]: audit 2022-04-28T21:09:15.917106+0000 mon.a (mon.0) 2503 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[59745]: cephadm 2022-04-28T21:09:15.307136+0000 mgr.y (mgr.24961) 2636 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:09:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[59745]: audit 2022-04-28T21:09:15.598177+0000 mon.a (mon.0) 2501 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[59745]: audit 2022-04-28T21:09:15.607776+0000 mon.a (mon.0) 2502 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[59745]: audit 2022-04-28T21:09:15.917106+0000 mon.a (mon.0) 2503 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[61152]: cephadm 2022-04-28T21:09:15.307136+0000 mgr.y (mgr.24961) 2636 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:09:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[61152]: audit 2022-04-28T21:09:15.598177+0000 mon.a (mon.0) 2501 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[61152]: audit 2022-04-28T21:09:15.607776+0000 mon.a (mon.0) 2502 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:16 smithi059 bash[61152]: audit 2022-04-28T21:09:15.917106+0000 mon.a (mon.0) 2503 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:09:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:17 smithi139 bash[43301]: cluster 2022-04-28T21:09:17.175148+0000 mgr.y (mgr.24961) 2637 : cluster [DBG] pgmap v2981: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:17 smithi059 bash[59745]: cluster 2022-04-28T21:09:17.175148+0000 mgr.y (mgr.24961) 2637 : cluster [DBG] pgmap v2981: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:17 smithi059 bash[61152]: cluster 2022-04-28T21:09:17.175148+0000 mgr.y (mgr.24961) 2637 : cluster [DBG] pgmap v2981: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:18 smithi139 bash[43301]: audit 2022-04-28T21:09:18.111648+0000 mon.a (mon.0) 2504 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:09:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:18 smithi139 bash[43301]: audit 2022-04-28T21:09:18.112159+0000 mon.a (mon.0) 2505 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:18 smithi139 bash[43301]: audit 2022-04-28T21:09:18.122614+0000 mon.a (mon.0) 2506 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:18 smithi139 bash[43301]: audit 2022-04-28T21:09:18.123444+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:18 smithi139 bash[43301]: audit 2022-04-28T21:09:18.124237+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:18 smithi139 bash[43301]: audit 2022-04-28T21:09:18.124968+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]: dispatch 2022-04-28T21:09:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[61152]: audit 2022-04-28T21:09:18.111648+0000 mon.a (mon.0) 2504 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:09:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[61152]: audit 2022-04-28T21:09:18.112159+0000 mon.a (mon.0) 2505 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:09:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[61152]: audit 2022-04-28T21:09:18.122614+0000 mon.a (mon.0) 2506 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:09:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[61152]: audit 2022-04-28T21:09:18.123444+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:09:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[61152]: audit 2022-04-28T21:09:18.124237+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:09:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[61152]: audit 2022-04-28T21:09:18.124968+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]: dispatch 2022-04-28T21:09:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[59745]: audit 2022-04-28T21:09:18.111648+0000 mon.a (mon.0) 2504 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:09:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[59745]: audit 2022-04-28T21:09:18.112159+0000 mon.a (mon.0) 2505 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:09:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[59745]: audit 2022-04-28T21:09:18.122614+0000 mon.a (mon.0) 2506 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:09:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[59745]: audit 2022-04-28T21:09:18.123444+0000 mon.a (mon.0) 2507 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:09:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[59745]: audit 2022-04-28T21:09:18.124237+0000 mon.a (mon.0) 2508 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:09:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:18 smithi059 bash[59745]: audit 2022-04-28T21:09:18.124968+0000 mon.a (mon.0) 2509 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]: dispatch 2022-04-28T21:09:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:19 smithi059 bash[61152]: cluster 2022-04-28T21:09:19.176192+0000 mgr.y (mgr.24961) 2638 : cluster [DBG] pgmap v2982: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:19 smithi059 bash[59745]: cluster 2022-04-28T21:09:19.176192+0000 mgr.y (mgr.24961) 2638 : cluster [DBG] pgmap v2982: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:20.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:19 smithi139 bash[43301]: cluster 2022-04-28T21:09:19.176192+0000 mgr.y (mgr.24961) 2638 : cluster [DBG] pgmap v2982: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:22.235 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:21.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:22 smithi059 bash[61152]: cluster 2022-04-28T21:09:21.177549+0000 mgr.y (mgr.24961) 2639 : cluster [DBG] pgmap v2983: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:22 smithi059 bash[59745]: cluster 2022-04-28T21:09:21.177549+0000 mgr.y (mgr.24961) 2639 : cluster [DBG] pgmap v2983: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:22 smithi139 bash[43301]: cluster 2022-04-28T21:09:21.177549+0000 mgr.y (mgr.24961) 2639 : cluster [DBG] pgmap v2983: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:23.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:24 smithi059 bash[59745]: cluster 2022-04-28T21:09:23.178261+0000 mgr.y (mgr.24961) 2640 : cluster [DBG] pgmap v2984: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:24.627 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:24 smithi059 bash[61152]: cluster 2022-04-28T21:09:23.178261+0000 mgr.y (mgr.24961) 2640 : cluster [DBG] pgmap v2984: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:24.628 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:24 smithi139 bash[43301]: cluster 2022-04-28T21:09:23.178261+0000 mgr.y (mgr.24961) 2640 : cluster [DBG] pgmap v2984: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:25 smithi059 bash[61152]: cluster 2022-04-28T21:09:25.179659+0000 mgr.y (mgr.24961) 2641 : cluster [DBG] pgmap v2985: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:25.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:25 smithi059 bash[59745]: cluster 2022-04-28T21:09:25.179659+0000 mgr.y (mgr.24961) 2641 : cluster [DBG] pgmap v2985: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:25.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:25 smithi139 bash[43301]: cluster 2022-04-28T21:09:25.179659+0000 mgr.y (mgr.24961) 2641 : cluster [DBG] pgmap v2985: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:28 smithi059 bash[61152]: cluster 2022-04-28T21:09:27.180649+0000 mgr.y (mgr.24961) 2642 : cluster [DBG] pgmap v2986: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:28 smithi059 bash[59745]: cluster 2022-04-28T21:09:27.180649+0000 mgr.y (mgr.24961) 2642 : cluster [DBG] pgmap v2986: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:28 smithi139 bash[43301]: cluster 2022-04-28T21:09:27.180649+0000 mgr.y (mgr.24961) 2642 : cluster [DBG] pgmap v2986: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:30 smithi059 bash[61152]: cluster 2022-04-28T21:09:29.181725+0000 mgr.y (mgr.24961) 2643 : cluster [DBG] pgmap v2987: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:30 smithi059 bash[59745]: cluster 2022-04-28T21:09:29.181725+0000 mgr.y (mgr.24961) 2643 : cluster [DBG] pgmap v2987: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:30 smithi139 bash[43301]: cluster 2022-04-28T21:09:29.181725+0000 mgr.y (mgr.24961) 2643 : cluster [DBG] pgmap v2987: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:32.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:32 smithi059 bash[59745]: cluster 2022-04-28T21:09:31.182917+0000 mgr.y (mgr.24961) 2644 : cluster [DBG] pgmap v2988: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:32.250 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:31.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:32 smithi059 bash[61152]: cluster 2022-04-28T21:09:31.182917+0000 mgr.y (mgr.24961) 2644 : cluster [DBG] pgmap v2988: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:32 smithi139 bash[43301]: cluster 2022-04-28T21:09:31.182917+0000 mgr.y (mgr.24961) 2644 : cluster [DBG] pgmap v2988: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:33.826Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:34.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:34 smithi059 bash[61152]: cluster 2022-04-28T21:09:33.183643+0000 mgr.y (mgr.24961) 2645 : cluster [DBG] pgmap v2989: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:34.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:34 smithi059 bash[59745]: cluster 2022-04-28T21:09:33.183643+0000 mgr.y (mgr.24961) 2645 : cluster [DBG] pgmap v2989: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:34 smithi139 bash[43301]: cluster 2022-04-28T21:09:33.183643+0000 mgr.y (mgr.24961) 2645 : cluster [DBG] pgmap v2989: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:35 smithi059 bash[61152]: cluster 2022-04-28T21:09:35.184998+0000 mgr.y (mgr.24961) 2646 : cluster [DBG] pgmap v2990: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:35 smithi059 bash[59745]: cluster 2022-04-28T21:09:35.184998+0000 mgr.y (mgr.24961) 2646 : cluster [DBG] pgmap v2990: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:35 smithi139 bash[43301]: cluster 2022-04-28T21:09:35.184998+0000 mgr.y (mgr.24961) 2646 : cluster [DBG] pgmap v2990: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:38 smithi059 bash[61152]: cluster 2022-04-28T21:09:37.186306+0000 mgr.y (mgr.24961) 2647 : cluster [DBG] pgmap v2991: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:38 smithi059 bash[59745]: cluster 2022-04-28T21:09:37.186306+0000 mgr.y (mgr.24961) 2647 : cluster [DBG] pgmap v2991: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:38 smithi139 bash[43301]: cluster 2022-04-28T21:09:37.186306+0000 mgr.y (mgr.24961) 2647 : cluster [DBG] pgmap v2991: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:40 smithi059 bash[61152]: cluster 2022-04-28T21:09:39.187448+0000 mgr.y (mgr.24961) 2648 : cluster [DBG] pgmap v2992: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:40 smithi059 bash[59745]: cluster 2022-04-28T21:09:39.187448+0000 mgr.y (mgr.24961) 2648 : cluster [DBG] pgmap v2992: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:40 smithi139 bash[43301]: cluster 2022-04-28T21:09:39.187448+0000 mgr.y (mgr.24961) 2648 : cluster [DBG] pgmap v2992: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:42.255 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:41.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:42 smithi059 bash[61152]: cluster 2022-04-28T21:09:41.188781+0000 mgr.y (mgr.24961) 2649 : cluster [DBG] pgmap v2993: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:42 smithi059 bash[59745]: cluster 2022-04-28T21:09:41.188781+0000 mgr.y (mgr.24961) 2649 : cluster [DBG] pgmap v2993: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:42 smithi139 bash[43301]: cluster 2022-04-28T21:09:41.188781+0000 mgr.y (mgr.24961) 2649 : cluster [DBG] pgmap v2993: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:43.828Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:44 smithi059 bash[61152]: cluster 2022-04-28T21:09:43.189482+0000 mgr.y (mgr.24961) 2650 : cluster [DBG] pgmap v2994: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:44 smithi059 bash[59745]: cluster 2022-04-28T21:09:43.189482+0000 mgr.y (mgr.24961) 2650 : cluster [DBG] pgmap v2994: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:44 smithi139 bash[43301]: cluster 2022-04-28T21:09:43.189482+0000 mgr.y (mgr.24961) 2650 : cluster [DBG] pgmap v2994: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:45 smithi059 bash[61152]: cluster 2022-04-28T21:09:45.190878+0000 mgr.y (mgr.24961) 2651 : cluster [DBG] pgmap v2995: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:45 smithi059 bash[59745]: cluster 2022-04-28T21:09:45.190878+0000 mgr.y (mgr.24961) 2651 : cluster [DBG] pgmap v2995: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:45 smithi139 bash[43301]: cluster 2022-04-28T21:09:45.190878+0000 mgr.y (mgr.24961) 2651 : cluster [DBG] pgmap v2995: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:48 smithi059 bash[61152]: cluster 2022-04-28T21:09:47.191953+0000 mgr.y (mgr.24961) 2652 : cluster [DBG] pgmap v2996: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:48 smithi059 bash[59745]: cluster 2022-04-28T21:09:47.191953+0000 mgr.y (mgr.24961) 2652 : cluster [DBG] pgmap v2996: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:48 smithi139 bash[43301]: cluster 2022-04-28T21:09:47.191953+0000 mgr.y (mgr.24961) 2652 : cluster [DBG] pgmap v2996: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:50 smithi059 bash[61152]: cluster 2022-04-28T21:09:49.193040+0000 mgr.y (mgr.24961) 2653 : cluster [DBG] pgmap v2997: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:50 smithi059 bash[59745]: cluster 2022-04-28T21:09:49.193040+0000 mgr.y (mgr.24961) 2653 : cluster [DBG] pgmap v2997: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:50 smithi139 bash[43301]: cluster 2022-04-28T21:09:49.193040+0000 mgr.y (mgr.24961) 2653 : cluster [DBG] pgmap v2997: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:52.261 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:51.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:52 smithi059 bash[61152]: cluster 2022-04-28T21:09:51.194309+0000 mgr.y (mgr.24961) 2654 : cluster [DBG] pgmap v2998: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:52 smithi059 bash[59745]: cluster 2022-04-28T21:09:51.194309+0000 mgr.y (mgr.24961) 2654 : cluster [DBG] pgmap v2998: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:52.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:52 smithi139 bash[43301]: cluster 2022-04-28T21:09:51.194309+0000 mgr.y (mgr.24961) 2654 : cluster [DBG] pgmap v2998: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:09:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:09:53.827Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:09:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:54 smithi059 bash[61152]: cluster 2022-04-28T21:09:53.194980+0000 mgr.y (mgr.24961) 2655 : cluster [DBG] pgmap v2999: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:54.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:54 smithi059 bash[59745]: cluster 2022-04-28T21:09:53.194980+0000 mgr.y (mgr.24961) 2655 : cluster [DBG] pgmap v2999: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:54 smithi139 bash[43301]: cluster 2022-04-28T21:09:53.194980+0000 mgr.y (mgr.24961) 2655 : cluster [DBG] pgmap v2999: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:55 smithi059 bash[61152]: cluster 2022-04-28T21:09:55.196378+0000 mgr.y (mgr.24961) 2656 : cluster [DBG] pgmap v3000: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:55 smithi059 bash[59745]: cluster 2022-04-28T21:09:55.196378+0000 mgr.y (mgr.24961) 2656 : cluster [DBG] pgmap v3000: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:55 smithi139 bash[43301]: cluster 2022-04-28T21:09:55.196378+0000 mgr.y (mgr.24961) 2656 : cluster [DBG] pgmap v3000: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:56 smithi059 bash[59745]: audit 2022-04-28T21:09:56.739135+0000 mon.c (mon.1) 33 : audit [DBG] from='client.? 172.21.15.59:0/1030352395' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T21:09:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:56 smithi059 bash[61152]: audit 2022-04-28T21:09:56.739135+0000 mon.c (mon.1) 33 : audit [DBG] from='client.? 172.21.15.59:0/1030352395' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T21:09:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:56 smithi139 bash[43301]: audit 2022-04-28T21:09:56.739135+0000 mon.c (mon.1) 33 : audit [DBG] from='client.? 172.21.15.59:0/1030352395' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T21:09:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:09:57 smithi059 bash[61152]: cluster 2022-04-28T21:09:57.197385+0000 mgr.y (mgr.24961) 2657 : cluster [DBG] pgmap v3001: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:09:57 smithi059 bash[59745]: cluster 2022-04-28T21:09:57.197385+0000 mgr.y (mgr.24961) 2657 : cluster [DBG] pgmap v3001: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:09:58.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:09:57 smithi139 bash[43301]: cluster 2022-04-28T21:09:57.197385+0000 mgr.y (mgr.24961) 2657 : cluster [DBG] pgmap v3001: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:00 smithi059 bash[61152]: cluster 2022-04-28T21:09:59.198481+0000 mgr.y (mgr.24961) 2658 : cluster [DBG] pgmap v3002: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:00 smithi059 bash[61152]: cluster 2022-04-28T21:10:00.000165+0000 mon.a (mon.0) 2510 : cluster [INF] overall HEALTH_OK 2022-04-28T21:10:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:00 smithi059 bash[59745]: cluster 2022-04-28T21:09:59.198481+0000 mgr.y (mgr.24961) 2658 : cluster [DBG] pgmap v3002: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:00 smithi059 bash[59745]: cluster 2022-04-28T21:10:00.000165+0000 mon.a (mon.0) 2510 : cluster [INF] overall HEALTH_OK 2022-04-28T21:10:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:00 smithi139 bash[43301]: cluster 2022-04-28T21:09:59.198481+0000 mgr.y (mgr.24961) 2658 : cluster [DBG] pgmap v3002: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:00 smithi139 bash[43301]: cluster 2022-04-28T21:10:00.000165+0000 mon.a (mon.0) 2510 : cluster [INF] overall HEALTH_OK 2022-04-28T21:10:02.262 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:01.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:02.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:02 smithi059 bash[59745]: cluster 2022-04-28T21:10:01.199616+0000 mgr.y (mgr.24961) 2659 : cluster [DBG] pgmap v3003: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:02 smithi059 bash[61152]: cluster 2022-04-28T21:10:01.199616+0000 mgr.y (mgr.24961) 2659 : cluster [DBG] pgmap v3003: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:02.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:02 smithi139 bash[43301]: cluster 2022-04-28T21:10:01.199616+0000 mgr.y (mgr.24961) 2659 : cluster [DBG] pgmap v3003: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:03.829Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:04 smithi059 bash[61152]: cluster 2022-04-28T21:10:03.200324+0000 mgr.y (mgr.24961) 2660 : cluster [DBG] pgmap v3004: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:04.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:04 smithi059 bash[59745]: cluster 2022-04-28T21:10:03.200324+0000 mgr.y (mgr.24961) 2660 : cluster [DBG] pgmap v3004: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:04 smithi139 bash[43301]: cluster 2022-04-28T21:10:03.200324+0000 mgr.y (mgr.24961) 2660 : cluster [DBG] pgmap v3004: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:05 smithi059 bash[59745]: cluster 2022-04-28T21:10:05.201686+0000 mgr.y (mgr.24961) 2661 : cluster [DBG] pgmap v3005: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:05 smithi059 bash[61152]: cluster 2022-04-28T21:10:05.201686+0000 mgr.y (mgr.24961) 2661 : cluster [DBG] pgmap v3005: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:05 smithi139 bash[43301]: cluster 2022-04-28T21:10:05.201686+0000 mgr.y (mgr.24961) 2661 : cluster [DBG] pgmap v3005: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:08.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:08 smithi059 bash[61152]: cluster 2022-04-28T21:10:07.202676+0000 mgr.y (mgr.24961) 2662 : cluster [DBG] pgmap v3006: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:08 smithi059 bash[59745]: cluster 2022-04-28T21:10:07.202676+0000 mgr.y (mgr.24961) 2662 : cluster [DBG] pgmap v3006: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:08.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:08 smithi139 bash[43301]: cluster 2022-04-28T21:10:07.202676+0000 mgr.y (mgr.24961) 2662 : cluster [DBG] pgmap v3006: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:10 smithi059 bash[61152]: cluster 2022-04-28T21:10:09.203904+0000 mgr.y (mgr.24961) 2663 : cluster [DBG] pgmap v3007: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:10 smithi059 bash[59745]: cluster 2022-04-28T21:10:09.203904+0000 mgr.y (mgr.24961) 2663 : cluster [DBG] pgmap v3007: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:10.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:10 smithi139 bash[43301]: cluster 2022-04-28T21:10:09.203904+0000 mgr.y (mgr.24961) 2663 : cluster [DBG] pgmap v3007: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:12.273 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:11.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:12.276 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:12 smithi059 bash[59745]: cluster 2022-04-28T21:10:11.205159+0000 mgr.y (mgr.24961) 2664 : cluster [DBG] pgmap v3008: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:12 smithi059 bash[61152]: cluster 2022-04-28T21:10:11.205159+0000 mgr.y (mgr.24961) 2664 : cluster [DBG] pgmap v3008: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:12 smithi139 bash[43301]: cluster 2022-04-28T21:10:11.205159+0000 mgr.y (mgr.24961) 2664 : cluster [DBG] pgmap v3008: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:13 smithi059 bash[61152]: cluster 2022-04-28T21:10:13.205884+0000 mgr.y (mgr.24961) 2665 : cluster [DBG] pgmap v3009: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:13 smithi059 bash[59745]: cluster 2022-04-28T21:10:13.205884+0000 mgr.y (mgr.24961) 2665 : cluster [DBG] pgmap v3009: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:13 smithi139 bash[43301]: cluster 2022-04-28T21:10:13.205884+0000 mgr.y (mgr.24961) 2665 : cluster [DBG] pgmap v3009: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:13.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:15 smithi059 bash[61152]: cluster 2022-04-28T21:10:15.207664+0000 mgr.y (mgr.24961) 2666 : cluster [DBG] pgmap v3010: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:15.905 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:15 smithi059 bash[59745]: cluster 2022-04-28T21:10:15.207664+0000 mgr.y (mgr.24961) 2666 : cluster [DBG] pgmap v3010: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:15.906 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:15 smithi139 bash[43301]: cluster 2022-04-28T21:10:15.207664+0000 mgr.y (mgr.24961) 2666 : cluster [DBG] pgmap v3010: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[59745]: audit 2022-04-28T21:10:15.923686+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:10:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[59745]: audit 2022-04-28T21:10:15.926893+0000 mon.a (mon.0) 2512 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:10:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[59745]: audit 2022-04-28T21:10:15.928898+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:10:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[59745]: cephadm 2022-04-28T21:10:15.930648+0000 mgr.y (mgr.24961) 2667 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:10:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[59745]: audit 2022-04-28T21:10:15.933212+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:10:16.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[61152]: audit 2022-04-28T21:10:15.923686+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:10:16.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[61152]: audit 2022-04-28T21:10:15.926893+0000 mon.a (mon.0) 2512 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:10:16.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[61152]: audit 2022-04-28T21:10:15.928898+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:10:16.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[61152]: cephadm 2022-04-28T21:10:15.930648+0000 mgr.y (mgr.24961) 2667 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:10:16.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:16 smithi059 bash[61152]: audit 2022-04-28T21:10:15.933212+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:10:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:16 smithi139 bash[43301]: audit 2022-04-28T21:10:15.923686+0000 mon.a (mon.0) 2511 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:10:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:16 smithi139 bash[43301]: audit 2022-04-28T21:10:15.926893+0000 mon.a (mon.0) 2512 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:10:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:16 smithi139 bash[43301]: audit 2022-04-28T21:10:15.928898+0000 mon.a (mon.0) 2513 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:10:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:16 smithi139 bash[43301]: cephadm 2022-04-28T21:10:15.930648+0000 mgr.y (mgr.24961) 2667 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:10:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:16 smithi139 bash[43301]: audit 2022-04-28T21:10:15.933212+0000 mon.a (mon.0) 2514 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:10:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:17 smithi059 bash[59745]: cluster 2022-04-28T21:10:17.208766+0000 mgr.y (mgr.24961) 2668 : cluster [DBG] pgmap v3011: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:17 smithi059 bash[61152]: cluster 2022-04-28T21:10:17.208766+0000 mgr.y (mgr.24961) 2668 : cluster [DBG] pgmap v3011: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:17 smithi139 bash[43301]: cluster 2022-04-28T21:10:17.208766+0000 mgr.y (mgr.24961) 2668 : cluster [DBG] pgmap v3011: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[61152]: audit 2022-04-28T21:10:18.126364+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[61152]: audit 2022-04-28T21:10:18.126894+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[61152]: audit 2022-04-28T21:10:18.136523+0000 mon.a (mon.0) 2517 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[61152]: audit 2022-04-28T21:10:18.137398+0000 mon.a (mon.0) 2518 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[61152]: audit 2022-04-28T21:10:18.138303+0000 mon.a (mon.0) 2519 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:10:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[61152]: audit 2022-04-28T21:10:18.138921+0000 mon.a (mon.0) 2520 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]: dispatch 2022-04-28T21:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[59745]: audit 2022-04-28T21:10:18.126364+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[59745]: audit 2022-04-28T21:10:18.126894+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[59745]: audit 2022-04-28T21:10:18.136523+0000 mon.a (mon.0) 2517 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[59745]: audit 2022-04-28T21:10:18.137398+0000 mon.a (mon.0) 2518 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:10:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[59745]: audit 2022-04-28T21:10:18.138303+0000 mon.a (mon.0) 2519 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:10:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:18 smithi059 bash[59745]: audit 2022-04-28T21:10:18.138921+0000 mon.a (mon.0) 2520 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]: dispatch 2022-04-28T21:10:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:18 smithi139 bash[43301]: audit 2022-04-28T21:10:18.126364+0000 mon.a (mon.0) 2515 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:18 smithi139 bash[43301]: audit 2022-04-28T21:10:18.126894+0000 mon.a (mon.0) 2516 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:18 smithi139 bash[43301]: audit 2022-04-28T21:10:18.136523+0000 mon.a (mon.0) 2517 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:18 smithi139 bash[43301]: audit 2022-04-28T21:10:18.137398+0000 mon.a (mon.0) 2518 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:10:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:18 smithi139 bash[43301]: audit 2022-04-28T21:10:18.138303+0000 mon.a (mon.0) 2519 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:10:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:18 smithi139 bash[43301]: audit 2022-04-28T21:10:18.138921+0000 mon.a (mon.0) 2520 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1f", "id": [4, 2]}]: dispatch 2022-04-28T21:10:19.776 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:19 smithi059 bash[59745]: cluster 2022-04-28T21:10:19.209918+0000 mgr.y (mgr.24961) 2669 : cluster [DBG] pgmap v3012: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:19.777 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:19 smithi059 bash[61152]: cluster 2022-04-28T21:10:19.209918+0000 mgr.y (mgr.24961) 2669 : cluster [DBG] pgmap v3012: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:19 smithi139 bash[43301]: cluster 2022-04-28T21:10:19.209918+0000 mgr.y (mgr.24961) 2669 : cluster [DBG] pgmap v3012: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:22.269 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:22 smithi059 bash[59745]: cluster 2022-04-28T21:10:21.211190+0000 mgr.y (mgr.24961) 2670 : cluster [DBG] pgmap v3013: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:22.270 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:21.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:22 smithi059 bash[61152]: cluster 2022-04-28T21:10:21.211190+0000 mgr.y (mgr.24961) 2670 : cluster [DBG] pgmap v3013: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:22 smithi139 bash[43301]: cluster 2022-04-28T21:10:21.211190+0000 mgr.y (mgr.24961) 2670 : cluster [DBG] pgmap v3013: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:23 smithi059 bash[61152]: audit 2022-04-28T21:10:23.129430+0000 mon.a (mon.0) 2521 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:10:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:23 smithi059 bash[59745]: audit 2022-04-28T21:10:23.129430+0000 mon.a (mon.0) 2521 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:10:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:23 smithi139 bash[43301]: audit 2022-04-28T21:10:23.129430+0000 mon.a (mon.0) 2521 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:10:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:23.830Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:24.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[59745]: cluster 2022-04-28T21:10:23.211904+0000 mgr.y (mgr.24961) 2671 : cluster [DBG] pgmap v3014: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[59745]: cephadm 2022-04-28T21:10:23.350432+0000 mgr.y (mgr.24961) 2672 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T21:10:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[59745]: audit 2022-04-28T21:10:23.361686+0000 mon.a (mon.0) 2522 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:10:24.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[59745]: audit 2022-04-28T21:10:23.648239+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:10:24.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[61152]: cluster 2022-04-28T21:10:23.211904+0000 mgr.y (mgr.24961) 2671 : cluster [DBG] pgmap v3014: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:24.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[61152]: cephadm 2022-04-28T21:10:23.350432+0000 mgr.y (mgr.24961) 2672 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T21:10:24.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[61152]: audit 2022-04-28T21:10:23.361686+0000 mon.a (mon.0) 2522 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:10:24.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:24 smithi059 bash[61152]: audit 2022-04-28T21:10:23.648239+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:10:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:24 smithi139 bash[43301]: cluster 2022-04-28T21:10:23.211904+0000 mgr.y (mgr.24961) 2671 : cluster [DBG] pgmap v3014: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:24 smithi139 bash[43301]: cephadm 2022-04-28T21:10:23.350432+0000 mgr.y (mgr.24961) 2672 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T21:10:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:24 smithi139 bash[43301]: audit 2022-04-28T21:10:23.361686+0000 mon.a (mon.0) 2522 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:10:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:24 smithi139 bash[43301]: audit 2022-04-28T21:10:23.648239+0000 mon.a (mon.0) 2523 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:10:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:25 smithi059 bash[61152]: cluster 2022-04-28T21:10:25.213274+0000 mgr.y (mgr.24961) 2673 : cluster [DBG] pgmap v3015: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:25.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:25 smithi059 bash[59745]: cluster 2022-04-28T21:10:25.213274+0000 mgr.y (mgr.24961) 2673 : cluster [DBG] pgmap v3015: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:25 smithi139 bash[43301]: cluster 2022-04-28T21:10:25.213274+0000 mgr.y (mgr.24961) 2673 : cluster [DBG] pgmap v3015: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:28 smithi059 bash[61152]: cluster 2022-04-28T21:10:27.214222+0000 mgr.y (mgr.24961) 2674 : cluster [DBG] pgmap v3016: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:28 smithi059 bash[59745]: cluster 2022-04-28T21:10:27.214222+0000 mgr.y (mgr.24961) 2674 : cluster [DBG] pgmap v3016: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:28.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:28 smithi139 bash[43301]: cluster 2022-04-28T21:10:27.214222+0000 mgr.y (mgr.24961) 2674 : cluster [DBG] pgmap v3016: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:30 smithi059 bash[61152]: cluster 2022-04-28T21:10:29.215320+0000 mgr.y (mgr.24961) 2675 : cluster [DBG] pgmap v3017: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:30 smithi059 bash[59745]: cluster 2022-04-28T21:10:29.215320+0000 mgr.y (mgr.24961) 2675 : cluster [DBG] pgmap v3017: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:30 smithi139 bash[43301]: cluster 2022-04-28T21:10:29.215320+0000 mgr.y (mgr.24961) 2675 : cluster [DBG] pgmap v3017: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:31 smithi059 bash[61152]: cluster 2022-04-28T21:10:31.216216+0000 mgr.y (mgr.24961) 2676 : cluster [DBG] pgmap v3018: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:31 smithi059 bash[59745]: cluster 2022-04-28T21:10:31.216216+0000 mgr.y (mgr.24961) 2676 : cluster [DBG] pgmap v3018: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:31 smithi139 bash[43301]: cluster 2022-04-28T21:10:31.216216+0000 mgr.y (mgr.24961) 2676 : cluster [DBG] pgmap v3018: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:31.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:33.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:34.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:34 smithi059 bash[59745]: cluster 2022-04-28T21:10:33.216923+0000 mgr.y (mgr.24961) 2677 : cluster [DBG] pgmap v3019: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:34.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:34 smithi059 bash[61152]: cluster 2022-04-28T21:10:33.216923+0000 mgr.y (mgr.24961) 2677 : cluster [DBG] pgmap v3019: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:34 smithi139 bash[43301]: cluster 2022-04-28T21:10:33.216923+0000 mgr.y (mgr.24961) 2677 : cluster [DBG] pgmap v3019: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:35 smithi059 bash[59745]: cluster 2022-04-28T21:10:35.218297+0000 mgr.y (mgr.24961) 2678 : cluster [DBG] pgmap v3020: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:35.869 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:35 smithi059 bash[61152]: cluster 2022-04-28T21:10:35.218297+0000 mgr.y (mgr.24961) 2678 : cluster [DBG] pgmap v3020: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:35 smithi139 bash[43301]: cluster 2022-04-28T21:10:35.218297+0000 mgr.y (mgr.24961) 2678 : cluster [DBG] pgmap v3020: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:38 smithi059 bash[61152]: cluster 2022-04-28T21:10:37.219349+0000 mgr.y (mgr.24961) 2679 : cluster [DBG] pgmap v3021: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:38 smithi059 bash[59745]: cluster 2022-04-28T21:10:37.219349+0000 mgr.y (mgr.24961) 2679 : cluster [DBG] pgmap v3021: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:38 smithi139 bash[43301]: cluster 2022-04-28T21:10:37.219349+0000 mgr.y (mgr.24961) 2679 : cluster [DBG] pgmap v3021: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:39 smithi059 bash[61152]: cluster 2022-04-28T21:10:39.220470+0000 mgr.y (mgr.24961) 2680 : cluster [DBG] pgmap v3022: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:39 smithi059 bash[59745]: cluster 2022-04-28T21:10:39.220470+0000 mgr.y (mgr.24961) 2680 : cluster [DBG] pgmap v3022: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:39 smithi139 bash[43301]: cluster 2022-04-28T21:10:39.220470+0000 mgr.y (mgr.24961) 2680 : cluster [DBG] pgmap v3022: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:42.280 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:41.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:42 smithi059 bash[61152]: cluster 2022-04-28T21:10:41.221727+0000 mgr.y (mgr.24961) 2681 : cluster [DBG] pgmap v3023: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:42 smithi059 bash[59745]: cluster 2022-04-28T21:10:41.221727+0000 mgr.y (mgr.24961) 2681 : cluster [DBG] pgmap v3023: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:42 smithi139 bash[43301]: cluster 2022-04-28T21:10:41.221727+0000 mgr.y (mgr.24961) 2681 : cluster [DBG] pgmap v3023: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:43 smithi059 bash[61152]: cluster 2022-04-28T21:10:43.222437+0000 mgr.y (mgr.24961) 2682 : cluster [DBG] pgmap v3024: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:43 smithi059 bash[59745]: cluster 2022-04-28T21:10:43.222437+0000 mgr.y (mgr.24961) 2682 : cluster [DBG] pgmap v3024: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:43 smithi139 bash[43301]: cluster 2022-04-28T21:10:43.222437+0000 mgr.y (mgr.24961) 2682 : cluster [DBG] pgmap v3024: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:44.088 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:43.832Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:45 smithi059 bash[61152]: cluster 2022-04-28T21:10:45.223876+0000 mgr.y (mgr.24961) 2683 : cluster [DBG] pgmap v3025: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:45 smithi059 bash[59745]: cluster 2022-04-28T21:10:45.223876+0000 mgr.y (mgr.24961) 2683 : cluster [DBG] pgmap v3025: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:45 smithi139 bash[43301]: cluster 2022-04-28T21:10:45.223876+0000 mgr.y (mgr.24961) 2683 : cluster [DBG] pgmap v3025: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:48 smithi059 bash[61152]: cluster 2022-04-28T21:10:47.224837+0000 mgr.y (mgr.24961) 2684 : cluster [DBG] pgmap v3026: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:48 smithi059 bash[59745]: cluster 2022-04-28T21:10:47.224837+0000 mgr.y (mgr.24961) 2684 : cluster [DBG] pgmap v3026: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:48.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:48 smithi139 bash[43301]: cluster 2022-04-28T21:10:47.224837+0000 mgr.y (mgr.24961) 2684 : cluster [DBG] pgmap v3026: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:49 smithi059 bash[61152]: cluster 2022-04-28T21:10:49.225979+0000 mgr.y (mgr.24961) 2685 : cluster [DBG] pgmap v3027: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:49 smithi059 bash[59745]: cluster 2022-04-28T21:10:49.225979+0000 mgr.y (mgr.24961) 2685 : cluster [DBG] pgmap v3027: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:49 smithi139 bash[43301]: cluster 2022-04-28T21:10:49.225979+0000 mgr.y (mgr.24961) 2685 : cluster [DBG] pgmap v3027: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:51 smithi059 bash[61152]: cluster 2022-04-28T21:10:51.227239+0000 mgr.y (mgr.24961) 2686 : cluster [DBG] pgmap v3028: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:51 smithi059 bash[59745]: cluster 2022-04-28T21:10:51.227239+0000 mgr.y (mgr.24961) 2686 : cluster [DBG] pgmap v3028: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:51 smithi139 bash[43301]: cluster 2022-04-28T21:10:51.227239+0000 mgr.y (mgr.24961) 2686 : cluster [DBG] pgmap v3028: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:51.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:53 smithi059 bash[61152]: cluster 2022-04-28T21:10:53.227971+0000 mgr.y (mgr.24961) 2687 : cluster [DBG] pgmap v3029: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:53.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:53 smithi059 bash[59745]: cluster 2022-04-28T21:10:53.227971+0000 mgr.y (mgr.24961) 2687 : cluster [DBG] pgmap v3029: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:53 smithi139 bash[43301]: cluster 2022-04-28T21:10:53.227971+0000 mgr.y (mgr.24961) 2687 : cluster [DBG] pgmap v3029: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:54.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:10:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:10:53.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:10:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:55 smithi059 bash[61152]: cluster 2022-04-28T21:10:55.229308+0000 mgr.y (mgr.24961) 2688 : cluster [DBG] pgmap v3030: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:55 smithi059 bash[59745]: cluster 2022-04-28T21:10:55.229308+0000 mgr.y (mgr.24961) 2688 : cluster [DBG] pgmap v3030: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:55 smithi139 bash[43301]: cluster 2022-04-28T21:10:55.229308+0000 mgr.y (mgr.24961) 2688 : cluster [DBG] pgmap v3030: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:57 smithi059 bash[61152]: cluster 2022-04-28T21:10:57.230430+0000 mgr.y (mgr.24961) 2689 : cluster [DBG] pgmap v3031: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:57 smithi059 bash[59745]: cluster 2022-04-28T21:10:57.230430+0000 mgr.y (mgr.24961) 2689 : cluster [DBG] pgmap v3031: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:57 smithi139 bash[43301]: cluster 2022-04-28T21:10:57.230430+0000 mgr.y (mgr.24961) 2689 : cluster [DBG] pgmap v3031: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:10:59 smithi059 bash[61152]: cluster 2022-04-28T21:10:59.231588+0000 mgr.y (mgr.24961) 2690 : cluster [DBG] pgmap v3032: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:10:59 smithi059 bash[59745]: cluster 2022-04-28T21:10:59.231588+0000 mgr.y (mgr.24961) 2690 : cluster [DBG] pgmap v3032: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:10:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:10:59 smithi139 bash[43301]: cluster 2022-04-28T21:10:59.231588+0000 mgr.y (mgr.24961) 2690 : cluster [DBG] pgmap v3032: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:01 smithi059 bash[61152]: cluster 2022-04-28T21:11:01.232832+0000 mgr.y (mgr.24961) 2691 : cluster [DBG] pgmap v3033: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:01 smithi059 bash[59745]: cluster 2022-04-28T21:11:01.232832+0000 mgr.y (mgr.24961) 2691 : cluster [DBG] pgmap v3033: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:01 smithi139 bash[43301]: cluster 2022-04-28T21:11:01.232832+0000 mgr.y (mgr.24961) 2691 : cluster [DBG] pgmap v3033: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:01.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:03 smithi059 bash[61152]: cluster 2022-04-28T21:11:03.233557+0000 mgr.y (mgr.24961) 2692 : cluster [DBG] pgmap v3034: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:03 smithi059 bash[59745]: cluster 2022-04-28T21:11:03.233557+0000 mgr.y (mgr.24961) 2692 : cluster [DBG] pgmap v3034: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:03 smithi139 bash[43301]: cluster 2022-04-28T21:11:03.233557+0000 mgr.y (mgr.24961) 2692 : cluster [DBG] pgmap v3034: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:04.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:03.831Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:05 smithi059 bash[61152]: cluster 2022-04-28T21:11:05.234907+0000 mgr.y (mgr.24961) 2693 : cluster [DBG] pgmap v3035: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:05 smithi059 bash[59745]: cluster 2022-04-28T21:11:05.234907+0000 mgr.y (mgr.24961) 2693 : cluster [DBG] pgmap v3035: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:05 smithi139 bash[43301]: cluster 2022-04-28T21:11:05.234907+0000 mgr.y (mgr.24961) 2693 : cluster [DBG] pgmap v3035: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:07.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:07 smithi059 bash[61152]: cluster 2022-04-28T21:11:07.235960+0000 mgr.y (mgr.24961) 2694 : cluster [DBG] pgmap v3036: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:07.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:07 smithi059 bash[59745]: cluster 2022-04-28T21:11:07.235960+0000 mgr.y (mgr.24961) 2694 : cluster [DBG] pgmap v3036: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:07 smithi139 bash[43301]: cluster 2022-04-28T21:11:07.235960+0000 mgr.y (mgr.24961) 2694 : cluster [DBG] pgmap v3036: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:09 smithi059 bash[61152]: audit 2022-04-28T21:11:09.012281+0000 mon.c (mon.1) 34 : audit [INF] from='client.? 172.21.15.59:0/4006700937' entity='client.admin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:11:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:09 smithi059 bash[59745]: audit 2022-04-28T21:11:09.012281+0000 mon.c (mon.1) 34 : audit [INF] from='client.? 172.21.15.59:0/4006700937' entity='client.admin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:11:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:09 smithi139 bash[43301]: audit 2022-04-28T21:11:09.012281+0000 mon.c (mon.1) 34 : audit [INF] from='client.? 172.21.15.59:0/4006700937' entity='client.admin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:11:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:10 smithi059 bash[61152]: cluster 2022-04-28T21:11:09.237111+0000 mgr.y (mgr.24961) 2695 : cluster [DBG] pgmap v3037: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:10.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:10 smithi059 bash[59745]: cluster 2022-04-28T21:11:09.237111+0000 mgr.y (mgr.24961) 2695 : cluster [DBG] pgmap v3037: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:10.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:10 smithi139 bash[43301]: cluster 2022-04-28T21:11:09.237111+0000 mgr.y (mgr.24961) 2695 : cluster [DBG] pgmap v3037: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:11 smithi059 bash[61152]: cluster 2022-04-28T21:11:11.238328+0000 mgr.y (mgr.24961) 2696 : cluster [DBG] pgmap v3038: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:11 smithi059 bash[59745]: cluster 2022-04-28T21:11:11.238328+0000 mgr.y (mgr.24961) 2696 : cluster [DBG] pgmap v3038: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:11 smithi139 bash[43301]: cluster 2022-04-28T21:11:11.238328+0000 mgr.y (mgr.24961) 2696 : cluster [DBG] pgmap v3038: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:11.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:13 smithi059 bash[61152]: cluster 2022-04-28T21:11:13.239058+0000 mgr.y (mgr.24961) 2697 : cluster [DBG] pgmap v3039: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:13 smithi059 bash[59745]: cluster 2022-04-28T21:11:13.239058+0000 mgr.y (mgr.24961) 2697 : cluster [DBG] pgmap v3039: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:13 smithi139 bash[43301]: cluster 2022-04-28T21:11:13.239058+0000 mgr.y (mgr.24961) 2697 : cluster [DBG] pgmap v3039: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:13.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:15 smithi059 bash[61152]: audit 2022-04-28T21:11:14.618365+0000 mon.a (mon.0) 2524 : audit [DBG] from='client.? 172.21.9.34:0/591956505' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T21:11:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:15 smithi059 bash[61152]: cluster 2022-04-28T21:11:15.240460+0000 mgr.y (mgr.24961) 2698 : cluster [DBG] pgmap v3040: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:15 smithi059 bash[59745]: audit 2022-04-28T21:11:14.618365+0000 mon.a (mon.0) 2524 : audit [DBG] from='client.? 172.21.9.34:0/591956505' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T21:11:15.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:15 smithi059 bash[59745]: cluster 2022-04-28T21:11:15.240460+0000 mgr.y (mgr.24961) 2698 : cluster [DBG] pgmap v3040: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:15 smithi139 bash[43301]: audit 2022-04-28T21:11:14.618365+0000 mon.a (mon.0) 2524 : audit [DBG] from='client.? 172.21.9.34:0/591956505' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2022-04-28T21:11:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:15 smithi139 bash[43301]: cluster 2022-04-28T21:11:15.240460+0000 mgr.y (mgr.24961) 2698 : cluster [DBG] pgmap v3040: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:17 smithi059 bash[61152]: cluster 2022-04-28T21:11:17.241438+0000 mgr.y (mgr.24961) 2699 : cluster [DBG] pgmap v3041: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:17 smithi059 bash[59745]: cluster 2022-04-28T21:11:17.241438+0000 mgr.y (mgr.24961) 2699 : cluster [DBG] pgmap v3041: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:17 smithi139 bash[43301]: cluster 2022-04-28T21:11:17.241438+0000 mgr.y (mgr.24961) 2699 : cluster [DBG] pgmap v3041: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:18 smithi139 bash[43301]: audit 2022-04-28T21:11:18.139078+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:11:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:18 smithi139 bash[43301]: audit 2022-04-28T21:11:18.139572+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:11:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:18 smithi139 bash[43301]: audit 2022-04-28T21:11:18.152028+0000 mon.a (mon.0) 2527 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:11:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:18 smithi139 bash[43301]: audit 2022-04-28T21:11:18.152711+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:11:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:18 smithi139 bash[43301]: audit 2022-04-28T21:11:18.153309+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:11:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:18 smithi139 bash[43301]: audit 2022-04-28T21:11:18.153874+0000 mon.a (mon.0) 2530 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:11:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[61152]: audit 2022-04-28T21:11:18.139078+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:11:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[61152]: audit 2022-04-28T21:11:18.139572+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:11:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[61152]: audit 2022-04-28T21:11:18.152028+0000 mon.a (mon.0) 2527 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:11:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[61152]: audit 2022-04-28T21:11:18.152711+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:11:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[61152]: audit 2022-04-28T21:11:18.153309+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:11:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[61152]: audit 2022-04-28T21:11:18.153874+0000 mon.a (mon.0) 2530 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:11:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[59745]: audit 2022-04-28T21:11:18.139078+0000 mon.a (mon.0) 2525 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:11:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[59745]: audit 2022-04-28T21:11:18.139572+0000 mon.a (mon.0) 2526 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:11:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[59745]: audit 2022-04-28T21:11:18.152028+0000 mon.a (mon.0) 2527 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:11:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[59745]: audit 2022-04-28T21:11:18.152711+0000 mon.a (mon.0) 2528 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:11:18.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[59745]: audit 2022-04-28T21:11:18.153309+0000 mon.a (mon.0) 2529 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:11:18.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:18 smithi059 bash[59745]: audit 2022-04-28T21:11:18.153874+0000 mon.a (mon.0) 2530 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:11:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:19 smithi139 bash[43301]: cluster 2022-04-28T21:11:19.242493+0000 mgr.y (mgr.24961) 2700 : cluster [DBG] pgmap v3042: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:19 smithi059 bash[61152]: cluster 2022-04-28T21:11:19.242493+0000 mgr.y (mgr.24961) 2700 : cluster [DBG] pgmap v3042: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:19 smithi059 bash[59745]: cluster 2022-04-28T21:11:19.242493+0000 mgr.y (mgr.24961) 2700 : cluster [DBG] pgmap v3042: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:21 smithi059 bash[61152]: cluster 2022-04-28T21:11:21.243766+0000 mgr.y (mgr.24961) 2701 : cluster [DBG] pgmap v3043: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:21 smithi059 bash[59745]: cluster 2022-04-28T21:11:21.243766+0000 mgr.y (mgr.24961) 2701 : cluster [DBG] pgmap v3043: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:21 smithi139 bash[43301]: cluster 2022-04-28T21:11:21.243766+0000 mgr.y (mgr.24961) 2701 : cluster [DBG] pgmap v3043: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:21.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:23 smithi059 bash[61152]: cluster 2022-04-28T21:11:23.244561+0000 mgr.y (mgr.24961) 2702 : cluster [DBG] pgmap v3044: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:23 smithi059 bash[59745]: cluster 2022-04-28T21:11:23.244561+0000 mgr.y (mgr.24961) 2702 : cluster [DBG] pgmap v3044: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:23 smithi139 bash[43301]: cluster 2022-04-28T21:11:23.244561+0000 mgr.y (mgr.24961) 2702 : cluster [DBG] pgmap v3044: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:23.833Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.656216+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.658951+0000 mon.a (mon.0) 2532 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.660940+0000 mon.a (mon.0) 2533 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: cephadm 2022-04-28T21:11:23.662852+0000 mgr.y (mgr.24961) 2703 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.665334+0000 mon.a (mon.0) 2534 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.952786+0000 mon.a (mon.0) 2535 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.953744+0000 mon.a (mon.0) 2536 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.954615+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.955445+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.956350+0000 mon.a (mon.0) 2539 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.957126+0000 mon.a (mon.0) 2540 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.957978+0000 mon.a (mon.0) 2541 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.958832+0000 mon.a (mon.0) 2542 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.959744+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:11:24.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:24 smithi139 bash[43301]: audit 2022-04-28T21:11:23.960579+0000 mon.a (mon.0) 2544 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.656216+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:11:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.658951+0000 mon.a (mon.0) 2532 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:11:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.660940+0000 mon.a (mon.0) 2533 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:11:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: cephadm 2022-04-28T21:11:23.662852+0000 mgr.y (mgr.24961) 2703 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:11:24.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.665334+0000 mon.a (mon.0) 2534 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:11:24.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.952786+0000 mon.a (mon.0) 2535 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.953744+0000 mon.a (mon.0) 2536 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.954615+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.955445+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.956350+0000 mon.a (mon.0) 2539 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.957126+0000 mon.a (mon.0) 2540 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.957978+0000 mon.a (mon.0) 2541 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.958832+0000 mon.a (mon.0) 2542 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.959744+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:11:24.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[59745]: audit 2022-04-28T21:11:23.960579+0000 mon.a (mon.0) 2544 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.656216+0000 mon.a (mon.0) 2531 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:11:24.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.658951+0000 mon.a (mon.0) 2532 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:11:24.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.660940+0000 mon.a (mon.0) 2533 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:11:24.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: cephadm 2022-04-28T21:11:23.662852+0000 mgr.y (mgr.24961) 2703 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:11:24.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.665334+0000 mon.a (mon.0) 2534 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:11:24.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.952786+0000 mon.a (mon.0) 2535 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.953744+0000 mon.a (mon.0) 2536 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.954615+0000 mon.a (mon.0) 2537 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.955445+0000 mon.a (mon.0) 2538 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.956350+0000 mon.a (mon.0) 2539 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.957126+0000 mon.a (mon.0) 2540 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.957978+0000 mon.a (mon.0) 2541 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:24.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.958832+0000 mon.a (mon.0) 2542 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:24.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.959744+0000 mon.a (mon.0) 2543 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:11:24.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:24 smithi059 bash[61152]: audit 2022-04-28T21:11:23.960579+0000 mon.a (mon.0) 2544 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:11:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:25 smithi059 bash[61152]: cluster 2022-04-28T21:11:25.245979+0000 mgr.y (mgr.24961) 2704 : cluster [DBG] pgmap v3045: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:25.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:25 smithi059 bash[59745]: cluster 2022-04-28T21:11:25.245979+0000 mgr.y (mgr.24961) 2704 : cluster [DBG] pgmap v3045: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:25 smithi139 bash[43301]: cluster 2022-04-28T21:11:25.245979+0000 mgr.y (mgr.24961) 2704 : cluster [DBG] pgmap v3045: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:27 smithi059 bash[61152]: cluster 2022-04-28T21:11:27.246951+0000 mgr.y (mgr.24961) 2705 : cluster [DBG] pgmap v3046: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:27 smithi059 bash[59745]: cluster 2022-04-28T21:11:27.246951+0000 mgr.y (mgr.24961) 2705 : cluster [DBG] pgmap v3046: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:27 smithi139 bash[43301]: cluster 2022-04-28T21:11:27.246951+0000 mgr.y (mgr.24961) 2705 : cluster [DBG] pgmap v3046: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:29 smithi059 bash[61152]: cluster 2022-04-28T21:11:29.248066+0000 mgr.y (mgr.24961) 2706 : cluster [DBG] pgmap v3047: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:29 smithi059 bash[59745]: cluster 2022-04-28T21:11:29.248066+0000 mgr.y (mgr.24961) 2706 : cluster [DBG] pgmap v3047: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:29 smithi139 bash[43301]: cluster 2022-04-28T21:11:29.248066+0000 mgr.y (mgr.24961) 2706 : cluster [DBG] pgmap v3047: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:30 smithi059 bash[61152]: audit 2022-04-28T21:11:30.641771+0000 mon.a (mon.0) 2545 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:11:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:30 smithi059 bash[59745]: audit 2022-04-28T21:11:30.641771+0000 mon.a (mon.0) 2545 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:11:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:30 smithi139 bash[43301]: audit 2022-04-28T21:11:30.641771+0000 mon.a (mon.0) 2545 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:11:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: cephadm 2022-04-28T21:11:30.855614+0000 mgr.y (mgr.24961) 2707 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T21:11:32.359 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:30.867354+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:11:32.359 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.175059+0000 mon.a (mon.0) 2547 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.176117+0000 mon.a (mon.0) 2548 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.177127+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.178063+0000 mon.a (mon.0) 2550 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.178955+0000 mon.a (mon.0) 2551 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.360 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.179914+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.180887+0000 mon.a (mon.0) 2553 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.181780+0000 mon.a (mon.0) 2554 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.182778+0000 mon.a (mon.0) 2555 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:11:32.366 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.183827+0000 mon.a (mon.0) 2556 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: audit 2022-04-28T21:11:31.193593+0000 mon.a (mon.0) 2557 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:11:32.367 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:31 smithi139 bash[43301]: cluster 2022-04-28T21:11:31.249354+0000 mgr.y (mgr.24961) 2708 : cluster [DBG] pgmap v3048: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:32.368 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:31.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:32.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: cephadm 2022-04-28T21:11:30.855614+0000 mgr.y (mgr.24961) 2707 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T21:11:32.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:30.867354+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:11:32.368 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.175059+0000 mon.a (mon.0) 2547 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.369 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.176117+0000 mon.a (mon.0) 2548 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.369 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.177127+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.369 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.178063+0000 mon.a (mon.0) 2550 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.369 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.178955+0000 mon.a (mon.0) 2551 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.369 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.179914+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.369 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.180887+0000 mon.a (mon.0) 2553 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.370 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.181780+0000 mon.a (mon.0) 2554 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.370 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.182778+0000 mon.a (mon.0) 2555 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:11:32.370 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.183827+0000 mon.a (mon.0) 2556 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.370 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: audit 2022-04-28T21:11:31.193593+0000 mon.a (mon.0) 2557 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:11:32.370 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[61152]: cluster 2022-04-28T21:11:31.249354+0000 mgr.y (mgr.24961) 2708 : cluster [DBG] pgmap v3048: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:32.371 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: cephadm 2022-04-28T21:11:30.855614+0000 mgr.y (mgr.24961) 2707 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T21:11:32.371 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:30.867354+0000 mon.a (mon.0) 2546 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:11:32.371 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.175059+0000 mon.a (mon.0) 2547 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.371 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.176117+0000 mon.a (mon.0) 2548 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.371 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.177127+0000 mon.a (mon.0) 2549 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.371 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.178063+0000 mon.a (mon.0) 2550 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.178955+0000 mon.a (mon.0) 2551 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.179914+0000 mon.a (mon.0) 2552 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.180887+0000 mon.a (mon.0) 2553 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.181780+0000 mon.a (mon.0) 2554 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.182778+0000 mon.a (mon.0) 2555 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.183827+0000 mon.a (mon.0) 2556 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:11:32.372 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: audit 2022-04-28T21:11:31.193593+0000 mon.a (mon.0) 2557 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:11:32.373 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:31 smithi059 bash[59745]: cluster 2022-04-28T21:11:31.249354+0000 mgr.y (mgr.24961) 2708 : cluster [DBG] pgmap v3048: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:33 smithi139 bash[43301]: cluster 2022-04-28T21:11:33.250067+0000 mgr.y (mgr.24961) 2709 : cluster [DBG] pgmap v3049: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:33.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:33 smithi059 bash[61152]: cluster 2022-04-28T21:11:33.250067+0000 mgr.y (mgr.24961) 2709 : cluster [DBG] pgmap v3049: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:33.836 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:33 smithi059 bash[59745]: cluster 2022-04-28T21:11:33.250067+0000 mgr.y (mgr.24961) 2709 : cluster [DBG] pgmap v3049: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:33.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:35 smithi059 bash[61152]: cluster 2022-04-28T21:11:35.251426+0000 mgr.y (mgr.24961) 2710 : cluster [DBG] pgmap v3050: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:35 smithi059 bash[59745]: cluster 2022-04-28T21:11:35.251426+0000 mgr.y (mgr.24961) 2710 : cluster [DBG] pgmap v3050: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:35 smithi139 bash[43301]: cluster 2022-04-28T21:11:35.251426+0000 mgr.y (mgr.24961) 2710 : cluster [DBG] pgmap v3050: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:37 smithi139 bash[43301]: cluster 2022-04-28T21:11:37.252437+0000 mgr.y (mgr.24961) 2711 : cluster [DBG] pgmap v3051: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:37 smithi059 bash[61152]: cluster 2022-04-28T21:11:37.252437+0000 mgr.y (mgr.24961) 2711 : cluster [DBG] pgmap v3051: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:37 smithi059 bash[59745]: cluster 2022-04-28T21:11:37.252437+0000 mgr.y (mgr.24961) 2711 : cluster [DBG] pgmap v3051: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:39 smithi139 bash[43301]: cluster 2022-04-28T21:11:39.253541+0000 mgr.y (mgr.24961) 2712 : cluster [DBG] pgmap v3052: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:39 smithi059 bash[59745]: cluster 2022-04-28T21:11:39.253541+0000 mgr.y (mgr.24961) 2712 : cluster [DBG] pgmap v3052: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:39 smithi059 bash[61152]: cluster 2022-04-28T21:11:39.253541+0000 mgr.y (mgr.24961) 2712 : cluster [DBG] pgmap v3052: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:41 smithi139 bash[43301]: cluster 2022-04-28T21:11:41.254808+0000 mgr.y (mgr.24961) 2713 : cluster [DBG] pgmap v3053: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:41 smithi059 bash[61152]: cluster 2022-04-28T21:11:41.254808+0000 mgr.y (mgr.24961) 2713 : cluster [DBG] pgmap v3053: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:41 smithi059 bash[59745]: cluster 2022-04-28T21:11:41.254808+0000 mgr.y (mgr.24961) 2713 : cluster [DBG] pgmap v3053: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:41.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:43 smithi139 bash[43301]: cluster 2022-04-28T21:11:43.255645+0000 mgr.y (mgr.24961) 2714 : cluster [DBG] pgmap v3054: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:43.835 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:43 smithi059 bash[59745]: cluster 2022-04-28T21:11:43.255645+0000 mgr.y (mgr.24961) 2714 : cluster [DBG] pgmap v3054: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:43.835 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:43 smithi059 bash[61152]: cluster 2022-04-28T21:11:43.255645+0000 mgr.y (mgr.24961) 2714 : cluster [DBG] pgmap v3054: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:11:44.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:43.834Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:45 smithi059 bash[61152]: cluster 2022-04-28T21:11:45.257167+0000 mgr.y (mgr.24961) 2715 : cluster [DBG] pgmap v3055: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:45 smithi059 bash[59745]: cluster 2022-04-28T21:11:45.257167+0000 mgr.y (mgr.24961) 2715 : cluster [DBG] pgmap v3055: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:45 smithi139 bash[43301]: cluster 2022-04-28T21:11:45.257167+0000 mgr.y (mgr.24961) 2715 : cluster [DBG] pgmap v3055: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:47 smithi139 bash[43301]: cluster 2022-04-28T21:11:47.258258+0000 mgr.y (mgr.24961) 2716 : cluster [DBG] pgmap v3056: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:47 smithi059 bash[61152]: cluster 2022-04-28T21:11:47.258258+0000 mgr.y (mgr.24961) 2716 : cluster [DBG] pgmap v3056: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:47 smithi059 bash[59745]: cluster 2022-04-28T21:11:47.258258+0000 mgr.y (mgr.24961) 2716 : cluster [DBG] pgmap v3056: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:49 smithi139 bash[43301]: cluster 2022-04-28T21:11:49.259393+0000 mgr.y (mgr.24961) 2717 : cluster [DBG] pgmap v3057: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:49 smithi059 bash[61152]: cluster 2022-04-28T21:11:49.259393+0000 mgr.y (mgr.24961) 2717 : cluster [DBG] pgmap v3057: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:49 smithi059 bash[59745]: cluster 2022-04-28T21:11:49.259393+0000 mgr.y (mgr.24961) 2717 : cluster [DBG] pgmap v3057: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:11:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:51 smithi139 bash[43301]: cluster 2022-04-28T21:11:51.260656+0000 mgr.y (mgr.24961) 2718 : cluster [DBG] pgmap v3058: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:51 smithi059 bash[61152]: cluster 2022-04-28T21:11:51.260656+0000 mgr.y (mgr.24961) 2718 : cluster [DBG] pgmap v3058: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:51 smithi059 bash[59745]: cluster 2022-04-28T21:11:51.260656+0000 mgr.y (mgr.24961) 2718 : cluster [DBG] pgmap v3058: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:51.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:53 smithi139 bash[43301]: cluster 2022-04-28T21:11:53.261389+0000 mgr.y (mgr.24961) 2719 : cluster [DBG] pgmap v3059: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:53.837 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:53 smithi059 bash[61152]: cluster 2022-04-28T21:11:53.261389+0000 mgr.y (mgr.24961) 2719 : cluster [DBG] pgmap v3059: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:53.838 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:11:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:11:53.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:11:53.838 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:53 smithi059 bash[59745]: cluster 2022-04-28T21:11:53.261389+0000 mgr.y (mgr.24961) 2719 : cluster [DBG] pgmap v3059: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:55 smithi059 bash[61152]: cluster 2022-04-28T21:11:55.262794+0000 mgr.y (mgr.24961) 2720 : cluster [DBG] pgmap v3060: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:55 smithi059 bash[59745]: cluster 2022-04-28T21:11:55.262794+0000 mgr.y (mgr.24961) 2720 : cluster [DBG] pgmap v3060: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:55 smithi139 bash[43301]: cluster 2022-04-28T21:11:55.262794+0000 mgr.y (mgr.24961) 2720 : cluster [DBG] pgmap v3060: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:57 smithi139 bash[43301]: cluster 2022-04-28T21:11:57.263807+0000 mgr.y (mgr.24961) 2721 : cluster [DBG] pgmap v3061: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:57 smithi059 bash[61152]: cluster 2022-04-28T21:11:57.263807+0000 mgr.y (mgr.24961) 2721 : cluster [DBG] pgmap v3061: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:57 smithi059 bash[59745]: cluster 2022-04-28T21:11:57.263807+0000 mgr.y (mgr.24961) 2721 : cluster [DBG] pgmap v3061: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:11:59 smithi139 bash[43301]: cluster 2022-04-28T21:11:59.264832+0000 mgr.y (mgr.24961) 2722 : cluster [DBG] pgmap v3062: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:11:59 smithi059 bash[61152]: cluster 2022-04-28T21:11:59.264832+0000 mgr.y (mgr.24961) 2722 : cluster [DBG] pgmap v3062: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:11:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:11:59 smithi059 bash[59745]: cluster 2022-04-28T21:11:59.264832+0000 mgr.y (mgr.24961) 2722 : cluster [DBG] pgmap v3062: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:12:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:01 smithi139 bash[43301]: cluster 2022-04-28T21:12:01.266065+0000 mgr.y (mgr.24961) 2723 : cluster [DBG] pgmap v3063: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:12:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:01 smithi059 bash[61152]: cluster 2022-04-28T21:12:01.266065+0000 mgr.y (mgr.24961) 2723 : cluster [DBG] pgmap v3063: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:12:01.906 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:01 smithi059 bash[59745]: cluster 2022-04-28T21:12:01.266065+0000 mgr.y (mgr.24961) 2723 : cluster [DBG] pgmap v3063: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:12:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:01.944Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:03 smithi139 bash[43301]: cluster 2022-04-28T21:12:03.266778+0000 mgr.y (mgr.24961) 2724 : cluster [DBG] pgmap v3064: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:12:03.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:03 smithi059 bash[61152]: cluster 2022-04-28T21:12:03.266778+0000 mgr.y (mgr.24961) 2724 : cluster [DBG] pgmap v3064: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:12:03.839 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:03.837Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:03.839 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:03 smithi059 bash[59745]: cluster 2022-04-28T21:12:03.266778+0000 mgr.y (mgr.24961) 2724 : cluster [DBG] pgmap v3064: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:12:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:05 smithi059 bash[61152]: cluster 2022-04-28T21:12:05.268202+0000 mgr.y (mgr.24961) 2725 : cluster [DBG] pgmap v3065: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:12:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:05 smithi059 bash[59745]: cluster 2022-04-28T21:12:05.268202+0000 mgr.y (mgr.24961) 2725 : cluster [DBG] pgmap v3065: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:12:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:05 smithi139 bash[43301]: cluster 2022-04-28T21:12:05.268202+0000 mgr.y (mgr.24961) 2725 : cluster [DBG] pgmap v3065: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2022-04-28T21:12:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:07 smithi139 bash[43301]: cluster 2022-04-28T21:12:07.269184+0000 mgr.y (mgr.24961) 2726 : cluster [DBG] pgmap v3066: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:07 smithi059 bash[59745]: cluster 2022-04-28T21:12:07.269184+0000 mgr.y (mgr.24961) 2726 : cluster [DBG] pgmap v3066: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:07.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:07 smithi059 bash[61152]: cluster 2022-04-28T21:12:07.269184+0000 mgr.y (mgr.24961) 2726 : cluster [DBG] pgmap v3066: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:09 smithi139 bash[43301]: cluster 2022-04-28T21:12:09.270280+0000 mgr.y (mgr.24961) 2727 : cluster [DBG] pgmap v3067: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:09 smithi059 bash[61152]: cluster 2022-04-28T21:12:09.270280+0000 mgr.y (mgr.24961) 2727 : cluster [DBG] pgmap v3067: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:09 smithi059 bash[59745]: cluster 2022-04-28T21:12:09.270280+0000 mgr.y (mgr.24961) 2727 : cluster [DBG] pgmap v3067: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:11 smithi139 bash[43301]: cluster 2022-04-28T21:12:11.271603+0000 mgr.y (mgr.24961) 2728 : cluster [DBG] pgmap v3068: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:11 smithi059 bash[61152]: cluster 2022-04-28T21:12:11.271603+0000 mgr.y (mgr.24961) 2728 : cluster [DBG] pgmap v3068: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:11 smithi059 bash[59745]: cluster 2022-04-28T21:12:11.271603+0000 mgr.y (mgr.24961) 2728 : cluster [DBG] pgmap v3068: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:11.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:13 smithi139 bash[43301]: cluster 2022-04-28T21:12:13.272354+0000 mgr.y (mgr.24961) 2729 : cluster [DBG] pgmap v3069: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:13.837 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:13 smithi059 bash[59745]: cluster 2022-04-28T21:12:13.272354+0000 mgr.y (mgr.24961) 2729 : cluster [DBG] pgmap v3069: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:13.838 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:13 smithi059 bash[61152]: cluster 2022-04-28T21:12:13.272354+0000 mgr.y (mgr.24961) 2729 : cluster [DBG] pgmap v3069: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:13.836Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:15 smithi059 bash[61152]: cluster 2022-04-28T21:12:15.273700+0000 mgr.y (mgr.24961) 2730 : cluster [DBG] pgmap v3070: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:15 smithi059 bash[59745]: cluster 2022-04-28T21:12:15.273700+0000 mgr.y (mgr.24961) 2730 : cluster [DBG] pgmap v3070: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:15 smithi139 bash[43301]: cluster 2022-04-28T21:12:15.273700+0000 mgr.y (mgr.24961) 2730 : cluster [DBG] pgmap v3070: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:17 smithi139 bash[43301]: cluster 2022-04-28T21:12:17.274739+0000 mgr.y (mgr.24961) 2731 : cluster [DBG] pgmap v3071: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:17.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:17 smithi059 bash[59745]: cluster 2022-04-28T21:12:17.274739+0000 mgr.y (mgr.24961) 2731 : cluster [DBG] pgmap v3071: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:17.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:17 smithi059 bash[61152]: cluster 2022-04-28T21:12:17.274739+0000 mgr.y (mgr.24961) 2731 : cluster [DBG] pgmap v3071: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:18 smithi139 bash[43301]: audit 2022-04-28T21:12:18.152040+0000 mon.a (mon.0) 2558 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:12:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:18 smithi139 bash[43301]: audit 2022-04-28T21:12:18.152613+0000 mon.a (mon.0) 2559 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:12:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:18 smithi139 bash[43301]: audit 2022-04-28T21:12:18.164423+0000 mon.a (mon.0) 2560 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:18 smithi139 bash[43301]: audit 2022-04-28T21:12:18.165169+0000 mon.a (mon.0) 2561 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:18 smithi139 bash[43301]: audit 2022-04-28T21:12:18.165717+0000 mon.a (mon.0) 2562 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:18 smithi139 bash[43301]: audit 2022-04-28T21:12:18.166268+0000 mon.a (mon.0) 2563 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:12:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[61152]: audit 2022-04-28T21:12:18.152040+0000 mon.a (mon.0) 2558 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[61152]: audit 2022-04-28T21:12:18.152613+0000 mon.a (mon.0) 2559 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[61152]: audit 2022-04-28T21:12:18.164423+0000 mon.a (mon.0) 2560 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[61152]: audit 2022-04-28T21:12:18.165169+0000 mon.a (mon.0) 2561 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[61152]: audit 2022-04-28T21:12:18.165717+0000 mon.a (mon.0) 2562 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[61152]: audit 2022-04-28T21:12:18.166268+0000 mon.a (mon.0) 2563 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:12:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[59745]: audit 2022-04-28T21:12:18.152040+0000 mon.a (mon.0) 2558 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:12:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[59745]: audit 2022-04-28T21:12:18.152613+0000 mon.a (mon.0) 2559 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:12:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[59745]: audit 2022-04-28T21:12:18.164423+0000 mon.a (mon.0) 2560 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[59745]: audit 2022-04-28T21:12:18.165169+0000 mon.a (mon.0) 2561 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[59745]: audit 2022-04-28T21:12:18.165717+0000 mon.a (mon.0) 2562 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:12:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:18 smithi059 bash[59745]: audit 2022-04-28T21:12:18.166268+0000 mon.a (mon.0) 2563 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:12:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:19 smithi059 bash[61152]: cluster 2022-04-28T21:12:19.275864+0000 mgr.y (mgr.24961) 2732 : cluster [DBG] pgmap v3072: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:19.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:19 smithi059 bash[59745]: cluster 2022-04-28T21:12:19.275864+0000 mgr.y (mgr.24961) 2732 : cluster [DBG] pgmap v3072: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:19 smithi139 bash[43301]: cluster 2022-04-28T21:12:19.275864+0000 mgr.y (mgr.24961) 2732 : cluster [DBG] pgmap v3072: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:21 smithi139 bash[43301]: cluster 2022-04-28T21:12:21.277111+0000 mgr.y (mgr.24961) 2733 : cluster [DBG] pgmap v3073: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:21.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:21 smithi059 bash[61152]: cluster 2022-04-28T21:12:21.277111+0000 mgr.y (mgr.24961) 2733 : cluster [DBG] pgmap v3073: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:21 smithi059 bash[59745]: cluster 2022-04-28T21:12:21.277111+0000 mgr.y (mgr.24961) 2733 : cluster [DBG] pgmap v3073: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:21.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:23 smithi139 bash[43301]: cluster 2022-04-28T21:12:23.277823+0000 mgr.y (mgr.24961) 2734 : cluster [DBG] pgmap v3074: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:23.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:23 smithi059 bash[61152]: cluster 2022-04-28T21:12:23.277823+0000 mgr.y (mgr.24961) 2734 : cluster [DBG] pgmap v3074: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:23.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:23 smithi059 bash[59745]: cluster 2022-04-28T21:12:23.277823+0000 mgr.y (mgr.24961) 2734 : cluster [DBG] pgmap v3074: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:23.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:25 smithi059 bash[61152]: cluster 2022-04-28T21:12:25.279180+0000 mgr.y (mgr.24961) 2735 : cluster [DBG] pgmap v3075: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:25.857 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:25 smithi059 bash[59745]: cluster 2022-04-28T21:12:25.279180+0000 mgr.y (mgr.24961) 2735 : cluster [DBG] pgmap v3075: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:25.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:25 smithi139 bash[43301]: cluster 2022-04-28T21:12:25.279180+0000 mgr.y (mgr.24961) 2735 : cluster [DBG] pgmap v3075: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:27 smithi139 bash[43301]: cluster 2022-04-28T21:12:27.280135+0000 mgr.y (mgr.24961) 2736 : cluster [DBG] pgmap v3076: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:27 smithi059 bash[61152]: cluster 2022-04-28T21:12:27.280135+0000 mgr.y (mgr.24961) 2736 : cluster [DBG] pgmap v3076: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:27 smithi059 bash[59745]: cluster 2022-04-28T21:12:27.280135+0000 mgr.y (mgr.24961) 2736 : cluster [DBG] pgmap v3076: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:29 smithi139 bash[43301]: cluster 2022-04-28T21:12:29.281229+0000 mgr.y (mgr.24961) 2737 : cluster [DBG] pgmap v3077: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:29 smithi059 bash[61152]: cluster 2022-04-28T21:12:29.281229+0000 mgr.y (mgr.24961) 2737 : cluster [DBG] pgmap v3077: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:29 smithi059 bash[59745]: cluster 2022-04-28T21:12:29.281229+0000 mgr.y (mgr.24961) 2737 : cluster [DBG] pgmap v3077: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[59745]: audit 2022-04-28T21:12:31.203459+0000 mon.a (mon.0) 2564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:12:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[59745]: audit 2022-04-28T21:12:31.206292+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:12:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[59745]: audit 2022-04-28T21:12:31.208175+0000 mon.a (mon.0) 2566 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:12:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[61152]: audit 2022-04-28T21:12:31.203459+0000 mon.a (mon.0) 2564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:12:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[61152]: audit 2022-04-28T21:12:31.206292+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:12:31.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[61152]: audit 2022-04-28T21:12:31.208175+0000 mon.a (mon.0) 2566 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:12:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:31 smithi139 bash[43301]: audit 2022-04-28T21:12:31.203459+0000 mon.a (mon.0) 2564 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:12:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:31 smithi139 bash[43301]: audit 2022-04-28T21:12:31.206292+0000 mon.a (mon.0) 2565 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:12:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:31 smithi139 bash[43301]: audit 2022-04-28T21:12:31.208175+0000 mon.a (mon.0) 2566 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:12:32.268 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:31.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:32.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:32 smithi059 bash[59745]: cephadm 2022-04-28T21:12:31.209797+0000 mgr.y (mgr.24961) 2738 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:12:32.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:32 smithi059 bash[59745]: cluster 2022-04-28T21:12:31.282509+0000 mgr.y (mgr.24961) 2739 : cluster [DBG] pgmap v3078: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:32.326 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:32 smithi059 bash[59745]: audit 2022-04-28T21:12:31.541361+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:12:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:32 smithi059 bash[61152]: cephadm 2022-04-28T21:12:31.209797+0000 mgr.y (mgr.24961) 2738 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:12:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:32 smithi059 bash[61152]: cluster 2022-04-28T21:12:31.282509+0000 mgr.y (mgr.24961) 2739 : cluster [DBG] pgmap v3078: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:32 smithi059 bash[61152]: audit 2022-04-28T21:12:31.541361+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:12:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:32 smithi139 bash[43301]: cephadm 2022-04-28T21:12:31.209797+0000 mgr.y (mgr.24961) 2738 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:12:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:32 smithi139 bash[43301]: cluster 2022-04-28T21:12:31.282509+0000 mgr.y (mgr.24961) 2739 : cluster [DBG] pgmap v3078: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:32 smithi139 bash[43301]: audit 2022-04-28T21:12:31.541361+0000 mon.a (mon.0) 2567 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:12:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:33 smithi139 bash[43301]: cluster 2022-04-28T21:12:33.283225+0000 mgr.y (mgr.24961) 2740 : cluster [DBG] pgmap v3079: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:33.839 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:33 smithi059 bash[61152]: cluster 2022-04-28T21:12:33.283225+0000 mgr.y (mgr.24961) 2740 : cluster [DBG] pgmap v3079: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:33.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:33 smithi059 bash[59745]: cluster 2022-04-28T21:12:33.283225+0000 mgr.y (mgr.24961) 2740 : cluster [DBG] pgmap v3079: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:34.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:33.838Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:35 smithi059 bash[61152]: cluster 2022-04-28T21:12:35.284629+0000 mgr.y (mgr.24961) 2741 : cluster [DBG] pgmap v3080: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:35 smithi059 bash[59745]: cluster 2022-04-28T21:12:35.284629+0000 mgr.y (mgr.24961) 2741 : cluster [DBG] pgmap v3080: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:35 smithi139 bash[43301]: cluster 2022-04-28T21:12:35.284629+0000 mgr.y (mgr.24961) 2741 : cluster [DBG] pgmap v3080: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:37 smithi139 bash[43301]: cluster 2022-04-28T21:12:37.285616+0000 mgr.y (mgr.24961) 2742 : cluster [DBG] pgmap v3081: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:37 smithi059 bash[61152]: cluster 2022-04-28T21:12:37.285616+0000 mgr.y (mgr.24961) 2742 : cluster [DBG] pgmap v3081: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:37 smithi059 bash[59745]: cluster 2022-04-28T21:12:37.285616+0000 mgr.y (mgr.24961) 2742 : cluster [DBG] pgmap v3081: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:39 smithi139 bash[43301]: cluster 2022-04-28T21:12:39.286706+0000 mgr.y (mgr.24961) 2743 : cluster [DBG] pgmap v3082: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:39 smithi059 bash[61152]: cluster 2022-04-28T21:12:39.286706+0000 mgr.y (mgr.24961) 2743 : cluster [DBG] pgmap v3082: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:39.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:39 smithi059 bash[59745]: cluster 2022-04-28T21:12:39.286706+0000 mgr.y (mgr.24961) 2743 : cluster [DBG] pgmap v3082: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:41 smithi139 bash[43301]: cluster 2022-04-28T21:12:41.287969+0000 mgr.y (mgr.24961) 2744 : cluster [DBG] pgmap v3083: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:41 smithi059 bash[61152]: cluster 2022-04-28T21:12:41.287969+0000 mgr.y (mgr.24961) 2744 : cluster [DBG] pgmap v3083: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:41 smithi059 bash[59745]: cluster 2022-04-28T21:12:41.287969+0000 mgr.y (mgr.24961) 2744 : cluster [DBG] pgmap v3083: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:41.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:43 smithi139 bash[43301]: cluster 2022-04-28T21:12:43.288664+0000 mgr.y (mgr.24961) 2745 : cluster [DBG] pgmap v3084: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:43 smithi059 bash[61152]: cluster 2022-04-28T21:12:43.288664+0000 mgr.y (mgr.24961) 2745 : cluster [DBG] pgmap v3084: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:43.841 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:43.839Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:43 smithi059 bash[59745]: cluster 2022-04-28T21:12:43.288664+0000 mgr.y (mgr.24961) 2745 : cluster [DBG] pgmap v3084: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:45 smithi059 bash[61152]: cluster 2022-04-28T21:12:45.290055+0000 mgr.y (mgr.24961) 2746 : cluster [DBG] pgmap v3085: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:45 smithi059 bash[59745]: cluster 2022-04-28T21:12:45.290055+0000 mgr.y (mgr.24961) 2746 : cluster [DBG] pgmap v3085: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:45 smithi139 bash[43301]: cluster 2022-04-28T21:12:45.290055+0000 mgr.y (mgr.24961) 2746 : cluster [DBG] pgmap v3085: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:47 smithi139 bash[43301]: cluster 2022-04-28T21:12:47.290984+0000 mgr.y (mgr.24961) 2747 : cluster [DBG] pgmap v3086: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:47 smithi059 bash[61152]: cluster 2022-04-28T21:12:47.290984+0000 mgr.y (mgr.24961) 2747 : cluster [DBG] pgmap v3086: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:47 smithi059 bash[59745]: cluster 2022-04-28T21:12:47.290984+0000 mgr.y (mgr.24961) 2747 : cluster [DBG] pgmap v3086: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:49 smithi139 bash[43301]: cluster 2022-04-28T21:12:49.292107+0000 mgr.y (mgr.24961) 2748 : cluster [DBG] pgmap v3087: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:49 smithi059 bash[59745]: cluster 2022-04-28T21:12:49.292107+0000 mgr.y (mgr.24961) 2748 : cluster [DBG] pgmap v3087: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:49.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:49 smithi059 bash[61152]: cluster 2022-04-28T21:12:49.292107+0000 mgr.y (mgr.24961) 2748 : cluster [DBG] pgmap v3087: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:51 smithi139 bash[43301]: cluster 2022-04-28T21:12:51.293338+0000 mgr.y (mgr.24961) 2749 : cluster [DBG] pgmap v3088: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:51 smithi059 bash[61152]: cluster 2022-04-28T21:12:51.293338+0000 mgr.y (mgr.24961) 2749 : cluster [DBG] pgmap v3088: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:51 smithi059 bash[59745]: cluster 2022-04-28T21:12:51.293338+0000 mgr.y (mgr.24961) 2749 : cluster [DBG] pgmap v3088: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:51.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:53 smithi139 bash[43301]: cluster 2022-04-28T21:12:53.294054+0000 mgr.y (mgr.24961) 2750 : cluster [DBG] pgmap v3089: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:53 smithi059 bash[61152]: cluster 2022-04-28T21:12:53.294054+0000 mgr.y (mgr.24961) 2750 : cluster [DBG] pgmap v3089: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:53 smithi059 bash[59745]: cluster 2022-04-28T21:12:53.294054+0000 mgr.y (mgr.24961) 2750 : cluster [DBG] pgmap v3089: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:12:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:12:53.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:12:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:55 smithi059 bash[61152]: cluster 2022-04-28T21:12:55.295398+0000 mgr.y (mgr.24961) 2751 : cluster [DBG] pgmap v3090: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:55 smithi059 bash[59745]: cluster 2022-04-28T21:12:55.295398+0000 mgr.y (mgr.24961) 2751 : cluster [DBG] pgmap v3090: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:55 smithi139 bash[43301]: cluster 2022-04-28T21:12:55.295398+0000 mgr.y (mgr.24961) 2751 : cluster [DBG] pgmap v3090: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:57 smithi139 bash[43301]: cluster 2022-04-28T21:12:57.296345+0000 mgr.y (mgr.24961) 2752 : cluster [DBG] pgmap v3091: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:57 smithi059 bash[61152]: cluster 2022-04-28T21:12:57.296345+0000 mgr.y (mgr.24961) 2752 : cluster [DBG] pgmap v3091: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:57 smithi059 bash[59745]: cluster 2022-04-28T21:12:57.296345+0000 mgr.y (mgr.24961) 2752 : cluster [DBG] pgmap v3091: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:12:59 smithi139 bash[43301]: cluster 2022-04-28T21:12:59.297431+0000 mgr.y (mgr.24961) 2753 : cluster [DBG] pgmap v3092: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:12:59 smithi059 bash[61152]: cluster 2022-04-28T21:12:59.297431+0000 mgr.y (mgr.24961) 2753 : cluster [DBG] pgmap v3092: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:12:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:12:59 smithi059 bash[59745]: cluster 2022-04-28T21:12:59.297431+0000 mgr.y (mgr.24961) 2753 : cluster [DBG] pgmap v3092: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:01 smithi139 bash[43301]: cluster 2022-04-28T21:13:01.298658+0000 mgr.y (mgr.24961) 2754 : cluster [DBG] pgmap v3093: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:01 smithi059 bash[61152]: cluster 2022-04-28T21:13:01.298658+0000 mgr.y (mgr.24961) 2754 : cluster [DBG] pgmap v3093: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:01 smithi059 bash[59745]: cluster 2022-04-28T21:13:01.298658+0000 mgr.y (mgr.24961) 2754 : cluster [DBG] pgmap v3093: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:01.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:03 smithi139 bash[43301]: cluster 2022-04-28T21:13:03.299364+0000 mgr.y (mgr.24961) 2755 : cluster [DBG] pgmap v3094: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:03 smithi059 bash[61152]: cluster 2022-04-28T21:13:03.299364+0000 mgr.y (mgr.24961) 2755 : cluster [DBG] pgmap v3094: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:04.017 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:03 smithi059 bash[59745]: cluster 2022-04-28T21:13:03.299364+0000 mgr.y (mgr.24961) 2755 : cluster [DBG] pgmap v3094: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:03.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:05 smithi059 bash[61152]: cluster 2022-04-28T21:13:05.300753+0000 mgr.y (mgr.24961) 2756 : cluster [DBG] pgmap v3095: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:05 smithi059 bash[59745]: cluster 2022-04-28T21:13:05.300753+0000 mgr.y (mgr.24961) 2756 : cluster [DBG] pgmap v3095: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:05 smithi139 bash[43301]: cluster 2022-04-28T21:13:05.300753+0000 mgr.y (mgr.24961) 2756 : cluster [DBG] pgmap v3095: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:07 smithi139 bash[43301]: cluster 2022-04-28T21:13:07.301753+0000 mgr.y (mgr.24961) 2757 : cluster [DBG] pgmap v3096: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:07 smithi059 bash[61152]: cluster 2022-04-28T21:13:07.301753+0000 mgr.y (mgr.24961) 2757 : cluster [DBG] pgmap v3096: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:07.943 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:07 smithi059 bash[59745]: cluster 2022-04-28T21:13:07.301753+0000 mgr.y (mgr.24961) 2757 : cluster [DBG] pgmap v3096: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:09 smithi139 bash[43301]: cluster 2022-04-28T21:13:09.302828+0000 mgr.y (mgr.24961) 2758 : cluster [DBG] pgmap v3097: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:09 smithi059 bash[61152]: cluster 2022-04-28T21:13:09.302828+0000 mgr.y (mgr.24961) 2758 : cluster [DBG] pgmap v3097: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:09 smithi059 bash[59745]: cluster 2022-04-28T21:13:09.302828+0000 mgr.y (mgr.24961) 2758 : cluster [DBG] pgmap v3097: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:11 smithi139 bash[43301]: cluster 2022-04-28T21:13:11.304090+0000 mgr.y (mgr.24961) 2759 : cluster [DBG] pgmap v3098: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:11 smithi059 bash[61152]: cluster 2022-04-28T21:13:11.304090+0000 mgr.y (mgr.24961) 2759 : cluster [DBG] pgmap v3098: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:11 smithi059 bash[59745]: cluster 2022-04-28T21:13:11.304090+0000 mgr.y (mgr.24961) 2759 : cluster [DBG] pgmap v3098: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:11.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:13 smithi139 bash[43301]: cluster 2022-04-28T21:13:13.304796+0000 mgr.y (mgr.24961) 2760 : cluster [DBG] pgmap v3099: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:13 smithi059 bash[61152]: cluster 2022-04-28T21:13:13.304796+0000 mgr.y (mgr.24961) 2760 : cluster [DBG] pgmap v3099: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:13 smithi059 bash[59745]: cluster 2022-04-28T21:13:13.304796+0000 mgr.y (mgr.24961) 2760 : cluster [DBG] pgmap v3099: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:13.841Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:15 smithi059 bash[61152]: cluster 2022-04-28T21:13:15.306157+0000 mgr.y (mgr.24961) 2761 : cluster [DBG] pgmap v3100: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:15 smithi059 bash[59745]: cluster 2022-04-28T21:13:15.306157+0000 mgr.y (mgr.24961) 2761 : cluster [DBG] pgmap v3100: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:15 smithi139 bash[43301]: cluster 2022-04-28T21:13:15.306157+0000 mgr.y (mgr.24961) 2761 : cluster [DBG] pgmap v3100: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:17 smithi059 bash[61152]: cluster 2022-04-28T21:13:17.307123+0000 mgr.y (mgr.24961) 2762 : cluster [DBG] pgmap v3101: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:17 smithi059 bash[59745]: cluster 2022-04-28T21:13:17.307123+0000 mgr.y (mgr.24961) 2762 : cluster [DBG] pgmap v3101: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:17 smithi139 bash[43301]: cluster 2022-04-28T21:13:17.307123+0000 mgr.y (mgr.24961) 2762 : cluster [DBG] pgmap v3101: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[61152]: audit 2022-04-28T21:13:18.160041+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:13:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[61152]: audit 2022-04-28T21:13:18.165654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:13:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[61152]: audit 2022-04-28T21:13:18.179810+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:13:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[61152]: audit 2022-04-28T21:13:18.180451+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:13:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[61152]: audit 2022-04-28T21:13:18.180756+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:13:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[61152]: audit 2022-04-28T21:13:18.181076+0000 mon.a (mon.0) 2573 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:13:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[59745]: audit 2022-04-28T21:13:18.160041+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:13:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[59745]: audit 2022-04-28T21:13:18.165654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:13:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[59745]: audit 2022-04-28T21:13:18.179810+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:13:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[59745]: audit 2022-04-28T21:13:18.180451+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:13:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[59745]: audit 2022-04-28T21:13:18.180756+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:13:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:18 smithi059 bash[59745]: audit 2022-04-28T21:13:18.181076+0000 mon.a (mon.0) 2573 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:13:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:18 smithi139 bash[43301]: audit 2022-04-28T21:13:18.160041+0000 mon.a (mon.0) 2568 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:13:18.898 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:18 smithi139 bash[43301]: audit 2022-04-28T21:13:18.165654+0000 mon.a (mon.0) 2569 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:13:18.899 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:18 smithi139 bash[43301]: audit 2022-04-28T21:13:18.179810+0000 mon.a (mon.0) 2570 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:13:18.899 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:18 smithi139 bash[43301]: audit 2022-04-28T21:13:18.180451+0000 mon.a (mon.0) 2571 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:13:18.899 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:18 smithi139 bash[43301]: audit 2022-04-28T21:13:18.180756+0000 mon.a (mon.0) 2572 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:13:18.899 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:18 smithi139 bash[43301]: audit 2022-04-28T21:13:18.181076+0000 mon.a (mon.0) 2573 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:13:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:19 smithi059 bash[61152]: cluster 2022-04-28T21:13:19.308261+0000 mgr.y (mgr.24961) 2763 : cluster [DBG] pgmap v3102: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:19.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:19 smithi059 bash[59745]: cluster 2022-04-28T21:13:19.308261+0000 mgr.y (mgr.24961) 2763 : cluster [DBG] pgmap v3102: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:19 smithi139 bash[43301]: cluster 2022-04-28T21:13:19.308261+0000 mgr.y (mgr.24961) 2763 : cluster [DBG] pgmap v3102: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:21.949Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:22 smithi139 bash[43301]: cluster 2022-04-28T21:13:21.309489+0000 mgr.y (mgr.24961) 2764 : cluster [DBG] pgmap v3103: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:22 smithi059 bash[61152]: cluster 2022-04-28T21:13:21.309489+0000 mgr.y (mgr.24961) 2764 : cluster [DBG] pgmap v3103: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:22 smithi059 bash[59745]: cluster 2022-04-28T21:13:21.309489+0000 mgr.y (mgr.24961) 2764 : cluster [DBG] pgmap v3103: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:23.840Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:24 smithi059 bash[59745]: cluster 2022-04-28T21:13:23.310225+0000 mgr.y (mgr.24961) 2765 : cluster [DBG] pgmap v3104: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:24 smithi059 bash[61152]: cluster 2022-04-28T21:13:23.310225+0000 mgr.y (mgr.24961) 2765 : cluster [DBG] pgmap v3104: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:24.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:24 smithi139 bash[43301]: cluster 2022-04-28T21:13:23.310225+0000 mgr.y (mgr.24961) 2765 : cluster [DBG] pgmap v3104: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:26 smithi059 bash[61152]: cluster 2022-04-28T21:13:25.311665+0000 mgr.y (mgr.24961) 2766 : cluster [DBG] pgmap v3105: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:26 smithi059 bash[59745]: cluster 2022-04-28T21:13:25.311665+0000 mgr.y (mgr.24961) 2766 : cluster [DBG] pgmap v3105: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:26.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:26 smithi139 bash[43301]: cluster 2022-04-28T21:13:25.311665+0000 mgr.y (mgr.24961) 2766 : cluster [DBG] pgmap v3105: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:28 smithi059 bash[61152]: cluster 2022-04-28T21:13:27.312642+0000 mgr.y (mgr.24961) 2767 : cluster [DBG] pgmap v3106: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:28 smithi059 bash[59745]: cluster 2022-04-28T21:13:27.312642+0000 mgr.y (mgr.24961) 2767 : cluster [DBG] pgmap v3106: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:28.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:28 smithi139 bash[43301]: cluster 2022-04-28T21:13:27.312642+0000 mgr.y (mgr.24961) 2767 : cluster [DBG] pgmap v3106: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:30 smithi059 bash[61152]: cluster 2022-04-28T21:13:29.313749+0000 mgr.y (mgr.24961) 2768 : cluster [DBG] pgmap v3107: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:30 smithi059 bash[59745]: cluster 2022-04-28T21:13:29.313749+0000 mgr.y (mgr.24961) 2768 : cluster [DBG] pgmap v3107: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:30.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:30 smithi139 bash[43301]: cluster 2022-04-28T21:13:29.313749+0000 mgr.y (mgr.24961) 2768 : cluster [DBG] pgmap v3107: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:31.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[59745]: cluster 2022-04-28T21:13:31.315316+0000 mgr.y (mgr.24961) 2769 : cluster [DBG] pgmap v3108: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[59745]: audit 2022-04-28T21:13:31.549172+0000 mon.a (mon.0) 2574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:13:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[59745]: audit 2022-04-28T21:13:31.552077+0000 mon.a (mon.0) 2575 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:13:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[59745]: audit 2022-04-28T21:13:31.554034+0000 mon.a (mon.0) 2576 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:13:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[59745]: cephadm 2022-04-28T21:13:31.555892+0000 mgr.y (mgr.24961) 2770 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:13:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[59745]: audit 2022-04-28T21:13:31.884119+0000 mon.a (mon.0) 2577 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:13:32.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[61152]: cluster 2022-04-28T21:13:31.315316+0000 mgr.y (mgr.24961) 2769 : cluster [DBG] pgmap v3108: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:32.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[61152]: audit 2022-04-28T21:13:31.549172+0000 mon.a (mon.0) 2574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:13:32.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[61152]: audit 2022-04-28T21:13:31.552077+0000 mon.a (mon.0) 2575 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:13:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[61152]: audit 2022-04-28T21:13:31.554034+0000 mon.a (mon.0) 2576 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:13:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[61152]: cephadm 2022-04-28T21:13:31.555892+0000 mgr.y (mgr.24961) 2770 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:13:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:32 smithi059 bash[61152]: audit 2022-04-28T21:13:31.884119+0000 mon.a (mon.0) 2577 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:13:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:32 smithi139 bash[43301]: cluster 2022-04-28T21:13:31.315316+0000 mgr.y (mgr.24961) 2769 : cluster [DBG] pgmap v3108: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:32 smithi139 bash[43301]: audit 2022-04-28T21:13:31.549172+0000 mon.a (mon.0) 2574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:13:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:32 smithi139 bash[43301]: audit 2022-04-28T21:13:31.552077+0000 mon.a (mon.0) 2575 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:13:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:32 smithi139 bash[43301]: audit 2022-04-28T21:13:31.554034+0000 mon.a (mon.0) 2576 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:13:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:32 smithi139 bash[43301]: cephadm 2022-04-28T21:13:31.555892+0000 mgr.y (mgr.24961) 2770 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:13:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:32 smithi139 bash[43301]: audit 2022-04-28T21:13:31.884119+0000 mon.a (mon.0) 2577 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:13:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:33.842Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:34 smithi059 bash[61152]: cluster 2022-04-28T21:13:33.316129+0000 mgr.y (mgr.24961) 2771 : cluster [DBG] pgmap v3109: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:34 smithi059 bash[59745]: cluster 2022-04-28T21:13:33.316129+0000 mgr.y (mgr.24961) 2771 : cluster [DBG] pgmap v3109: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:34.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:34 smithi139 bash[43301]: cluster 2022-04-28T21:13:33.316129+0000 mgr.y (mgr.24961) 2771 : cluster [DBG] pgmap v3109: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:36 smithi059 bash[61152]: cluster 2022-04-28T21:13:35.317541+0000 mgr.y (mgr.24961) 2772 : cluster [DBG] pgmap v3110: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:36 smithi059 bash[59745]: cluster 2022-04-28T21:13:35.317541+0000 mgr.y (mgr.24961) 2772 : cluster [DBG] pgmap v3110: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:36 smithi139 bash[43301]: cluster 2022-04-28T21:13:35.317541+0000 mgr.y (mgr.24961) 2772 : cluster [DBG] pgmap v3110: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:38 smithi059 bash[61152]: cluster 2022-04-28T21:13:37.318560+0000 mgr.y (mgr.24961) 2773 : cluster [DBG] pgmap v3111: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:38 smithi059 bash[59745]: cluster 2022-04-28T21:13:37.318560+0000 mgr.y (mgr.24961) 2773 : cluster [DBG] pgmap v3111: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:38.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:38 smithi139 bash[43301]: cluster 2022-04-28T21:13:37.318560+0000 mgr.y (mgr.24961) 2773 : cluster [DBG] pgmap v3111: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:40 smithi059 bash[59745]: cluster 2022-04-28T21:13:39.319671+0000 mgr.y (mgr.24961) 2774 : cluster [DBG] pgmap v3112: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:40 smithi059 bash[61152]: cluster 2022-04-28T21:13:39.319671+0000 mgr.y (mgr.24961) 2774 : cluster [DBG] pgmap v3112: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:40 smithi139 bash[43301]: cluster 2022-04-28T21:13:39.319671+0000 mgr.y (mgr.24961) 2774 : cluster [DBG] pgmap v3112: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:41.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:42 smithi059 bash[61152]: cluster 2022-04-28T21:13:41.320914+0000 mgr.y (mgr.24961) 2775 : cluster [DBG] pgmap v3113: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:42 smithi059 bash[59745]: cluster 2022-04-28T21:13:41.320914+0000 mgr.y (mgr.24961) 2775 : cluster [DBG] pgmap v3113: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:42 smithi139 bash[43301]: cluster 2022-04-28T21:13:41.320914+0000 mgr.y (mgr.24961) 2775 : cluster [DBG] pgmap v3113: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:43.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:44 smithi059 bash[61152]: cluster 2022-04-28T21:13:43.321653+0000 mgr.y (mgr.24961) 2776 : cluster [DBG] pgmap v3114: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:44 smithi059 bash[59745]: cluster 2022-04-28T21:13:43.321653+0000 mgr.y (mgr.24961) 2776 : cluster [DBG] pgmap v3114: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:44.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:44 smithi139 bash[43301]: cluster 2022-04-28T21:13:43.321653+0000 mgr.y (mgr.24961) 2776 : cluster [DBG] pgmap v3114: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:46 smithi059 bash[61152]: cluster 2022-04-28T21:13:45.323045+0000 mgr.y (mgr.24961) 2777 : cluster [DBG] pgmap v3115: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:46 smithi059 bash[59745]: cluster 2022-04-28T21:13:45.323045+0000 mgr.y (mgr.24961) 2777 : cluster [DBG] pgmap v3115: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:46 smithi139 bash[43301]: cluster 2022-04-28T21:13:45.323045+0000 mgr.y (mgr.24961) 2777 : cluster [DBG] pgmap v3115: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:48 smithi059 bash[61152]: cluster 2022-04-28T21:13:47.324018+0000 mgr.y (mgr.24961) 2778 : cluster [DBG] pgmap v3116: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:48 smithi059 bash[59745]: cluster 2022-04-28T21:13:47.324018+0000 mgr.y (mgr.24961) 2778 : cluster [DBG] pgmap v3116: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:48 smithi139 bash[43301]: cluster 2022-04-28T21:13:47.324018+0000 mgr.y (mgr.24961) 2778 : cluster [DBG] pgmap v3116: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:50 smithi059 bash[61152]: cluster 2022-04-28T21:13:49.325113+0000 mgr.y (mgr.24961) 2779 : cluster [DBG] pgmap v3117: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:50.870 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:50 smithi059 bash[59745]: cluster 2022-04-28T21:13:49.325113+0000 mgr.y (mgr.24961) 2779 : cluster [DBG] pgmap v3117: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:50.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:50 smithi139 bash[43301]: cluster 2022-04-28T21:13:49.325113+0000 mgr.y (mgr.24961) 2779 : cluster [DBG] pgmap v3117: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:51.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:52 smithi059 bash[61152]: cluster 2022-04-28T21:13:51.326351+0000 mgr.y (mgr.24961) 2780 : cluster [DBG] pgmap v3118: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:52 smithi059 bash[59745]: cluster 2022-04-28T21:13:51.326351+0000 mgr.y (mgr.24961) 2780 : cluster [DBG] pgmap v3118: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:52 smithi139 bash[43301]: cluster 2022-04-28T21:13:51.326351+0000 mgr.y (mgr.24961) 2780 : cluster [DBG] pgmap v3118: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:13:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:13:53.843Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:13:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:54 smithi059 bash[61152]: cluster 2022-04-28T21:13:53.327080+0000 mgr.y (mgr.24961) 2781 : cluster [DBG] pgmap v3119: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:54.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:54 smithi059 bash[59745]: cluster 2022-04-28T21:13:53.327080+0000 mgr.y (mgr.24961) 2781 : cluster [DBG] pgmap v3119: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:54.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:54 smithi139 bash[43301]: cluster 2022-04-28T21:13:53.327080+0000 mgr.y (mgr.24961) 2781 : cluster [DBG] pgmap v3119: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:56 smithi059 bash[61152]: cluster 2022-04-28T21:13:55.328129+0000 mgr.y (mgr.24961) 2782 : cluster [DBG] pgmap v3120: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:56 smithi059 bash[59745]: cluster 2022-04-28T21:13:55.328129+0000 mgr.y (mgr.24961) 2782 : cluster [DBG] pgmap v3120: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:56 smithi139 bash[43301]: cluster 2022-04-28T21:13:55.328129+0000 mgr.y (mgr.24961) 2782 : cluster [DBG] pgmap v3120: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:13:58 smithi059 bash[61152]: cluster 2022-04-28T21:13:57.329055+0000 mgr.y (mgr.24961) 2783 : cluster [DBG] pgmap v3121: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:13:58 smithi059 bash[59745]: cluster 2022-04-28T21:13:57.329055+0000 mgr.y (mgr.24961) 2783 : cluster [DBG] pgmap v3121: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:13:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:13:58 smithi139 bash[43301]: cluster 2022-04-28T21:13:57.329055+0000 mgr.y (mgr.24961) 2783 : cluster [DBG] pgmap v3121: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:00 smithi059 bash[61152]: cluster 2022-04-28T21:13:59.330187+0000 mgr.y (mgr.24961) 2784 : cluster [DBG] pgmap v3122: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:00 smithi059 bash[59745]: cluster 2022-04-28T21:13:59.330187+0000 mgr.y (mgr.24961) 2784 : cluster [DBG] pgmap v3122: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:00 smithi139 bash[43301]: cluster 2022-04-28T21:13:59.330187+0000 mgr.y (mgr.24961) 2784 : cluster [DBG] pgmap v3122: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:01.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:02 smithi059 bash[61152]: cluster 2022-04-28T21:14:01.331437+0000 mgr.y (mgr.24961) 2785 : cluster [DBG] pgmap v3123: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:02 smithi059 bash[59745]: cluster 2022-04-28T21:14:01.331437+0000 mgr.y (mgr.24961) 2785 : cluster [DBG] pgmap v3123: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:02 smithi139 bash[43301]: cluster 2022-04-28T21:14:01.331437+0000 mgr.y (mgr.24961) 2785 : cluster [DBG] pgmap v3123: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:03.844Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:04.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:04 smithi059 bash[61152]: cluster 2022-04-28T21:14:03.332132+0000 mgr.y (mgr.24961) 2786 : cluster [DBG] pgmap v3124: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:04.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:04 smithi059 bash[59745]: cluster 2022-04-28T21:14:03.332132+0000 mgr.y (mgr.24961) 2786 : cluster [DBG] pgmap v3124: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:04 smithi139 bash[43301]: cluster 2022-04-28T21:14:03.332132+0000 mgr.y (mgr.24961) 2786 : cluster [DBG] pgmap v3124: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:06 smithi059 bash[61152]: cluster 2022-04-28T21:14:05.333511+0000 mgr.y (mgr.24961) 2787 : cluster [DBG] pgmap v3125: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:06 smithi059 bash[59745]: cluster 2022-04-28T21:14:05.333511+0000 mgr.y (mgr.24961) 2787 : cluster [DBG] pgmap v3125: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:06 smithi139 bash[43301]: cluster 2022-04-28T21:14:05.333511+0000 mgr.y (mgr.24961) 2787 : cluster [DBG] pgmap v3125: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:08 smithi059 bash[61152]: cluster 2022-04-28T21:14:07.334482+0000 mgr.y (mgr.24961) 2788 : cluster [DBG] pgmap v3126: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:08 smithi059 bash[59745]: cluster 2022-04-28T21:14:07.334482+0000 mgr.y (mgr.24961) 2788 : cluster [DBG] pgmap v3126: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:08.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:08 smithi139 bash[43301]: cluster 2022-04-28T21:14:07.334482+0000 mgr.y (mgr.24961) 2788 : cluster [DBG] pgmap v3126: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:10.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:10 smithi059 bash[59745]: cluster 2022-04-28T21:14:09.335576+0000 mgr.y (mgr.24961) 2789 : cluster [DBG] pgmap v3127: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:10 smithi059 bash[61152]: cluster 2022-04-28T21:14:09.335576+0000 mgr.y (mgr.24961) 2789 : cluster [DBG] pgmap v3127: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:10.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:10 smithi139 bash[43301]: cluster 2022-04-28T21:14:09.335576+0000 mgr.y (mgr.24961) 2789 : cluster [DBG] pgmap v3127: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:11.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:12 smithi059 bash[61152]: cluster 2022-04-28T21:14:11.336790+0000 mgr.y (mgr.24961) 2790 : cluster [DBG] pgmap v3128: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:12.858 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:12 smithi059 bash[59745]: cluster 2022-04-28T21:14:11.336790+0000 mgr.y (mgr.24961) 2790 : cluster [DBG] pgmap v3128: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:12.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:12 smithi139 bash[43301]: cluster 2022-04-28T21:14:11.336790+0000 mgr.y (mgr.24961) 2790 : cluster [DBG] pgmap v3128: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:13.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:14.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:14 smithi139 bash[43301]: cluster 2022-04-28T21:14:13.337485+0000 mgr.y (mgr.24961) 2791 : cluster [DBG] pgmap v3129: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:14 smithi059 bash[61152]: cluster 2022-04-28T21:14:13.337485+0000 mgr.y (mgr.24961) 2791 : cluster [DBG] pgmap v3129: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:14 smithi059 bash[59745]: cluster 2022-04-28T21:14:13.337485+0000 mgr.y (mgr.24961) 2791 : cluster [DBG] pgmap v3129: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:16.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:16 smithi139 bash[43301]: cluster 2022-04-28T21:14:15.338852+0000 mgr.y (mgr.24961) 2792 : cluster [DBG] pgmap v3130: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:16 smithi059 bash[61152]: cluster 2022-04-28T21:14:15.338852+0000 mgr.y (mgr.24961) 2792 : cluster [DBG] pgmap v3130: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:17.111 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:16 smithi059 bash[59745]: cluster 2022-04-28T21:14:15.338852+0000 mgr.y (mgr.24961) 2792 : cluster [DBG] pgmap v3130: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: cluster 2022-04-28T21:14:17.339844+0000 mgr.y (mgr.24961) 2793 : cluster [DBG] pgmap v3131: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:18.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: audit 2022-04-28T21:14:18.184467+0000 mon.a (mon.0) 2578 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: audit 2022-04-28T21:14:18.186231+0000 mon.a (mon.0) 2579 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: audit 2022-04-28T21:14:18.197906+0000 mon.a (mon.0) 2580 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: audit 2022-04-28T21:14:18.198388+0000 mon.a (mon.0) 2581 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: audit 2022-04-28T21:14:18.198817+0000 mon.a (mon.0) 2582 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:14:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:18 smithi139 bash[43301]: audit 2022-04-28T21:14:18.199215+0000 mon.a (mon.0) 2583 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:14:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: cluster 2022-04-28T21:14:17.339844+0000 mgr.y (mgr.24961) 2793 : cluster [DBG] pgmap v3131: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: audit 2022-04-28T21:14:18.184467+0000 mon.a (mon.0) 2578 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:14:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: audit 2022-04-28T21:14:18.186231+0000 mon.a (mon.0) 2579 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:14:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: audit 2022-04-28T21:14:18.197906+0000 mon.a (mon.0) 2580 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:14:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: audit 2022-04-28T21:14:18.198388+0000 mon.a (mon.0) 2581 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:14:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: audit 2022-04-28T21:14:18.198817+0000 mon.a (mon.0) 2582 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:14:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[61152]: audit 2022-04-28T21:14:18.199215+0000 mon.a (mon.0) 2583 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:14:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: cluster 2022-04-28T21:14:17.339844+0000 mgr.y (mgr.24961) 2793 : cluster [DBG] pgmap v3131: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: audit 2022-04-28T21:14:18.184467+0000 mon.a (mon.0) 2578 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:14:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: audit 2022-04-28T21:14:18.186231+0000 mon.a (mon.0) 2579 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:14:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: audit 2022-04-28T21:14:18.197906+0000 mon.a (mon.0) 2580 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:14:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: audit 2022-04-28T21:14:18.198388+0000 mon.a (mon.0) 2581 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:14:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: audit 2022-04-28T21:14:18.198817+0000 mon.a (mon.0) 2582 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:14:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:18 smithi059 bash[59745]: audit 2022-04-28T21:14:18.199215+0000 mon.a (mon.0) 2583 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:14:20.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:20 smithi139 bash[43301]: cluster 2022-04-28T21:14:19.340891+0000 mgr.y (mgr.24961) 2794 : cluster [DBG] pgmap v3132: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:20 smithi059 bash[61152]: cluster 2022-04-28T21:14:19.340891+0000 mgr.y (mgr.24961) 2794 : cluster [DBG] pgmap v3132: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:20 smithi059 bash[59745]: cluster 2022-04-28T21:14:19.340891+0000 mgr.y (mgr.24961) 2794 : cluster [DBG] pgmap v3132: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:21.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:22.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:22 smithi139 bash[43301]: cluster 2022-04-28T21:14:21.342221+0000 mgr.y (mgr.24961) 2795 : cluster [DBG] pgmap v3133: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:22 smithi059 bash[61152]: cluster 2022-04-28T21:14:21.342221+0000 mgr.y (mgr.24961) 2795 : cluster [DBG] pgmap v3133: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:22 smithi059 bash[59745]: cluster 2022-04-28T21:14:21.342221+0000 mgr.y (mgr.24961) 2795 : cluster [DBG] pgmap v3133: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:23.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:24 smithi059 bash[61152]: cluster 2022-04-28T21:14:23.342917+0000 mgr.y (mgr.24961) 2796 : cluster [DBG] pgmap v3134: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:24 smithi059 bash[59745]: cluster 2022-04-28T21:14:23.342917+0000 mgr.y (mgr.24961) 2796 : cluster [DBG] pgmap v3134: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:25.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:24 smithi139 bash[43301]: cluster 2022-04-28T21:14:23.342917+0000 mgr.y (mgr.24961) 2796 : cluster [DBG] pgmap v3134: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:26 smithi059 bash[61152]: cluster 2022-04-28T21:14:25.344322+0000 mgr.y (mgr.24961) 2797 : cluster [DBG] pgmap v3135: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:27.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:26 smithi059 bash[59745]: cluster 2022-04-28T21:14:25.344322+0000 mgr.y (mgr.24961) 2797 : cluster [DBG] pgmap v3135: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:26 smithi139 bash[43301]: cluster 2022-04-28T21:14:25.344322+0000 mgr.y (mgr.24961) 2797 : cluster [DBG] pgmap v3135: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:28 smithi059 bash[61152]: cluster 2022-04-28T21:14:27.345175+0000 mgr.y (mgr.24961) 2798 : cluster [DBG] pgmap v3136: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:28 smithi059 bash[59745]: cluster 2022-04-28T21:14:27.345175+0000 mgr.y (mgr.24961) 2798 : cluster [DBG] pgmap v3136: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:28 smithi139 bash[43301]: cluster 2022-04-28T21:14:27.345175+0000 mgr.y (mgr.24961) 2798 : cluster [DBG] pgmap v3136: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:30 smithi059 bash[61152]: cluster 2022-04-28T21:14:29.346261+0000 mgr.y (mgr.24961) 2799 : cluster [DBG] pgmap v3137: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:30 smithi059 bash[59745]: cluster 2022-04-28T21:14:29.346261+0000 mgr.y (mgr.24961) 2799 : cluster [DBG] pgmap v3137: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:30 smithi139 bash[43301]: cluster 2022-04-28T21:14:29.346261+0000 mgr.y (mgr.24961) 2799 : cluster [DBG] pgmap v3137: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:31.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[59745]: cluster 2022-04-28T21:14:31.347539+0000 mgr.y (mgr.24961) 2800 : cluster [DBG] pgmap v3138: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[59745]: audit 2022-04-28T21:14:31.892460+0000 mon.a (mon.0) 2584 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:14:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[59745]: audit 2022-04-28T21:14:31.895508+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:14:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[59745]: audit 2022-04-28T21:14:31.897665+0000 mon.a (mon.0) 2586 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:14:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[59745]: cephadm 2022-04-28T21:14:31.899520+0000 mgr.y (mgr.24961) 2801 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:14:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[59745]: audit 2022-04-28T21:14:32.233379+0000 mon.a (mon.0) 2587 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:14:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[61152]: cluster 2022-04-28T21:14:31.347539+0000 mgr.y (mgr.24961) 2800 : cluster [DBG] pgmap v3138: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[61152]: audit 2022-04-28T21:14:31.892460+0000 mon.a (mon.0) 2584 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:14:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[61152]: audit 2022-04-28T21:14:31.895508+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:14:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[61152]: audit 2022-04-28T21:14:31.897665+0000 mon.a (mon.0) 2586 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:14:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[61152]: cephadm 2022-04-28T21:14:31.899520+0000 mgr.y (mgr.24961) 2801 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:14:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:32 smithi059 bash[61152]: audit 2022-04-28T21:14:32.233379+0000 mon.a (mon.0) 2587 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:14:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:32 smithi139 bash[43301]: cluster 2022-04-28T21:14:31.347539+0000 mgr.y (mgr.24961) 2800 : cluster [DBG] pgmap v3138: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:32 smithi139 bash[43301]: audit 2022-04-28T21:14:31.892460+0000 mon.a (mon.0) 2584 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:14:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:32 smithi139 bash[43301]: audit 2022-04-28T21:14:31.895508+0000 mon.a (mon.0) 2585 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:14:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:32 smithi139 bash[43301]: audit 2022-04-28T21:14:31.897665+0000 mon.a (mon.0) 2586 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:14:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:32 smithi139 bash[43301]: cephadm 2022-04-28T21:14:31.899520+0000 mgr.y (mgr.24961) 2801 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:14:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:32 smithi139 bash[43301]: audit 2022-04-28T21:14:32.233379+0000 mon.a (mon.0) 2587 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:14:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:33.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:35.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:34 smithi059 bash[59745]: cluster 2022-04-28T21:14:33.348245+0000 mgr.y (mgr.24961) 2802 : cluster [DBG] pgmap v3139: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:34 smithi059 bash[61152]: cluster 2022-04-28T21:14:33.348245+0000 mgr.y (mgr.24961) 2802 : cluster [DBG] pgmap v3139: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:34 smithi139 bash[43301]: cluster 2022-04-28T21:14:33.348245+0000 mgr.y (mgr.24961) 2802 : cluster [DBG] pgmap v3139: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:36 smithi059 bash[61152]: cluster 2022-04-28T21:14:35.349689+0000 mgr.y (mgr.24961) 2803 : cluster [DBG] pgmap v3140: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:37.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:36 smithi059 bash[59745]: cluster 2022-04-28T21:14:35.349689+0000 mgr.y (mgr.24961) 2803 : cluster [DBG] pgmap v3140: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:37.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:36 smithi139 bash[43301]: cluster 2022-04-28T21:14:35.349689+0000 mgr.y (mgr.24961) 2803 : cluster [DBG] pgmap v3140: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:38 smithi059 bash[61152]: cluster 2022-04-28T21:14:37.350665+0000 mgr.y (mgr.24961) 2804 : cluster [DBG] pgmap v3141: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:38 smithi059 bash[59745]: cluster 2022-04-28T21:14:37.350665+0000 mgr.y (mgr.24961) 2804 : cluster [DBG] pgmap v3141: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:38 smithi139 bash[43301]: cluster 2022-04-28T21:14:37.350665+0000 mgr.y (mgr.24961) 2804 : cluster [DBG] pgmap v3141: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:41.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:40 smithi059 bash[61152]: cluster 2022-04-28T21:14:39.351812+0000 mgr.y (mgr.24961) 2805 : cluster [DBG] pgmap v3142: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:41.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:40 smithi059 bash[59745]: cluster 2022-04-28T21:14:39.351812+0000 mgr.y (mgr.24961) 2805 : cluster [DBG] pgmap v3142: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:40 smithi139 bash[43301]: cluster 2022-04-28T21:14:39.351812+0000 mgr.y (mgr.24961) 2805 : cluster [DBG] pgmap v3142: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:41.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:43.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:42 smithi059 bash[61152]: cluster 2022-04-28T21:14:41.353038+0000 mgr.y (mgr.24961) 2806 : cluster [DBG] pgmap v3143: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:42 smithi059 bash[59745]: cluster 2022-04-28T21:14:41.353038+0000 mgr.y (mgr.24961) 2806 : cluster [DBG] pgmap v3143: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:42 smithi139 bash[43301]: cluster 2022-04-28T21:14:41.353038+0000 mgr.y (mgr.24961) 2806 : cluster [DBG] pgmap v3143: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:43.845Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:44 smithi059 bash[61152]: cluster 2022-04-28T21:14:43.353743+0000 mgr.y (mgr.24961) 2807 : cluster [DBG] pgmap v3144: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:45.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:44 smithi059 bash[59745]: cluster 2022-04-28T21:14:43.353743+0000 mgr.y (mgr.24961) 2807 : cluster [DBG] pgmap v3144: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:44 smithi139 bash[43301]: cluster 2022-04-28T21:14:43.353743+0000 mgr.y (mgr.24961) 2807 : cluster [DBG] pgmap v3144: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:46 smithi059 bash[61152]: cluster 2022-04-28T21:14:45.355136+0000 mgr.y (mgr.24961) 2808 : cluster [DBG] pgmap v3145: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:46 smithi059 bash[59745]: cluster 2022-04-28T21:14:45.355136+0000 mgr.y (mgr.24961) 2808 : cluster [DBG] pgmap v3145: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:46 smithi139 bash[43301]: cluster 2022-04-28T21:14:45.355136+0000 mgr.y (mgr.24961) 2808 : cluster [DBG] pgmap v3145: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:48 smithi059 bash[61152]: cluster 2022-04-28T21:14:47.356131+0000 mgr.y (mgr.24961) 2809 : cluster [DBG] pgmap v3146: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:48 smithi059 bash[59745]: cluster 2022-04-28T21:14:47.356131+0000 mgr.y (mgr.24961) 2809 : cluster [DBG] pgmap v3146: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:48 smithi139 bash[43301]: cluster 2022-04-28T21:14:47.356131+0000 mgr.y (mgr.24961) 2809 : cluster [DBG] pgmap v3146: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:50 smithi059 bash[61152]: cluster 2022-04-28T21:14:49.357329+0000 mgr.y (mgr.24961) 2810 : cluster [DBG] pgmap v3147: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:50 smithi059 bash[59745]: cluster 2022-04-28T21:14:49.357329+0000 mgr.y (mgr.24961) 2810 : cluster [DBG] pgmap v3147: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:50 smithi139 bash[43301]: cluster 2022-04-28T21:14:49.357329+0000 mgr.y (mgr.24961) 2810 : cluster [DBG] pgmap v3147: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:51.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:52 smithi059 bash[61152]: cluster 2022-04-28T21:14:51.358628+0000 mgr.y (mgr.24961) 2811 : cluster [DBG] pgmap v3148: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:53.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:52 smithi059 bash[59745]: cluster 2022-04-28T21:14:51.358628+0000 mgr.y (mgr.24961) 2811 : cluster [DBG] pgmap v3148: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:52 smithi139 bash[43301]: cluster 2022-04-28T21:14:51.358628+0000 mgr.y (mgr.24961) 2811 : cluster [DBG] pgmap v3148: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:14:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:14:53.846Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:14:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:54 smithi059 bash[61152]: cluster 2022-04-28T21:14:53.359291+0000 mgr.y (mgr.24961) 2812 : cluster [DBG] pgmap v3149: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:54 smithi059 bash[59745]: cluster 2022-04-28T21:14:53.359291+0000 mgr.y (mgr.24961) 2812 : cluster [DBG] pgmap v3149: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:54 smithi139 bash[43301]: cluster 2022-04-28T21:14:53.359291+0000 mgr.y (mgr.24961) 2812 : cluster [DBG] pgmap v3149: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:56 smithi059 bash[61152]: cluster 2022-04-28T21:14:55.360662+0000 mgr.y (mgr.24961) 2813 : cluster [DBG] pgmap v3150: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:56 smithi059 bash[59745]: cluster 2022-04-28T21:14:55.360662+0000 mgr.y (mgr.24961) 2813 : cluster [DBG] pgmap v3150: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:56 smithi139 bash[43301]: cluster 2022-04-28T21:14:55.360662+0000 mgr.y (mgr.24961) 2813 : cluster [DBG] pgmap v3150: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:14:58 smithi059 bash[61152]: cluster 2022-04-28T21:14:57.361651+0000 mgr.y (mgr.24961) 2814 : cluster [DBG] pgmap v3151: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:59.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:14:58 smithi059 bash[59745]: cluster 2022-04-28T21:14:57.361651+0000 mgr.y (mgr.24961) 2814 : cluster [DBG] pgmap v3151: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:14:59.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:14:58 smithi139 bash[43301]: cluster 2022-04-28T21:14:57.361651+0000 mgr.y (mgr.24961) 2814 : cluster [DBG] pgmap v3151: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:00 smithi059 bash[59745]: cluster 2022-04-28T21:14:59.362630+0000 mgr.y (mgr.24961) 2815 : cluster [DBG] pgmap v3152: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:01.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:00 smithi059 bash[61152]: cluster 2022-04-28T21:14:59.362630+0000 mgr.y (mgr.24961) 2815 : cluster [DBG] pgmap v3152: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:00 smithi139 bash[43301]: cluster 2022-04-28T21:14:59.362630+0000 mgr.y (mgr.24961) 2815 : cluster [DBG] pgmap v3152: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:01.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:02 smithi059 bash[61152]: cluster 2022-04-28T21:15:01.363900+0000 mgr.y (mgr.24961) 2816 : cluster [DBG] pgmap v3153: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:02 smithi059 bash[59745]: cluster 2022-04-28T21:15:01.363900+0000 mgr.y (mgr.24961) 2816 : cluster [DBG] pgmap v3153: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:02 smithi139 bash[43301]: cluster 2022-04-28T21:15:01.363900+0000 mgr.y (mgr.24961) 2816 : cluster [DBG] pgmap v3153: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:03.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:05.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:04 smithi059 bash[59745]: cluster 2022-04-28T21:15:03.364620+0000 mgr.y (mgr.24961) 2817 : cluster [DBG] pgmap v3154: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:05.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:04 smithi059 bash[61152]: cluster 2022-04-28T21:15:03.364620+0000 mgr.y (mgr.24961) 2817 : cluster [DBG] pgmap v3154: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:05.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:04 smithi139 bash[43301]: cluster 2022-04-28T21:15:03.364620+0000 mgr.y (mgr.24961) 2817 : cluster [DBG] pgmap v3154: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:06 smithi059 bash[61152]: cluster 2022-04-28T21:15:05.366154+0000 mgr.y (mgr.24961) 2818 : cluster [DBG] pgmap v3155: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:07.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:06 smithi059 bash[59745]: cluster 2022-04-28T21:15:05.366154+0000 mgr.y (mgr.24961) 2818 : cluster [DBG] pgmap v3155: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:07.158 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:06 smithi139 bash[43301]: cluster 2022-04-28T21:15:05.366154+0000 mgr.y (mgr.24961) 2818 : cluster [DBG] pgmap v3155: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:08 smithi059 bash[61152]: cluster 2022-04-28T21:15:07.367164+0000 mgr.y (mgr.24961) 2819 : cluster [DBG] pgmap v3156: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:08 smithi059 bash[59745]: cluster 2022-04-28T21:15:07.367164+0000 mgr.y (mgr.24961) 2819 : cluster [DBG] pgmap v3156: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:08 smithi139 bash[43301]: cluster 2022-04-28T21:15:07.367164+0000 mgr.y (mgr.24961) 2819 : cluster [DBG] pgmap v3156: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:10 smithi059 bash[61152]: cluster 2022-04-28T21:15:09.368339+0000 mgr.y (mgr.24961) 2820 : cluster [DBG] pgmap v3157: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:10 smithi059 bash[59745]: cluster 2022-04-28T21:15:09.368339+0000 mgr.y (mgr.24961) 2820 : cluster [DBG] pgmap v3157: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:10 smithi139 bash[43301]: cluster 2022-04-28T21:15:09.368339+0000 mgr.y (mgr.24961) 2820 : cluster [DBG] pgmap v3157: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:11.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:12 smithi059 bash[61152]: cluster 2022-04-28T21:15:11.369589+0000 mgr.y (mgr.24961) 2821 : cluster [DBG] pgmap v3158: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:13.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:12 smithi059 bash[59745]: cluster 2022-04-28T21:15:11.369589+0000 mgr.y (mgr.24961) 2821 : cluster [DBG] pgmap v3158: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:12 smithi139 bash[43301]: cluster 2022-04-28T21:15:11.369589+0000 mgr.y (mgr.24961) 2821 : cluster [DBG] pgmap v3158: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:13.848Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:14 smithi059 bash[61152]: cluster 2022-04-28T21:15:13.370326+0000 mgr.y (mgr.24961) 2822 : cluster [DBG] pgmap v3159: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:14 smithi059 bash[59745]: cluster 2022-04-28T21:15:13.370326+0000 mgr.y (mgr.24961) 2822 : cluster [DBG] pgmap v3159: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:15.131 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:14 smithi139 bash[43301]: cluster 2022-04-28T21:15:13.370326+0000 mgr.y (mgr.24961) 2822 : cluster [DBG] pgmap v3159: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:16 smithi059 bash[61152]: cluster 2022-04-28T21:15:15.371759+0000 mgr.y (mgr.24961) 2823 : cluster [DBG] pgmap v3160: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:16 smithi059 bash[59745]: cluster 2022-04-28T21:15:15.371759+0000 mgr.y (mgr.24961) 2823 : cluster [DBG] pgmap v3160: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:16 smithi139 bash[43301]: cluster 2022-04-28T21:15:15.371759+0000 mgr.y (mgr.24961) 2823 : cluster [DBG] pgmap v3160: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: cluster 2022-04-28T21:15:17.372724+0000 mgr.y (mgr.24961) 2824 : cluster [DBG] pgmap v3161: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: audit 2022-04-28T21:15:18.201450+0000 mon.a (mon.0) 2588 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:15:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: audit 2022-04-28T21:15:18.202332+0000 mon.a (mon.0) 2589 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:15:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: audit 2022-04-28T21:15:18.212274+0000 mon.a (mon.0) 2590 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:15:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: audit 2022-04-28T21:15:18.212732+0000 mon.a (mon.0) 2591 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:15:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: audit 2022-04-28T21:15:18.213133+0000 mon.a (mon.0) 2592 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:15:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[61152]: audit 2022-04-28T21:15:18.213539+0000 mon.a (mon.0) 2593 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.14", "id": [4, 0]}]: dispatch 2022-04-28T21:15:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: cluster 2022-04-28T21:15:17.372724+0000 mgr.y (mgr.24961) 2824 : cluster [DBG] pgmap v3161: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: audit 2022-04-28T21:15:18.201450+0000 mon.a (mon.0) 2588 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:15:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: audit 2022-04-28T21:15:18.202332+0000 mon.a (mon.0) 2589 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:15:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: audit 2022-04-28T21:15:18.212274+0000 mon.a (mon.0) 2590 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:15:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: audit 2022-04-28T21:15:18.212732+0000 mon.a (mon.0) 2591 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:15:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: audit 2022-04-28T21:15:18.213133+0000 mon.a (mon.0) 2592 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:15:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:18 smithi059 bash[59745]: audit 2022-04-28T21:15:18.213539+0000 mon.a (mon.0) 2593 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.14", "id": [4, 0]}]: dispatch 2022-04-28T21:15:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: cluster 2022-04-28T21:15:17.372724+0000 mgr.y (mgr.24961) 2824 : cluster [DBG] pgmap v3161: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:19.149 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: audit 2022-04-28T21:15:18.201450+0000 mon.a (mon.0) 2588 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:15:19.150 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: audit 2022-04-28T21:15:18.202332+0000 mon.a (mon.0) 2589 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:15:19.151 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: audit 2022-04-28T21:15:18.212274+0000 mon.a (mon.0) 2590 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:15:19.151 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: audit 2022-04-28T21:15:18.212732+0000 mon.a (mon.0) 2591 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:15:19.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: audit 2022-04-28T21:15:18.213133+0000 mon.a (mon.0) 2592 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:15:19.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:18 smithi139 bash[43301]: audit 2022-04-28T21:15:18.213539+0000 mon.a (mon.0) 2593 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.14", "id": [4, 0]}]: dispatch 2022-04-28T21:15:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:20 smithi059 bash[59745]: cluster 2022-04-28T21:15:19.373878+0000 mgr.y (mgr.24961) 2825 : cluster [DBG] pgmap v3162: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:20 smithi059 bash[61152]: cluster 2022-04-28T21:15:19.373878+0000 mgr.y (mgr.24961) 2825 : cluster [DBG] pgmap v3162: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:20 smithi139 bash[43301]: cluster 2022-04-28T21:15:19.373878+0000 mgr.y (mgr.24961) 2825 : cluster [DBG] pgmap v3162: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:21.958Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:22 smithi059 bash[61152]: cluster 2022-04-28T21:15:21.375072+0000 mgr.y (mgr.24961) 2826 : cluster [DBG] pgmap v3163: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:23.118 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:22 smithi059 bash[59745]: cluster 2022-04-28T21:15:21.375072+0000 mgr.y (mgr.24961) 2826 : cluster [DBG] pgmap v3163: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:22 smithi139 bash[43301]: cluster 2022-04-28T21:15:21.375072+0000 mgr.y (mgr.24961) 2826 : cluster [DBG] pgmap v3163: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:23.847Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:24 smithi059 bash[61152]: cluster 2022-04-28T21:15:23.375836+0000 mgr.y (mgr.24961) 2827 : cluster [DBG] pgmap v3164: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:25.124 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:24 smithi059 bash[59745]: cluster 2022-04-28T21:15:23.375836+0000 mgr.y (mgr.24961) 2827 : cluster [DBG] pgmap v3164: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:25.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:24 smithi139 bash[43301]: cluster 2022-04-28T21:15:23.375836+0000 mgr.y (mgr.24961) 2827 : cluster [DBG] pgmap v3164: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:26 smithi059 bash[61152]: cluster 2022-04-28T21:15:25.377203+0000 mgr.y (mgr.24961) 2828 : cluster [DBG] pgmap v3165: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:27.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:26 smithi059 bash[59745]: cluster 2022-04-28T21:15:25.377203+0000 mgr.y (mgr.24961) 2828 : cluster [DBG] pgmap v3165: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:26 smithi139 bash[43301]: cluster 2022-04-28T21:15:25.377203+0000 mgr.y (mgr.24961) 2828 : cluster [DBG] pgmap v3165: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:28 smithi059 bash[61152]: cluster 2022-04-28T21:15:27.378159+0000 mgr.y (mgr.24961) 2829 : cluster [DBG] pgmap v3166: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:28 smithi059 bash[59745]: cluster 2022-04-28T21:15:27.378159+0000 mgr.y (mgr.24961) 2829 : cluster [DBG] pgmap v3166: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:28 smithi139 bash[43301]: cluster 2022-04-28T21:15:27.378159+0000 mgr.y (mgr.24961) 2829 : cluster [DBG] pgmap v3166: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:30 smithi059 bash[61152]: cluster 2022-04-28T21:15:29.379282+0000 mgr.y (mgr.24961) 2830 : cluster [DBG] pgmap v3167: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:30 smithi059 bash[59745]: cluster 2022-04-28T21:15:29.379282+0000 mgr.y (mgr.24961) 2830 : cluster [DBG] pgmap v3167: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:30 smithi139 bash[43301]: cluster 2022-04-28T21:15:29.379282+0000 mgr.y (mgr.24961) 2830 : cluster [DBG] pgmap v3167: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:31.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[59745]: cluster 2022-04-28T21:15:31.380560+0000 mgr.y (mgr.24961) 2831 : cluster [DBG] pgmap v3168: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[59745]: audit 2022-04-28T21:15:32.243517+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:15:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[59745]: audit 2022-04-28T21:15:32.246258+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:15:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[59745]: audit 2022-04-28T21:15:32.248147+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:15:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[59745]: cephadm 2022-04-28T21:15:32.249879+0000 mgr.y (mgr.24961) 2832 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:15:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[59745]: audit 2022-04-28T21:15:32.552247+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:15:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[61152]: cluster 2022-04-28T21:15:31.380560+0000 mgr.y (mgr.24961) 2831 : cluster [DBG] pgmap v3168: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[61152]: audit 2022-04-28T21:15:32.243517+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:15:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[61152]: audit 2022-04-28T21:15:32.246258+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:15:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[61152]: audit 2022-04-28T21:15:32.248147+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:15:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[61152]: cephadm 2022-04-28T21:15:32.249879+0000 mgr.y (mgr.24961) 2832 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:15:33.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:32 smithi059 bash[61152]: audit 2022-04-28T21:15:32.552247+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:15:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:32 smithi139 bash[43301]: cluster 2022-04-28T21:15:31.380560+0000 mgr.y (mgr.24961) 2831 : cluster [DBG] pgmap v3168: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:32 smithi139 bash[43301]: audit 2022-04-28T21:15:32.243517+0000 mon.a (mon.0) 2594 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:15:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:32 smithi139 bash[43301]: audit 2022-04-28T21:15:32.246258+0000 mon.a (mon.0) 2595 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:15:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:32 smithi139 bash[43301]: audit 2022-04-28T21:15:32.248147+0000 mon.a (mon.0) 2596 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:15:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:32 smithi139 bash[43301]: cephadm 2022-04-28T21:15:32.249879+0000 mgr.y (mgr.24961) 2832 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:15:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:32 smithi139 bash[43301]: audit 2022-04-28T21:15:32.552247+0000 mon.a (mon.0) 2597 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:15:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:33.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:35.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:34 smithi059 bash[61152]: cluster 2022-04-28T21:15:33.381293+0000 mgr.y (mgr.24961) 2833 : cluster [DBG] pgmap v3169: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:35.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:34 smithi059 bash[59745]: cluster 2022-04-28T21:15:33.381293+0000 mgr.y (mgr.24961) 2833 : cluster [DBG] pgmap v3169: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:34 smithi139 bash[43301]: cluster 2022-04-28T21:15:33.381293+0000 mgr.y (mgr.24961) 2833 : cluster [DBG] pgmap v3169: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:36 smithi059 bash[61152]: cluster 2022-04-28T21:15:35.382694+0000 mgr.y (mgr.24961) 2834 : cluster [DBG] pgmap v3170: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:37.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:36 smithi059 bash[59745]: cluster 2022-04-28T21:15:35.382694+0000 mgr.y (mgr.24961) 2834 : cluster [DBG] pgmap v3170: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:37.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:36 smithi139 bash[43301]: cluster 2022-04-28T21:15:35.382694+0000 mgr.y (mgr.24961) 2834 : cluster [DBG] pgmap v3170: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:39.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:38 smithi059 bash[59745]: cluster 2022-04-28T21:15:37.383644+0000 mgr.y (mgr.24961) 2835 : cluster [DBG] pgmap v3171: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:38 smithi059 bash[61152]: cluster 2022-04-28T21:15:37.383644+0000 mgr.y (mgr.24961) 2835 : cluster [DBG] pgmap v3171: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:38 smithi139 bash[43301]: cluster 2022-04-28T21:15:37.383644+0000 mgr.y (mgr.24961) 2835 : cluster [DBG] pgmap v3171: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:40 smithi139 bash[43301]: cluster 2022-04-28T21:15:39.384775+0000 mgr.y (mgr.24961) 2836 : cluster [DBG] pgmap v3172: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:40 smithi059 bash[61152]: cluster 2022-04-28T21:15:39.384775+0000 mgr.y (mgr.24961) 2836 : cluster [DBG] pgmap v3172: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:40 smithi059 bash[59745]: cluster 2022-04-28T21:15:39.384775+0000 mgr.y (mgr.24961) 2836 : cluster [DBG] pgmap v3172: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:41.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:42 smithi139 bash[43301]: cluster 2022-04-28T21:15:41.386034+0000 mgr.y (mgr.24961) 2837 : cluster [DBG] pgmap v3173: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:42 smithi059 bash[59745]: cluster 2022-04-28T21:15:41.386034+0000 mgr.y (mgr.24961) 2837 : cluster [DBG] pgmap v3173: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:42 smithi059 bash[61152]: cluster 2022-04-28T21:15:41.386034+0000 mgr.y (mgr.24961) 2837 : cluster [DBG] pgmap v3173: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:43.850Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:44 smithi139 bash[43301]: cluster 2022-04-28T21:15:43.386749+0000 mgr.y (mgr.24961) 2838 : cluster [DBG] pgmap v3174: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:44 smithi059 bash[59745]: cluster 2022-04-28T21:15:43.386749+0000 mgr.y (mgr.24961) 2838 : cluster [DBG] pgmap v3174: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:44 smithi059 bash[61152]: cluster 2022-04-28T21:15:43.386749+0000 mgr.y (mgr.24961) 2838 : cluster [DBG] pgmap v3174: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:46 smithi139 bash[43301]: cluster 2022-04-28T21:15:45.388191+0000 mgr.y (mgr.24961) 2839 : cluster [DBG] pgmap v3175: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:46 smithi059 bash[59745]: cluster 2022-04-28T21:15:45.388191+0000 mgr.y (mgr.24961) 2839 : cluster [DBG] pgmap v3175: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:46 smithi059 bash[61152]: cluster 2022-04-28T21:15:45.388191+0000 mgr.y (mgr.24961) 2839 : cluster [DBG] pgmap v3175: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:48 smithi139 bash[43301]: cluster 2022-04-28T21:15:47.389122+0000 mgr.y (mgr.24961) 2840 : cluster [DBG] pgmap v3176: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:48 smithi059 bash[59745]: cluster 2022-04-28T21:15:47.389122+0000 mgr.y (mgr.24961) 2840 : cluster [DBG] pgmap v3176: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:48 smithi059 bash[61152]: cluster 2022-04-28T21:15:47.389122+0000 mgr.y (mgr.24961) 2840 : cluster [DBG] pgmap v3176: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:50 smithi059 bash[61152]: cluster 2022-04-28T21:15:49.390257+0000 mgr.y (mgr.24961) 2841 : cluster [DBG] pgmap v3177: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:50 smithi059 bash[59745]: cluster 2022-04-28T21:15:49.390257+0000 mgr.y (mgr.24961) 2841 : cluster [DBG] pgmap v3177: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:50 smithi139 bash[43301]: cluster 2022-04-28T21:15:49.390257+0000 mgr.y (mgr.24961) 2841 : cluster [DBG] pgmap v3177: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:51.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:52 smithi059 bash[61152]: cluster 2022-04-28T21:15:51.391533+0000 mgr.y (mgr.24961) 2842 : cluster [DBG] pgmap v3178: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:52 smithi059 bash[59745]: cluster 2022-04-28T21:15:51.391533+0000 mgr.y (mgr.24961) 2842 : cluster [DBG] pgmap v3178: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:52 smithi139 bash[43301]: cluster 2022-04-28T21:15:51.391533+0000 mgr.y (mgr.24961) 2842 : cluster [DBG] pgmap v3178: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:15:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:15:53.849Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:15:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:54 smithi059 bash[59745]: cluster 2022-04-28T21:15:53.392245+0000 mgr.y (mgr.24961) 2843 : cluster [DBG] pgmap v3179: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:54 smithi059 bash[61152]: cluster 2022-04-28T21:15:53.392245+0000 mgr.y (mgr.24961) 2843 : cluster [DBG] pgmap v3179: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:54 smithi139 bash[43301]: cluster 2022-04-28T21:15:53.392245+0000 mgr.y (mgr.24961) 2843 : cluster [DBG] pgmap v3179: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:56 smithi059 bash[61152]: cluster 2022-04-28T21:15:55.393650+0000 mgr.y (mgr.24961) 2844 : cluster [DBG] pgmap v3180: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:56 smithi059 bash[59745]: cluster 2022-04-28T21:15:55.393650+0000 mgr.y (mgr.24961) 2844 : cluster [DBG] pgmap v3180: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:56 smithi139 bash[43301]: cluster 2022-04-28T21:15:55.393650+0000 mgr.y (mgr.24961) 2844 : cluster [DBG] pgmap v3180: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:15:58 smithi059 bash[59745]: cluster 2022-04-28T21:15:57.394634+0000 mgr.y (mgr.24961) 2845 : cluster [DBG] pgmap v3181: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:15:58 smithi059 bash[61152]: cluster 2022-04-28T21:15:57.394634+0000 mgr.y (mgr.24961) 2845 : cluster [DBG] pgmap v3181: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:15:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:15:58 smithi139 bash[43301]: cluster 2022-04-28T21:15:57.394634+0000 mgr.y (mgr.24961) 2845 : cluster [DBG] pgmap v3181: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:00 smithi059 bash[59745]: cluster 2022-04-28T21:15:59.395797+0000 mgr.y (mgr.24961) 2846 : cluster [DBG] pgmap v3182: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:00 smithi059 bash[61152]: cluster 2022-04-28T21:15:59.395797+0000 mgr.y (mgr.24961) 2846 : cluster [DBG] pgmap v3182: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:00 smithi139 bash[43301]: cluster 2022-04-28T21:15:59.395797+0000 mgr.y (mgr.24961) 2846 : cluster [DBG] pgmap v3182: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:01.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:02 smithi059 bash[61152]: cluster 2022-04-28T21:16:01.397039+0000 mgr.y (mgr.24961) 2847 : cluster [DBG] pgmap v3183: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:02 smithi059 bash[59745]: cluster 2022-04-28T21:16:01.397039+0000 mgr.y (mgr.24961) 2847 : cluster [DBG] pgmap v3183: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:02 smithi139 bash[43301]: cluster 2022-04-28T21:16:01.397039+0000 mgr.y (mgr.24961) 2847 : cluster [DBG] pgmap v3183: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:03.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:04 smithi059 bash[61152]: cluster 2022-04-28T21:16:03.397768+0000 mgr.y (mgr.24961) 2848 : cluster [DBG] pgmap v3184: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:04 smithi059 bash[59745]: cluster 2022-04-28T21:16:03.397768+0000 mgr.y (mgr.24961) 2848 : cluster [DBG] pgmap v3184: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:04 smithi139 bash[43301]: cluster 2022-04-28T21:16:03.397768+0000 mgr.y (mgr.24961) 2848 : cluster [DBG] pgmap v3184: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:06 smithi059 bash[59745]: cluster 2022-04-28T21:16:05.399149+0000 mgr.y (mgr.24961) 2849 : cluster [DBG] pgmap v3185: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:07.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:06 smithi059 bash[61152]: cluster 2022-04-28T21:16:05.399149+0000 mgr.y (mgr.24961) 2849 : cluster [DBG] pgmap v3185: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:06 smithi139 bash[43301]: cluster 2022-04-28T21:16:05.399149+0000 mgr.y (mgr.24961) 2849 : cluster [DBG] pgmap v3185: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:08 smithi059 bash[61152]: cluster 2022-04-28T21:16:07.400175+0000 mgr.y (mgr.24961) 2850 : cluster [DBG] pgmap v3186: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:08 smithi059 bash[59745]: cluster 2022-04-28T21:16:07.400175+0000 mgr.y (mgr.24961) 2850 : cluster [DBG] pgmap v3186: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:08 smithi139 bash[43301]: cluster 2022-04-28T21:16:07.400175+0000 mgr.y (mgr.24961) 2850 : cluster [DBG] pgmap v3186: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:10 smithi059 bash[59745]: cluster 2022-04-28T21:16:09.401277+0000 mgr.y (mgr.24961) 2851 : cluster [DBG] pgmap v3187: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:10 smithi059 bash[61152]: cluster 2022-04-28T21:16:09.401277+0000 mgr.y (mgr.24961) 2851 : cluster [DBG] pgmap v3187: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:10 smithi139 bash[43301]: cluster 2022-04-28T21:16:09.401277+0000 mgr.y (mgr.24961) 2851 : cluster [DBG] pgmap v3187: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:11.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:12 smithi059 bash[61152]: cluster 2022-04-28T21:16:11.402545+0000 mgr.y (mgr.24961) 2852 : cluster [DBG] pgmap v3188: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:12 smithi059 bash[59745]: cluster 2022-04-28T21:16:11.402545+0000 mgr.y (mgr.24961) 2852 : cluster [DBG] pgmap v3188: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:12 smithi139 bash[43301]: cluster 2022-04-28T21:16:11.402545+0000 mgr.y (mgr.24961) 2852 : cluster [DBG] pgmap v3188: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:13.852Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:14 smithi059 bash[59745]: cluster 2022-04-28T21:16:13.403265+0000 mgr.y (mgr.24961) 2853 : cluster [DBG] pgmap v3189: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:15.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:14 smithi059 bash[61152]: cluster 2022-04-28T21:16:13.403265+0000 mgr.y (mgr.24961) 2853 : cluster [DBG] pgmap v3189: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:14 smithi139 bash[43301]: cluster 2022-04-28T21:16:13.403265+0000 mgr.y (mgr.24961) 2853 : cluster [DBG] pgmap v3189: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:16 smithi059 bash[59745]: cluster 2022-04-28T21:16:15.404652+0000 mgr.y (mgr.24961) 2854 : cluster [DBG] pgmap v3190: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:16 smithi059 bash[61152]: cluster 2022-04-28T21:16:15.404652+0000 mgr.y (mgr.24961) 2854 : cluster [DBG] pgmap v3190: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:16 smithi139 bash[43301]: cluster 2022-04-28T21:16:15.404652+0000 mgr.y (mgr.24961) 2854 : cluster [DBG] pgmap v3190: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: cluster 2022-04-28T21:16:17.405670+0000 mgr.y (mgr.24961) 2855 : cluster [DBG] pgmap v3191: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: audit 2022-04-28T21:16:18.211811+0000 mon.a (mon.0) 2598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:16:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: audit 2022-04-28T21:16:18.212173+0000 mon.a (mon.0) 2599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:16:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: audit 2022-04-28T21:16:18.221943+0000 mon.a (mon.0) 2600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: audit 2022-04-28T21:16:18.222400+0000 mon.a (mon.0) 2601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: audit 2022-04-28T21:16:18.222711+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[59745]: audit 2022-04-28T21:16:18.223083+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:16:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: cluster 2022-04-28T21:16:17.405670+0000 mgr.y (mgr.24961) 2855 : cluster [DBG] pgmap v3191: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: audit 2022-04-28T21:16:18.211811+0000 mon.a (mon.0) 2598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:16:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: audit 2022-04-28T21:16:18.212173+0000 mon.a (mon.0) 2599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:16:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: audit 2022-04-28T21:16:18.221943+0000 mon.a (mon.0) 2600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: audit 2022-04-28T21:16:18.222400+0000 mon.a (mon.0) 2601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: audit 2022-04-28T21:16:18.222711+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:18 smithi059 bash[61152]: audit 2022-04-28T21:16:18.223083+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:16:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: cluster 2022-04-28T21:16:17.405670+0000 mgr.y (mgr.24961) 2855 : cluster [DBG] pgmap v3191: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: audit 2022-04-28T21:16:18.211811+0000 mon.a (mon.0) 2598 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:16:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: audit 2022-04-28T21:16:18.212173+0000 mon.a (mon.0) 2599 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:16:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: audit 2022-04-28T21:16:18.221943+0000 mon.a (mon.0) 2600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: audit 2022-04-28T21:16:18.222400+0000 mon.a (mon.0) 2601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: audit 2022-04-28T21:16:18.222711+0000 mon.a (mon.0) 2602 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:16:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:18 smithi139 bash[43301]: audit 2022-04-28T21:16:18.223083+0000 mon.a (mon.0) 2603 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:16:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:20 smithi059 bash[61152]: cluster 2022-04-28T21:16:19.406779+0000 mgr.y (mgr.24961) 2856 : cluster [DBG] pgmap v3192: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:20 smithi059 bash[59745]: cluster 2022-04-28T21:16:19.406779+0000 mgr.y (mgr.24961) 2856 : cluster [DBG] pgmap v3192: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:20 smithi139 bash[43301]: cluster 2022-04-28T21:16:19.406779+0000 mgr.y (mgr.24961) 2856 : cluster [DBG] pgmap v3192: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:21.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:22 smithi059 bash[59745]: cluster 2022-04-28T21:16:21.408115+0000 mgr.y (mgr.24961) 2857 : cluster [DBG] pgmap v3193: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:23.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:22 smithi059 bash[61152]: cluster 2022-04-28T21:16:21.408115+0000 mgr.y (mgr.24961) 2857 : cluster [DBG] pgmap v3193: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:22 smithi139 bash[43301]: cluster 2022-04-28T21:16:21.408115+0000 mgr.y (mgr.24961) 2857 : cluster [DBG] pgmap v3193: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:23.853Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:24 smithi059 bash[59745]: cluster 2022-04-28T21:16:23.408855+0000 mgr.y (mgr.24961) 2858 : cluster [DBG] pgmap v3194: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:24 smithi059 bash[61152]: cluster 2022-04-28T21:16:23.408855+0000 mgr.y (mgr.24961) 2858 : cluster [DBG] pgmap v3194: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:24 smithi139 bash[43301]: cluster 2022-04-28T21:16:23.408855+0000 mgr.y (mgr.24961) 2858 : cluster [DBG] pgmap v3194: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:26 smithi059 bash[61152]: cluster 2022-04-28T21:16:25.410284+0000 mgr.y (mgr.24961) 2859 : cluster [DBG] pgmap v3195: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:26 smithi059 bash[59745]: cluster 2022-04-28T21:16:25.410284+0000 mgr.y (mgr.24961) 2859 : cluster [DBG] pgmap v3195: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:26 smithi139 bash[43301]: cluster 2022-04-28T21:16:25.410284+0000 mgr.y (mgr.24961) 2859 : cluster [DBG] pgmap v3195: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:28 smithi059 bash[59745]: cluster 2022-04-28T21:16:27.411288+0000 mgr.y (mgr.24961) 2860 : cluster [DBG] pgmap v3196: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:28 smithi059 bash[61152]: cluster 2022-04-28T21:16:27.411288+0000 mgr.y (mgr.24961) 2860 : cluster [DBG] pgmap v3196: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:28 smithi139 bash[43301]: cluster 2022-04-28T21:16:27.411288+0000 mgr.y (mgr.24961) 2860 : cluster [DBG] pgmap v3196: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:30 smithi059 bash[59745]: cluster 2022-04-28T21:16:29.412395+0000 mgr.y (mgr.24961) 2861 : cluster [DBG] pgmap v3197: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:30 smithi059 bash[61152]: cluster 2022-04-28T21:16:29.412395+0000 mgr.y (mgr.24961) 2861 : cluster [DBG] pgmap v3197: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:30 smithi139 bash[43301]: cluster 2022-04-28T21:16:29.412395+0000 mgr.y (mgr.24961) 2861 : cluster [DBG] pgmap v3197: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:31.964Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[59745]: cluster 2022-04-28T21:16:31.413656+0000 mgr.y (mgr.24961) 2862 : cluster [DBG] pgmap v3198: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[59745]: audit 2022-04-28T21:16:32.559285+0000 mon.a (mon.0) 2604 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:16:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[59745]: audit 2022-04-28T21:16:32.562225+0000 mon.a (mon.0) 2605 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:16:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[59745]: audit 2022-04-28T21:16:32.564125+0000 mon.a (mon.0) 2606 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:16:33.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[59745]: audit 2022-04-28T21:16:32.865001+0000 mon.a (mon.0) 2607 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:16:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[61152]: cluster 2022-04-28T21:16:31.413656+0000 mgr.y (mgr.24961) 2862 : cluster [DBG] pgmap v3198: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[61152]: audit 2022-04-28T21:16:32.559285+0000 mon.a (mon.0) 2604 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:16:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[61152]: audit 2022-04-28T21:16:32.562225+0000 mon.a (mon.0) 2605 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:16:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[61152]: audit 2022-04-28T21:16:32.564125+0000 mon.a (mon.0) 2606 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:16:33.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:32 smithi059 bash[61152]: audit 2022-04-28T21:16:32.865001+0000 mon.a (mon.0) 2607 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:16:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:32 smithi139 bash[43301]: cluster 2022-04-28T21:16:31.413656+0000 mgr.y (mgr.24961) 2862 : cluster [DBG] pgmap v3198: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:32 smithi139 bash[43301]: audit 2022-04-28T21:16:32.559285+0000 mon.a (mon.0) 2604 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:16:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:32 smithi139 bash[43301]: audit 2022-04-28T21:16:32.562225+0000 mon.a (mon.0) 2605 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:16:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:32 smithi139 bash[43301]: audit 2022-04-28T21:16:32.564125+0000 mon.a (mon.0) 2606 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:16:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:32 smithi139 bash[43301]: audit 2022-04-28T21:16:32.865001+0000 mon.a (mon.0) 2607 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:16:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:33 smithi059 bash[61152]: cephadm 2022-04-28T21:16:32.565861+0000 mgr.y (mgr.24961) 2863 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:16:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:33.854Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:33 smithi059 bash[59745]: cephadm 2022-04-28T21:16:32.565861+0000 mgr.y (mgr.24961) 2863 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:16:34.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:33 smithi139 bash[43301]: cephadm 2022-04-28T21:16:32.565861+0000 mgr.y (mgr.24961) 2863 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:16:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:34 smithi059 bash[61152]: cluster 2022-04-28T21:16:33.414362+0000 mgr.y (mgr.24961) 2864 : cluster [DBG] pgmap v3199: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:34 smithi059 bash[59745]: cluster 2022-04-28T21:16:33.414362+0000 mgr.y (mgr.24961) 2864 : cluster [DBG] pgmap v3199: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:34 smithi139 bash[43301]: cluster 2022-04-28T21:16:33.414362+0000 mgr.y (mgr.24961) 2864 : cluster [DBG] pgmap v3199: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:37 smithi059 bash[61152]: cluster 2022-04-28T21:16:35.415780+0000 mgr.y (mgr.24961) 2865 : cluster [DBG] pgmap v3200: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:37 smithi059 bash[59745]: cluster 2022-04-28T21:16:35.415780+0000 mgr.y (mgr.24961) 2865 : cluster [DBG] pgmap v3200: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:37 smithi139 bash[43301]: cluster 2022-04-28T21:16:35.415780+0000 mgr.y (mgr.24961) 2865 : cluster [DBG] pgmap v3200: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:39 smithi059 bash[61152]: cluster 2022-04-28T21:16:37.416736+0000 mgr.y (mgr.24961) 2866 : cluster [DBG] pgmap v3201: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:39 smithi059 bash[59745]: cluster 2022-04-28T21:16:37.416736+0000 mgr.y (mgr.24961) 2866 : cluster [DBG] pgmap v3201: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:39 smithi139 bash[43301]: cluster 2022-04-28T21:16:37.416736+0000 mgr.y (mgr.24961) 2866 : cluster [DBG] pgmap v3201: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:41 smithi059 bash[61152]: cluster 2022-04-28T21:16:39.417847+0000 mgr.y (mgr.24961) 2867 : cluster [DBG] pgmap v3202: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:41 smithi059 bash[59745]: cluster 2022-04-28T21:16:39.417847+0000 mgr.y (mgr.24961) 2867 : cluster [DBG] pgmap v3202: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:41 smithi139 bash[43301]: cluster 2022-04-28T21:16:39.417847+0000 mgr.y (mgr.24961) 2867 : cluster [DBG] pgmap v3202: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:41.963Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:43 smithi059 bash[61152]: cluster 2022-04-28T21:16:41.419107+0000 mgr.y (mgr.24961) 2868 : cluster [DBG] pgmap v3203: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:43 smithi059 bash[59745]: cluster 2022-04-28T21:16:41.419107+0000 mgr.y (mgr.24961) 2868 : cluster [DBG] pgmap v3203: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:43 smithi139 bash[43301]: cluster 2022-04-28T21:16:41.419107+0000 mgr.y (mgr.24961) 2868 : cluster [DBG] pgmap v3203: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:43.855Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:45 smithi059 bash[59745]: cluster 2022-04-28T21:16:43.419845+0000 mgr.y (mgr.24961) 2869 : cluster [DBG] pgmap v3204: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:45 smithi059 bash[61152]: cluster 2022-04-28T21:16:43.419845+0000 mgr.y (mgr.24961) 2869 : cluster [DBG] pgmap v3204: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:45 smithi139 bash[43301]: cluster 2022-04-28T21:16:43.419845+0000 mgr.y (mgr.24961) 2869 : cluster [DBG] pgmap v3204: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:47 smithi059 bash[61152]: cluster 2022-04-28T21:16:45.421185+0000 mgr.y (mgr.24961) 2870 : cluster [DBG] pgmap v3205: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:47 smithi059 bash[59745]: cluster 2022-04-28T21:16:45.421185+0000 mgr.y (mgr.24961) 2870 : cluster [DBG] pgmap v3205: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:47 smithi139 bash[43301]: cluster 2022-04-28T21:16:45.421185+0000 mgr.y (mgr.24961) 2870 : cluster [DBG] pgmap v3205: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:49 smithi059 bash[59745]: cluster 2022-04-28T21:16:47.422165+0000 mgr.y (mgr.24961) 2871 : cluster [DBG] pgmap v3206: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:49 smithi059 bash[61152]: cluster 2022-04-28T21:16:47.422165+0000 mgr.y (mgr.24961) 2871 : cluster [DBG] pgmap v3206: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:49 smithi139 bash[43301]: cluster 2022-04-28T21:16:47.422165+0000 mgr.y (mgr.24961) 2871 : cluster [DBG] pgmap v3206: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:51 smithi059 bash[61152]: cluster 2022-04-28T21:16:49.423394+0000 mgr.y (mgr.24961) 2872 : cluster [DBG] pgmap v3207: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:51 smithi059 bash[59745]: cluster 2022-04-28T21:16:49.423394+0000 mgr.y (mgr.24961) 2872 : cluster [DBG] pgmap v3207: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:51 smithi139 bash[43301]: cluster 2022-04-28T21:16:49.423394+0000 mgr.y (mgr.24961) 2872 : cluster [DBG] pgmap v3207: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:51.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:53 smithi059 bash[61152]: cluster 2022-04-28T21:16:51.424722+0000 mgr.y (mgr.24961) 2873 : cluster [DBG] pgmap v3208: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:53 smithi059 bash[59745]: cluster 2022-04-28T21:16:51.424722+0000 mgr.y (mgr.24961) 2873 : cluster [DBG] pgmap v3208: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:53 smithi139 bash[43301]: cluster 2022-04-28T21:16:51.424722+0000 mgr.y (mgr.24961) 2873 : cluster [DBG] pgmap v3208: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:16:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:16:53.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:16:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:55 smithi059 bash[61152]: cluster 2022-04-28T21:16:53.425485+0000 mgr.y (mgr.24961) 2874 : cluster [DBG] pgmap v3209: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:55 smithi059 bash[59745]: cluster 2022-04-28T21:16:53.425485+0000 mgr.y (mgr.24961) 2874 : cluster [DBG] pgmap v3209: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:55 smithi139 bash[43301]: cluster 2022-04-28T21:16:53.425485+0000 mgr.y (mgr.24961) 2874 : cluster [DBG] pgmap v3209: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:57 smithi059 bash[61152]: cluster 2022-04-28T21:16:55.426868+0000 mgr.y (mgr.24961) 2875 : cluster [DBG] pgmap v3210: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:57.625 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:57 smithi059 bash[59745]: cluster 2022-04-28T21:16:55.426868+0000 mgr.y (mgr.24961) 2875 : cluster [DBG] pgmap v3210: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:57.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:57 smithi139 bash[43301]: cluster 2022-04-28T21:16:55.426868+0000 mgr.y (mgr.24961) 2875 : cluster [DBG] pgmap v3210: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:16:59 smithi059 bash[61152]: cluster 2022-04-28T21:16:57.427887+0000 mgr.y (mgr.24961) 2876 : cluster [DBG] pgmap v3211: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:16:59 smithi059 bash[59745]: cluster 2022-04-28T21:16:57.427887+0000 mgr.y (mgr.24961) 2876 : cluster [DBG] pgmap v3211: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:16:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:16:59 smithi139 bash[43301]: cluster 2022-04-28T21:16:57.427887+0000 mgr.y (mgr.24961) 2876 : cluster [DBG] pgmap v3211: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:01.332 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:01 smithi059 bash[61152]: cluster 2022-04-28T21:16:59.429027+0000 mgr.y (mgr.24961) 2877 : cluster [DBG] pgmap v3212: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:01.332 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:01 smithi059 bash[59745]: cluster 2022-04-28T21:16:59.429027+0000 mgr.y (mgr.24961) 2877 : cluster [DBG] pgmap v3212: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:01 smithi139 bash[43301]: cluster 2022-04-28T21:16:59.429027+0000 mgr.y (mgr.24961) 2877 : cluster [DBG] pgmap v3212: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:01.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:03 smithi059 bash[61152]: cluster 2022-04-28T21:17:01.430345+0000 mgr.y (mgr.24961) 2878 : cluster [DBG] pgmap v3213: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:03 smithi059 bash[59745]: cluster 2022-04-28T21:17:01.430345+0000 mgr.y (mgr.24961) 2878 : cluster [DBG] pgmap v3213: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:03 smithi139 bash[43301]: cluster 2022-04-28T21:17:01.430345+0000 mgr.y (mgr.24961) 2878 : cluster [DBG] pgmap v3213: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:03.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:05 smithi059 bash[61152]: cluster 2022-04-28T21:17:03.431099+0000 mgr.y (mgr.24961) 2879 : cluster [DBG] pgmap v3214: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:05 smithi059 bash[59745]: cluster 2022-04-28T21:17:03.431099+0000 mgr.y (mgr.24961) 2879 : cluster [DBG] pgmap v3214: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:05 smithi139 bash[43301]: cluster 2022-04-28T21:17:03.431099+0000 mgr.y (mgr.24961) 2879 : cluster [DBG] pgmap v3214: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:07 smithi059 bash[61152]: cluster 2022-04-28T21:17:05.432648+0000 mgr.y (mgr.24961) 2880 : cluster [DBG] pgmap v3215: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:07 smithi059 bash[59745]: cluster 2022-04-28T21:17:05.432648+0000 mgr.y (mgr.24961) 2880 : cluster [DBG] pgmap v3215: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:07.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:07 smithi139 bash[43301]: cluster 2022-04-28T21:17:05.432648+0000 mgr.y (mgr.24961) 2880 : cluster [DBG] pgmap v3215: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:09 smithi059 bash[61152]: cluster 2022-04-28T21:17:07.433648+0000 mgr.y (mgr.24961) 2881 : cluster [DBG] pgmap v3216: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:09 smithi059 bash[59745]: cluster 2022-04-28T21:17:07.433648+0000 mgr.y (mgr.24961) 2881 : cluster [DBG] pgmap v3216: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:09 smithi139 bash[43301]: cluster 2022-04-28T21:17:07.433648+0000 mgr.y (mgr.24961) 2881 : cluster [DBG] pgmap v3216: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:11 smithi139 bash[43301]: cluster 2022-04-28T21:17:09.434806+0000 mgr.y (mgr.24961) 2882 : cluster [DBG] pgmap v3217: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:11 smithi059 bash[61152]: cluster 2022-04-28T21:17:09.434806+0000 mgr.y (mgr.24961) 2882 : cluster [DBG] pgmap v3217: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:11 smithi059 bash[59745]: cluster 2022-04-28T21:17:09.434806+0000 mgr.y (mgr.24961) 2882 : cluster [DBG] pgmap v3217: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:11.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:13 smithi139 bash[43301]: cluster 2022-04-28T21:17:11.436113+0000 mgr.y (mgr.24961) 2883 : cluster [DBG] pgmap v3218: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:13 smithi059 bash[61152]: cluster 2022-04-28T21:17:11.436113+0000 mgr.y (mgr.24961) 2883 : cluster [DBG] pgmap v3218: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:13 smithi059 bash[59745]: cluster 2022-04-28T21:17:11.436113+0000 mgr.y (mgr.24961) 2883 : cluster [DBG] pgmap v3218: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:13.856Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:15 smithi139 bash[43301]: cluster 2022-04-28T21:17:13.436898+0000 mgr.y (mgr.24961) 2884 : cluster [DBG] pgmap v3219: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:15 smithi059 bash[59745]: cluster 2022-04-28T21:17:13.436898+0000 mgr.y (mgr.24961) 2884 : cluster [DBG] pgmap v3219: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:15 smithi059 bash[61152]: cluster 2022-04-28T21:17:13.436898+0000 mgr.y (mgr.24961) 2884 : cluster [DBG] pgmap v3219: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:17 smithi139 bash[43301]: cluster 2022-04-28T21:17:15.438346+0000 mgr.y (mgr.24961) 2885 : cluster [DBG] pgmap v3220: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:17 smithi059 bash[61152]: cluster 2022-04-28T21:17:15.438346+0000 mgr.y (mgr.24961) 2885 : cluster [DBG] pgmap v3220: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:17 smithi059 bash[59745]: cluster 2022-04-28T21:17:15.438346+0000 mgr.y (mgr.24961) 2885 : cluster [DBG] pgmap v3220: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: cluster 2022-04-28T21:17:17.439315+0000 mgr.y (mgr.24961) 2886 : cluster [DBG] pgmap v3221: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: audit 2022-04-28T21:17:18.226003+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:17:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: audit 2022-04-28T21:17:18.226567+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:17:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: audit 2022-04-28T21:17:18.236931+0000 mon.a (mon.0) 2610 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:17:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: audit 2022-04-28T21:17:18.237503+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:17:19.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: audit 2022-04-28T21:17:18.237923+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:17:19.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:19 smithi139 bash[43301]: audit 2022-04-28T21:17:18.238368+0000 mon.a (mon.0) 2613 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:17:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: cluster 2022-04-28T21:17:17.439315+0000 mgr.y (mgr.24961) 2886 : cluster [DBG] pgmap v3221: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: audit 2022-04-28T21:17:18.226003+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:17:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: audit 2022-04-28T21:17:18.226567+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:17:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: audit 2022-04-28T21:17:18.236931+0000 mon.a (mon.0) 2610 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:17:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: audit 2022-04-28T21:17:18.237503+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:17:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: audit 2022-04-28T21:17:18.237923+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:17:19.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[61152]: audit 2022-04-28T21:17:18.238368+0000 mon.a (mon.0) 2613 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:17:19.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: cluster 2022-04-28T21:17:17.439315+0000 mgr.y (mgr.24961) 2886 : cluster [DBG] pgmap v3221: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:19.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: audit 2022-04-28T21:17:18.226003+0000 mon.a (mon.0) 2608 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:17:19.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: audit 2022-04-28T21:17:18.226567+0000 mon.a (mon.0) 2609 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:17:19.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: audit 2022-04-28T21:17:18.236931+0000 mon.a (mon.0) 2610 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:17:19.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: audit 2022-04-28T21:17:18.237503+0000 mon.a (mon.0) 2611 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:17:19.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: audit 2022-04-28T21:17:18.237923+0000 mon.a (mon.0) 2612 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:17:19.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:19 smithi059 bash[59745]: audit 2022-04-28T21:17:18.238368+0000 mon.a (mon.0) 2613 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:17:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:21 smithi139 bash[43301]: cluster 2022-04-28T21:17:19.440389+0000 mgr.y (mgr.24961) 2887 : cluster [DBG] pgmap v3222: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:21 smithi059 bash[61152]: cluster 2022-04-28T21:17:19.440389+0000 mgr.y (mgr.24961) 2887 : cluster [DBG] pgmap v3222: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:21 smithi059 bash[59745]: cluster 2022-04-28T21:17:19.440389+0000 mgr.y (mgr.24961) 2887 : cluster [DBG] pgmap v3222: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:21.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:23 smithi059 bash[61152]: cluster 2022-04-28T21:17:21.441653+0000 mgr.y (mgr.24961) 2888 : cluster [DBG] pgmap v3223: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:23 smithi059 bash[59745]: cluster 2022-04-28T21:17:21.441653+0000 mgr.y (mgr.24961) 2888 : cluster [DBG] pgmap v3223: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:23 smithi139 bash[43301]: cluster 2022-04-28T21:17:21.441653+0000 mgr.y (mgr.24961) 2888 : cluster [DBG] pgmap v3223: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:23.857Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:25 smithi059 bash[61152]: cluster 2022-04-28T21:17:23.442394+0000 mgr.y (mgr.24961) 2889 : cluster [DBG] pgmap v3224: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:25.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:25 smithi059 bash[59745]: cluster 2022-04-28T21:17:23.442394+0000 mgr.y (mgr.24961) 2889 : cluster [DBG] pgmap v3224: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:25 smithi139 bash[43301]: cluster 2022-04-28T21:17:23.442394+0000 mgr.y (mgr.24961) 2889 : cluster [DBG] pgmap v3224: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:27 smithi059 bash[61152]: cluster 2022-04-28T21:17:25.443874+0000 mgr.y (mgr.24961) 2890 : cluster [DBG] pgmap v3225: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:27 smithi059 bash[59745]: cluster 2022-04-28T21:17:25.443874+0000 mgr.y (mgr.24961) 2890 : cluster [DBG] pgmap v3225: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:27 smithi139 bash[43301]: cluster 2022-04-28T21:17:25.443874+0000 mgr.y (mgr.24961) 2890 : cluster [DBG] pgmap v3225: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:29 smithi059 bash[61152]: cluster 2022-04-28T21:17:27.444862+0000 mgr.y (mgr.24961) 2891 : cluster [DBG] pgmap v3226: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:29 smithi059 bash[59745]: cluster 2022-04-28T21:17:27.444862+0000 mgr.y (mgr.24961) 2891 : cluster [DBG] pgmap v3226: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:29 smithi139 bash[43301]: cluster 2022-04-28T21:17:27.444862+0000 mgr.y (mgr.24961) 2891 : cluster [DBG] pgmap v3226: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:31 smithi059 bash[61152]: cluster 2022-04-28T21:17:29.446098+0000 mgr.y (mgr.24961) 2892 : cluster [DBG] pgmap v3227: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:31.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:31 smithi059 bash[59745]: cluster 2022-04-28T21:17:29.446098+0000 mgr.y (mgr.24961) 2892 : cluster [DBG] pgmap v3227: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:31.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:31 smithi139 bash[43301]: cluster 2022-04-28T21:17:29.446098+0000 mgr.y (mgr.24961) 2892 : cluster [DBG] pgmap v3227: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:31.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:33.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[61152]: cluster 2022-04-28T21:17:31.447417+0000 mgr.y (mgr.24961) 2893 : cluster [DBG] pgmap v3228: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[61152]: audit 2022-04-28T21:17:32.873294+0000 mon.a (mon.0) 2614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:17:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[61152]: audit 2022-04-28T21:17:32.876283+0000 mon.a (mon.0) 2615 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:17:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[61152]: audit 2022-04-28T21:17:32.878269+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:17:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[61152]: audit 2022-04-28T21:17:32.881543+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:17:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[61152]: audit 2022-04-28T21:17:32.882152+0000 mon.a (mon.0) 2618 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:17:33.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[59745]: cluster 2022-04-28T21:17:31.447417+0000 mgr.y (mgr.24961) 2893 : cluster [DBG] pgmap v3228: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:33.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[59745]: audit 2022-04-28T21:17:32.873294+0000 mon.a (mon.0) 2614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:17:33.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[59745]: audit 2022-04-28T21:17:32.876283+0000 mon.a (mon.0) 2615 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:17:33.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[59745]: audit 2022-04-28T21:17:32.878269+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:17:33.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[59745]: audit 2022-04-28T21:17:32.881543+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:17:33.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[59745]: audit 2022-04-28T21:17:32.882152+0000 mon.a (mon.0) 2618 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:17:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:33 smithi139 bash[43301]: cluster 2022-04-28T21:17:31.447417+0000 mgr.y (mgr.24961) 2893 : cluster [DBG] pgmap v3228: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:33 smithi139 bash[43301]: audit 2022-04-28T21:17:32.873294+0000 mon.a (mon.0) 2614 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:17:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:33 smithi139 bash[43301]: audit 2022-04-28T21:17:32.876283+0000 mon.a (mon.0) 2615 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:17:33.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:33 smithi139 bash[43301]: audit 2022-04-28T21:17:32.878269+0000 mon.a (mon.0) 2616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:17:33.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:33 smithi139 bash[43301]: audit 2022-04-28T21:17:32.881543+0000 mon.a (mon.0) 2617 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:17:33.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:33 smithi139 bash[43301]: audit 2022-04-28T21:17:32.882152+0000 mon.a (mon.0) 2618 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:17:34.156 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:33.858Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:34.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:34 smithi059 bash[59745]: cephadm 2022-04-28T21:17:32.880101+0000 mgr.y (mgr.24961) 2894 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:17:34.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:34 smithi059 bash[61152]: cephadm 2022-04-28T21:17:32.880101+0000 mgr.y (mgr.24961) 2894 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:17:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:34 smithi139 bash[43301]: cephadm 2022-04-28T21:17:32.880101+0000 mgr.y (mgr.24961) 2894 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:17:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:35 smithi059 bash[61152]: cluster 2022-04-28T21:17:33.448184+0000 mgr.y (mgr.24961) 2895 : cluster [DBG] pgmap v3229: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:35 smithi059 bash[59745]: cluster 2022-04-28T21:17:33.448184+0000 mgr.y (mgr.24961) 2895 : cluster [DBG] pgmap v3229: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:35 smithi139 bash[43301]: cluster 2022-04-28T21:17:33.448184+0000 mgr.y (mgr.24961) 2895 : cluster [DBG] pgmap v3229: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:37 smithi059 bash[59745]: cluster 2022-04-28T21:17:35.449599+0000 mgr.y (mgr.24961) 2896 : cluster [DBG] pgmap v3230: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:37 smithi059 bash[59745]: audit 2022-04-28T21:17:37.020581+0000 mon.a (mon.0) 2619 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:37 smithi059 bash[61152]: cluster 2022-04-28T21:17:35.449599+0000 mgr.y (mgr.24961) 2896 : cluster [DBG] pgmap v3230: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:37 smithi059 bash[61152]: audit 2022-04-28T21:17:37.020581+0000 mon.a (mon.0) 2619 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:37 smithi139 bash[43301]: cluster 2022-04-28T21:17:35.449599+0000 mgr.y (mgr.24961) 2896 : cluster [DBG] pgmap v3230: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:37 smithi139 bash[43301]: audit 2022-04-28T21:17:37.020581+0000 mon.a (mon.0) 2619 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:38 smithi059 bash[61152]: audit 2022-04-28T21:17:37.203706+0000 mon.a (mon.0) 2620 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:38 smithi059 bash[61152]: audit 2022-04-28T21:17:37.493235+0000 mon.a (mon.0) 2621 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:38 smithi059 bash[59745]: audit 2022-04-28T21:17:37.203706+0000 mon.a (mon.0) 2620 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:38 smithi059 bash[59745]: audit 2022-04-28T21:17:37.493235+0000 mon.a (mon.0) 2621 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:38 smithi139 bash[43301]: audit 2022-04-28T21:17:37.203706+0000 mon.a (mon.0) 2620 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:38 smithi139 bash[43301]: audit 2022-04-28T21:17:37.493235+0000 mon.a (mon.0) 2621 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:17:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:39 smithi059 bash[61152]: cluster 2022-04-28T21:17:37.450608+0000 mgr.y (mgr.24961) 2897 : cluster [DBG] pgmap v3231: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:39 smithi059 bash[59745]: cluster 2022-04-28T21:17:37.450608+0000 mgr.y (mgr.24961) 2897 : cluster [DBG] pgmap v3231: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:39 smithi139 bash[43301]: cluster 2022-04-28T21:17:37.450608+0000 mgr.y (mgr.24961) 2897 : cluster [DBG] pgmap v3231: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:41 smithi059 bash[61152]: cluster 2022-04-28T21:17:39.451715+0000 mgr.y (mgr.24961) 2898 : cluster [DBG] pgmap v3232: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:41 smithi059 bash[59745]: cluster 2022-04-28T21:17:39.451715+0000 mgr.y (mgr.24961) 2898 : cluster [DBG] pgmap v3232: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:41 smithi139 bash[43301]: cluster 2022-04-28T21:17:39.451715+0000 mgr.y (mgr.24961) 2898 : cluster [DBG] pgmap v3232: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:41.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:43 smithi059 bash[61152]: cluster 2022-04-28T21:17:41.452511+0000 mgr.y (mgr.24961) 2899 : cluster [DBG] pgmap v3233: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:43 smithi059 bash[59745]: cluster 2022-04-28T21:17:41.452511+0000 mgr.y (mgr.24961) 2899 : cluster [DBG] pgmap v3233: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:43 smithi139 bash[43301]: cluster 2022-04-28T21:17:41.452511+0000 mgr.y (mgr.24961) 2899 : cluster [DBG] pgmap v3233: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:43.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:45 smithi059 bash[59745]: cluster 2022-04-28T21:17:43.453215+0000 mgr.y (mgr.24961) 2900 : cluster [DBG] pgmap v3234: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:45 smithi059 bash[61152]: cluster 2022-04-28T21:17:43.453215+0000 mgr.y (mgr.24961) 2900 : cluster [DBG] pgmap v3234: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:45 smithi139 bash[43301]: cluster 2022-04-28T21:17:43.453215+0000 mgr.y (mgr.24961) 2900 : cluster [DBG] pgmap v3234: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:47 smithi059 bash[61152]: cluster 2022-04-28T21:17:45.454641+0000 mgr.y (mgr.24961) 2901 : cluster [DBG] pgmap v3235: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:47 smithi059 bash[59745]: cluster 2022-04-28T21:17:45.454641+0000 mgr.y (mgr.24961) 2901 : cluster [DBG] pgmap v3235: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:47 smithi139 bash[43301]: cluster 2022-04-28T21:17:45.454641+0000 mgr.y (mgr.24961) 2901 : cluster [DBG] pgmap v3235: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:49 smithi059 bash[61152]: cluster 2022-04-28T21:17:47.455773+0000 mgr.y (mgr.24961) 2902 : cluster [DBG] pgmap v3236: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:49.635 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:49 smithi059 bash[59745]: cluster 2022-04-28T21:17:47.455773+0000 mgr.y (mgr.24961) 2902 : cluster [DBG] pgmap v3236: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:49.635 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:49 smithi139 bash[43301]: cluster 2022-04-28T21:17:47.455773+0000 mgr.y (mgr.24961) 2902 : cluster [DBG] pgmap v3236: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:51.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:51 smithi059 bash[61152]: cluster 2022-04-28T21:17:49.456932+0000 mgr.y (mgr.24961) 2903 : cluster [DBG] pgmap v3237: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:51.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:51 smithi059 bash[59745]: cluster 2022-04-28T21:17:49.456932+0000 mgr.y (mgr.24961) 2903 : cluster [DBG] pgmap v3237: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:51 smithi139 bash[43301]: cluster 2022-04-28T21:17:49.456932+0000 mgr.y (mgr.24961) 2903 : cluster [DBG] pgmap v3237: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:51.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:53.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:53 smithi059 bash[59745]: cluster 2022-04-28T21:17:51.458212+0000 mgr.y (mgr.24961) 2904 : cluster [DBG] pgmap v3238: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:53.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:53 smithi059 bash[61152]: cluster 2022-04-28T21:17:51.458212+0000 mgr.y (mgr.24961) 2904 : cluster [DBG] pgmap v3238: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:53.631 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:53 smithi139 bash[43301]: cluster 2022-04-28T21:17:51.458212+0000 mgr.y (mgr.24961) 2904 : cluster [DBG] pgmap v3238: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:17:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:17:53.859Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:17:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:55 smithi059 bash[61152]: cluster 2022-04-28T21:17:53.458951+0000 mgr.y (mgr.24961) 2905 : cluster [DBG] pgmap v3239: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:55 smithi059 bash[59745]: cluster 2022-04-28T21:17:53.458951+0000 mgr.y (mgr.24961) 2905 : cluster [DBG] pgmap v3239: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:55 smithi139 bash[43301]: cluster 2022-04-28T21:17:53.458951+0000 mgr.y (mgr.24961) 2905 : cluster [DBG] pgmap v3239: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:57 smithi059 bash[61152]: cluster 2022-04-28T21:17:55.460501+0000 mgr.y (mgr.24961) 2906 : cluster [DBG] pgmap v3240: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:57 smithi059 bash[59745]: cluster 2022-04-28T21:17:55.460501+0000 mgr.y (mgr.24961) 2906 : cluster [DBG] pgmap v3240: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:57 smithi139 bash[43301]: cluster 2022-04-28T21:17:55.460501+0000 mgr.y (mgr.24961) 2906 : cluster [DBG] pgmap v3240: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:17:59 smithi059 bash[61152]: cluster 2022-04-28T21:17:57.461546+0000 mgr.y (mgr.24961) 2907 : cluster [DBG] pgmap v3241: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:17:59 smithi059 bash[59745]: cluster 2022-04-28T21:17:57.461546+0000 mgr.y (mgr.24961) 2907 : cluster [DBG] pgmap v3241: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:17:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:17:59 smithi139 bash[43301]: cluster 2022-04-28T21:17:57.461546+0000 mgr.y (mgr.24961) 2907 : cluster [DBG] pgmap v3241: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:01 smithi059 bash[61152]: cluster 2022-04-28T21:17:59.462691+0000 mgr.y (mgr.24961) 2908 : cluster [DBG] pgmap v3242: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:01.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:01 smithi059 bash[59745]: cluster 2022-04-28T21:17:59.462691+0000 mgr.y (mgr.24961) 2908 : cluster [DBG] pgmap v3242: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:01 smithi139 bash[43301]: cluster 2022-04-28T21:17:59.462691+0000 mgr.y (mgr.24961) 2908 : cluster [DBG] pgmap v3242: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:01.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:03 smithi059 bash[61152]: cluster 2022-04-28T21:18:01.463993+0000 mgr.y (mgr.24961) 2909 : cluster [DBG] pgmap v3243: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:03 smithi059 bash[59745]: cluster 2022-04-28T21:18:01.463993+0000 mgr.y (mgr.24961) 2909 : cluster [DBG] pgmap v3243: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:03 smithi139 bash[43301]: cluster 2022-04-28T21:18:01.463993+0000 mgr.y (mgr.24961) 2909 : cluster [DBG] pgmap v3243: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:03.860Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:05 smithi059 bash[61152]: cluster 2022-04-28T21:18:03.464826+0000 mgr.y (mgr.24961) 2910 : cluster [DBG] pgmap v3244: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:05 smithi059 bash[59745]: cluster 2022-04-28T21:18:03.464826+0000 mgr.y (mgr.24961) 2910 : cluster [DBG] pgmap v3244: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:05 smithi139 bash[43301]: cluster 2022-04-28T21:18:03.464826+0000 mgr.y (mgr.24961) 2910 : cluster [DBG] pgmap v3244: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:07 smithi059 bash[61152]: cluster 2022-04-28T21:18:05.466290+0000 mgr.y (mgr.24961) 2911 : cluster [DBG] pgmap v3245: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:07 smithi059 bash[59745]: cluster 2022-04-28T21:18:05.466290+0000 mgr.y (mgr.24961) 2911 : cluster [DBG] pgmap v3245: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:07 smithi139 bash[43301]: cluster 2022-04-28T21:18:05.466290+0000 mgr.y (mgr.24961) 2911 : cluster [DBG] pgmap v3245: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:09 smithi059 bash[61152]: cluster 2022-04-28T21:18:07.467284+0000 mgr.y (mgr.24961) 2912 : cluster [DBG] pgmap v3246: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:09 smithi059 bash[59745]: cluster 2022-04-28T21:18:07.467284+0000 mgr.y (mgr.24961) 2912 : cluster [DBG] pgmap v3246: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:09 smithi139 bash[43301]: cluster 2022-04-28T21:18:07.467284+0000 mgr.y (mgr.24961) 2912 : cluster [DBG] pgmap v3246: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:11 smithi059 bash[61152]: cluster 2022-04-28T21:18:09.468433+0000 mgr.y (mgr.24961) 2913 : cluster [DBG] pgmap v3247: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:11 smithi059 bash[59745]: cluster 2022-04-28T21:18:09.468433+0000 mgr.y (mgr.24961) 2913 : cluster [DBG] pgmap v3247: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:11 smithi139 bash[43301]: cluster 2022-04-28T21:18:09.468433+0000 mgr.y (mgr.24961) 2913 : cluster [DBG] pgmap v3247: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:11.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:13 smithi059 bash[61152]: cluster 2022-04-28T21:18:11.469698+0000 mgr.y (mgr.24961) 2914 : cluster [DBG] pgmap v3248: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:13 smithi059 bash[59745]: cluster 2022-04-28T21:18:11.469698+0000 mgr.y (mgr.24961) 2914 : cluster [DBG] pgmap v3248: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:13 smithi139 bash[43301]: cluster 2022-04-28T21:18:11.469698+0000 mgr.y (mgr.24961) 2914 : cluster [DBG] pgmap v3248: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:13.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:15 smithi059 bash[61152]: cluster 2022-04-28T21:18:13.470381+0000 mgr.y (mgr.24961) 2915 : cluster [DBG] pgmap v3249: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:15 smithi059 bash[59745]: cluster 2022-04-28T21:18:13.470381+0000 mgr.y (mgr.24961) 2915 : cluster [DBG] pgmap v3249: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:15 smithi139 bash[43301]: cluster 2022-04-28T21:18:13.470381+0000 mgr.y (mgr.24961) 2915 : cluster [DBG] pgmap v3249: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:17 smithi059 bash[61152]: cluster 2022-04-28T21:18:15.471817+0000 mgr.y (mgr.24961) 2916 : cluster [DBG] pgmap v3250: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:17.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:17 smithi059 bash[59745]: cluster 2022-04-28T21:18:15.471817+0000 mgr.y (mgr.24961) 2916 : cluster [DBG] pgmap v3250: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:17.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:17 smithi139 bash[43301]: cluster 2022-04-28T21:18:15.471817+0000 mgr.y (mgr.24961) 2916 : cluster [DBG] pgmap v3250: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[61152]: audit 2022-04-28T21:18:18.239098+0000 mon.a (mon.0) 2622 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:18:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[61152]: audit 2022-04-28T21:18:18.239699+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:18:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[61152]: audit 2022-04-28T21:18:18.252355+0000 mon.a (mon.0) 2624 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:18:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[61152]: audit 2022-04-28T21:18:18.253094+0000 mon.a (mon.0) 2625 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:18:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[61152]: audit 2022-04-28T21:18:18.253754+0000 mon.a (mon.0) 2626 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:18:18.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[61152]: audit 2022-04-28T21:18:18.254410+0000 mon.a (mon.0) 2627 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T21:18:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[59745]: audit 2022-04-28T21:18:18.239098+0000 mon.a (mon.0) 2622 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:18:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[59745]: audit 2022-04-28T21:18:18.239699+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:18:18.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[59745]: audit 2022-04-28T21:18:18.252355+0000 mon.a (mon.0) 2624 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:18:18.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[59745]: audit 2022-04-28T21:18:18.253094+0000 mon.a (mon.0) 2625 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:18:18.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[59745]: audit 2022-04-28T21:18:18.253754+0000 mon.a (mon.0) 2626 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:18:18.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:18 smithi059 bash[59745]: audit 2022-04-28T21:18:18.254410+0000 mon.a (mon.0) 2627 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T21:18:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:18 smithi139 bash[43301]: audit 2022-04-28T21:18:18.239098+0000 mon.a (mon.0) 2622 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:18:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:18 smithi139 bash[43301]: audit 2022-04-28T21:18:18.239699+0000 mon.a (mon.0) 2623 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:18:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:18 smithi139 bash[43301]: audit 2022-04-28T21:18:18.252355+0000 mon.a (mon.0) 2624 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:18:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:18 smithi139 bash[43301]: audit 2022-04-28T21:18:18.253094+0000 mon.a (mon.0) 2625 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:18:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:18 smithi139 bash[43301]: audit 2022-04-28T21:18:18.253754+0000 mon.a (mon.0) 2626 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:18:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:18 smithi139 bash[43301]: audit 2022-04-28T21:18:18.254410+0000 mon.a (mon.0) 2627 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T21:18:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:19 smithi059 bash[61152]: cluster 2022-04-28T21:18:17.472803+0000 mgr.y (mgr.24961) 2917 : cluster [DBG] pgmap v3251: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:19 smithi059 bash[59745]: cluster 2022-04-28T21:18:17.472803+0000 mgr.y (mgr.24961) 2917 : cluster [DBG] pgmap v3251: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:19 smithi139 bash[43301]: cluster 2022-04-28T21:18:17.472803+0000 mgr.y (mgr.24961) 2917 : cluster [DBG] pgmap v3251: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:21 smithi059 bash[61152]: cluster 2022-04-28T21:18:19.473925+0000 mgr.y (mgr.24961) 2918 : cluster [DBG] pgmap v3252: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:21 smithi059 bash[59745]: cluster 2022-04-28T21:18:19.473925+0000 mgr.y (mgr.24961) 2918 : cluster [DBG] pgmap v3252: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:21 smithi139 bash[43301]: cluster 2022-04-28T21:18:19.473925+0000 mgr.y (mgr.24961) 2918 : cluster [DBG] pgmap v3252: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:21.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:23 smithi059 bash[61152]: cluster 2022-04-28T21:18:21.475180+0000 mgr.y (mgr.24961) 2919 : cluster [DBG] pgmap v3253: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:23.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:23 smithi059 bash[59745]: cluster 2022-04-28T21:18:21.475180+0000 mgr.y (mgr.24961) 2919 : cluster [DBG] pgmap v3253: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:23 smithi139 bash[43301]: cluster 2022-04-28T21:18:21.475180+0000 mgr.y (mgr.24961) 2919 : cluster [DBG] pgmap v3253: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:23.861Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:25.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:25 smithi139 bash[43301]: cluster 2022-04-28T21:18:23.475931+0000 mgr.y (mgr.24961) 2920 : cluster [DBG] pgmap v3254: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:25.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:25 smithi059 bash[61152]: cluster 2022-04-28T21:18:23.475931+0000 mgr.y (mgr.24961) 2920 : cluster [DBG] pgmap v3254: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:25 smithi059 bash[59745]: cluster 2022-04-28T21:18:23.475931+0000 mgr.y (mgr.24961) 2920 : cluster [DBG] pgmap v3254: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:27 smithi139 bash[43301]: cluster 2022-04-28T21:18:25.477361+0000 mgr.y (mgr.24961) 2921 : cluster [DBG] pgmap v3255: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:27 smithi059 bash[61152]: cluster 2022-04-28T21:18:25.477361+0000 mgr.y (mgr.24961) 2921 : cluster [DBG] pgmap v3255: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:27 smithi059 bash[59745]: cluster 2022-04-28T21:18:25.477361+0000 mgr.y (mgr.24961) 2921 : cluster [DBG] pgmap v3255: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:29 smithi139 bash[43301]: cluster 2022-04-28T21:18:27.478347+0000 mgr.y (mgr.24961) 2922 : cluster [DBG] pgmap v3256: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:29 smithi059 bash[61152]: cluster 2022-04-28T21:18:27.478347+0000 mgr.y (mgr.24961) 2922 : cluster [DBG] pgmap v3256: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:29.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:29 smithi059 bash[59745]: cluster 2022-04-28T21:18:27.478347+0000 mgr.y (mgr.24961) 2922 : cluster [DBG] pgmap v3256: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:31 smithi139 bash[43301]: cluster 2022-04-28T21:18:29.479615+0000 mgr.y (mgr.24961) 2923 : cluster [DBG] pgmap v3257: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:31 smithi059 bash[61152]: cluster 2022-04-28T21:18:29.479615+0000 mgr.y (mgr.24961) 2923 : cluster [DBG] pgmap v3257: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:31 smithi059 bash[59745]: cluster 2022-04-28T21:18:29.479615+0000 mgr.y (mgr.24961) 2923 : cluster [DBG] pgmap v3257: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:32.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:31.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:33 smithi139 bash[43301]: cluster 2022-04-28T21:18:31.480987+0000 mgr.y (mgr.24961) 2924 : cluster [DBG] pgmap v3258: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:33 smithi059 bash[61152]: cluster 2022-04-28T21:18:31.480987+0000 mgr.y (mgr.24961) 2924 : cluster [DBG] pgmap v3258: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:33 smithi059 bash[59745]: cluster 2022-04-28T21:18:31.480987+0000 mgr.y (mgr.24961) 2924 : cluster [DBG] pgmap v3258: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:33.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:35 smithi059 bash[61152]: cluster 2022-04-28T21:18:33.481755+0000 mgr.y (mgr.24961) 2925 : cluster [DBG] pgmap v3259: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:35 smithi059 bash[59745]: cluster 2022-04-28T21:18:33.481755+0000 mgr.y (mgr.24961) 2925 : cluster [DBG] pgmap v3259: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:35 smithi139 bash[43301]: cluster 2022-04-28T21:18:33.481755+0000 mgr.y (mgr.24961) 2925 : cluster [DBG] pgmap v3259: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:36 smithi059 bash[61152]: cluster 2022-04-28T21:18:35.483163+0000 mgr.y (mgr.24961) 2926 : cluster [DBG] pgmap v3260: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:36 smithi059 bash[59745]: cluster 2022-04-28T21:18:35.483163+0000 mgr.y (mgr.24961) 2926 : cluster [DBG] pgmap v3260: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:36.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:36 smithi139 bash[43301]: cluster 2022-04-28T21:18:35.483163+0000 mgr.y (mgr.24961) 2926 : cluster [DBG] pgmap v3260: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:37 smithi059 bash[61152]: audit 2022-04-28T21:18:37.502923+0000 mon.a (mon.0) 2628 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:18:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:37 smithi059 bash[61152]: audit 2022-04-28T21:18:37.505720+0000 mon.a (mon.0) 2629 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:18:37.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:37 smithi059 bash[61152]: audit 2022-04-28T21:18:37.507576+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:18:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:37 smithi059 bash[59745]: audit 2022-04-28T21:18:37.502923+0000 mon.a (mon.0) 2628 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:18:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:37 smithi059 bash[59745]: audit 2022-04-28T21:18:37.505720+0000 mon.a (mon.0) 2629 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:18:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:37 smithi059 bash[59745]: audit 2022-04-28T21:18:37.507576+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:18:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:37 smithi139 bash[43301]: audit 2022-04-28T21:18:37.502923+0000 mon.a (mon.0) 2628 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:18:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:37 smithi139 bash[43301]: audit 2022-04-28T21:18:37.505720+0000 mon.a (mon.0) 2629 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:18:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:37 smithi139 bash[43301]: audit 2022-04-28T21:18:37.507576+0000 mon.a (mon.0) 2630 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:18:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:38 smithi059 bash[61152]: cluster 2022-04-28T21:18:37.484166+0000 mgr.y (mgr.24961) 2927 : cluster [DBG] pgmap v3261: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:38 smithi059 bash[61152]: cephadm 2022-04-28T21:18:37.509180+0000 mgr.y (mgr.24961) 2928 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:18:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:38 smithi059 bash[61152]: audit 2022-04-28T21:18:37.813796+0000 mon.a (mon.0) 2631 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:18:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:38 smithi059 bash[59745]: cluster 2022-04-28T21:18:37.484166+0000 mgr.y (mgr.24961) 2927 : cluster [DBG] pgmap v3261: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:38 smithi059 bash[59745]: cephadm 2022-04-28T21:18:37.509180+0000 mgr.y (mgr.24961) 2928 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:18:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:38 smithi059 bash[59745]: audit 2022-04-28T21:18:37.813796+0000 mon.a (mon.0) 2631 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:18:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:38 smithi139 bash[43301]: cluster 2022-04-28T21:18:37.484166+0000 mgr.y (mgr.24961) 2927 : cluster [DBG] pgmap v3261: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:38 smithi139 bash[43301]: cephadm 2022-04-28T21:18:37.509180+0000 mgr.y (mgr.24961) 2928 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:18:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:38 smithi139 bash[43301]: audit 2022-04-28T21:18:37.813796+0000 mon.a (mon.0) 2631 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:18:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:40 smithi059 bash[59745]: cluster 2022-04-28T21:18:39.485280+0000 mgr.y (mgr.24961) 2929 : cluster [DBG] pgmap v3262: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:41.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:40 smithi059 bash[61152]: cluster 2022-04-28T21:18:39.485280+0000 mgr.y (mgr.24961) 2929 : cluster [DBG] pgmap v3262: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:40 smithi139 bash[43301]: cluster 2022-04-28T21:18:39.485280+0000 mgr.y (mgr.24961) 2929 : cluster [DBG] pgmap v3262: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:41.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:43.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:42 smithi059 bash[61152]: cluster 2022-04-28T21:18:41.486597+0000 mgr.y (mgr.24961) 2930 : cluster [DBG] pgmap v3263: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:42 smithi059 bash[59745]: cluster 2022-04-28T21:18:41.486597+0000 mgr.y (mgr.24961) 2930 : cluster [DBG] pgmap v3263: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:42 smithi139 bash[43301]: cluster 2022-04-28T21:18:41.486597+0000 mgr.y (mgr.24961) 2930 : cluster [DBG] pgmap v3263: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:43.863Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:44 smithi059 bash[61152]: cluster 2022-04-28T21:18:43.487431+0000 mgr.y (mgr.24961) 2931 : cluster [DBG] pgmap v3264: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:45.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:44 smithi059 bash[59745]: cluster 2022-04-28T21:18:43.487431+0000 mgr.y (mgr.24961) 2931 : cluster [DBG] pgmap v3264: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:44 smithi139 bash[43301]: cluster 2022-04-28T21:18:43.487431+0000 mgr.y (mgr.24961) 2931 : cluster [DBG] pgmap v3264: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:46 smithi059 bash[61152]: cluster 2022-04-28T21:18:45.488876+0000 mgr.y (mgr.24961) 2932 : cluster [DBG] pgmap v3265: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:46 smithi059 bash[59745]: cluster 2022-04-28T21:18:45.488876+0000 mgr.y (mgr.24961) 2932 : cluster [DBG] pgmap v3265: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:46 smithi139 bash[43301]: cluster 2022-04-28T21:18:45.488876+0000 mgr.y (mgr.24961) 2932 : cluster [DBG] pgmap v3265: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:48 smithi139 bash[43301]: cluster 2022-04-28T21:18:47.489830+0000 mgr.y (mgr.24961) 2933 : cluster [DBG] pgmap v3266: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:48 smithi059 bash[61152]: cluster 2022-04-28T21:18:47.489830+0000 mgr.y (mgr.24961) 2933 : cluster [DBG] pgmap v3266: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:49.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:48 smithi059 bash[59745]: cluster 2022-04-28T21:18:47.489830+0000 mgr.y (mgr.24961) 2933 : cluster [DBG] pgmap v3266: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:50 smithi139 bash[43301]: cluster 2022-04-28T21:18:49.490944+0000 mgr.y (mgr.24961) 2934 : cluster [DBG] pgmap v3267: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:50 smithi059 bash[61152]: cluster 2022-04-28T21:18:49.490944+0000 mgr.y (mgr.24961) 2934 : cluster [DBG] pgmap v3267: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:50 smithi059 bash[59745]: cluster 2022-04-28T21:18:49.490944+0000 mgr.y (mgr.24961) 2934 : cluster [DBG] pgmap v3267: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:51.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:52 smithi139 bash[43301]: cluster 2022-04-28T21:18:51.492207+0000 mgr.y (mgr.24961) 2935 : cluster [DBG] pgmap v3268: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:52 smithi059 bash[61152]: cluster 2022-04-28T21:18:51.492207+0000 mgr.y (mgr.24961) 2935 : cluster [DBG] pgmap v3268: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:52 smithi059 bash[59745]: cluster 2022-04-28T21:18:51.492207+0000 mgr.y (mgr.24961) 2935 : cluster [DBG] pgmap v3268: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:18:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:18:53.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:18:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:54 smithi139 bash[43301]: cluster 2022-04-28T21:18:53.492880+0000 mgr.y (mgr.24961) 2936 : cluster [DBG] pgmap v3269: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:54 smithi059 bash[61152]: cluster 2022-04-28T21:18:53.492880+0000 mgr.y (mgr.24961) 2936 : cluster [DBG] pgmap v3269: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:54 smithi059 bash[59745]: cluster 2022-04-28T21:18:53.492880+0000 mgr.y (mgr.24961) 2936 : cluster [DBG] pgmap v3269: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:56 smithi139 bash[43301]: cluster 2022-04-28T21:18:55.494270+0000 mgr.y (mgr.24961) 2937 : cluster [DBG] pgmap v3270: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:56 smithi059 bash[61152]: cluster 2022-04-28T21:18:55.494270+0000 mgr.y (mgr.24961) 2937 : cluster [DBG] pgmap v3270: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:56 smithi059 bash[59745]: cluster 2022-04-28T21:18:55.494270+0000 mgr.y (mgr.24961) 2937 : cluster [DBG] pgmap v3270: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:18:58 smithi059 bash[61152]: cluster 2022-04-28T21:18:57.495266+0000 mgr.y (mgr.24961) 2938 : cluster [DBG] pgmap v3271: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:18:58 smithi059 bash[59745]: cluster 2022-04-28T21:18:57.495266+0000 mgr.y (mgr.24961) 2938 : cluster [DBG] pgmap v3271: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:18:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:18:58 smithi139 bash[43301]: cluster 2022-04-28T21:18:57.495266+0000 mgr.y (mgr.24961) 2938 : cluster [DBG] pgmap v3271: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:00 smithi059 bash[61152]: cluster 2022-04-28T21:18:59.496385+0000 mgr.y (mgr.24961) 2939 : cluster [DBG] pgmap v3272: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:00 smithi059 bash[59745]: cluster 2022-04-28T21:18:59.496385+0000 mgr.y (mgr.24961) 2939 : cluster [DBG] pgmap v3272: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:00 smithi139 bash[43301]: cluster 2022-04-28T21:18:59.496385+0000 mgr.y (mgr.24961) 2939 : cluster [DBG] pgmap v3272: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:01.971Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:02 smithi059 bash[61152]: cluster 2022-04-28T21:19:01.497653+0000 mgr.y (mgr.24961) 2940 : cluster [DBG] pgmap v3273: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:02 smithi059 bash[59745]: cluster 2022-04-28T21:19:01.497653+0000 mgr.y (mgr.24961) 2940 : cluster [DBG] pgmap v3273: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:02 smithi139 bash[43301]: cluster 2022-04-28T21:19:01.497653+0000 mgr.y (mgr.24961) 2940 : cluster [DBG] pgmap v3273: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:03.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:04 smithi059 bash[59745]: cluster 2022-04-28T21:19:03.498409+0000 mgr.y (mgr.24961) 2941 : cluster [DBG] pgmap v3274: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:05.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:04 smithi059 bash[61152]: cluster 2022-04-28T21:19:03.498409+0000 mgr.y (mgr.24961) 2941 : cluster [DBG] pgmap v3274: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:04 smithi139 bash[43301]: cluster 2022-04-28T21:19:03.498409+0000 mgr.y (mgr.24961) 2941 : cluster [DBG] pgmap v3274: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:06 smithi059 bash[61152]: cluster 2022-04-28T21:19:05.499816+0000 mgr.y (mgr.24961) 2942 : cluster [DBG] pgmap v3275: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:06 smithi059 bash[59745]: cluster 2022-04-28T21:19:05.499816+0000 mgr.y (mgr.24961) 2942 : cluster [DBG] pgmap v3275: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:06 smithi139 bash[43301]: cluster 2022-04-28T21:19:05.499816+0000 mgr.y (mgr.24961) 2942 : cluster [DBG] pgmap v3275: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:08 smithi059 bash[61152]: cluster 2022-04-28T21:19:07.500893+0000 mgr.y (mgr.24961) 2943 : cluster [DBG] pgmap v3276: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:08 smithi059 bash[59745]: cluster 2022-04-28T21:19:07.500893+0000 mgr.y (mgr.24961) 2943 : cluster [DBG] pgmap v3276: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:08 smithi139 bash[43301]: cluster 2022-04-28T21:19:07.500893+0000 mgr.y (mgr.24961) 2943 : cluster [DBG] pgmap v3276: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:10 smithi059 bash[61152]: cluster 2022-04-28T21:19:09.502120+0000 mgr.y (mgr.24961) 2944 : cluster [DBG] pgmap v3277: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:11.401 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:10 smithi059 bash[59745]: cluster 2022-04-28T21:19:09.502120+0000 mgr.y (mgr.24961) 2944 : cluster [DBG] pgmap v3277: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:11.402 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:10 smithi139 bash[43301]: cluster 2022-04-28T21:19:09.502120+0000 mgr.y (mgr.24961) 2944 : cluster [DBG] pgmap v3277: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:11.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:12 smithi059 bash[61152]: cluster 2022-04-28T21:19:11.503388+0000 mgr.y (mgr.24961) 2945 : cluster [DBG] pgmap v3278: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:12 smithi059 bash[59745]: cluster 2022-04-28T21:19:11.503388+0000 mgr.y (mgr.24961) 2945 : cluster [DBG] pgmap v3278: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:12 smithi139 bash[43301]: cluster 2022-04-28T21:19:11.503388+0000 mgr.y (mgr.24961) 2945 : cluster [DBG] pgmap v3278: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:13.865Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:14 smithi059 bash[61152]: cluster 2022-04-28T21:19:13.504129+0000 mgr.y (mgr.24961) 2946 : cluster [DBG] pgmap v3279: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:14 smithi059 bash[59745]: cluster 2022-04-28T21:19:13.504129+0000 mgr.y (mgr.24961) 2946 : cluster [DBG] pgmap v3279: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:14 smithi139 bash[43301]: cluster 2022-04-28T21:19:13.504129+0000 mgr.y (mgr.24961) 2946 : cluster [DBG] pgmap v3279: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:16 smithi059 bash[61152]: cluster 2022-04-28T21:19:15.505496+0000 mgr.y (mgr.24961) 2947 : cluster [DBG] pgmap v3280: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:16 smithi059 bash[59745]: cluster 2022-04-28T21:19:15.505496+0000 mgr.y (mgr.24961) 2947 : cluster [DBG] pgmap v3280: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:16 smithi139 bash[43301]: cluster 2022-04-28T21:19:15.505496+0000 mgr.y (mgr.24961) 2947 : cluster [DBG] pgmap v3280: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: cluster 2022-04-28T21:19:17.506462+0000 mgr.y (mgr.24961) 2948 : cluster [DBG] pgmap v3281: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: audit 2022-04-28T21:19:18.254354+0000 mon.a (mon.0) 2632 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:19:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: audit 2022-04-28T21:19:18.255133+0000 mon.a (mon.0) 2633 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:19:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: audit 2022-04-28T21:19:18.267123+0000 mon.a (mon.0) 2634 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:19:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: audit 2022-04-28T21:19:18.267881+0000 mon.a (mon.0) 2635 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:19:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: audit 2022-04-28T21:19:18.268604+0000 mon.a (mon.0) 2636 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:19:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[61152]: audit 2022-04-28T21:19:18.269282+0000 mon.a (mon.0) 2637 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:19:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: cluster 2022-04-28T21:19:17.506462+0000 mgr.y (mgr.24961) 2948 : cluster [DBG] pgmap v3281: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: audit 2022-04-28T21:19:18.254354+0000 mon.a (mon.0) 2632 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:19:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: audit 2022-04-28T21:19:18.255133+0000 mon.a (mon.0) 2633 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:19:19.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: audit 2022-04-28T21:19:18.267123+0000 mon.a (mon.0) 2634 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:19:19.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: audit 2022-04-28T21:19:18.267881+0000 mon.a (mon.0) 2635 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:19:19.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: audit 2022-04-28T21:19:18.268604+0000 mon.a (mon.0) 2636 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:19:19.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:18 smithi059 bash[59745]: audit 2022-04-28T21:19:18.269282+0000 mon.a (mon.0) 2637 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:19:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: cluster 2022-04-28T21:19:17.506462+0000 mgr.y (mgr.24961) 2948 : cluster [DBG] pgmap v3281: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: audit 2022-04-28T21:19:18.254354+0000 mon.a (mon.0) 2632 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:19:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: audit 2022-04-28T21:19:18.255133+0000 mon.a (mon.0) 2633 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:19:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: audit 2022-04-28T21:19:18.267123+0000 mon.a (mon.0) 2634 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:19:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: audit 2022-04-28T21:19:18.267881+0000 mon.a (mon.0) 2635 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:19:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: audit 2022-04-28T21:19:18.268604+0000 mon.a (mon.0) 2636 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:19:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:18 smithi139 bash[43301]: audit 2022-04-28T21:19:18.269282+0000 mon.a (mon.0) 2637 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:19:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:20 smithi059 bash[61152]: cluster 2022-04-28T21:19:19.507608+0000 mgr.y (mgr.24961) 2949 : cluster [DBG] pgmap v3282: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:20 smithi059 bash[59745]: cluster 2022-04-28T21:19:19.507608+0000 mgr.y (mgr.24961) 2949 : cluster [DBG] pgmap v3282: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:20 smithi139 bash[43301]: cluster 2022-04-28T21:19:19.507608+0000 mgr.y (mgr.24961) 2949 : cluster [DBG] pgmap v3282: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:21.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:22 smithi059 bash[61152]: cluster 2022-04-28T21:19:21.508853+0000 mgr.y (mgr.24961) 2950 : cluster [DBG] pgmap v3283: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:22 smithi059 bash[59745]: cluster 2022-04-28T21:19:21.508853+0000 mgr.y (mgr.24961) 2950 : cluster [DBG] pgmap v3283: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:22 smithi139 bash[43301]: cluster 2022-04-28T21:19:21.508853+0000 mgr.y (mgr.24961) 2950 : cluster [DBG] pgmap v3283: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:23.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:24 smithi059 bash[61152]: cluster 2022-04-28T21:19:23.509538+0000 mgr.y (mgr.24961) 2951 : cluster [DBG] pgmap v3284: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:24 smithi059 bash[59745]: cluster 2022-04-28T21:19:23.509538+0000 mgr.y (mgr.24961) 2951 : cluster [DBG] pgmap v3284: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:24 smithi139 bash[43301]: cluster 2022-04-28T21:19:23.509538+0000 mgr.y (mgr.24961) 2951 : cluster [DBG] pgmap v3284: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:26 smithi059 bash[61152]: cluster 2022-04-28T21:19:25.510928+0000 mgr.y (mgr.24961) 2952 : cluster [DBG] pgmap v3285: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:26 smithi059 bash[59745]: cluster 2022-04-28T21:19:25.510928+0000 mgr.y (mgr.24961) 2952 : cluster [DBG] pgmap v3285: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:26 smithi139 bash[43301]: cluster 2022-04-28T21:19:25.510928+0000 mgr.y (mgr.24961) 2952 : cluster [DBG] pgmap v3285: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:28 smithi059 bash[61152]: cluster 2022-04-28T21:19:27.511951+0000 mgr.y (mgr.24961) 2953 : cluster [DBG] pgmap v3286: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:28 smithi059 bash[59745]: cluster 2022-04-28T21:19:27.511951+0000 mgr.y (mgr.24961) 2953 : cluster [DBG] pgmap v3286: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:28 smithi139 bash[43301]: cluster 2022-04-28T21:19:27.511951+0000 mgr.y (mgr.24961) 2953 : cluster [DBG] pgmap v3286: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:30 smithi059 bash[59745]: cluster 2022-04-28T21:19:29.513104+0000 mgr.y (mgr.24961) 2954 : cluster [DBG] pgmap v3287: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:30 smithi059 bash[61152]: cluster 2022-04-28T21:19:29.513104+0000 mgr.y (mgr.24961) 2954 : cluster [DBG] pgmap v3287: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:30 smithi139 bash[43301]: cluster 2022-04-28T21:19:29.513104+0000 mgr.y (mgr.24961) 2954 : cluster [DBG] pgmap v3287: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:31.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:32 smithi059 bash[59745]: cluster 2022-04-28T21:19:31.514354+0000 mgr.y (mgr.24961) 2955 : cluster [DBG] pgmap v3288: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:32 smithi059 bash[61152]: cluster 2022-04-28T21:19:31.514354+0000 mgr.y (mgr.24961) 2955 : cluster [DBG] pgmap v3288: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:32 smithi139 bash[43301]: cluster 2022-04-28T21:19:31.514354+0000 mgr.y (mgr.24961) 2955 : cluster [DBG] pgmap v3288: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:33.866Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:34 smithi059 bash[61152]: cluster 2022-04-28T21:19:33.515094+0000 mgr.y (mgr.24961) 2956 : cluster [DBG] pgmap v3289: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:34 smithi059 bash[59745]: cluster 2022-04-28T21:19:33.515094+0000 mgr.y (mgr.24961) 2956 : cluster [DBG] pgmap v3289: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:34 smithi139 bash[43301]: cluster 2022-04-28T21:19:33.515094+0000 mgr.y (mgr.24961) 2956 : cluster [DBG] pgmap v3289: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:36 smithi059 bash[61152]: cluster 2022-04-28T21:19:35.516520+0000 mgr.y (mgr.24961) 2957 : cluster [DBG] pgmap v3290: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:37.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:36 smithi059 bash[59745]: cluster 2022-04-28T21:19:35.516520+0000 mgr.y (mgr.24961) 2957 : cluster [DBG] pgmap v3290: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:36 smithi139 bash[43301]: cluster 2022-04-28T21:19:35.516520+0000 mgr.y (mgr.24961) 2957 : cluster [DBG] pgmap v3290: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:37 smithi059 bash[59745]: audit 2022-04-28T21:19:37.823048+0000 mon.a (mon.0) 2638 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:19:38.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:37 smithi059 bash[59745]: audit 2022-04-28T21:19:37.825781+0000 mon.a (mon.0) 2639 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:19:38.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:37 smithi059 bash[59745]: audit 2022-04-28T21:19:37.827526+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:19:38.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:37 smithi059 bash[61152]: audit 2022-04-28T21:19:37.823048+0000 mon.a (mon.0) 2638 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:19:38.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:37 smithi059 bash[61152]: audit 2022-04-28T21:19:37.825781+0000 mon.a (mon.0) 2639 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:19:38.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:37 smithi059 bash[61152]: audit 2022-04-28T21:19:37.827526+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:19:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:37 smithi139 bash[43301]: audit 2022-04-28T21:19:37.823048+0000 mon.a (mon.0) 2638 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:19:38.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:37 smithi139 bash[43301]: audit 2022-04-28T21:19:37.825781+0000 mon.a (mon.0) 2639 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:19:38.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:37 smithi139 bash[43301]: audit 2022-04-28T21:19:37.827526+0000 mon.a (mon.0) 2640 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:19:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[61152]: cluster 2022-04-28T21:19:37.517511+0000 mgr.y (mgr.24961) 2958 : cluster [DBG] pgmap v3291: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[61152]: cephadm 2022-04-28T21:19:37.829131+0000 mgr.y (mgr.24961) 2959 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:19:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[61152]: audit 2022-04-28T21:19:38.135277+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[61152]: audit 2022-04-28T21:19:38.142636+0000 mon.a (mon.0) 2642 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[61152]: audit 2022-04-28T21:19:38.431735+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[59745]: cluster 2022-04-28T21:19:37.517511+0000 mgr.y (mgr.24961) 2958 : cluster [DBG] pgmap v3291: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[59745]: cephadm 2022-04-28T21:19:37.829131+0000 mgr.y (mgr.24961) 2959 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:19:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[59745]: audit 2022-04-28T21:19:38.135277+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[59745]: audit 2022-04-28T21:19:38.142636+0000 mon.a (mon.0) 2642 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:39 smithi059 bash[59745]: audit 2022-04-28T21:19:38.431735+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:39 smithi139 bash[43301]: cluster 2022-04-28T21:19:37.517511+0000 mgr.y (mgr.24961) 2958 : cluster [DBG] pgmap v3291: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:39 smithi139 bash[43301]: cephadm 2022-04-28T21:19:37.829131+0000 mgr.y (mgr.24961) 2959 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:19:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:39 smithi139 bash[43301]: audit 2022-04-28T21:19:38.135277+0000 mon.a (mon.0) 2641 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:39 smithi139 bash[43301]: audit 2022-04-28T21:19:38.142636+0000 mon.a (mon.0) 2642 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:39 smithi139 bash[43301]: audit 2022-04-28T21:19:38.431735+0000 mon.a (mon.0) 2643 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:19:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:41 smithi059 bash[61152]: cluster 2022-04-28T21:19:39.518651+0000 mgr.y (mgr.24961) 2960 : cluster [DBG] pgmap v3292: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:41 smithi059 bash[59745]: cluster 2022-04-28T21:19:39.518651+0000 mgr.y (mgr.24961) 2960 : cluster [DBG] pgmap v3292: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:41 smithi139 bash[43301]: cluster 2022-04-28T21:19:39.518651+0000 mgr.y (mgr.24961) 2960 : cluster [DBG] pgmap v3292: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:41.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:43 smithi059 bash[61152]: cluster 2022-04-28T21:19:41.520000+0000 mgr.y (mgr.24961) 2961 : cluster [DBG] pgmap v3293: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:43 smithi059 bash[59745]: cluster 2022-04-28T21:19:41.520000+0000 mgr.y (mgr.24961) 2961 : cluster [DBG] pgmap v3293: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:43 smithi139 bash[43301]: cluster 2022-04-28T21:19:41.520000+0000 mgr.y (mgr.24961) 2961 : cluster [DBG] pgmap v3293: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:43.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:45 smithi059 bash[61152]: cluster 2022-04-28T21:19:43.520692+0000 mgr.y (mgr.24961) 2962 : cluster [DBG] pgmap v3294: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:45 smithi059 bash[59745]: cluster 2022-04-28T21:19:43.520692+0000 mgr.y (mgr.24961) 2962 : cluster [DBG] pgmap v3294: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:45 smithi139 bash[43301]: cluster 2022-04-28T21:19:43.520692+0000 mgr.y (mgr.24961) 2962 : cluster [DBG] pgmap v3294: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:47 smithi059 bash[61152]: cluster 2022-04-28T21:19:45.522141+0000 mgr.y (mgr.24961) 2963 : cluster [DBG] pgmap v3295: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:47 smithi059 bash[59745]: cluster 2022-04-28T21:19:45.522141+0000 mgr.y (mgr.24961) 2963 : cluster [DBG] pgmap v3295: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:47 smithi139 bash[43301]: cluster 2022-04-28T21:19:45.522141+0000 mgr.y (mgr.24961) 2963 : cluster [DBG] pgmap v3295: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:49 smithi059 bash[61152]: cluster 2022-04-28T21:19:47.523118+0000 mgr.y (mgr.24961) 2964 : cluster [DBG] pgmap v3296: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:49 smithi059 bash[59745]: cluster 2022-04-28T21:19:47.523118+0000 mgr.y (mgr.24961) 2964 : cluster [DBG] pgmap v3296: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:49 smithi139 bash[43301]: cluster 2022-04-28T21:19:47.523118+0000 mgr.y (mgr.24961) 2964 : cluster [DBG] pgmap v3296: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:51.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:51 smithi059 bash[61152]: cluster 2022-04-28T21:19:49.524318+0000 mgr.y (mgr.24961) 2965 : cluster [DBG] pgmap v3297: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:51.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:51 smithi059 bash[59745]: cluster 2022-04-28T21:19:49.524318+0000 mgr.y (mgr.24961) 2965 : cluster [DBG] pgmap v3297: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:51 smithi139 bash[43301]: cluster 2022-04-28T21:19:49.524318+0000 mgr.y (mgr.24961) 2965 : cluster [DBG] pgmap v3297: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:51.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:53 smithi059 bash[61152]: cluster 2022-04-28T21:19:51.525605+0000 mgr.y (mgr.24961) 2966 : cluster [DBG] pgmap v3298: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:53 smithi059 bash[59745]: cluster 2022-04-28T21:19:51.525605+0000 mgr.y (mgr.24961) 2966 : cluster [DBG] pgmap v3298: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:53 smithi139 bash[43301]: cluster 2022-04-28T21:19:51.525605+0000 mgr.y (mgr.24961) 2966 : cluster [DBG] pgmap v3298: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:19:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:19:53.867Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:19:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:55 smithi059 bash[61152]: cluster 2022-04-28T21:19:53.526450+0000 mgr.y (mgr.24961) 2967 : cluster [DBG] pgmap v3299: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:55 smithi059 bash[59745]: cluster 2022-04-28T21:19:53.526450+0000 mgr.y (mgr.24961) 2967 : cluster [DBG] pgmap v3299: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:55 smithi139 bash[43301]: cluster 2022-04-28T21:19:53.526450+0000 mgr.y (mgr.24961) 2967 : cluster [DBG] pgmap v3299: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:57.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:57 smithi059 bash[61152]: cluster 2022-04-28T21:19:55.527955+0000 mgr.y (mgr.24961) 2968 : cluster [DBG] pgmap v3300: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:57 smithi059 bash[59745]: cluster 2022-04-28T21:19:55.527955+0000 mgr.y (mgr.24961) 2968 : cluster [DBG] pgmap v3300: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:57 smithi139 bash[43301]: cluster 2022-04-28T21:19:55.527955+0000 mgr.y (mgr.24961) 2968 : cluster [DBG] pgmap v3300: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:19:59 smithi059 bash[61152]: cluster 2022-04-28T21:19:57.528931+0000 mgr.y (mgr.24961) 2969 : cluster [DBG] pgmap v3301: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:19:59 smithi059 bash[59745]: cluster 2022-04-28T21:19:57.528931+0000 mgr.y (mgr.24961) 2969 : cluster [DBG] pgmap v3301: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:19:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:19:59 smithi139 bash[43301]: cluster 2022-04-28T21:19:57.528931+0000 mgr.y (mgr.24961) 2969 : cluster [DBG] pgmap v3301: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:00 smithi139 bash[43301]: cluster 2022-04-28T21:19:59.530074+0000 mgr.y (mgr.24961) 2970 : cluster [DBG] pgmap v3302: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:00 smithi139 bash[43301]: cluster 2022-04-28T21:20:00.000125+0000 mon.a (mon.0) 2644 : cluster [INF] overall HEALTH_OK 2022-04-28T21:20:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:00 smithi059 bash[61152]: cluster 2022-04-28T21:19:59.530074+0000 mgr.y (mgr.24961) 2970 : cluster [DBG] pgmap v3302: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:00 smithi059 bash[61152]: cluster 2022-04-28T21:20:00.000125+0000 mon.a (mon.0) 2644 : cluster [INF] overall HEALTH_OK 2022-04-28T21:20:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:00 smithi059 bash[59745]: cluster 2022-04-28T21:19:59.530074+0000 mgr.y (mgr.24961) 2970 : cluster [DBG] pgmap v3302: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:00 smithi059 bash[59745]: cluster 2022-04-28T21:20:00.000125+0000 mon.a (mon.0) 2644 : cluster [INF] overall HEALTH_OK 2022-04-28T21:20:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:01.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:02 smithi139 bash[43301]: cluster 2022-04-28T21:20:01.531332+0000 mgr.y (mgr.24961) 2971 : cluster [DBG] pgmap v3303: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:02 smithi059 bash[61152]: cluster 2022-04-28T21:20:01.531332+0000 mgr.y (mgr.24961) 2971 : cluster [DBG] pgmap v3303: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:02 smithi059 bash[59745]: cluster 2022-04-28T21:20:01.531332+0000 mgr.y (mgr.24961) 2971 : cluster [DBG] pgmap v3303: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:03.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:04 smithi139 bash[43301]: cluster 2022-04-28T21:20:03.532087+0000 mgr.y (mgr.24961) 2972 : cluster [DBG] pgmap v3304: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:04 smithi059 bash[61152]: cluster 2022-04-28T21:20:03.532087+0000 mgr.y (mgr.24961) 2972 : cluster [DBG] pgmap v3304: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:05.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:04 smithi059 bash[59745]: cluster 2022-04-28T21:20:03.532087+0000 mgr.y (mgr.24961) 2972 : cluster [DBG] pgmap v3304: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:06 smithi139 bash[43301]: cluster 2022-04-28T21:20:05.533447+0000 mgr.y (mgr.24961) 2973 : cluster [DBG] pgmap v3305: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:06 smithi059 bash[59745]: cluster 2022-04-28T21:20:05.533447+0000 mgr.y (mgr.24961) 2973 : cluster [DBG] pgmap v3305: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:07.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:06 smithi059 bash[61152]: cluster 2022-04-28T21:20:05.533447+0000 mgr.y (mgr.24961) 2973 : cluster [DBG] pgmap v3305: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:08 smithi059 bash[61152]: cluster 2022-04-28T21:20:07.534412+0000 mgr.y (mgr.24961) 2974 : cluster [DBG] pgmap v3306: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:08 smithi059 bash[59745]: cluster 2022-04-28T21:20:07.534412+0000 mgr.y (mgr.24961) 2974 : cluster [DBG] pgmap v3306: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:08 smithi139 bash[43301]: cluster 2022-04-28T21:20:07.534412+0000 mgr.y (mgr.24961) 2974 : cluster [DBG] pgmap v3306: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:10 smithi059 bash[61152]: cluster 2022-04-28T21:20:09.535657+0000 mgr.y (mgr.24961) 2975 : cluster [DBG] pgmap v3307: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:10 smithi059 bash[59745]: cluster 2022-04-28T21:20:09.535657+0000 mgr.y (mgr.24961) 2975 : cluster [DBG] pgmap v3307: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:10 smithi139 bash[43301]: cluster 2022-04-28T21:20:09.535657+0000 mgr.y (mgr.24961) 2975 : cluster [DBG] pgmap v3307: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:11.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:12 smithi059 bash[61152]: cluster 2022-04-28T21:20:11.536858+0000 mgr.y (mgr.24961) 2976 : cluster [DBG] pgmap v3308: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:12 smithi059 bash[59745]: cluster 2022-04-28T21:20:11.536858+0000 mgr.y (mgr.24961) 2976 : cluster [DBG] pgmap v3308: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:12 smithi139 bash[43301]: cluster 2022-04-28T21:20:11.536858+0000 mgr.y (mgr.24961) 2976 : cluster [DBG] pgmap v3308: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:13.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:14 smithi059 bash[61152]: cluster 2022-04-28T21:20:13.537574+0000 mgr.y (mgr.24961) 2977 : cluster [DBG] pgmap v3309: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:14 smithi059 bash[59745]: cluster 2022-04-28T21:20:13.537574+0000 mgr.y (mgr.24961) 2977 : cluster [DBG] pgmap v3309: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:14 smithi139 bash[43301]: cluster 2022-04-28T21:20:13.537574+0000 mgr.y (mgr.24961) 2977 : cluster [DBG] pgmap v3309: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:16 smithi059 bash[61152]: cluster 2022-04-28T21:20:15.538971+0000 mgr.y (mgr.24961) 2978 : cluster [DBG] pgmap v3310: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:16 smithi059 bash[59745]: cluster 2022-04-28T21:20:15.538971+0000 mgr.y (mgr.24961) 2978 : cluster [DBG] pgmap v3310: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:17.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:16 smithi139 bash[43301]: cluster 2022-04-28T21:20:15.538971+0000 mgr.y (mgr.24961) 2978 : cluster [DBG] pgmap v3310: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: cluster 2022-04-28T21:20:17.540004+0000 mgr.y (mgr.24961) 2979 : cluster [DBG] pgmap v3311: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: audit 2022-04-28T21:20:18.266818+0000 mon.a (mon.0) 2645 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:20:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: audit 2022-04-28T21:20:18.267666+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:20:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: audit 2022-04-28T21:20:18.279775+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:20:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: audit 2022-04-28T21:20:18.280600+0000 mon.a (mon.0) 2648 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:20:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: audit 2022-04-28T21:20:18.281297+0000 mon.a (mon.0) 2649 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:20:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[59745]: audit 2022-04-28T21:20:18.281989+0000 mon.a (mon.0) 2650 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:20:19.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: cluster 2022-04-28T21:20:17.540004+0000 mgr.y (mgr.24961) 2979 : cluster [DBG] pgmap v3311: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:19.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: audit 2022-04-28T21:20:18.266818+0000 mon.a (mon.0) 2645 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:20:19.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: audit 2022-04-28T21:20:18.267666+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:20:19.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: audit 2022-04-28T21:20:18.279775+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:20:19.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: audit 2022-04-28T21:20:18.280600+0000 mon.a (mon.0) 2648 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:20:19.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: audit 2022-04-28T21:20:18.281297+0000 mon.a (mon.0) 2649 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:20:19.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:18 smithi059 bash[61152]: audit 2022-04-28T21:20:18.281989+0000 mon.a (mon.0) 2650 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:20:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: cluster 2022-04-28T21:20:17.540004+0000 mgr.y (mgr.24961) 2979 : cluster [DBG] pgmap v3311: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: audit 2022-04-28T21:20:18.266818+0000 mon.a (mon.0) 2645 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:20:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: audit 2022-04-28T21:20:18.267666+0000 mon.a (mon.0) 2646 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:20:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: audit 2022-04-28T21:20:18.279775+0000 mon.a (mon.0) 2647 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:20:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: audit 2022-04-28T21:20:18.280600+0000 mon.a (mon.0) 2648 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:20:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: audit 2022-04-28T21:20:18.281297+0000 mon.a (mon.0) 2649 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:20:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:18 smithi139 bash[43301]: audit 2022-04-28T21:20:18.281989+0000 mon.a (mon.0) 2650 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:20:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:20 smithi059 bash[61152]: cluster 2022-04-28T21:20:19.541189+0000 mgr.y (mgr.24961) 2980 : cluster [DBG] pgmap v3312: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:20 smithi059 bash[59745]: cluster 2022-04-28T21:20:19.541189+0000 mgr.y (mgr.24961) 2980 : cluster [DBG] pgmap v3312: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:20 smithi139 bash[43301]: cluster 2022-04-28T21:20:19.541189+0000 mgr.y (mgr.24961) 2980 : cluster [DBG] pgmap v3312: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:21.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:22 smithi059 bash[61152]: cluster 2022-04-28T21:20:21.542422+0000 mgr.y (mgr.24961) 2981 : cluster [DBG] pgmap v3313: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:22 smithi059 bash[59745]: cluster 2022-04-28T21:20:21.542422+0000 mgr.y (mgr.24961) 2981 : cluster [DBG] pgmap v3313: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:22 smithi139 bash[43301]: cluster 2022-04-28T21:20:21.542422+0000 mgr.y (mgr.24961) 2981 : cluster [DBG] pgmap v3313: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:23.868Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:24 smithi059 bash[61152]: cluster 2022-04-28T21:20:23.543281+0000 mgr.y (mgr.24961) 2982 : cluster [DBG] pgmap v3314: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:25.108 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:24 smithi059 bash[59745]: cluster 2022-04-28T21:20:23.543281+0000 mgr.y (mgr.24961) 2982 : cluster [DBG] pgmap v3314: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:25.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:24 smithi139 bash[43301]: cluster 2022-04-28T21:20:23.543281+0000 mgr.y (mgr.24961) 2982 : cluster [DBG] pgmap v3314: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:26 smithi059 bash[61152]: cluster 2022-04-28T21:20:25.544702+0000 mgr.y (mgr.24961) 2983 : cluster [DBG] pgmap v3315: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:27.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:26 smithi059 bash[59745]: cluster 2022-04-28T21:20:25.544702+0000 mgr.y (mgr.24961) 2983 : cluster [DBG] pgmap v3315: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:26 smithi139 bash[43301]: cluster 2022-04-28T21:20:25.544702+0000 mgr.y (mgr.24961) 2983 : cluster [DBG] pgmap v3315: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:29.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:28 smithi059 bash[61152]: cluster 2022-04-28T21:20:27.545690+0000 mgr.y (mgr.24961) 2984 : cluster [DBG] pgmap v3316: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:28 smithi059 bash[59745]: cluster 2022-04-28T21:20:27.545690+0000 mgr.y (mgr.24961) 2984 : cluster [DBG] pgmap v3316: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:28 smithi139 bash[43301]: cluster 2022-04-28T21:20:27.545690+0000 mgr.y (mgr.24961) 2984 : cluster [DBG] pgmap v3316: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:30 smithi059 bash[61152]: cluster 2022-04-28T21:20:29.546760+0000 mgr.y (mgr.24961) 2985 : cluster [DBG] pgmap v3317: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:30 smithi059 bash[59745]: cluster 2022-04-28T21:20:29.546760+0000 mgr.y (mgr.24961) 2985 : cluster [DBG] pgmap v3317: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:30 smithi139 bash[43301]: cluster 2022-04-28T21:20:29.546760+0000 mgr.y (mgr.24961) 2985 : cluster [DBG] pgmap v3317: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:31.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:32 smithi059 bash[61152]: cluster 2022-04-28T21:20:31.548076+0000 mgr.y (mgr.24961) 2986 : cluster [DBG] pgmap v3318: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:32 smithi059 bash[59745]: cluster 2022-04-28T21:20:31.548076+0000 mgr.y (mgr.24961) 2986 : cluster [DBG] pgmap v3318: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:32 smithi139 bash[43301]: cluster 2022-04-28T21:20:31.548076+0000 mgr.y (mgr.24961) 2986 : cluster [DBG] pgmap v3318: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:33.869Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:34 smithi059 bash[61152]: cluster 2022-04-28T21:20:33.548843+0000 mgr.y (mgr.24961) 2987 : cluster [DBG] pgmap v3319: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:35.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:34 smithi059 bash[59745]: cluster 2022-04-28T21:20:33.548843+0000 mgr.y (mgr.24961) 2987 : cluster [DBG] pgmap v3319: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:34 smithi139 bash[43301]: cluster 2022-04-28T21:20:33.548843+0000 mgr.y (mgr.24961) 2987 : cluster [DBG] pgmap v3319: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:36 smithi059 bash[61152]: cluster 2022-04-28T21:20:35.550243+0000 mgr.y (mgr.24961) 2988 : cluster [DBG] pgmap v3320: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:37.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:36 smithi059 bash[59745]: cluster 2022-04-28T21:20:35.550243+0000 mgr.y (mgr.24961) 2988 : cluster [DBG] pgmap v3320: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:36 smithi139 bash[43301]: cluster 2022-04-28T21:20:35.550243+0000 mgr.y (mgr.24961) 2988 : cluster [DBG] pgmap v3320: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[61152]: cluster 2022-04-28T21:20:37.551236+0000 mgr.y (mgr.24961) 2989 : cluster [DBG] pgmap v3321: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[61152]: audit 2022-04-28T21:20:38.438211+0000 mon.a (mon.0) 2651 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:20:39.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[61152]: audit 2022-04-28T21:20:38.441181+0000 mon.a (mon.0) 2652 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:20:39.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[61152]: audit 2022-04-28T21:20:38.442915+0000 mon.a (mon.0) 2653 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:20:39.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[59745]: cluster 2022-04-28T21:20:37.551236+0000 mgr.y (mgr.24961) 2989 : cluster [DBG] pgmap v3321: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:39.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[59745]: audit 2022-04-28T21:20:38.438211+0000 mon.a (mon.0) 2651 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:20:39.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[59745]: audit 2022-04-28T21:20:38.441181+0000 mon.a (mon.0) 2652 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:20:39.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:38 smithi059 bash[59745]: audit 2022-04-28T21:20:38.442915+0000 mon.a (mon.0) 2653 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:20:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:38 smithi139 bash[43301]: cluster 2022-04-28T21:20:37.551236+0000 mgr.y (mgr.24961) 2989 : cluster [DBG] pgmap v3321: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:38 smithi139 bash[43301]: audit 2022-04-28T21:20:38.438211+0000 mon.a (mon.0) 2651 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:20:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:38 smithi139 bash[43301]: audit 2022-04-28T21:20:38.441181+0000 mon.a (mon.0) 2652 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:20:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:38 smithi139 bash[43301]: audit 2022-04-28T21:20:38.442915+0000 mon.a (mon.0) 2653 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:20:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:39 smithi059 bash[61152]: cephadm 2022-04-28T21:20:38.444522+0000 mgr.y (mgr.24961) 2990 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:20:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:39 smithi059 bash[61152]: audit 2022-04-28T21:20:38.750339+0000 mon.a (mon.0) 2654 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:20:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:39 smithi059 bash[59745]: cephadm 2022-04-28T21:20:38.444522+0000 mgr.y (mgr.24961) 2990 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:20:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:39 smithi059 bash[59745]: audit 2022-04-28T21:20:38.750339+0000 mon.a (mon.0) 2654 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:20:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:39 smithi139 bash[43301]: cephadm 2022-04-28T21:20:38.444522+0000 mgr.y (mgr.24961) 2990 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:20:40.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:39 smithi139 bash[43301]: audit 2022-04-28T21:20:38.750339+0000 mon.a (mon.0) 2654 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:20:41.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:40 smithi059 bash[61152]: cluster 2022-04-28T21:20:39.552386+0000 mgr.y (mgr.24961) 2991 : cluster [DBG] pgmap v3322: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:40 smithi059 bash[59745]: cluster 2022-04-28T21:20:39.552386+0000 mgr.y (mgr.24961) 2991 : cluster [DBG] pgmap v3322: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:40 smithi139 bash[43301]: cluster 2022-04-28T21:20:39.552386+0000 mgr.y (mgr.24961) 2991 : cluster [DBG] pgmap v3322: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:41.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:43.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:42 smithi059 bash[61152]: cluster 2022-04-28T21:20:41.553631+0000 mgr.y (mgr.24961) 2992 : cluster [DBG] pgmap v3323: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:42 smithi059 bash[59745]: cluster 2022-04-28T21:20:41.553631+0000 mgr.y (mgr.24961) 2992 : cluster [DBG] pgmap v3323: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:42 smithi139 bash[43301]: cluster 2022-04-28T21:20:41.553631+0000 mgr.y (mgr.24961) 2992 : cluster [DBG] pgmap v3323: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:43.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:44 smithi059 bash[61152]: cluster 2022-04-28T21:20:43.554428+0000 mgr.y (mgr.24961) 2993 : cluster [DBG] pgmap v3324: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:45.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:44 smithi059 bash[59745]: cluster 2022-04-28T21:20:43.554428+0000 mgr.y (mgr.24961) 2993 : cluster [DBG] pgmap v3324: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:44 smithi139 bash[43301]: cluster 2022-04-28T21:20:43.554428+0000 mgr.y (mgr.24961) 2993 : cluster [DBG] pgmap v3324: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:46 smithi059 bash[61152]: cluster 2022-04-28T21:20:45.555840+0000 mgr.y (mgr.24961) 2994 : cluster [DBG] pgmap v3325: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:46 smithi059 bash[59745]: cluster 2022-04-28T21:20:45.555840+0000 mgr.y (mgr.24961) 2994 : cluster [DBG] pgmap v3325: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:46 smithi139 bash[43301]: cluster 2022-04-28T21:20:45.555840+0000 mgr.y (mgr.24961) 2994 : cluster [DBG] pgmap v3325: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:48 smithi059 bash[61152]: cluster 2022-04-28T21:20:47.556835+0000 mgr.y (mgr.24961) 2995 : cluster [DBG] pgmap v3326: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:48 smithi059 bash[59745]: cluster 2022-04-28T21:20:47.556835+0000 mgr.y (mgr.24961) 2995 : cluster [DBG] pgmap v3326: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:48 smithi139 bash[43301]: cluster 2022-04-28T21:20:47.556835+0000 mgr.y (mgr.24961) 2995 : cluster [DBG] pgmap v3326: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:50 smithi059 bash[61152]: cluster 2022-04-28T21:20:49.557936+0000 mgr.y (mgr.24961) 2996 : cluster [DBG] pgmap v3327: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:50 smithi059 bash[59745]: cluster 2022-04-28T21:20:49.557936+0000 mgr.y (mgr.24961) 2996 : cluster [DBG] pgmap v3327: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:50 smithi139 bash[43301]: cluster 2022-04-28T21:20:49.557936+0000 mgr.y (mgr.24961) 2996 : cluster [DBG] pgmap v3327: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:51.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:52 smithi059 bash[61152]: cluster 2022-04-28T21:20:51.559190+0000 mgr.y (mgr.24961) 2997 : cluster [DBG] pgmap v3328: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:52 smithi059 bash[59745]: cluster 2022-04-28T21:20:51.559190+0000 mgr.y (mgr.24961) 2997 : cluster [DBG] pgmap v3328: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:52 smithi139 bash[43301]: cluster 2022-04-28T21:20:51.559190+0000 mgr.y (mgr.24961) 2997 : cluster [DBG] pgmap v3328: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:20:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:20:53.871Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:20:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:54 smithi059 bash[61152]: cluster 2022-04-28T21:20:53.559943+0000 mgr.y (mgr.24961) 2998 : cluster [DBG] pgmap v3329: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:54 smithi059 bash[59745]: cluster 2022-04-28T21:20:53.559943+0000 mgr.y (mgr.24961) 2998 : cluster [DBG] pgmap v3329: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:54 smithi139 bash[43301]: cluster 2022-04-28T21:20:53.559943+0000 mgr.y (mgr.24961) 2998 : cluster [DBG] pgmap v3329: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:56 smithi059 bash[61152]: cluster 2022-04-28T21:20:55.561265+0000 mgr.y (mgr.24961) 2999 : cluster [DBG] pgmap v3330: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:56 smithi059 bash[59745]: cluster 2022-04-28T21:20:55.561265+0000 mgr.y (mgr.24961) 2999 : cluster [DBG] pgmap v3330: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:56 smithi139 bash[43301]: cluster 2022-04-28T21:20:55.561265+0000 mgr.y (mgr.24961) 2999 : cluster [DBG] pgmap v3330: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:20:58 smithi059 bash[61152]: cluster 2022-04-28T21:20:57.562240+0000 mgr.y (mgr.24961) 3000 : cluster [DBG] pgmap v3331: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:59.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:20:58 smithi059 bash[59745]: cluster 2022-04-28T21:20:57.562240+0000 mgr.y (mgr.24961) 3000 : cluster [DBG] pgmap v3331: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:20:59.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:20:58 smithi139 bash[43301]: cluster 2022-04-28T21:20:57.562240+0000 mgr.y (mgr.24961) 3000 : cluster [DBG] pgmap v3331: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:00 smithi059 bash[61152]: cluster 2022-04-28T21:20:59.563350+0000 mgr.y (mgr.24961) 3001 : cluster [DBG] pgmap v3332: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:00 smithi059 bash[59745]: cluster 2022-04-28T21:20:59.563350+0000 mgr.y (mgr.24961) 3001 : cluster [DBG] pgmap v3332: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:00 smithi139 bash[43301]: cluster 2022-04-28T21:20:59.563350+0000 mgr.y (mgr.24961) 3001 : cluster [DBG] pgmap v3332: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:01.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:02 smithi059 bash[61152]: cluster 2022-04-28T21:21:01.564615+0000 mgr.y (mgr.24961) 3002 : cluster [DBG] pgmap v3333: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:02 smithi059 bash[59745]: cluster 2022-04-28T21:21:01.564615+0000 mgr.y (mgr.24961) 3002 : cluster [DBG] pgmap v3333: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:02 smithi139 bash[43301]: cluster 2022-04-28T21:21:01.564615+0000 mgr.y (mgr.24961) 3002 : cluster [DBG] pgmap v3333: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:03.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:04 smithi059 bash[61152]: cluster 2022-04-28T21:21:03.565350+0000 mgr.y (mgr.24961) 3003 : cluster [DBG] pgmap v3334: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:04 smithi059 bash[59745]: cluster 2022-04-28T21:21:03.565350+0000 mgr.y (mgr.24961) 3003 : cluster [DBG] pgmap v3334: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:05.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:04 smithi139 bash[43301]: cluster 2022-04-28T21:21:03.565350+0000 mgr.y (mgr.24961) 3003 : cluster [DBG] pgmap v3334: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:06 smithi059 bash[61152]: cluster 2022-04-28T21:21:05.566793+0000 mgr.y (mgr.24961) 3004 : cluster [DBG] pgmap v3335: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:07.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:06 smithi059 bash[59745]: cluster 2022-04-28T21:21:05.566793+0000 mgr.y (mgr.24961) 3004 : cluster [DBG] pgmap v3335: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:06 smithi139 bash[43301]: cluster 2022-04-28T21:21:05.566793+0000 mgr.y (mgr.24961) 3004 : cluster [DBG] pgmap v3335: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:08 smithi139 bash[43301]: cluster 2022-04-28T21:21:07.567820+0000 mgr.y (mgr.24961) 3005 : cluster [DBG] pgmap v3336: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:08 smithi059 bash[61152]: cluster 2022-04-28T21:21:07.567820+0000 mgr.y (mgr.24961) 3005 : cluster [DBG] pgmap v3336: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:08 smithi059 bash[59745]: cluster 2022-04-28T21:21:07.567820+0000 mgr.y (mgr.24961) 3005 : cluster [DBG] pgmap v3336: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:10 smithi139 bash[43301]: cluster 2022-04-28T21:21:09.568898+0000 mgr.y (mgr.24961) 3006 : cluster [DBG] pgmap v3337: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:10 smithi059 bash[61152]: cluster 2022-04-28T21:21:09.568898+0000 mgr.y (mgr.24961) 3006 : cluster [DBG] pgmap v3337: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:10 smithi059 bash[59745]: cluster 2022-04-28T21:21:09.568898+0000 mgr.y (mgr.24961) 3006 : cluster [DBG] pgmap v3337: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:11.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:12 smithi139 bash[43301]: cluster 2022-04-28T21:21:11.570145+0000 mgr.y (mgr.24961) 3007 : cluster [DBG] pgmap v3338: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:12 smithi059 bash[61152]: cluster 2022-04-28T21:21:11.570145+0000 mgr.y (mgr.24961) 3007 : cluster [DBG] pgmap v3338: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:12 smithi059 bash[59745]: cluster 2022-04-28T21:21:11.570145+0000 mgr.y (mgr.24961) 3007 : cluster [DBG] pgmap v3338: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:13.870Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:14 smithi139 bash[43301]: cluster 2022-04-28T21:21:13.570866+0000 mgr.y (mgr.24961) 3008 : cluster [DBG] pgmap v3339: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:14 smithi059 bash[59745]: cluster 2022-04-28T21:21:13.570866+0000 mgr.y (mgr.24961) 3008 : cluster [DBG] pgmap v3339: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:15.377 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:14 smithi059 bash[61152]: cluster 2022-04-28T21:21:13.570866+0000 mgr.y (mgr.24961) 3008 : cluster [DBG] pgmap v3339: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:17.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:16 smithi139 bash[43301]: cluster 2022-04-28T21:21:15.572339+0000 mgr.y (mgr.24961) 3009 : cluster [DBG] pgmap v3340: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:16 smithi059 bash[61152]: cluster 2022-04-28T21:21:15.572339+0000 mgr.y (mgr.24961) 3009 : cluster [DBG] pgmap v3340: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:16 smithi059 bash[59745]: cluster 2022-04-28T21:21:15.572339+0000 mgr.y (mgr.24961) 3009 : cluster [DBG] pgmap v3340: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: cluster 2022-04-28T21:21:17.573270+0000 mgr.y (mgr.24961) 3010 : cluster [DBG] pgmap v3341: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: audit 2022-04-28T21:21:18.280643+0000 mon.a (mon.0) 2655 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:21:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: audit 2022-04-28T21:21:18.281143+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:21:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: audit 2022-04-28T21:21:18.292273+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:21:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: audit 2022-04-28T21:21:18.293331+0000 mon.a (mon.0) 2658 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:21:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: audit 2022-04-28T21:21:18.293885+0000 mon.a (mon.0) 2659 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:21:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[61152]: audit 2022-04-28T21:21:18.294225+0000 mon.a (mon.0) 2660 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:21:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: cluster 2022-04-28T21:21:17.573270+0000 mgr.y (mgr.24961) 3010 : cluster [DBG] pgmap v3341: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: audit 2022-04-28T21:21:18.280643+0000 mon.a (mon.0) 2655 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:21:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: audit 2022-04-28T21:21:18.281143+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:21:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: audit 2022-04-28T21:21:18.292273+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:21:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: audit 2022-04-28T21:21:18.293331+0000 mon.a (mon.0) 2658 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:21:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: audit 2022-04-28T21:21:18.293885+0000 mon.a (mon.0) 2659 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:21:19.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:18 smithi059 bash[59745]: audit 2022-04-28T21:21:18.294225+0000 mon.a (mon.0) 2660 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:21:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: cluster 2022-04-28T21:21:17.573270+0000 mgr.y (mgr.24961) 3010 : cluster [DBG] pgmap v3341: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: audit 2022-04-28T21:21:18.280643+0000 mon.a (mon.0) 2655 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:21:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: audit 2022-04-28T21:21:18.281143+0000 mon.a (mon.0) 2656 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:21:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: audit 2022-04-28T21:21:18.292273+0000 mon.a (mon.0) 2657 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:21:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: audit 2022-04-28T21:21:18.293331+0000 mon.a (mon.0) 2658 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:21:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: audit 2022-04-28T21:21:18.293885+0000 mon.a (mon.0) 2659 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:21:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:18 smithi139 bash[43301]: audit 2022-04-28T21:21:18.294225+0000 mon.a (mon.0) 2660 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:21:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:20 smithi059 bash[61152]: cluster 2022-04-28T21:21:19.574380+0000 mgr.y (mgr.24961) 3011 : cluster [DBG] pgmap v3342: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:21.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:20 smithi059 bash[59745]: cluster 2022-04-28T21:21:19.574380+0000 mgr.y (mgr.24961) 3011 : cluster [DBG] pgmap v3342: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:20 smithi139 bash[43301]: cluster 2022-04-28T21:21:19.574380+0000 mgr.y (mgr.24961) 3011 : cluster [DBG] pgmap v3342: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:21.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:22 smithi059 bash[59745]: cluster 2022-04-28T21:21:21.575806+0000 mgr.y (mgr.24961) 3012 : cluster [DBG] pgmap v3343: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:22 smithi059 bash[61152]: cluster 2022-04-28T21:21:21.575806+0000 mgr.y (mgr.24961) 3012 : cluster [DBG] pgmap v3343: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:22 smithi139 bash[43301]: cluster 2022-04-28T21:21:21.575806+0000 mgr.y (mgr.24961) 3012 : cluster [DBG] pgmap v3343: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:23.872Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:25.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:24 smithi059 bash[61152]: cluster 2022-04-28T21:21:23.576576+0000 mgr.y (mgr.24961) 3013 : cluster [DBG] pgmap v3344: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:24 smithi059 bash[59745]: cluster 2022-04-28T21:21:23.576576+0000 mgr.y (mgr.24961) 3013 : cluster [DBG] pgmap v3344: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:24 smithi139 bash[43301]: cluster 2022-04-28T21:21:23.576576+0000 mgr.y (mgr.24961) 3013 : cluster [DBG] pgmap v3344: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:26 smithi059 bash[61152]: cluster 2022-04-28T21:21:25.577963+0000 mgr.y (mgr.24961) 3014 : cluster [DBG] pgmap v3345: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:26 smithi059 bash[59745]: cluster 2022-04-28T21:21:25.577963+0000 mgr.y (mgr.24961) 3014 : cluster [DBG] pgmap v3345: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:26 smithi139 bash[43301]: cluster 2022-04-28T21:21:25.577963+0000 mgr.y (mgr.24961) 3014 : cluster [DBG] pgmap v3345: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:28 smithi059 bash[61152]: cluster 2022-04-28T21:21:27.578968+0000 mgr.y (mgr.24961) 3015 : cluster [DBG] pgmap v3346: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:29.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:28 smithi059 bash[59745]: cluster 2022-04-28T21:21:27.578968+0000 mgr.y (mgr.24961) 3015 : cluster [DBG] pgmap v3346: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:28 smithi139 bash[43301]: cluster 2022-04-28T21:21:27.578968+0000 mgr.y (mgr.24961) 3015 : cluster [DBG] pgmap v3346: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:30 smithi059 bash[59745]: cluster 2022-04-28T21:21:29.580096+0000 mgr.y (mgr.24961) 3016 : cluster [DBG] pgmap v3347: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:30 smithi059 bash[61152]: cluster 2022-04-28T21:21:29.580096+0000 mgr.y (mgr.24961) 3016 : cluster [DBG] pgmap v3347: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:30 smithi139 bash[43301]: cluster 2022-04-28T21:21:29.580096+0000 mgr.y (mgr.24961) 3016 : cluster [DBG] pgmap v3347: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:31.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:32 smithi059 bash[61152]: cluster 2022-04-28T21:21:31.581340+0000 mgr.y (mgr.24961) 3017 : cluster [DBG] pgmap v3348: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:32 smithi059 bash[59745]: cluster 2022-04-28T21:21:31.581340+0000 mgr.y (mgr.24961) 3017 : cluster [DBG] pgmap v3348: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:32 smithi139 bash[43301]: cluster 2022-04-28T21:21:31.581340+0000 mgr.y (mgr.24961) 3017 : cluster [DBG] pgmap v3348: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:33.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:34 smithi059 bash[61152]: cluster 2022-04-28T21:21:33.582081+0000 mgr.y (mgr.24961) 3018 : cluster [DBG] pgmap v3349: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:34 smithi059 bash[59745]: cluster 2022-04-28T21:21:33.582081+0000 mgr.y (mgr.24961) 3018 : cluster [DBG] pgmap v3349: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:34 smithi139 bash[43301]: cluster 2022-04-28T21:21:33.582081+0000 mgr.y (mgr.24961) 3018 : cluster [DBG] pgmap v3349: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:36 smithi059 bash[61152]: cluster 2022-04-28T21:21:35.583475+0000 mgr.y (mgr.24961) 3019 : cluster [DBG] pgmap v3350: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:36 smithi059 bash[59745]: cluster 2022-04-28T21:21:35.583475+0000 mgr.y (mgr.24961) 3019 : cluster [DBG] pgmap v3350: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:36 smithi139 bash[43301]: cluster 2022-04-28T21:21:35.583475+0000 mgr.y (mgr.24961) 3019 : cluster [DBG] pgmap v3350: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[59745]: cluster 2022-04-28T21:21:37.584763+0000 mgr.y (mgr.24961) 3020 : cluster [DBG] pgmap v3351: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[59745]: audit 2022-04-28T21:21:38.759027+0000 mon.a (mon.0) 2661 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:21:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[59745]: audit 2022-04-28T21:21:38.761981+0000 mon.a (mon.0) 2662 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:21:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[59745]: audit 2022-04-28T21:21:38.763998+0000 mon.a (mon.0) 2663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:21:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[61152]: cluster 2022-04-28T21:21:37.584763+0000 mgr.y (mgr.24961) 3020 : cluster [DBG] pgmap v3351: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[61152]: audit 2022-04-28T21:21:38.759027+0000 mon.a (mon.0) 2661 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:21:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[61152]: audit 2022-04-28T21:21:38.761981+0000 mon.a (mon.0) 2662 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:21:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:38 smithi059 bash[61152]: audit 2022-04-28T21:21:38.763998+0000 mon.a (mon.0) 2663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:21:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:38 smithi139 bash[43301]: cluster 2022-04-28T21:21:37.584763+0000 mgr.y (mgr.24961) 3020 : cluster [DBG] pgmap v3351: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:38 smithi139 bash[43301]: audit 2022-04-28T21:21:38.759027+0000 mon.a (mon.0) 2661 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:21:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:38 smithi139 bash[43301]: audit 2022-04-28T21:21:38.761981+0000 mon.a (mon.0) 2662 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:21:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:38 smithi139 bash[43301]: audit 2022-04-28T21:21:38.763998+0000 mon.a (mon.0) 2663 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:21:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: cephadm 2022-04-28T21:21:38.765793+0000 mgr.y (mgr.24961) 3021 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:21:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.052722+0000 mon.a (mon.0) 2664 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.053440+0000 mon.a (mon.0) 2665 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.054018+0000 mon.a (mon.0) 2666 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.054570+0000 mon.a (mon.0) 2667 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.055112+0000 mon.a (mon.0) 2668 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.055635+0000 mon.a (mon.0) 2669 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.056213+0000 mon.a (mon.0) 2670 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.056762+0000 mon.a (mon.0) 2671 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.057363+0000 mon.a (mon.0) 2672 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:21:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.058058+0000 mon.a (mon.0) 2673 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.062388+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.063001+0000 mon.a (mon.0) 2675 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.063620+0000 mon.a (mon.0) 2676 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.064245+0000 mon.a (mon.0) 2677 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: cephadm 2022-04-28T21:21:38.765793+0000 mgr.y (mgr.24961) 3021 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:21:40.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.052722+0000 mon.a (mon.0) 2664 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.053440+0000 mon.a (mon.0) 2665 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.054018+0000 mon.a (mon.0) 2666 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.054570+0000 mon.a (mon.0) 2667 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.055112+0000 mon.a (mon.0) 2668 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.055635+0000 mon.a (mon.0) 2669 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.056213+0000 mon.a (mon.0) 2670 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.056762+0000 mon.a (mon.0) 2671 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.057363+0000 mon.a (mon.0) 2672 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:21:40.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.058058+0000 mon.a (mon.0) 2673 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.348 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.062388+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.063001+0000 mon.a (mon.0) 2675 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.063620+0000 mon.a (mon.0) 2676 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.064245+0000 mon.a (mon.0) 2677 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.064869+0000 mon.a (mon.0) 2678 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.350 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.065480+0000 mon.a (mon.0) 2679 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.066055+0000 mon.a (mon.0) 2680 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.066647+0000 mon.a (mon.0) 2681 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.351 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.067280+0000 mon.a (mon.0) 2682 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:21:40.352 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.067911+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.352 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[61152]: audit 2022-04-28T21:21:39.074764+0000 mon.a (mon.0) 2684 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:21:40.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.064869+0000 mon.a (mon.0) 2678 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.065480+0000 mon.a (mon.0) 2679 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.353 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.066055+0000 mon.a (mon.0) 2680 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.066647+0000 mon.a (mon.0) 2681 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.354 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.067280+0000 mon.a (mon.0) 2682 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:21:40.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.067911+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.355 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:39 smithi059 bash[59745]: audit 2022-04-28T21:21:39.074764+0000 mon.a (mon.0) 2684 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:21:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: cephadm 2022-04-28T21:21:38.765793+0000 mgr.y (mgr.24961) 3021 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:21:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.052722+0000 mon.a (mon.0) 2664 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.053440+0000 mon.a (mon.0) 2665 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.054018+0000 mon.a (mon.0) 2666 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.054570+0000 mon.a (mon.0) 2667 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.055112+0000 mon.a (mon.0) 2668 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.055635+0000 mon.a (mon.0) 2669 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.056213+0000 mon.a (mon.0) 2670 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.056762+0000 mon.a (mon.0) 2671 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.057363+0000 mon.a (mon.0) 2672 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:21:40.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.058058+0000 mon.a (mon.0) 2673 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.062388+0000 mon.a (mon.0) 2674 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.374 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.063001+0000 mon.a (mon.0) 2675 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.063620+0000 mon.a (mon.0) 2676 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.064245+0000 mon.a (mon.0) 2677 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.064869+0000 mon.a (mon.0) 2678 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.065480+0000 mon.a (mon.0) 2679 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.066055+0000 mon.a (mon.0) 2680 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:21:40.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.066647+0000 mon.a (mon.0) 2681 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.067280+0000 mon.a (mon.0) 2682 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:21:40.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.067911+0000 mon.a (mon.0) 2683 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:21:40.377 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:39 smithi139 bash[43301]: audit 2022-04-28T21:21:39.074764+0000 mon.a (mon.0) 2684 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:21:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:41 smithi059 bash[61152]: cluster 2022-04-28T21:21:39.586023+0000 mgr.y (mgr.24961) 3022 : cluster [DBG] pgmap v3352: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:41 smithi059 bash[59745]: cluster 2022-04-28T21:21:39.586023+0000 mgr.y (mgr.24961) 3022 : cluster [DBG] pgmap v3352: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:41 smithi139 bash[43301]: cluster 2022-04-28T21:21:39.586023+0000 mgr.y (mgr.24961) 3022 : cluster [DBG] pgmap v3352: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:41.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:43 smithi059 bash[61152]: cluster 2022-04-28T21:21:41.587356+0000 mgr.y (mgr.24961) 3023 : cluster [DBG] pgmap v3353: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:43 smithi059 bash[59745]: cluster 2022-04-28T21:21:41.587356+0000 mgr.y (mgr.24961) 3023 : cluster [DBG] pgmap v3353: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:43 smithi139 bash[43301]: cluster 2022-04-28T21:21:41.587356+0000 mgr.y (mgr.24961) 3023 : cluster [DBG] pgmap v3353: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:43.873Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:45.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:45 smithi059 bash[61152]: cluster 2022-04-28T21:21:43.588203+0000 mgr.y (mgr.24961) 3024 : cluster [DBG] pgmap v3354: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:45.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:45 smithi059 bash[59745]: cluster 2022-04-28T21:21:43.588203+0000 mgr.y (mgr.24961) 3024 : cluster [DBG] pgmap v3354: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:45 smithi139 bash[43301]: cluster 2022-04-28T21:21:43.588203+0000 mgr.y (mgr.24961) 3024 : cluster [DBG] pgmap v3354: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:47 smithi139 bash[43301]: cluster 2022-04-28T21:21:45.589726+0000 mgr.y (mgr.24961) 3025 : cluster [DBG] pgmap v3355: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:47 smithi059 bash[61152]: cluster 2022-04-28T21:21:45.589726+0000 mgr.y (mgr.24961) 3025 : cluster [DBG] pgmap v3355: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:47 smithi059 bash[59745]: cluster 2022-04-28T21:21:45.589726+0000 mgr.y (mgr.24961) 3025 : cluster [DBG] pgmap v3355: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:49 smithi139 bash[43301]: cluster 2022-04-28T21:21:47.590706+0000 mgr.y (mgr.24961) 3026 : cluster [DBG] pgmap v3356: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:49 smithi059 bash[61152]: cluster 2022-04-28T21:21:47.590706+0000 mgr.y (mgr.24961) 3026 : cluster [DBG] pgmap v3356: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:49 smithi059 bash[59745]: cluster 2022-04-28T21:21:47.590706+0000 mgr.y (mgr.24961) 3026 : cluster [DBG] pgmap v3356: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:51 smithi139 bash[43301]: cluster 2022-04-28T21:21:49.591895+0000 mgr.y (mgr.24961) 3027 : cluster [DBG] pgmap v3357: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:51 smithi059 bash[61152]: cluster 2022-04-28T21:21:49.591895+0000 mgr.y (mgr.24961) 3027 : cluster [DBG] pgmap v3357: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:51 smithi059 bash[59745]: cluster 2022-04-28T21:21:49.591895+0000 mgr.y (mgr.24961) 3027 : cluster [DBG] pgmap v3357: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:51.980Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:53 smithi139 bash[43301]: cluster 2022-04-28T21:21:51.593174+0000 mgr.y (mgr.24961) 3028 : cluster [DBG] pgmap v3358: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:53 smithi059 bash[61152]: cluster 2022-04-28T21:21:51.593174+0000 mgr.y (mgr.24961) 3028 : cluster [DBG] pgmap v3358: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:53.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:53 smithi059 bash[59745]: cluster 2022-04-28T21:21:51.593174+0000 mgr.y (mgr.24961) 3028 : cluster [DBG] pgmap v3358: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:21:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:21:53.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:21:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:55 smithi059 bash[61152]: cluster 2022-04-28T21:21:53.593926+0000 mgr.y (mgr.24961) 3029 : cluster [DBG] pgmap v3359: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:55 smithi059 bash[59745]: cluster 2022-04-28T21:21:53.593926+0000 mgr.y (mgr.24961) 3029 : cluster [DBG] pgmap v3359: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:55 smithi139 bash[43301]: cluster 2022-04-28T21:21:53.593926+0000 mgr.y (mgr.24961) 3029 : cluster [DBG] pgmap v3359: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:57 smithi059 bash[61152]: cluster 2022-04-28T21:21:55.595331+0000 mgr.y (mgr.24961) 3030 : cluster [DBG] pgmap v3360: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:57 smithi059 bash[59745]: cluster 2022-04-28T21:21:55.595331+0000 mgr.y (mgr.24961) 3030 : cluster [DBG] pgmap v3360: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:57 smithi139 bash[43301]: cluster 2022-04-28T21:21:55.595331+0000 mgr.y (mgr.24961) 3030 : cluster [DBG] pgmap v3360: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:21:59 smithi059 bash[61152]: cluster 2022-04-28T21:21:57.596368+0000 mgr.y (mgr.24961) 3031 : cluster [DBG] pgmap v3361: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:21:59 smithi059 bash[59745]: cluster 2022-04-28T21:21:57.596368+0000 mgr.y (mgr.24961) 3031 : cluster [DBG] pgmap v3361: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:21:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:21:59 smithi139 bash[43301]: cluster 2022-04-28T21:21:57.596368+0000 mgr.y (mgr.24961) 3031 : cluster [DBG] pgmap v3361: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:01 smithi059 bash[61152]: cluster 2022-04-28T21:21:59.597686+0000 mgr.y (mgr.24961) 3032 : cluster [DBG] pgmap v3362: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:01.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:01 smithi059 bash[59745]: cluster 2022-04-28T21:21:59.597686+0000 mgr.y (mgr.24961) 3032 : cluster [DBG] pgmap v3362: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:01 smithi139 bash[43301]: cluster 2022-04-28T21:21:59.597686+0000 mgr.y (mgr.24961) 3032 : cluster [DBG] pgmap v3362: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:01.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:03 smithi059 bash[61152]: cluster 2022-04-28T21:22:01.598971+0000 mgr.y (mgr.24961) 3033 : cluster [DBG] pgmap v3363: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:03 smithi059 bash[59745]: cluster 2022-04-28T21:22:01.598971+0000 mgr.y (mgr.24961) 3033 : cluster [DBG] pgmap v3363: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:03 smithi139 bash[43301]: cluster 2022-04-28T21:22:01.598971+0000 mgr.y (mgr.24961) 3033 : cluster [DBG] pgmap v3363: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:03.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:05 smithi059 bash[61152]: cluster 2022-04-28T21:22:03.599720+0000 mgr.y (mgr.24961) 3034 : cluster [DBG] pgmap v3364: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:05 smithi059 bash[59745]: cluster 2022-04-28T21:22:03.599720+0000 mgr.y (mgr.24961) 3034 : cluster [DBG] pgmap v3364: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:05 smithi139 bash[43301]: cluster 2022-04-28T21:22:03.599720+0000 mgr.y (mgr.24961) 3034 : cluster [DBG] pgmap v3364: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:07 smithi059 bash[59745]: cluster 2022-04-28T21:22:05.601258+0000 mgr.y (mgr.24961) 3035 : cluster [DBG] pgmap v3365: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:07 smithi059 bash[61152]: cluster 2022-04-28T21:22:05.601258+0000 mgr.y (mgr.24961) 3035 : cluster [DBG] pgmap v3365: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:07 smithi139 bash[43301]: cluster 2022-04-28T21:22:05.601258+0000 mgr.y (mgr.24961) 3035 : cluster [DBG] pgmap v3365: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:09 smithi059 bash[61152]: cluster 2022-04-28T21:22:07.602283+0000 mgr.y (mgr.24961) 3036 : cluster [DBG] pgmap v3366: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:09 smithi059 bash[59745]: cluster 2022-04-28T21:22:07.602283+0000 mgr.y (mgr.24961) 3036 : cluster [DBG] pgmap v3366: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:09 smithi139 bash[43301]: cluster 2022-04-28T21:22:07.602283+0000 mgr.y (mgr.24961) 3036 : cluster [DBG] pgmap v3366: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:11 smithi059 bash[61152]: cluster 2022-04-28T21:22:09.603417+0000 mgr.y (mgr.24961) 3037 : cluster [DBG] pgmap v3367: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:11 smithi059 bash[59745]: cluster 2022-04-28T21:22:09.603417+0000 mgr.y (mgr.24961) 3037 : cluster [DBG] pgmap v3367: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:11 smithi139 bash[43301]: cluster 2022-04-28T21:22:09.603417+0000 mgr.y (mgr.24961) 3037 : cluster [DBG] pgmap v3367: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:11.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:13 smithi059 bash[61152]: cluster 2022-04-28T21:22:11.604678+0000 mgr.y (mgr.24961) 3038 : cluster [DBG] pgmap v3368: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:13 smithi059 bash[59745]: cluster 2022-04-28T21:22:11.604678+0000 mgr.y (mgr.24961) 3038 : cluster [DBG] pgmap v3368: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:13 smithi139 bash[43301]: cluster 2022-04-28T21:22:11.604678+0000 mgr.y (mgr.24961) 3038 : cluster [DBG] pgmap v3368: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:13.874Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:15 smithi059 bash[61152]: cluster 2022-04-28T21:22:13.605410+0000 mgr.y (mgr.24961) 3039 : cluster [DBG] pgmap v3369: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:15 smithi059 bash[59745]: cluster 2022-04-28T21:22:13.605410+0000 mgr.y (mgr.24961) 3039 : cluster [DBG] pgmap v3369: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:15 smithi139 bash[43301]: cluster 2022-04-28T21:22:13.605410+0000 mgr.y (mgr.24961) 3039 : cluster [DBG] pgmap v3369: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:17 smithi059 bash[61152]: cluster 2022-04-28T21:22:15.606885+0000 mgr.y (mgr.24961) 3040 : cluster [DBG] pgmap v3370: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:17.622 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:17 smithi059 bash[59745]: cluster 2022-04-28T21:22:15.606885+0000 mgr.y (mgr.24961) 3040 : cluster [DBG] pgmap v3370: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:17.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:17 smithi139 bash[43301]: cluster 2022-04-28T21:22:15.606885+0000 mgr.y (mgr.24961) 3040 : cluster [DBG] pgmap v3370: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:19.467 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: cluster 2022-04-28T21:22:17.607940+0000 mgr.y (mgr.24961) 3041 : cluster [DBG] pgmap v3371: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:19.467 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: audit 2022-04-28T21:22:18.291329+0000 mon.a (mon.0) 2685 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:22:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: cluster 2022-04-28T21:22:17.607940+0000 mgr.y (mgr.24961) 3041 : cluster [DBG] pgmap v3371: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: audit 2022-04-28T21:22:18.291329+0000 mon.a (mon.0) 2685 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:22:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: audit 2022-04-28T21:22:18.291818+0000 mon.a (mon.0) 2686 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:22:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: audit 2022-04-28T21:22:18.303893+0000 mon.a (mon.0) 2687 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:22:19.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: audit 2022-04-28T21:22:18.304403+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:22:19.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: audit 2022-04-28T21:22:18.304917+0000 mon.a (mon.0) 2689 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:22:19.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[61152]: audit 2022-04-28T21:22:18.305398+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:22:19.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: cluster 2022-04-28T21:22:17.607940+0000 mgr.y (mgr.24961) 3041 : cluster [DBG] pgmap v3371: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:19.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: audit 2022-04-28T21:22:18.291329+0000 mon.a (mon.0) 2685 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:22:19.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: audit 2022-04-28T21:22:18.291818+0000 mon.a (mon.0) 2686 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:22:19.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: audit 2022-04-28T21:22:18.303893+0000 mon.a (mon.0) 2687 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:22:19.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: audit 2022-04-28T21:22:18.304403+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:22:19.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: audit 2022-04-28T21:22:18.304917+0000 mon.a (mon.0) 2689 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:22:19.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:19 smithi059 bash[59745]: audit 2022-04-28T21:22:18.305398+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:22:19.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: audit 2022-04-28T21:22:18.291818+0000 mon.a (mon.0) 2686 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:22:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: audit 2022-04-28T21:22:18.303893+0000 mon.a (mon.0) 2687 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:22:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: audit 2022-04-28T21:22:18.304403+0000 mon.a (mon.0) 2688 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:22:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: audit 2022-04-28T21:22:18.304917+0000 mon.a (mon.0) 2689 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:22:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:19 smithi139 bash[43301]: audit 2022-04-28T21:22:18.305398+0000 mon.a (mon.0) 2690 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:22:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:21 smithi059 bash[61152]: cluster 2022-04-28T21:22:19.609012+0000 mgr.y (mgr.24961) 3042 : cluster [DBG] pgmap v3372: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:21 smithi059 bash[59745]: cluster 2022-04-28T21:22:19.609012+0000 mgr.y (mgr.24961) 3042 : cluster [DBG] pgmap v3372: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:21 smithi139 bash[43301]: cluster 2022-04-28T21:22:19.609012+0000 mgr.y (mgr.24961) 3042 : cluster [DBG] pgmap v3372: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:21.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:23 smithi059 bash[61152]: cluster 2022-04-28T21:22:21.610261+0000 mgr.y (mgr.24961) 3043 : cluster [DBG] pgmap v3373: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:23 smithi059 bash[59745]: cluster 2022-04-28T21:22:21.610261+0000 mgr.y (mgr.24961) 3043 : cluster [DBG] pgmap v3373: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:23 smithi139 bash[43301]: cluster 2022-04-28T21:22:21.610261+0000 mgr.y (mgr.24961) 3043 : cluster [DBG] pgmap v3373: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:23.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:25 smithi059 bash[61152]: cluster 2022-04-28T21:22:23.610926+0000 mgr.y (mgr.24961) 3044 : cluster [DBG] pgmap v3374: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:25 smithi059 bash[59745]: cluster 2022-04-28T21:22:23.610926+0000 mgr.y (mgr.24961) 3044 : cluster [DBG] pgmap v3374: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:25 smithi139 bash[43301]: cluster 2022-04-28T21:22:23.610926+0000 mgr.y (mgr.24961) 3044 : cluster [DBG] pgmap v3374: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:27 smithi059 bash[59745]: cluster 2022-04-28T21:22:25.612425+0000 mgr.y (mgr.24961) 3045 : cluster [DBG] pgmap v3375: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:27.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:27 smithi059 bash[61152]: cluster 2022-04-28T21:22:25.612425+0000 mgr.y (mgr.24961) 3045 : cluster [DBG] pgmap v3375: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:27 smithi139 bash[43301]: cluster 2022-04-28T21:22:25.612425+0000 mgr.y (mgr.24961) 3045 : cluster [DBG] pgmap v3375: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:29 smithi059 bash[59745]: cluster 2022-04-28T21:22:27.613400+0000 mgr.y (mgr.24961) 3046 : cluster [DBG] pgmap v3376: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:29 smithi059 bash[61152]: cluster 2022-04-28T21:22:27.613400+0000 mgr.y (mgr.24961) 3046 : cluster [DBG] pgmap v3376: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:29 smithi139 bash[43301]: cluster 2022-04-28T21:22:27.613400+0000 mgr.y (mgr.24961) 3046 : cluster [DBG] pgmap v3376: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:31 smithi059 bash[61152]: cluster 2022-04-28T21:22:29.614539+0000 mgr.y (mgr.24961) 3047 : cluster [DBG] pgmap v3377: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:31 smithi059 bash[59745]: cluster 2022-04-28T21:22:29.614539+0000 mgr.y (mgr.24961) 3047 : cluster [DBG] pgmap v3377: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:31 smithi139 bash[43301]: cluster 2022-04-28T21:22:29.614539+0000 mgr.y (mgr.24961) 3047 : cluster [DBG] pgmap v3377: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:31.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:33.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:33 smithi059 bash[61152]: cluster 2022-04-28T21:22:31.615772+0000 mgr.y (mgr.24961) 3048 : cluster [DBG] pgmap v3378: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:33.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:33 smithi059 bash[59745]: cluster 2022-04-28T21:22:31.615772+0000 mgr.y (mgr.24961) 3048 : cluster [DBG] pgmap v3378: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:33 smithi139 bash[43301]: cluster 2022-04-28T21:22:31.615772+0000 mgr.y (mgr.24961) 3048 : cluster [DBG] pgmap v3378: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:33.876Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:35 smithi059 bash[61152]: cluster 2022-04-28T21:22:33.616501+0000 mgr.y (mgr.24961) 3049 : cluster [DBG] pgmap v3379: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:35 smithi059 bash[59745]: cluster 2022-04-28T21:22:33.616501+0000 mgr.y (mgr.24961) 3049 : cluster [DBG] pgmap v3379: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:35 smithi139 bash[43301]: cluster 2022-04-28T21:22:33.616501+0000 mgr.y (mgr.24961) 3049 : cluster [DBG] pgmap v3379: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:37 smithi059 bash[61152]: cluster 2022-04-28T21:22:35.617974+0000 mgr.y (mgr.24961) 3050 : cluster [DBG] pgmap v3380: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:37 smithi059 bash[59745]: cluster 2022-04-28T21:22:35.617974+0000 mgr.y (mgr.24961) 3050 : cluster [DBG] pgmap v3380: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:37 smithi139 bash[43301]: cluster 2022-04-28T21:22:35.617974+0000 mgr.y (mgr.24961) 3050 : cluster [DBG] pgmap v3380: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[61152]: cluster 2022-04-28T21:22:37.619028+0000 mgr.y (mgr.24961) 3051 : cluster [DBG] pgmap v3381: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[61152]: audit 2022-04-28T21:22:39.084142+0000 mon.a (mon.0) 2691 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:22:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[61152]: audit 2022-04-28T21:22:39.087178+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:22:39.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[61152]: audit 2022-04-28T21:22:39.089379+0000 mon.a (mon.0) 2693 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:22:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[59745]: cluster 2022-04-28T21:22:37.619028+0000 mgr.y (mgr.24961) 3051 : cluster [DBG] pgmap v3381: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[59745]: audit 2022-04-28T21:22:39.084142+0000 mon.a (mon.0) 2691 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:22:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[59745]: audit 2022-04-28T21:22:39.087178+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:22:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:39 smithi059 bash[59745]: audit 2022-04-28T21:22:39.089379+0000 mon.a (mon.0) 2693 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:22:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:39 smithi139 bash[43301]: cluster 2022-04-28T21:22:37.619028+0000 mgr.y (mgr.24961) 3051 : cluster [DBG] pgmap v3381: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:39 smithi139 bash[43301]: audit 2022-04-28T21:22:39.084142+0000 mon.a (mon.0) 2691 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:22:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:39 smithi139 bash[43301]: audit 2022-04-28T21:22:39.087178+0000 mon.a (mon.0) 2692 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:22:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:39 smithi139 bash[43301]: audit 2022-04-28T21:22:39.089379+0000 mon.a (mon.0) 2693 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:22:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:40 smithi059 bash[61152]: cephadm 2022-04-28T21:22:39.091222+0000 mgr.y (mgr.24961) 3052 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:22:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:40 smithi059 bash[61152]: audit 2022-04-28T21:22:39.420259+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:22:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:40 smithi059 bash[59745]: cephadm 2022-04-28T21:22:39.091222+0000 mgr.y (mgr.24961) 3052 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:22:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:40 smithi059 bash[59745]: audit 2022-04-28T21:22:39.420259+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:22:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:40 smithi139 bash[43301]: cephadm 2022-04-28T21:22:39.091222+0000 mgr.y (mgr.24961) 3052 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:22:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:40 smithi139 bash[43301]: audit 2022-04-28T21:22:39.420259+0000 mon.a (mon.0) 2694 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:22:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:41 smithi059 bash[61152]: cluster 2022-04-28T21:22:39.620210+0000 mgr.y (mgr.24961) 3053 : cluster [DBG] pgmap v3382: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:41 smithi059 bash[59745]: cluster 2022-04-28T21:22:39.620210+0000 mgr.y (mgr.24961) 3053 : cluster [DBG] pgmap v3382: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:41 smithi139 bash[43301]: cluster 2022-04-28T21:22:39.620210+0000 mgr.y (mgr.24961) 3053 : cluster [DBG] pgmap v3382: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:41.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:42 smithi059 bash[61152]: cluster 2022-04-28T21:22:41.621501+0000 mgr.y (mgr.24961) 3054 : cluster [DBG] pgmap v3383: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:42 smithi059 bash[59745]: cluster 2022-04-28T21:22:41.621501+0000 mgr.y (mgr.24961) 3054 : cluster [DBG] pgmap v3383: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:42.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:42 smithi139 bash[43301]: cluster 2022-04-28T21:22:41.621501+0000 mgr.y (mgr.24961) 3054 : cluster [DBG] pgmap v3383: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:43.875Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:44 smithi059 bash[61152]: cluster 2022-04-28T21:22:43.622234+0000 mgr.y (mgr.24961) 3055 : cluster [DBG] pgmap v3384: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:45.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:44 smithi059 bash[59745]: cluster 2022-04-28T21:22:43.622234+0000 mgr.y (mgr.24961) 3055 : cluster [DBG] pgmap v3384: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:44 smithi139 bash[43301]: cluster 2022-04-28T21:22:43.622234+0000 mgr.y (mgr.24961) 3055 : cluster [DBG] pgmap v3384: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:46 smithi059 bash[61152]: cluster 2022-04-28T21:22:45.623660+0000 mgr.y (mgr.24961) 3056 : cluster [DBG] pgmap v3385: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:46 smithi059 bash[59745]: cluster 2022-04-28T21:22:45.623660+0000 mgr.y (mgr.24961) 3056 : cluster [DBG] pgmap v3385: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:46 smithi139 bash[43301]: cluster 2022-04-28T21:22:45.623660+0000 mgr.y (mgr.24961) 3056 : cluster [DBG] pgmap v3385: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:48 smithi059 bash[61152]: cluster 2022-04-28T21:22:47.624653+0000 mgr.y (mgr.24961) 3057 : cluster [DBG] pgmap v3386: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:49.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:48 smithi059 bash[59745]: cluster 2022-04-28T21:22:47.624653+0000 mgr.y (mgr.24961) 3057 : cluster [DBG] pgmap v3386: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:48 smithi139 bash[43301]: cluster 2022-04-28T21:22:47.624653+0000 mgr.y (mgr.24961) 3057 : cluster [DBG] pgmap v3386: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:50 smithi059 bash[61152]: cluster 2022-04-28T21:22:49.625789+0000 mgr.y (mgr.24961) 3058 : cluster [DBG] pgmap v3387: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:50 smithi059 bash[59745]: cluster 2022-04-28T21:22:49.625789+0000 mgr.y (mgr.24961) 3058 : cluster [DBG] pgmap v3387: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:50 smithi139 bash[43301]: cluster 2022-04-28T21:22:49.625789+0000 mgr.y (mgr.24961) 3058 : cluster [DBG] pgmap v3387: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:51.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:52 smithi059 bash[61152]: cluster 2022-04-28T21:22:51.627055+0000 mgr.y (mgr.24961) 3059 : cluster [DBG] pgmap v3388: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:52 smithi059 bash[59745]: cluster 2022-04-28T21:22:51.627055+0000 mgr.y (mgr.24961) 3059 : cluster [DBG] pgmap v3388: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:52 smithi139 bash[43301]: cluster 2022-04-28T21:22:51.627055+0000 mgr.y (mgr.24961) 3059 : cluster [DBG] pgmap v3388: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:22:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:22:53.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:22:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:54 smithi059 bash[61152]: cluster 2022-04-28T21:22:53.627926+0000 mgr.y (mgr.24961) 3060 : cluster [DBG] pgmap v3389: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:54 smithi059 bash[59745]: cluster 2022-04-28T21:22:53.627926+0000 mgr.y (mgr.24961) 3060 : cluster [DBG] pgmap v3389: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:54 smithi139 bash[43301]: cluster 2022-04-28T21:22:53.627926+0000 mgr.y (mgr.24961) 3060 : cluster [DBG] pgmap v3389: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:56 smithi059 bash[61152]: cluster 2022-04-28T21:22:55.629316+0000 mgr.y (mgr.24961) 3061 : cluster [DBG] pgmap v3390: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:56 smithi059 bash[59745]: cluster 2022-04-28T21:22:55.629316+0000 mgr.y (mgr.24961) 3061 : cluster [DBG] pgmap v3390: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:56 smithi139 bash[43301]: cluster 2022-04-28T21:22:55.629316+0000 mgr.y (mgr.24961) 3061 : cluster [DBG] pgmap v3390: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:22:58 smithi059 bash[61152]: cluster 2022-04-28T21:22:57.630314+0000 mgr.y (mgr.24961) 3062 : cluster [DBG] pgmap v3391: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:59.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:22:58 smithi059 bash[59745]: cluster 2022-04-28T21:22:57.630314+0000 mgr.y (mgr.24961) 3062 : cluster [DBG] pgmap v3391: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:22:59.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:22:58 smithi139 bash[43301]: cluster 2022-04-28T21:22:57.630314+0000 mgr.y (mgr.24961) 3062 : cluster [DBG] pgmap v3391: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:00 smithi059 bash[61152]: cluster 2022-04-28T21:22:59.631447+0000 mgr.y (mgr.24961) 3063 : cluster [DBG] pgmap v3392: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:00 smithi059 bash[59745]: cluster 2022-04-28T21:22:59.631447+0000 mgr.y (mgr.24961) 3063 : cluster [DBG] pgmap v3392: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:00 smithi139 bash[43301]: cluster 2022-04-28T21:22:59.631447+0000 mgr.y (mgr.24961) 3063 : cluster [DBG] pgmap v3392: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:01.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:02 smithi059 bash[61152]: cluster 2022-04-28T21:23:01.632741+0000 mgr.y (mgr.24961) 3064 : cluster [DBG] pgmap v3393: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:02 smithi059 bash[59745]: cluster 2022-04-28T21:23:01.632741+0000 mgr.y (mgr.24961) 3064 : cluster [DBG] pgmap v3393: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:02 smithi139 bash[43301]: cluster 2022-04-28T21:23:01.632741+0000 mgr.y (mgr.24961) 3064 : cluster [DBG] pgmap v3393: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:03.877Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:04 smithi059 bash[61152]: cluster 2022-04-28T21:23:03.633419+0000 mgr.y (mgr.24961) 3065 : cluster [DBG] pgmap v3394: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:05.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:04 smithi059 bash[59745]: cluster 2022-04-28T21:23:03.633419+0000 mgr.y (mgr.24961) 3065 : cluster [DBG] pgmap v3394: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:05.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:04 smithi139 bash[43301]: cluster 2022-04-28T21:23:03.633419+0000 mgr.y (mgr.24961) 3065 : cluster [DBG] pgmap v3394: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:06 smithi059 bash[61152]: cluster 2022-04-28T21:23:05.634826+0000 mgr.y (mgr.24961) 3066 : cluster [DBG] pgmap v3395: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:06 smithi059 bash[59745]: cluster 2022-04-28T21:23:05.634826+0000 mgr.y (mgr.24961) 3066 : cluster [DBG] pgmap v3395: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:06 smithi139 bash[43301]: cluster 2022-04-28T21:23:05.634826+0000 mgr.y (mgr.24961) 3066 : cluster [DBG] pgmap v3395: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:08 smithi059 bash[61152]: cluster 2022-04-28T21:23:07.635856+0000 mgr.y (mgr.24961) 3067 : cluster [DBG] pgmap v3396: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:08 smithi059 bash[59745]: cluster 2022-04-28T21:23:07.635856+0000 mgr.y (mgr.24961) 3067 : cluster [DBG] pgmap v3396: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:08 smithi139 bash[43301]: cluster 2022-04-28T21:23:07.635856+0000 mgr.y (mgr.24961) 3067 : cluster [DBG] pgmap v3396: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:10 smithi059 bash[61152]: cluster 2022-04-28T21:23:09.636965+0000 mgr.y (mgr.24961) 3068 : cluster [DBG] pgmap v3397: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:10 smithi059 bash[59745]: cluster 2022-04-28T21:23:09.636965+0000 mgr.y (mgr.24961) 3068 : cluster [DBG] pgmap v3397: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:10 smithi139 bash[43301]: cluster 2022-04-28T21:23:09.636965+0000 mgr.y (mgr.24961) 3068 : cluster [DBG] pgmap v3397: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:11.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:12 smithi059 bash[61152]: cluster 2022-04-28T21:23:11.638370+0000 mgr.y (mgr.24961) 3069 : cluster [DBG] pgmap v3398: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:12 smithi059 bash[59745]: cluster 2022-04-28T21:23:11.638370+0000 mgr.y (mgr.24961) 3069 : cluster [DBG] pgmap v3398: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:12 smithi139 bash[43301]: cluster 2022-04-28T21:23:11.638370+0000 mgr.y (mgr.24961) 3069 : cluster [DBG] pgmap v3398: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:13.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:14 smithi059 bash[59745]: cluster 2022-04-28T21:23:13.639121+0000 mgr.y (mgr.24961) 3070 : cluster [DBG] pgmap v3399: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:14 smithi059 bash[61152]: cluster 2022-04-28T21:23:13.639121+0000 mgr.y (mgr.24961) 3070 : cluster [DBG] pgmap v3399: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:14 smithi139 bash[43301]: cluster 2022-04-28T21:23:13.639121+0000 mgr.y (mgr.24961) 3070 : cluster [DBG] pgmap v3399: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:16 smithi059 bash[61152]: cluster 2022-04-28T21:23:15.640632+0000 mgr.y (mgr.24961) 3071 : cluster [DBG] pgmap v3400: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:16 smithi059 bash[59745]: cluster 2022-04-28T21:23:15.640632+0000 mgr.y (mgr.24961) 3071 : cluster [DBG] pgmap v3400: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:16 smithi139 bash[43301]: cluster 2022-04-28T21:23:15.640632+0000 mgr.y (mgr.24961) 3071 : cluster [DBG] pgmap v3400: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: cluster 2022-04-28T21:23:17.641617+0000 mgr.y (mgr.24961) 3072 : cluster [DBG] pgmap v3401: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: audit 2022-04-28T21:23:18.304326+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:23:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: audit 2022-04-28T21:23:18.311748+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:23:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: audit 2022-04-28T21:23:18.316983+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:23:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: audit 2022-04-28T21:23:18.319642+0000 mon.a (mon.0) 2698 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:23:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: audit 2022-04-28T21:23:18.320194+0000 mon.a (mon.0) 2699 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:23:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[61152]: audit 2022-04-28T21:23:18.320711+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:23:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: cluster 2022-04-28T21:23:17.641617+0000 mgr.y (mgr.24961) 3072 : cluster [DBG] pgmap v3401: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: audit 2022-04-28T21:23:18.304326+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:23:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: audit 2022-04-28T21:23:18.311748+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:23:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: audit 2022-04-28T21:23:18.316983+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:23:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: audit 2022-04-28T21:23:18.319642+0000 mon.a (mon.0) 2698 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:23:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: audit 2022-04-28T21:23:18.320194+0000 mon.a (mon.0) 2699 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:23:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:18 smithi059 bash[59745]: audit 2022-04-28T21:23:18.320711+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:23:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: cluster 2022-04-28T21:23:17.641617+0000 mgr.y (mgr.24961) 3072 : cluster [DBG] pgmap v3401: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: audit 2022-04-28T21:23:18.304326+0000 mon.a (mon.0) 2695 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:23:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: audit 2022-04-28T21:23:18.311748+0000 mon.a (mon.0) 2696 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:23:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: audit 2022-04-28T21:23:18.316983+0000 mon.a (mon.0) 2697 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:23:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: audit 2022-04-28T21:23:18.319642+0000 mon.a (mon.0) 2698 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:23:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: audit 2022-04-28T21:23:18.320194+0000 mon.a (mon.0) 2699 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:23:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:18 smithi139 bash[43301]: audit 2022-04-28T21:23:18.320711+0000 mon.a (mon.0) 2700 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:23:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:20 smithi059 bash[61152]: cluster 2022-04-28T21:23:19.642725+0000 mgr.y (mgr.24961) 3073 : cluster [DBG] pgmap v3402: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:20 smithi059 bash[59745]: cluster 2022-04-28T21:23:19.642725+0000 mgr.y (mgr.24961) 3073 : cluster [DBG] pgmap v3402: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:20 smithi139 bash[43301]: cluster 2022-04-28T21:23:19.642725+0000 mgr.y (mgr.24961) 3073 : cluster [DBG] pgmap v3402: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:21.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:22 smithi059 bash[61152]: cluster 2022-04-28T21:23:21.644002+0000 mgr.y (mgr.24961) 3074 : cluster [DBG] pgmap v3403: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:22 smithi059 bash[59745]: cluster 2022-04-28T21:23:21.644002+0000 mgr.y (mgr.24961) 3074 : cluster [DBG] pgmap v3403: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:23.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:22 smithi139 bash[43301]: cluster 2022-04-28T21:23:21.644002+0000 mgr.y (mgr.24961) 3074 : cluster [DBG] pgmap v3403: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:23.878Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:24 smithi059 bash[61152]: cluster 2022-04-28T21:23:23.644709+0000 mgr.y (mgr.24961) 3075 : cluster [DBG] pgmap v3404: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:24 smithi059 bash[59745]: cluster 2022-04-28T21:23:23.644709+0000 mgr.y (mgr.24961) 3075 : cluster [DBG] pgmap v3404: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:25.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:24 smithi139 bash[43301]: cluster 2022-04-28T21:23:23.644709+0000 mgr.y (mgr.24961) 3075 : cluster [DBG] pgmap v3404: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:26 smithi059 bash[61152]: cluster 2022-04-28T21:23:25.646126+0000 mgr.y (mgr.24961) 3076 : cluster [DBG] pgmap v3405: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:27.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:26 smithi059 bash[59745]: cluster 2022-04-28T21:23:25.646126+0000 mgr.y (mgr.24961) 3076 : cluster [DBG] pgmap v3405: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:26 smithi139 bash[43301]: cluster 2022-04-28T21:23:25.646126+0000 mgr.y (mgr.24961) 3076 : cluster [DBG] pgmap v3405: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:28 smithi059 bash[61152]: cluster 2022-04-28T21:23:27.647102+0000 mgr.y (mgr.24961) 3077 : cluster [DBG] pgmap v3406: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:28 smithi059 bash[59745]: cluster 2022-04-28T21:23:27.647102+0000 mgr.y (mgr.24961) 3077 : cluster [DBG] pgmap v3406: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:28 smithi139 bash[43301]: cluster 2022-04-28T21:23:27.647102+0000 mgr.y (mgr.24961) 3077 : cluster [DBG] pgmap v3406: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:30 smithi059 bash[61152]: cluster 2022-04-28T21:23:29.648226+0000 mgr.y (mgr.24961) 3078 : cluster [DBG] pgmap v3407: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:30 smithi059 bash[59745]: cluster 2022-04-28T21:23:29.648226+0000 mgr.y (mgr.24961) 3078 : cluster [DBG] pgmap v3407: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:30 smithi139 bash[43301]: cluster 2022-04-28T21:23:29.648226+0000 mgr.y (mgr.24961) 3078 : cluster [DBG] pgmap v3407: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:31.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:32 smithi059 bash[61152]: cluster 2022-04-28T21:23:31.649491+0000 mgr.y (mgr.24961) 3079 : cluster [DBG] pgmap v3408: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:32 smithi059 bash[59745]: cluster 2022-04-28T21:23:31.649491+0000 mgr.y (mgr.24961) 3079 : cluster [DBG] pgmap v3408: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:32 smithi139 bash[43301]: cluster 2022-04-28T21:23:31.649491+0000 mgr.y (mgr.24961) 3079 : cluster [DBG] pgmap v3408: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:33.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:35.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:34 smithi059 bash[61152]: cluster 2022-04-28T21:23:33.650315+0000 mgr.y (mgr.24961) 3080 : cluster [DBG] pgmap v3409: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:35.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:34 smithi059 bash[59745]: cluster 2022-04-28T21:23:33.650315+0000 mgr.y (mgr.24961) 3080 : cluster [DBG] pgmap v3409: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:34 smithi139 bash[43301]: cluster 2022-04-28T21:23:33.650315+0000 mgr.y (mgr.24961) 3080 : cluster [DBG] pgmap v3409: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:36 smithi059 bash[61152]: cluster 2022-04-28T21:23:35.651762+0000 mgr.y (mgr.24961) 3081 : cluster [DBG] pgmap v3410: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:37.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:36 smithi059 bash[59745]: cluster 2022-04-28T21:23:35.651762+0000 mgr.y (mgr.24961) 3081 : cluster [DBG] pgmap v3410: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:37.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:36 smithi139 bash[43301]: cluster 2022-04-28T21:23:35.651762+0000 mgr.y (mgr.24961) 3081 : cluster [DBG] pgmap v3410: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:38 smithi059 bash[61152]: cluster 2022-04-28T21:23:37.652744+0000 mgr.y (mgr.24961) 3082 : cluster [DBG] pgmap v3411: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:39.120 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:38 smithi059 bash[59745]: cluster 2022-04-28T21:23:37.652744+0000 mgr.y (mgr.24961) 3082 : cluster [DBG] pgmap v3411: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:39.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:38 smithi139 bash[43301]: cluster 2022-04-28T21:23:37.652744+0000 mgr.y (mgr.24961) 3082 : cluster [DBG] pgmap v3411: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[59745]: audit 2022-04-28T21:23:39.427231+0000 mon.a (mon.0) 2701 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:23:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[59745]: audit 2022-04-28T21:23:39.430066+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:23:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[59745]: audit 2022-04-28T21:23:39.431985+0000 mon.a (mon.0) 2703 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:23:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[59745]: audit 2022-04-28T21:23:39.767900+0000 mon.a (mon.0) 2704 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:23:40.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[61152]: audit 2022-04-28T21:23:39.427231+0000 mon.a (mon.0) 2701 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:23:40.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[61152]: audit 2022-04-28T21:23:39.430066+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:23:40.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[61152]: audit 2022-04-28T21:23:39.431985+0000 mon.a (mon.0) 2703 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:23:40.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:39 smithi059 bash[61152]: audit 2022-04-28T21:23:39.767900+0000 mon.a (mon.0) 2704 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:23:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:39 smithi139 bash[43301]: audit 2022-04-28T21:23:39.427231+0000 mon.a (mon.0) 2701 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:23:40.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:39 smithi139 bash[43301]: audit 2022-04-28T21:23:39.430066+0000 mon.a (mon.0) 2702 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:23:40.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:39 smithi139 bash[43301]: audit 2022-04-28T21:23:39.431985+0000 mon.a (mon.0) 2703 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:23:40.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:39 smithi139 bash[43301]: audit 2022-04-28T21:23:39.767900+0000 mon.a (mon.0) 2704 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:23:41.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:40 smithi059 bash[61152]: cephadm 2022-04-28T21:23:39.433661+0000 mgr.y (mgr.24961) 3083 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:23:41.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:40 smithi059 bash[61152]: cluster 2022-04-28T21:23:39.653981+0000 mgr.y (mgr.24961) 3084 : cluster [DBG] pgmap v3412: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:40 smithi059 bash[59745]: cephadm 2022-04-28T21:23:39.433661+0000 mgr.y (mgr.24961) 3083 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:23:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:40 smithi059 bash[59745]: cluster 2022-04-28T21:23:39.653981+0000 mgr.y (mgr.24961) 3084 : cluster [DBG] pgmap v3412: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:40 smithi139 bash[43301]: cephadm 2022-04-28T21:23:39.433661+0000 mgr.y (mgr.24961) 3083 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:23:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:40 smithi139 bash[43301]: cluster 2022-04-28T21:23:39.653981+0000 mgr.y (mgr.24961) 3084 : cluster [DBG] pgmap v3412: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:41.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:43.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:42 smithi139 bash[43301]: cluster 2022-04-28T21:23:41.655198+0000 mgr.y (mgr.24961) 3085 : cluster [DBG] pgmap v3413: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:42 smithi059 bash[61152]: cluster 2022-04-28T21:23:41.655198+0000 mgr.y (mgr.24961) 3085 : cluster [DBG] pgmap v3413: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:43.388 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:42 smithi059 bash[59745]: cluster 2022-04-28T21:23:41.655198+0000 mgr.y (mgr.24961) 3085 : cluster [DBG] pgmap v3413: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:43.879Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:44 smithi139 bash[43301]: cluster 2022-04-28T21:23:43.655984+0000 mgr.y (mgr.24961) 3086 : cluster [DBG] pgmap v3414: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:44 smithi059 bash[61152]: cluster 2022-04-28T21:23:43.655984+0000 mgr.y (mgr.24961) 3086 : cluster [DBG] pgmap v3414: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:44 smithi059 bash[59745]: cluster 2022-04-28T21:23:43.655984+0000 mgr.y (mgr.24961) 3086 : cluster [DBG] pgmap v3414: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:46 smithi139 bash[43301]: cluster 2022-04-28T21:23:45.657398+0000 mgr.y (mgr.24961) 3087 : cluster [DBG] pgmap v3415: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:46 smithi059 bash[61152]: cluster 2022-04-28T21:23:45.657398+0000 mgr.y (mgr.24961) 3087 : cluster [DBG] pgmap v3415: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:46 smithi059 bash[59745]: cluster 2022-04-28T21:23:45.657398+0000 mgr.y (mgr.24961) 3087 : cluster [DBG] pgmap v3415: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:48 smithi139 bash[43301]: cluster 2022-04-28T21:23:47.658385+0000 mgr.y (mgr.24961) 3088 : cluster [DBG] pgmap v3416: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:48 smithi059 bash[59745]: cluster 2022-04-28T21:23:47.658385+0000 mgr.y (mgr.24961) 3088 : cluster [DBG] pgmap v3416: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:48 smithi059 bash[61152]: cluster 2022-04-28T21:23:47.658385+0000 mgr.y (mgr.24961) 3088 : cluster [DBG] pgmap v3416: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:50 smithi139 bash[43301]: cluster 2022-04-28T21:23:49.659514+0000 mgr.y (mgr.24961) 3089 : cluster [DBG] pgmap v3417: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:51.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:50 smithi059 bash[61152]: cluster 2022-04-28T21:23:49.659514+0000 mgr.y (mgr.24961) 3089 : cluster [DBG] pgmap v3417: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:51.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:50 smithi059 bash[59745]: cluster 2022-04-28T21:23:49.659514+0000 mgr.y (mgr.24961) 3089 : cluster [DBG] pgmap v3417: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:51.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:52 smithi059 bash[61152]: cluster 2022-04-28T21:23:51.660800+0000 mgr.y (mgr.24961) 3090 : cluster [DBG] pgmap v3418: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:52 smithi059 bash[59745]: cluster 2022-04-28T21:23:51.660800+0000 mgr.y (mgr.24961) 3090 : cluster [DBG] pgmap v3418: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:52 smithi139 bash[43301]: cluster 2022-04-28T21:23:51.660800+0000 mgr.y (mgr.24961) 3090 : cluster [DBG] pgmap v3418: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:23:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:23:53.880Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:23:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:54 smithi059 bash[59745]: cluster 2022-04-28T21:23:53.661540+0000 mgr.y (mgr.24961) 3091 : cluster [DBG] pgmap v3419: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:54 smithi059 bash[61152]: cluster 2022-04-28T21:23:53.661540+0000 mgr.y (mgr.24961) 3091 : cluster [DBG] pgmap v3419: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:55.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:54 smithi139 bash[43301]: cluster 2022-04-28T21:23:53.661540+0000 mgr.y (mgr.24961) 3091 : cluster [DBG] pgmap v3419: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:56 smithi059 bash[61152]: cluster 2022-04-28T21:23:55.662954+0000 mgr.y (mgr.24961) 3092 : cluster [DBG] pgmap v3420: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:56 smithi059 bash[59745]: cluster 2022-04-28T21:23:55.662954+0000 mgr.y (mgr.24961) 3092 : cluster [DBG] pgmap v3420: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:56 smithi139 bash[43301]: cluster 2022-04-28T21:23:55.662954+0000 mgr.y (mgr.24961) 3092 : cluster [DBG] pgmap v3420: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:23:58 smithi059 bash[61152]: cluster 2022-04-28T21:23:57.663976+0000 mgr.y (mgr.24961) 3093 : cluster [DBG] pgmap v3421: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:23:58 smithi059 bash[59745]: cluster 2022-04-28T21:23:57.663976+0000 mgr.y (mgr.24961) 3093 : cluster [DBG] pgmap v3421: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:23:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:23:58 smithi139 bash[43301]: cluster 2022-04-28T21:23:57.663976+0000 mgr.y (mgr.24961) 3093 : cluster [DBG] pgmap v3421: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:00 smithi059 bash[59745]: cluster 2022-04-28T21:23:59.665067+0000 mgr.y (mgr.24961) 3094 : cluster [DBG] pgmap v3422: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:00 smithi059 bash[61152]: cluster 2022-04-28T21:23:59.665067+0000 mgr.y (mgr.24961) 3094 : cluster [DBG] pgmap v3422: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:00 smithi139 bash[43301]: cluster 2022-04-28T21:23:59.665067+0000 mgr.y (mgr.24961) 3094 : cluster [DBG] pgmap v3422: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:01.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:02 smithi059 bash[61152]: cluster 2022-04-28T21:24:01.666367+0000 mgr.y (mgr.24961) 3095 : cluster [DBG] pgmap v3423: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:02 smithi059 bash[59745]: cluster 2022-04-28T21:24:01.666367+0000 mgr.y (mgr.24961) 3095 : cluster [DBG] pgmap v3423: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:02 smithi139 bash[43301]: cluster 2022-04-28T21:24:01.666367+0000 mgr.y (mgr.24961) 3095 : cluster [DBG] pgmap v3423: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:03.881Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:04 smithi059 bash[61152]: cluster 2022-04-28T21:24:03.667096+0000 mgr.y (mgr.24961) 3096 : cluster [DBG] pgmap v3424: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:04 smithi059 bash[59745]: cluster 2022-04-28T21:24:03.667096+0000 mgr.y (mgr.24961) 3096 : cluster [DBG] pgmap v3424: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:04 smithi139 bash[43301]: cluster 2022-04-28T21:24:03.667096+0000 mgr.y (mgr.24961) 3096 : cluster [DBG] pgmap v3424: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:06 smithi059 bash[59745]: cluster 2022-04-28T21:24:05.668532+0000 mgr.y (mgr.24961) 3097 : cluster [DBG] pgmap v3425: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:06 smithi059 bash[61152]: cluster 2022-04-28T21:24:05.668532+0000 mgr.y (mgr.24961) 3097 : cluster [DBG] pgmap v3425: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:06 smithi139 bash[43301]: cluster 2022-04-28T21:24:05.668532+0000 mgr.y (mgr.24961) 3097 : cluster [DBG] pgmap v3425: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:08 smithi059 bash[61152]: cluster 2022-04-28T21:24:07.669459+0000 mgr.y (mgr.24961) 3098 : cluster [DBG] pgmap v3426: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:08 smithi059 bash[59745]: cluster 2022-04-28T21:24:07.669459+0000 mgr.y (mgr.24961) 3098 : cluster [DBG] pgmap v3426: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:08 smithi139 bash[43301]: cluster 2022-04-28T21:24:07.669459+0000 mgr.y (mgr.24961) 3098 : cluster [DBG] pgmap v3426: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:10 smithi059 bash[61152]: cluster 2022-04-28T21:24:09.670569+0000 mgr.y (mgr.24961) 3099 : cluster [DBG] pgmap v3427: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:10 smithi059 bash[59745]: cluster 2022-04-28T21:24:09.670569+0000 mgr.y (mgr.24961) 3099 : cluster [DBG] pgmap v3427: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:10 smithi139 bash[43301]: cluster 2022-04-28T21:24:09.670569+0000 mgr.y (mgr.24961) 3099 : cluster [DBG] pgmap v3427: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:11.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:12 smithi059 bash[61152]: cluster 2022-04-28T21:24:11.671882+0000 mgr.y (mgr.24961) 3100 : cluster [DBG] pgmap v3428: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:12 smithi059 bash[59745]: cluster 2022-04-28T21:24:11.671882+0000 mgr.y (mgr.24961) 3100 : cluster [DBG] pgmap v3428: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:12 smithi139 bash[43301]: cluster 2022-04-28T21:24:11.671882+0000 mgr.y (mgr.24961) 3100 : cluster [DBG] pgmap v3428: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:13.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:14 smithi059 bash[61152]: cluster 2022-04-28T21:24:13.672646+0000 mgr.y (mgr.24961) 3101 : cluster [DBG] pgmap v3429: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:14 smithi059 bash[59745]: cluster 2022-04-28T21:24:13.672646+0000 mgr.y (mgr.24961) 3101 : cluster [DBG] pgmap v3429: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:14 smithi139 bash[43301]: cluster 2022-04-28T21:24:13.672646+0000 mgr.y (mgr.24961) 3101 : cluster [DBG] pgmap v3429: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:16 smithi059 bash[61152]: cluster 2022-04-28T21:24:15.674056+0000 mgr.y (mgr.24961) 3102 : cluster [DBG] pgmap v3430: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:16 smithi059 bash[59745]: cluster 2022-04-28T21:24:15.674056+0000 mgr.y (mgr.24961) 3102 : cluster [DBG] pgmap v3430: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:16 smithi139 bash[43301]: cluster 2022-04-28T21:24:15.674056+0000 mgr.y (mgr.24961) 3102 : cluster [DBG] pgmap v3430: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: cluster 2022-04-28T21:24:17.675144+0000 mgr.y (mgr.24961) 3103 : cluster [DBG] pgmap v3431: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: audit 2022-04-28T21:24:18.323269+0000 mon.a (mon.0) 2705 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:24:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: audit 2022-04-28T21:24:18.323863+0000 mon.a (mon.0) 2706 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:24:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: audit 2022-04-28T21:24:18.333756+0000 mon.a (mon.0) 2707 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: audit 2022-04-28T21:24:18.334527+0000 mon.a (mon.0) 2708 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: audit 2022-04-28T21:24:18.335125+0000 mon.a (mon.0) 2709 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[59745]: audit 2022-04-28T21:24:18.335702+0000 mon.a (mon.0) 2710 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T21:24:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: cluster 2022-04-28T21:24:17.675144+0000 mgr.y (mgr.24961) 3103 : cluster [DBG] pgmap v3431: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: audit 2022-04-28T21:24:18.323269+0000 mon.a (mon.0) 2705 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:24:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: audit 2022-04-28T21:24:18.323863+0000 mon.a (mon.0) 2706 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:24:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: audit 2022-04-28T21:24:18.333756+0000 mon.a (mon.0) 2707 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: audit 2022-04-28T21:24:18.334527+0000 mon.a (mon.0) 2708 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: audit 2022-04-28T21:24:18.335125+0000 mon.a (mon.0) 2709 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:18 smithi059 bash[61152]: audit 2022-04-28T21:24:18.335702+0000 mon.a (mon.0) 2710 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T21:24:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: cluster 2022-04-28T21:24:17.675144+0000 mgr.y (mgr.24961) 3103 : cluster [DBG] pgmap v3431: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: audit 2022-04-28T21:24:18.323269+0000 mon.a (mon.0) 2705 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:24:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: audit 2022-04-28T21:24:18.323863+0000 mon.a (mon.0) 2706 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:24:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: audit 2022-04-28T21:24:18.333756+0000 mon.a (mon.0) 2707 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: audit 2022-04-28T21:24:18.334527+0000 mon.a (mon.0) 2708 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: audit 2022-04-28T21:24:18.335125+0000 mon.a (mon.0) 2709 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:24:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:18 smithi139 bash[43301]: audit 2022-04-28T21:24:18.335702+0000 mon.a (mon.0) 2710 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T21:24:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:20 smithi059 bash[59745]: cluster 2022-04-28T21:24:19.676328+0000 mgr.y (mgr.24961) 3104 : cluster [DBG] pgmap v3432: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:20 smithi059 bash[61152]: cluster 2022-04-28T21:24:19.676328+0000 mgr.y (mgr.24961) 3104 : cluster [DBG] pgmap v3432: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:20 smithi139 bash[43301]: cluster 2022-04-28T21:24:19.676328+0000 mgr.y (mgr.24961) 3104 : cluster [DBG] pgmap v3432: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:21.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:22 smithi059 bash[61152]: cluster 2022-04-28T21:24:21.677569+0000 mgr.y (mgr.24961) 3105 : cluster [DBG] pgmap v3433: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:22 smithi059 bash[59745]: cluster 2022-04-28T21:24:21.677569+0000 mgr.y (mgr.24961) 3105 : cluster [DBG] pgmap v3433: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:22 smithi139 bash[43301]: cluster 2022-04-28T21:24:21.677569+0000 mgr.y (mgr.24961) 3105 : cluster [DBG] pgmap v3433: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:23.882Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:24 smithi059 bash[61152]: cluster 2022-04-28T21:24:23.678317+0000 mgr.y (mgr.24961) 3106 : cluster [DBG] pgmap v3434: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:24 smithi059 bash[59745]: cluster 2022-04-28T21:24:23.678317+0000 mgr.y (mgr.24961) 3106 : cluster [DBG] pgmap v3434: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:24 smithi139 bash[43301]: cluster 2022-04-28T21:24:23.678317+0000 mgr.y (mgr.24961) 3106 : cluster [DBG] pgmap v3434: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:26 smithi059 bash[61152]: cluster 2022-04-28T21:24:25.679764+0000 mgr.y (mgr.24961) 3107 : cluster [DBG] pgmap v3435: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:26 smithi059 bash[59745]: cluster 2022-04-28T21:24:25.679764+0000 mgr.y (mgr.24961) 3107 : cluster [DBG] pgmap v3435: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:26 smithi139 bash[43301]: cluster 2022-04-28T21:24:25.679764+0000 mgr.y (mgr.24961) 3107 : cluster [DBG] pgmap v3435: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:28 smithi059 bash[59745]: cluster 2022-04-28T21:24:27.680841+0000 mgr.y (mgr.24961) 3108 : cluster [DBG] pgmap v3436: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:29.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:28 smithi059 bash[61152]: cluster 2022-04-28T21:24:27.680841+0000 mgr.y (mgr.24961) 3108 : cluster [DBG] pgmap v3436: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:28 smithi139 bash[43301]: cluster 2022-04-28T21:24:27.680841+0000 mgr.y (mgr.24961) 3108 : cluster [DBG] pgmap v3436: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:31.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:30 smithi059 bash[61152]: cluster 2022-04-28T21:24:29.681983+0000 mgr.y (mgr.24961) 3109 : cluster [DBG] pgmap v3437: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:30 smithi059 bash[59745]: cluster 2022-04-28T21:24:29.681983+0000 mgr.y (mgr.24961) 3109 : cluster [DBG] pgmap v3437: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:30 smithi139 bash[43301]: cluster 2022-04-28T21:24:29.681983+0000 mgr.y (mgr.24961) 3109 : cluster [DBG] pgmap v3437: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:31.989Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:32 smithi059 bash[61152]: cluster 2022-04-28T21:24:31.683339+0000 mgr.y (mgr.24961) 3110 : cluster [DBG] pgmap v3438: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:32 smithi059 bash[59745]: cluster 2022-04-28T21:24:31.683339+0000 mgr.y (mgr.24961) 3110 : cluster [DBG] pgmap v3438: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:32 smithi139 bash[43301]: cluster 2022-04-28T21:24:31.683339+0000 mgr.y (mgr.24961) 3110 : cluster [DBG] pgmap v3438: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:33.883Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:34 smithi059 bash[61152]: cluster 2022-04-28T21:24:33.684065+0000 mgr.y (mgr.24961) 3111 : cluster [DBG] pgmap v3439: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:34 smithi059 bash[59745]: cluster 2022-04-28T21:24:33.684065+0000 mgr.y (mgr.24961) 3111 : cluster [DBG] pgmap v3439: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:34 smithi139 bash[43301]: cluster 2022-04-28T21:24:33.684065+0000 mgr.y (mgr.24961) 3111 : cluster [DBG] pgmap v3439: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:36 smithi059 bash[61152]: cluster 2022-04-28T21:24:35.685511+0000 mgr.y (mgr.24961) 3112 : cluster [DBG] pgmap v3440: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:36 smithi059 bash[59745]: cluster 2022-04-28T21:24:35.685511+0000 mgr.y (mgr.24961) 3112 : cluster [DBG] pgmap v3440: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:36 smithi139 bash[43301]: cluster 2022-04-28T21:24:35.685511+0000 mgr.y (mgr.24961) 3112 : cluster [DBG] pgmap v3440: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:38 smithi059 bash[61152]: cluster 2022-04-28T21:24:37.686536+0000 mgr.y (mgr.24961) 3113 : cluster [DBG] pgmap v3441: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:38 smithi059 bash[59745]: cluster 2022-04-28T21:24:37.686536+0000 mgr.y (mgr.24961) 3113 : cluster [DBG] pgmap v3441: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:38 smithi139 bash[43301]: cluster 2022-04-28T21:24:37.686536+0000 mgr.y (mgr.24961) 3113 : cluster [DBG] pgmap v3441: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[59745]: cluster 2022-04-28T21:24:39.687660+0000 mgr.y (mgr.24961) 3114 : cluster [DBG] pgmap v3442: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[59745]: audit 2022-04-28T21:24:39.773535+0000 mon.a (mon.0) 2711 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:24:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[59745]: audit 2022-04-28T21:24:39.775151+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:24:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[59745]: audit 2022-04-28T21:24:39.776128+0000 mon.a (mon.0) 2713 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:24:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[59745]: cephadm 2022-04-28T21:24:39.776975+0000 mgr.y (mgr.24961) 3115 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:24:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[59745]: audit 2022-04-28T21:24:40.088495+0000 mon.a (mon.0) 2714 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:24:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[61152]: cluster 2022-04-28T21:24:39.687660+0000 mgr.y (mgr.24961) 3114 : cluster [DBG] pgmap v3442: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[61152]: audit 2022-04-28T21:24:39.773535+0000 mon.a (mon.0) 2711 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:24:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[61152]: audit 2022-04-28T21:24:39.775151+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:24:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[61152]: audit 2022-04-28T21:24:39.776128+0000 mon.a (mon.0) 2713 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:24:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[61152]: cephadm 2022-04-28T21:24:39.776975+0000 mgr.y (mgr.24961) 3115 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:24:41.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:40 smithi059 bash[61152]: audit 2022-04-28T21:24:40.088495+0000 mon.a (mon.0) 2714 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:24:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:40 smithi139 bash[43301]: cluster 2022-04-28T21:24:39.687660+0000 mgr.y (mgr.24961) 3114 : cluster [DBG] pgmap v3442: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:41.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:40 smithi139 bash[43301]: audit 2022-04-28T21:24:39.773535+0000 mon.a (mon.0) 2711 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:24:41.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:40 smithi139 bash[43301]: audit 2022-04-28T21:24:39.775151+0000 mon.a (mon.0) 2712 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:24:41.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:40 smithi139 bash[43301]: audit 2022-04-28T21:24:39.776128+0000 mon.a (mon.0) 2713 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:24:41.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:40 smithi139 bash[43301]: cephadm 2022-04-28T21:24:39.776975+0000 mgr.y (mgr.24961) 3115 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:24:41.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:40 smithi139 bash[43301]: audit 2022-04-28T21:24:40.088495+0000 mon.a (mon.0) 2714 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:24:42.344 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:41.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:43.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:42 smithi059 bash[61152]: cluster 2022-04-28T21:24:41.688960+0000 mgr.y (mgr.24961) 3116 : cluster [DBG] pgmap v3443: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:43.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:42 smithi059 bash[59745]: cluster 2022-04-28T21:24:41.688960+0000 mgr.y (mgr.24961) 3116 : cluster [DBG] pgmap v3443: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:42 smithi139 bash[43301]: cluster 2022-04-28T21:24:41.688960+0000 mgr.y (mgr.24961) 3116 : cluster [DBG] pgmap v3443: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:43.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:44 smithi059 bash[61152]: cluster 2022-04-28T21:24:43.689690+0000 mgr.y (mgr.24961) 3117 : cluster [DBG] pgmap v3444: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:45.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:44 smithi059 bash[59745]: cluster 2022-04-28T21:24:43.689690+0000 mgr.y (mgr.24961) 3117 : cluster [DBG] pgmap v3444: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:44 smithi139 bash[43301]: cluster 2022-04-28T21:24:43.689690+0000 mgr.y (mgr.24961) 3117 : cluster [DBG] pgmap v3444: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:46 smithi059 bash[59745]: cluster 2022-04-28T21:24:45.691116+0000 mgr.y (mgr.24961) 3118 : cluster [DBG] pgmap v3445: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:46 smithi059 bash[61152]: cluster 2022-04-28T21:24:45.691116+0000 mgr.y (mgr.24961) 3118 : cluster [DBG] pgmap v3445: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:46 smithi139 bash[43301]: cluster 2022-04-28T21:24:45.691116+0000 mgr.y (mgr.24961) 3118 : cluster [DBG] pgmap v3445: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:48 smithi059 bash[61152]: cluster 2022-04-28T21:24:47.692159+0000 mgr.y (mgr.24961) 3119 : cluster [DBG] pgmap v3446: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:48 smithi059 bash[59745]: cluster 2022-04-28T21:24:47.692159+0000 mgr.y (mgr.24961) 3119 : cluster [DBG] pgmap v3446: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:48 smithi139 bash[43301]: cluster 2022-04-28T21:24:47.692159+0000 mgr.y (mgr.24961) 3119 : cluster [DBG] pgmap v3446: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:50 smithi059 bash[61152]: cluster 2022-04-28T21:24:49.693321+0000 mgr.y (mgr.24961) 3120 : cluster [DBG] pgmap v3447: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:50 smithi059 bash[59745]: cluster 2022-04-28T21:24:49.693321+0000 mgr.y (mgr.24961) 3120 : cluster [DBG] pgmap v3447: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:50 smithi139 bash[43301]: cluster 2022-04-28T21:24:49.693321+0000 mgr.y (mgr.24961) 3120 : cluster [DBG] pgmap v3447: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:51.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:52 smithi059 bash[59745]: cluster 2022-04-28T21:24:51.694577+0000 mgr.y (mgr.24961) 3121 : cluster [DBG] pgmap v3448: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:52 smithi059 bash[61152]: cluster 2022-04-28T21:24:51.694577+0000 mgr.y (mgr.24961) 3121 : cluster [DBG] pgmap v3448: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:52 smithi139 bash[43301]: cluster 2022-04-28T21:24:51.694577+0000 mgr.y (mgr.24961) 3121 : cluster [DBG] pgmap v3448: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:24:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:24:53.885Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:24:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:54 smithi059 bash[61152]: cluster 2022-04-28T21:24:53.695317+0000 mgr.y (mgr.24961) 3122 : cluster [DBG] pgmap v3449: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:54 smithi059 bash[59745]: cluster 2022-04-28T21:24:53.695317+0000 mgr.y (mgr.24961) 3122 : cluster [DBG] pgmap v3449: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:54 smithi139 bash[43301]: cluster 2022-04-28T21:24:53.695317+0000 mgr.y (mgr.24961) 3122 : cluster [DBG] pgmap v3449: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:56 smithi059 bash[61152]: cluster 2022-04-28T21:24:55.696696+0000 mgr.y (mgr.24961) 3123 : cluster [DBG] pgmap v3450: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:56 smithi059 bash[59745]: cluster 2022-04-28T21:24:55.696696+0000 mgr.y (mgr.24961) 3123 : cluster [DBG] pgmap v3450: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:57.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:56 smithi139 bash[43301]: cluster 2022-04-28T21:24:55.696696+0000 mgr.y (mgr.24961) 3123 : cluster [DBG] pgmap v3450: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:24:58 smithi059 bash[61152]: cluster 2022-04-28T21:24:57.697679+0000 mgr.y (mgr.24961) 3124 : cluster [DBG] pgmap v3451: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:59.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:24:58 smithi059 bash[59745]: cluster 2022-04-28T21:24:57.697679+0000 mgr.y (mgr.24961) 3124 : cluster [DBG] pgmap v3451: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:24:59.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:24:58 smithi139 bash[43301]: cluster 2022-04-28T21:24:57.697679+0000 mgr.y (mgr.24961) 3124 : cluster [DBG] pgmap v3451: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:00 smithi059 bash[61152]: cluster 2022-04-28T21:24:59.698816+0000 mgr.y (mgr.24961) 3125 : cluster [DBG] pgmap v3452: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:00 smithi059 bash[59745]: cluster 2022-04-28T21:24:59.698816+0000 mgr.y (mgr.24961) 3125 : cluster [DBG] pgmap v3452: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:00 smithi139 bash[43301]: cluster 2022-04-28T21:24:59.698816+0000 mgr.y (mgr.24961) 3125 : cluster [DBG] pgmap v3452: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:01.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:02 smithi059 bash[61152]: cluster 2022-04-28T21:25:01.700086+0000 mgr.y (mgr.24961) 3126 : cluster [DBG] pgmap v3453: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:02 smithi059 bash[59745]: cluster 2022-04-28T21:25:01.700086+0000 mgr.y (mgr.24961) 3126 : cluster [DBG] pgmap v3453: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:02 smithi139 bash[43301]: cluster 2022-04-28T21:25:01.700086+0000 mgr.y (mgr.24961) 3126 : cluster [DBG] pgmap v3453: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:03.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:05.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:04 smithi059 bash[61152]: cluster 2022-04-28T21:25:03.700855+0000 mgr.y (mgr.24961) 3127 : cluster [DBG] pgmap v3454: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:05.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:04 smithi059 bash[59745]: cluster 2022-04-28T21:25:03.700855+0000 mgr.y (mgr.24961) 3127 : cluster [DBG] pgmap v3454: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:05.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:04 smithi139 bash[43301]: cluster 2022-04-28T21:25:03.700855+0000 mgr.y (mgr.24961) 3127 : cluster [DBG] pgmap v3454: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:06 smithi059 bash[61152]: cluster 2022-04-28T21:25:05.702264+0000 mgr.y (mgr.24961) 3128 : cluster [DBG] pgmap v3455: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:06 smithi059 bash[59745]: cluster 2022-04-28T21:25:05.702264+0000 mgr.y (mgr.24961) 3128 : cluster [DBG] pgmap v3455: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:06 smithi139 bash[43301]: cluster 2022-04-28T21:25:05.702264+0000 mgr.y (mgr.24961) 3128 : cluster [DBG] pgmap v3455: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:08 smithi059 bash[59745]: cluster 2022-04-28T21:25:07.703251+0000 mgr.y (mgr.24961) 3129 : cluster [DBG] pgmap v3456: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:08 smithi059 bash[61152]: cluster 2022-04-28T21:25:07.703251+0000 mgr.y (mgr.24961) 3129 : cluster [DBG] pgmap v3456: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:08 smithi139 bash[43301]: cluster 2022-04-28T21:25:07.703251+0000 mgr.y (mgr.24961) 3129 : cluster [DBG] pgmap v3456: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:10 smithi059 bash[61152]: cluster 2022-04-28T21:25:09.704421+0000 mgr.y (mgr.24961) 3130 : cluster [DBG] pgmap v3457: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:10 smithi059 bash[59745]: cluster 2022-04-28T21:25:09.704421+0000 mgr.y (mgr.24961) 3130 : cluster [DBG] pgmap v3457: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:10 smithi139 bash[43301]: cluster 2022-04-28T21:25:09.704421+0000 mgr.y (mgr.24961) 3130 : cluster [DBG] pgmap v3457: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:11.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:12 smithi059 bash[61152]: cluster 2022-04-28T21:25:11.705649+0000 mgr.y (mgr.24961) 3131 : cluster [DBG] pgmap v3458: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:12 smithi059 bash[59745]: cluster 2022-04-28T21:25:11.705649+0000 mgr.y (mgr.24961) 3131 : cluster [DBG] pgmap v3458: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:12 smithi139 bash[43301]: cluster 2022-04-28T21:25:11.705649+0000 mgr.y (mgr.24961) 3131 : cluster [DBG] pgmap v3458: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:13.884Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:14 smithi059 bash[59745]: cluster 2022-04-28T21:25:13.706404+0000 mgr.y (mgr.24961) 3132 : cluster [DBG] pgmap v3459: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:14 smithi059 bash[61152]: cluster 2022-04-28T21:25:13.706404+0000 mgr.y (mgr.24961) 3132 : cluster [DBG] pgmap v3459: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:14 smithi139 bash[43301]: cluster 2022-04-28T21:25:13.706404+0000 mgr.y (mgr.24961) 3132 : cluster [DBG] pgmap v3459: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:16 smithi059 bash[61152]: cluster 2022-04-28T21:25:15.707840+0000 mgr.y (mgr.24961) 3133 : cluster [DBG] pgmap v3460: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:16 smithi059 bash[59745]: cluster 2022-04-28T21:25:15.707840+0000 mgr.y (mgr.24961) 3133 : cluster [DBG] pgmap v3460: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:16 smithi139 bash[43301]: cluster 2022-04-28T21:25:15.707840+0000 mgr.y (mgr.24961) 3133 : cluster [DBG] pgmap v3460: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: cluster 2022-04-28T21:25:17.708774+0000 mgr.y (mgr.24961) 3134 : cluster [DBG] pgmap v3461: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: audit 2022-04-28T21:25:18.337392+0000 mon.a (mon.0) 2715 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:25:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: audit 2022-04-28T21:25:18.338309+0000 mon.a (mon.0) 2716 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:25:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: audit 2022-04-28T21:25:18.349390+0000 mon.a (mon.0) 2717 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:25:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: audit 2022-04-28T21:25:18.349990+0000 mon.a (mon.0) 2718 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:25:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: audit 2022-04-28T21:25:18.350307+0000 mon.a (mon.0) 2719 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:25:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[61152]: audit 2022-04-28T21:25:18.350576+0000 mon.a (mon.0) 2720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:25:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: cluster 2022-04-28T21:25:17.708774+0000 mgr.y (mgr.24961) 3134 : cluster [DBG] pgmap v3461: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: audit 2022-04-28T21:25:18.337392+0000 mon.a (mon.0) 2715 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:25:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: audit 2022-04-28T21:25:18.338309+0000 mon.a (mon.0) 2716 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:25:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: audit 2022-04-28T21:25:18.349390+0000 mon.a (mon.0) 2717 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:25:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: audit 2022-04-28T21:25:18.349990+0000 mon.a (mon.0) 2718 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:25:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: audit 2022-04-28T21:25:18.350307+0000 mon.a (mon.0) 2719 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:25:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:18 smithi059 bash[59745]: audit 2022-04-28T21:25:18.350576+0000 mon.a (mon.0) 2720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:25:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: cluster 2022-04-28T21:25:17.708774+0000 mgr.y (mgr.24961) 3134 : cluster [DBG] pgmap v3461: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: audit 2022-04-28T21:25:18.337392+0000 mon.a (mon.0) 2715 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:25:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: audit 2022-04-28T21:25:18.338309+0000 mon.a (mon.0) 2716 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:25:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: audit 2022-04-28T21:25:18.349390+0000 mon.a (mon.0) 2717 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:25:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: audit 2022-04-28T21:25:18.349990+0000 mon.a (mon.0) 2718 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:25:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: audit 2022-04-28T21:25:18.350307+0000 mon.a (mon.0) 2719 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:25:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:18 smithi139 bash[43301]: audit 2022-04-28T21:25:18.350576+0000 mon.a (mon.0) 2720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:25:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:20 smithi059 bash[61152]: cluster 2022-04-28T21:25:19.709948+0000 mgr.y (mgr.24961) 3135 : cluster [DBG] pgmap v3462: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:20 smithi059 bash[59745]: cluster 2022-04-28T21:25:19.709948+0000 mgr.y (mgr.24961) 3135 : cluster [DBG] pgmap v3462: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:20 smithi139 bash[43301]: cluster 2022-04-28T21:25:19.709948+0000 mgr.y (mgr.24961) 3135 : cluster [DBG] pgmap v3462: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:21.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:22 smithi059 bash[61152]: cluster 2022-04-28T21:25:21.711207+0000 mgr.y (mgr.24961) 3136 : cluster [DBG] pgmap v3463: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:22 smithi059 bash[59745]: cluster 2022-04-28T21:25:21.711207+0000 mgr.y (mgr.24961) 3136 : cluster [DBG] pgmap v3463: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:22 smithi139 bash[43301]: cluster 2022-04-28T21:25:21.711207+0000 mgr.y (mgr.24961) 3136 : cluster [DBG] pgmap v3463: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:24.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:23.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:24 smithi059 bash[61152]: cluster 2022-04-28T21:25:23.712016+0000 mgr.y (mgr.24961) 3137 : cluster [DBG] pgmap v3464: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:24 smithi059 bash[59745]: cluster 2022-04-28T21:25:23.712016+0000 mgr.y (mgr.24961) 3137 : cluster [DBG] pgmap v3464: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:25.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:24 smithi139 bash[43301]: cluster 2022-04-28T21:25:23.712016+0000 mgr.y (mgr.24961) 3137 : cluster [DBG] pgmap v3464: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:27.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:26 smithi059 bash[59745]: cluster 2022-04-28T21:25:25.713425+0000 mgr.y (mgr.24961) 3138 : cluster [DBG] pgmap v3465: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:26 smithi059 bash[61152]: cluster 2022-04-28T21:25:25.713425+0000 mgr.y (mgr.24961) 3138 : cluster [DBG] pgmap v3465: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:26 smithi139 bash[43301]: cluster 2022-04-28T21:25:25.713425+0000 mgr.y (mgr.24961) 3138 : cluster [DBG] pgmap v3465: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:28 smithi059 bash[61152]: cluster 2022-04-28T21:25:27.714411+0000 mgr.y (mgr.24961) 3139 : cluster [DBG] pgmap v3466: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:28 smithi059 bash[59745]: cluster 2022-04-28T21:25:27.714411+0000 mgr.y (mgr.24961) 3139 : cluster [DBG] pgmap v3466: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:28 smithi139 bash[43301]: cluster 2022-04-28T21:25:27.714411+0000 mgr.y (mgr.24961) 3139 : cluster [DBG] pgmap v3466: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:30 smithi059 bash[61152]: cluster 2022-04-28T21:25:29.715531+0000 mgr.y (mgr.24961) 3140 : cluster [DBG] pgmap v3467: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:30 smithi059 bash[59745]: cluster 2022-04-28T21:25:29.715531+0000 mgr.y (mgr.24961) 3140 : cluster [DBG] pgmap v3467: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:30 smithi139 bash[43301]: cluster 2022-04-28T21:25:29.715531+0000 mgr.y (mgr.24961) 3140 : cluster [DBG] pgmap v3467: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:31.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:32 smithi139 bash[43301]: cluster 2022-04-28T21:25:31.716859+0000 mgr.y (mgr.24961) 3141 : cluster [DBG] pgmap v3468: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:32 smithi059 bash[61152]: cluster 2022-04-28T21:25:31.716859+0000 mgr.y (mgr.24961) 3141 : cluster [DBG] pgmap v3468: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:32 smithi059 bash[59745]: cluster 2022-04-28T21:25:31.716859+0000 mgr.y (mgr.24961) 3141 : cluster [DBG] pgmap v3468: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:33.887Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:34 smithi139 bash[43301]: cluster 2022-04-28T21:25:33.717608+0000 mgr.y (mgr.24961) 3142 : cluster [DBG] pgmap v3469: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:34 smithi059 bash[61152]: cluster 2022-04-28T21:25:33.717608+0000 mgr.y (mgr.24961) 3142 : cluster [DBG] pgmap v3469: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:34 smithi059 bash[59745]: cluster 2022-04-28T21:25:33.717608+0000 mgr.y (mgr.24961) 3142 : cluster [DBG] pgmap v3469: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:37.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:36 smithi139 bash[43301]: cluster 2022-04-28T21:25:35.718996+0000 mgr.y (mgr.24961) 3143 : cluster [DBG] pgmap v3470: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:36 smithi059 bash[61152]: cluster 2022-04-28T21:25:35.718996+0000 mgr.y (mgr.24961) 3143 : cluster [DBG] pgmap v3470: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:36 smithi059 bash[59745]: cluster 2022-04-28T21:25:35.718996+0000 mgr.y (mgr.24961) 3143 : cluster [DBG] pgmap v3470: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:38 smithi139 bash[43301]: cluster 2022-04-28T21:25:37.720000+0000 mgr.y (mgr.24961) 3144 : cluster [DBG] pgmap v3471: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:38 smithi059 bash[61152]: cluster 2022-04-28T21:25:37.720000+0000 mgr.y (mgr.24961) 3144 : cluster [DBG] pgmap v3471: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:38 smithi059 bash[59745]: cluster 2022-04-28T21:25:37.720000+0000 mgr.y (mgr.24961) 3144 : cluster [DBG] pgmap v3471: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[59745]: cluster 2022-04-28T21:25:39.721105+0000 mgr.y (mgr.24961) 3145 : cluster [DBG] pgmap v3472: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[59745]: audit 2022-04-28T21:25:40.097280+0000 mon.a (mon.0) 2721 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:25:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[59745]: audit 2022-04-28T21:25:40.100362+0000 mon.a (mon.0) 2722 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:25:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[59745]: audit 2022-04-28T21:25:40.102313+0000 mon.a (mon.0) 2723 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:25:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[59745]: cephadm 2022-04-28T21:25:40.104195+0000 mgr.y (mgr.24961) 3146 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:25:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[59745]: audit 2022-04-28T21:25:40.426826+0000 mon.a (mon.0) 2724 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:25:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[61152]: cluster 2022-04-28T21:25:39.721105+0000 mgr.y (mgr.24961) 3145 : cluster [DBG] pgmap v3472: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[61152]: audit 2022-04-28T21:25:40.097280+0000 mon.a (mon.0) 2721 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:25:41.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[61152]: audit 2022-04-28T21:25:40.100362+0000 mon.a (mon.0) 2722 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:25:41.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[61152]: audit 2022-04-28T21:25:40.102313+0000 mon.a (mon.0) 2723 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:25:41.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[61152]: cephadm 2022-04-28T21:25:40.104195+0000 mgr.y (mgr.24961) 3146 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:25:41.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:40 smithi059 bash[61152]: audit 2022-04-28T21:25:40.426826+0000 mon.a (mon.0) 2724 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:25:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:40 smithi139 bash[43301]: cluster 2022-04-28T21:25:39.721105+0000 mgr.y (mgr.24961) 3145 : cluster [DBG] pgmap v3472: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:40 smithi139 bash[43301]: audit 2022-04-28T21:25:40.097280+0000 mon.a (mon.0) 2721 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:25:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:40 smithi139 bash[43301]: audit 2022-04-28T21:25:40.100362+0000 mon.a (mon.0) 2722 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:25:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:40 smithi139 bash[43301]: audit 2022-04-28T21:25:40.102313+0000 mon.a (mon.0) 2723 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:25:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:40 smithi139 bash[43301]: cephadm 2022-04-28T21:25:40.104195+0000 mgr.y (mgr.24961) 3146 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:25:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:40 smithi139 bash[43301]: audit 2022-04-28T21:25:40.426826+0000 mon.a (mon.0) 2724 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:25:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:41.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:42 smithi059 bash[61152]: cluster 2022-04-28T21:25:41.722355+0000 mgr.y (mgr.24961) 3147 : cluster [DBG] pgmap v3473: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:42 smithi059 bash[59745]: cluster 2022-04-28T21:25:41.722355+0000 mgr.y (mgr.24961) 3147 : cluster [DBG] pgmap v3473: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:42 smithi139 bash[43301]: cluster 2022-04-28T21:25:41.722355+0000 mgr.y (mgr.24961) 3147 : cluster [DBG] pgmap v3473: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:43.888Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:44 smithi059 bash[59745]: cluster 2022-04-28T21:25:43.723115+0000 mgr.y (mgr.24961) 3148 : cluster [DBG] pgmap v3474: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:44 smithi059 bash[61152]: cluster 2022-04-28T21:25:43.723115+0000 mgr.y (mgr.24961) 3148 : cluster [DBG] pgmap v3474: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:44 smithi139 bash[43301]: cluster 2022-04-28T21:25:43.723115+0000 mgr.y (mgr.24961) 3148 : cluster [DBG] pgmap v3474: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:46 smithi059 bash[61152]: cluster 2022-04-28T21:25:45.724555+0000 mgr.y (mgr.24961) 3149 : cluster [DBG] pgmap v3475: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:46 smithi059 bash[59745]: cluster 2022-04-28T21:25:45.724555+0000 mgr.y (mgr.24961) 3149 : cluster [DBG] pgmap v3475: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:46 smithi139 bash[43301]: cluster 2022-04-28T21:25:45.724555+0000 mgr.y (mgr.24961) 3149 : cluster [DBG] pgmap v3475: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:48 smithi059 bash[59745]: cluster 2022-04-28T21:25:47.725550+0000 mgr.y (mgr.24961) 3150 : cluster [DBG] pgmap v3476: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:48 smithi059 bash[61152]: cluster 2022-04-28T21:25:47.725550+0000 mgr.y (mgr.24961) 3150 : cluster [DBG] pgmap v3476: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:48 smithi139 bash[43301]: cluster 2022-04-28T21:25:47.725550+0000 mgr.y (mgr.24961) 3150 : cluster [DBG] pgmap v3476: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:50 smithi059 bash[59745]: cluster 2022-04-28T21:25:49.726685+0000 mgr.y (mgr.24961) 3151 : cluster [DBG] pgmap v3477: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:50 smithi059 bash[61152]: cluster 2022-04-28T21:25:49.726685+0000 mgr.y (mgr.24961) 3151 : cluster [DBG] pgmap v3477: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:50 smithi139 bash[43301]: cluster 2022-04-28T21:25:49.726685+0000 mgr.y (mgr.24961) 3151 : cluster [DBG] pgmap v3477: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:51.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:52 smithi059 bash[59745]: cluster 2022-04-28T21:25:51.728000+0000 mgr.y (mgr.24961) 3152 : cluster [DBG] pgmap v3478: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:52 smithi059 bash[61152]: cluster 2022-04-28T21:25:51.728000+0000 mgr.y (mgr.24961) 3152 : cluster [DBG] pgmap v3478: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:52 smithi139 bash[43301]: cluster 2022-04-28T21:25:51.728000+0000 mgr.y (mgr.24961) 3152 : cluster [DBG] pgmap v3478: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:25:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:25:53.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:25:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:54 smithi059 bash[61152]: cluster 2022-04-28T21:25:53.728727+0000 mgr.y (mgr.24961) 3153 : cluster [DBG] pgmap v3479: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:55.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:54 smithi059 bash[59745]: cluster 2022-04-28T21:25:53.728727+0000 mgr.y (mgr.24961) 3153 : cluster [DBG] pgmap v3479: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:54 smithi139 bash[43301]: cluster 2022-04-28T21:25:53.728727+0000 mgr.y (mgr.24961) 3153 : cluster [DBG] pgmap v3479: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:56 smithi059 bash[61152]: cluster 2022-04-28T21:25:55.730158+0000 mgr.y (mgr.24961) 3154 : cluster [DBG] pgmap v3480: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:56 smithi059 bash[59745]: cluster 2022-04-28T21:25:55.730158+0000 mgr.y (mgr.24961) 3154 : cluster [DBG] pgmap v3480: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:56 smithi139 bash[43301]: cluster 2022-04-28T21:25:55.730158+0000 mgr.y (mgr.24961) 3154 : cluster [DBG] pgmap v3480: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:25:58 smithi059 bash[59745]: cluster 2022-04-28T21:25:57.731131+0000 mgr.y (mgr.24961) 3155 : cluster [DBG] pgmap v3481: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:25:58 smithi059 bash[61152]: cluster 2022-04-28T21:25:57.731131+0000 mgr.y (mgr.24961) 3155 : cluster [DBG] pgmap v3481: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:25:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:25:58 smithi139 bash[43301]: cluster 2022-04-28T21:25:57.731131+0000 mgr.y (mgr.24961) 3155 : cluster [DBG] pgmap v3481: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:00 smithi059 bash[59745]: cluster 2022-04-28T21:25:59.732381+0000 mgr.y (mgr.24961) 3156 : cluster [DBG] pgmap v3482: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:00 smithi059 bash[61152]: cluster 2022-04-28T21:25:59.732381+0000 mgr.y (mgr.24961) 3156 : cluster [DBG] pgmap v3482: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:00 smithi139 bash[43301]: cluster 2022-04-28T21:25:59.732381+0000 mgr.y (mgr.24961) 3156 : cluster [DBG] pgmap v3482: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:01.994Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:02 smithi059 bash[59745]: cluster 2022-04-28T21:26:01.733703+0000 mgr.y (mgr.24961) 3157 : cluster [DBG] pgmap v3483: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:02 smithi059 bash[61152]: cluster 2022-04-28T21:26:01.733703+0000 mgr.y (mgr.24961) 3157 : cluster [DBG] pgmap v3483: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:03.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:02 smithi139 bash[43301]: cluster 2022-04-28T21:26:01.733703+0000 mgr.y (mgr.24961) 3157 : cluster [DBG] pgmap v3483: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:03.889Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:04 smithi059 bash[59745]: cluster 2022-04-28T21:26:03.734378+0000 mgr.y (mgr.24961) 3158 : cluster [DBG] pgmap v3484: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:04 smithi059 bash[61152]: cluster 2022-04-28T21:26:03.734378+0000 mgr.y (mgr.24961) 3158 : cluster [DBG] pgmap v3484: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:04 smithi139 bash[43301]: cluster 2022-04-28T21:26:03.734378+0000 mgr.y (mgr.24961) 3158 : cluster [DBG] pgmap v3484: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:06 smithi059 bash[61152]: cluster 2022-04-28T21:26:05.735823+0000 mgr.y (mgr.24961) 3159 : cluster [DBG] pgmap v3485: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:06 smithi059 bash[59745]: cluster 2022-04-28T21:26:05.735823+0000 mgr.y (mgr.24961) 3159 : cluster [DBG] pgmap v3485: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:06 smithi139 bash[43301]: cluster 2022-04-28T21:26:05.735823+0000 mgr.y (mgr.24961) 3159 : cluster [DBG] pgmap v3485: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:08 smithi059 bash[59745]: cluster 2022-04-28T21:26:07.736786+0000 mgr.y (mgr.24961) 3160 : cluster [DBG] pgmap v3486: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:08 smithi059 bash[61152]: cluster 2022-04-28T21:26:07.736786+0000 mgr.y (mgr.24961) 3160 : cluster [DBG] pgmap v3486: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:08 smithi139 bash[43301]: cluster 2022-04-28T21:26:07.736786+0000 mgr.y (mgr.24961) 3160 : cluster [DBG] pgmap v3486: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:10 smithi059 bash[59745]: cluster 2022-04-28T21:26:09.737882+0000 mgr.y (mgr.24961) 3161 : cluster [DBG] pgmap v3487: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:10 smithi059 bash[61152]: cluster 2022-04-28T21:26:09.737882+0000 mgr.y (mgr.24961) 3161 : cluster [DBG] pgmap v3487: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:10 smithi139 bash[43301]: cluster 2022-04-28T21:26:09.737882+0000 mgr.y (mgr.24961) 3161 : cluster [DBG] pgmap v3487: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:12.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:11 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:11.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:12 smithi059 bash[59745]: cluster 2022-04-28T21:26:11.739160+0000 mgr.y (mgr.24961) 3162 : cluster [DBG] pgmap v3488: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:12 smithi059 bash[61152]: cluster 2022-04-28T21:26:11.739160+0000 mgr.y (mgr.24961) 3162 : cluster [DBG] pgmap v3488: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:12 smithi139 bash[43301]: cluster 2022-04-28T21:26:11.739160+0000 mgr.y (mgr.24961) 3162 : cluster [DBG] pgmap v3488: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:13.891Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:14 smithi059 bash[61152]: cluster 2022-04-28T21:26:13.739925+0000 mgr.y (mgr.24961) 3163 : cluster [DBG] pgmap v3489: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:15.190 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:14 smithi059 bash[59745]: cluster 2022-04-28T21:26:13.739925+0000 mgr.y (mgr.24961) 3163 : cluster [DBG] pgmap v3489: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:15.191 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:14 smithi139 bash[43301]: cluster 2022-04-28T21:26:13.739925+0000 mgr.y (mgr.24961) 3163 : cluster [DBG] pgmap v3489: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:16 smithi059 bash[59745]: cluster 2022-04-28T21:26:15.741300+0000 mgr.y (mgr.24961) 3164 : cluster [DBG] pgmap v3490: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:16 smithi059 bash[61152]: cluster 2022-04-28T21:26:15.741300+0000 mgr.y (mgr.24961) 3164 : cluster [DBG] pgmap v3490: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:16 smithi139 bash[43301]: cluster 2022-04-28T21:26:15.741300+0000 mgr.y (mgr.24961) 3164 : cluster [DBG] pgmap v3490: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: cluster 2022-04-28T21:26:17.742286+0000 mgr.y (mgr.24961) 3165 : cluster [DBG] pgmap v3491: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: audit 2022-04-28T21:26:18.350997+0000 mon.a (mon.0) 2725 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:26:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: audit 2022-04-28T21:26:18.351250+0000 mon.a (mon.0) 2726 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:26:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: audit 2022-04-28T21:26:18.364124+0000 mon.a (mon.0) 2727 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:26:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: audit 2022-04-28T21:26:18.364376+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:26:19.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: audit 2022-04-28T21:26:18.364945+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:26:19.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[59745]: audit 2022-04-28T21:26:18.365526+0000 mon.a (mon.0) 2730 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:26:19.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: cluster 2022-04-28T21:26:17.742286+0000 mgr.y (mgr.24961) 3165 : cluster [DBG] pgmap v3491: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:19.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: audit 2022-04-28T21:26:18.350997+0000 mon.a (mon.0) 2725 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:26:19.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: audit 2022-04-28T21:26:18.351250+0000 mon.a (mon.0) 2726 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:26:19.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: audit 2022-04-28T21:26:18.364124+0000 mon.a (mon.0) 2727 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:26:19.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: audit 2022-04-28T21:26:18.364376+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:26:19.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: audit 2022-04-28T21:26:18.364945+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:26:19.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:18 smithi059 bash[61152]: audit 2022-04-28T21:26:18.365526+0000 mon.a (mon.0) 2730 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:26:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: cluster 2022-04-28T21:26:17.742286+0000 mgr.y (mgr.24961) 3165 : cluster [DBG] pgmap v3491: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: audit 2022-04-28T21:26:18.350997+0000 mon.a (mon.0) 2725 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:26:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: audit 2022-04-28T21:26:18.351250+0000 mon.a (mon.0) 2726 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:26:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: audit 2022-04-28T21:26:18.364124+0000 mon.a (mon.0) 2727 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:26:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: audit 2022-04-28T21:26:18.364376+0000 mon.a (mon.0) 2728 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:26:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: audit 2022-04-28T21:26:18.364945+0000 mon.a (mon.0) 2729 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:26:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:18 smithi139 bash[43301]: audit 2022-04-28T21:26:18.365526+0000 mon.a (mon.0) 2730 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T21:26:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:20 smithi059 bash[59745]: cluster 2022-04-28T21:26:19.743752+0000 mgr.y (mgr.24961) 3166 : cluster [DBG] pgmap v3492: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:20 smithi059 bash[61152]: cluster 2022-04-28T21:26:19.743752+0000 mgr.y (mgr.24961) 3166 : cluster [DBG] pgmap v3492: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:20 smithi139 bash[43301]: cluster 2022-04-28T21:26:19.743752+0000 mgr.y (mgr.24961) 3166 : cluster [DBG] pgmap v3492: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:21.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:22 smithi059 bash[59745]: cluster 2022-04-28T21:26:21.744941+0000 mgr.y (mgr.24961) 3167 : cluster [DBG] pgmap v3493: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:22 smithi059 bash[61152]: cluster 2022-04-28T21:26:21.744941+0000 mgr.y (mgr.24961) 3167 : cluster [DBG] pgmap v3493: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:22 smithi139 bash[43301]: cluster 2022-04-28T21:26:21.744941+0000 mgr.y (mgr.24961) 3167 : cluster [DBG] pgmap v3493: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:23.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:25.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:24 smithi059 bash[59745]: cluster 2022-04-28T21:26:23.745689+0000 mgr.y (mgr.24961) 3168 : cluster [DBG] pgmap v3494: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:25.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:24 smithi059 bash[61152]: cluster 2022-04-28T21:26:23.745689+0000 mgr.y (mgr.24961) 3168 : cluster [DBG] pgmap v3494: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:25.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:24 smithi139 bash[43301]: cluster 2022-04-28T21:26:23.745689+0000 mgr.y (mgr.24961) 3168 : cluster [DBG] pgmap v3494: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:27.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:26 smithi059 bash[59745]: cluster 2022-04-28T21:26:25.747104+0000 mgr.y (mgr.24961) 3169 : cluster [DBG] pgmap v3495: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:27.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:26 smithi059 bash[61152]: cluster 2022-04-28T21:26:25.747104+0000 mgr.y (mgr.24961) 3169 : cluster [DBG] pgmap v3495: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:26 smithi139 bash[43301]: cluster 2022-04-28T21:26:25.747104+0000 mgr.y (mgr.24961) 3169 : cluster [DBG] pgmap v3495: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:29.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:28 smithi059 bash[59745]: cluster 2022-04-28T21:26:27.748188+0000 mgr.y (mgr.24961) 3170 : cluster [DBG] pgmap v3496: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:28 smithi059 bash[61152]: cluster 2022-04-28T21:26:27.748188+0000 mgr.y (mgr.24961) 3170 : cluster [DBG] pgmap v3496: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:28 smithi139 bash[43301]: cluster 2022-04-28T21:26:27.748188+0000 mgr.y (mgr.24961) 3170 : cluster [DBG] pgmap v3496: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:30 smithi059 bash[59745]: cluster 2022-04-28T21:26:29.749364+0000 mgr.y (mgr.24961) 3171 : cluster [DBG] pgmap v3497: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:30 smithi059 bash[61152]: cluster 2022-04-28T21:26:29.749364+0000 mgr.y (mgr.24961) 3171 : cluster [DBG] pgmap v3497: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:30 smithi139 bash[43301]: cluster 2022-04-28T21:26:29.749364+0000 mgr.y (mgr.24961) 3171 : cluster [DBG] pgmap v3497: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:31 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:31.997Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:32 smithi059 bash[59745]: cluster 2022-04-28T21:26:31.750722+0000 mgr.y (mgr.24961) 3172 : cluster [DBG] pgmap v3498: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:32 smithi059 bash[61152]: cluster 2022-04-28T21:26:31.750722+0000 mgr.y (mgr.24961) 3172 : cluster [DBG] pgmap v3498: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:32 smithi139 bash[43301]: cluster 2022-04-28T21:26:31.750722+0000 mgr.y (mgr.24961) 3172 : cluster [DBG] pgmap v3498: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:33.892Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:35.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:34 smithi059 bash[59745]: cluster 2022-04-28T21:26:33.751472+0000 mgr.y (mgr.24961) 3173 : cluster [DBG] pgmap v3499: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:34 smithi059 bash[61152]: cluster 2022-04-28T21:26:33.751472+0000 mgr.y (mgr.24961) 3173 : cluster [DBG] pgmap v3499: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:34 smithi139 bash[43301]: cluster 2022-04-28T21:26:33.751472+0000 mgr.y (mgr.24961) 3173 : cluster [DBG] pgmap v3499: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:37.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:36 smithi059 bash[59745]: cluster 2022-04-28T21:26:35.752851+0000 mgr.y (mgr.24961) 3174 : cluster [DBG] pgmap v3500: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:37.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:36 smithi059 bash[61152]: cluster 2022-04-28T21:26:35.752851+0000 mgr.y (mgr.24961) 3174 : cluster [DBG] pgmap v3500: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:37.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:36 smithi139 bash[43301]: cluster 2022-04-28T21:26:35.752851+0000 mgr.y (mgr.24961) 3174 : cluster [DBG] pgmap v3500: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:39.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:38 smithi059 bash[61152]: cluster 2022-04-28T21:26:37.753815+0000 mgr.y (mgr.24961) 3175 : cluster [DBG] pgmap v3501: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:39.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:38 smithi059 bash[59745]: cluster 2022-04-28T21:26:37.753815+0000 mgr.y (mgr.24961) 3175 : cluster [DBG] pgmap v3501: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:38 smithi139 bash[43301]: cluster 2022-04-28T21:26:37.753815+0000 mgr.y (mgr.24961) 3175 : cluster [DBG] pgmap v3501: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[59745]: cluster 2022-04-28T21:26:39.754865+0000 mgr.y (mgr.24961) 3176 : cluster [DBG] pgmap v3502: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[59745]: audit 2022-04-28T21:26:40.436180+0000 mon.a (mon.0) 2731 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:26:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[59745]: audit 2022-04-28T21:26:40.439173+0000 mon.a (mon.0) 2732 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:26:41.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[59745]: audit 2022-04-28T21:26:40.441190+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:26:41.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[59745]: cephadm 2022-04-28T21:26:40.443043+0000 mgr.y (mgr.24961) 3177 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:26:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[61152]: cluster 2022-04-28T21:26:39.754865+0000 mgr.y (mgr.24961) 3176 : cluster [DBG] pgmap v3502: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[61152]: audit 2022-04-28T21:26:40.436180+0000 mon.a (mon.0) 2731 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:26:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[61152]: audit 2022-04-28T21:26:40.439173+0000 mon.a (mon.0) 2732 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:26:41.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[61152]: audit 2022-04-28T21:26:40.441190+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:26:41.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:40 smithi059 bash[61152]: cephadm 2022-04-28T21:26:40.443043+0000 mgr.y (mgr.24961) 3177 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:26:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:40 smithi139 bash[43301]: cluster 2022-04-28T21:26:39.754865+0000 mgr.y (mgr.24961) 3176 : cluster [DBG] pgmap v3502: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:41.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:40 smithi139 bash[43301]: audit 2022-04-28T21:26:40.436180+0000 mon.a (mon.0) 2731 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:26:41.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:40 smithi139 bash[43301]: audit 2022-04-28T21:26:40.439173+0000 mon.a (mon.0) 2732 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:26:41.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:40 smithi139 bash[43301]: audit 2022-04-28T21:26:40.441190+0000 mon.a (mon.0) 2733 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:26:41.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:40 smithi139 bash[43301]: cephadm 2022-04-28T21:26:40.443043+0000 mgr.y (mgr.24961) 3177 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:26:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:41 smithi059 bash[61152]: audit 2022-04-28T21:26:40.767820+0000 mon.a (mon.0) 2734 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:26:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:41 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:41.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:41 smithi059 bash[59745]: audit 2022-04-28T21:26:40.767820+0000 mon.a (mon.0) 2734 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:26:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:41 smithi139 bash[43301]: audit 2022-04-28T21:26:40.767820+0000 mon.a (mon.0) 2734 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:26:43.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:42 smithi059 bash[61152]: cluster 2022-04-28T21:26:41.756068+0000 mgr.y (mgr.24961) 3178 : cluster [DBG] pgmap v3503: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:42 smithi059 bash[59745]: cluster 2022-04-28T21:26:41.756068+0000 mgr.y (mgr.24961) 3178 : cluster [DBG] pgmap v3503: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:42 smithi139 bash[43301]: cluster 2022-04-28T21:26:41.756068+0000 mgr.y (mgr.24961) 3178 : cluster [DBG] pgmap v3503: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:43.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:44 smithi059 bash[61152]: cluster 2022-04-28T21:26:43.756815+0000 mgr.y (mgr.24961) 3179 : cluster [DBG] pgmap v3504: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:45.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:44 smithi059 bash[59745]: cluster 2022-04-28T21:26:43.756815+0000 mgr.y (mgr.24961) 3179 : cluster [DBG] pgmap v3504: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:44 smithi139 bash[43301]: cluster 2022-04-28T21:26:43.756815+0000 mgr.y (mgr.24961) 3179 : cluster [DBG] pgmap v3504: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:46 smithi059 bash[61152]: cluster 2022-04-28T21:26:45.758178+0000 mgr.y (mgr.24961) 3180 : cluster [DBG] pgmap v3505: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:47.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:46 smithi059 bash[59745]: cluster 2022-04-28T21:26:45.758178+0000 mgr.y (mgr.24961) 3180 : cluster [DBG] pgmap v3505: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:47.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:46 smithi139 bash[43301]: cluster 2022-04-28T21:26:45.758178+0000 mgr.y (mgr.24961) 3180 : cluster [DBG] pgmap v3505: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:48 smithi059 bash[61152]: cluster 2022-04-28T21:26:47.759155+0000 mgr.y (mgr.24961) 3181 : cluster [DBG] pgmap v3506: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:48 smithi059 bash[59745]: cluster 2022-04-28T21:26:47.759155+0000 mgr.y (mgr.24961) 3181 : cluster [DBG] pgmap v3506: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:49.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:48 smithi139 bash[43301]: cluster 2022-04-28T21:26:47.759155+0000 mgr.y (mgr.24961) 3181 : cluster [DBG] pgmap v3506: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:50 smithi059 bash[61152]: cluster 2022-04-28T21:26:49.760488+0000 mgr.y (mgr.24961) 3182 : cluster [DBG] pgmap v3507: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:50 smithi059 bash[59745]: cluster 2022-04-28T21:26:49.760488+0000 mgr.y (mgr.24961) 3182 : cluster [DBG] pgmap v3507: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:50 smithi139 bash[43301]: cluster 2022-04-28T21:26:49.760488+0000 mgr.y (mgr.24961) 3182 : cluster [DBG] pgmap v3507: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:51 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:51.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:52 smithi059 bash[61152]: cluster 2022-04-28T21:26:51.761706+0000 mgr.y (mgr.24961) 3183 : cluster [DBG] pgmap v3508: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:52 smithi059 bash[59745]: cluster 2022-04-28T21:26:51.761706+0000 mgr.y (mgr.24961) 3183 : cluster [DBG] pgmap v3508: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:52 smithi139 bash[43301]: cluster 2022-04-28T21:26:51.761706+0000 mgr.y (mgr.24961) 3183 : cluster [DBG] pgmap v3508: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:26:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:26:53.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:26:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:54 smithi059 bash[61152]: cluster 2022-04-28T21:26:53.762424+0000 mgr.y (mgr.24961) 3184 : cluster [DBG] pgmap v3509: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:54 smithi059 bash[59745]: cluster 2022-04-28T21:26:53.762424+0000 mgr.y (mgr.24961) 3184 : cluster [DBG] pgmap v3509: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:54 smithi139 bash[43301]: cluster 2022-04-28T21:26:53.762424+0000 mgr.y (mgr.24961) 3184 : cluster [DBG] pgmap v3509: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:56 smithi059 bash[59745]: cluster 2022-04-28T21:26:55.763834+0000 mgr.y (mgr.24961) 3185 : cluster [DBG] pgmap v3510: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:57.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:56 smithi059 bash[61152]: cluster 2022-04-28T21:26:55.763834+0000 mgr.y (mgr.24961) 3185 : cluster [DBG] pgmap v3510: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:57.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:56 smithi139 bash[43301]: cluster 2022-04-28T21:26:55.763834+0000 mgr.y (mgr.24961) 3185 : cluster [DBG] pgmap v3510: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:26:58 smithi059 bash[61152]: cluster 2022-04-28T21:26:57.764829+0000 mgr.y (mgr.24961) 3186 : cluster [DBG] pgmap v3511: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:59.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:26:58 smithi059 bash[59745]: cluster 2022-04-28T21:26:57.764829+0000 mgr.y (mgr.24961) 3186 : cluster [DBG] pgmap v3511: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:26:59.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:26:58 smithi139 bash[43301]: cluster 2022-04-28T21:26:57.764829+0000 mgr.y (mgr.24961) 3186 : cluster [DBG] pgmap v3511: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:00 smithi059 bash[59745]: cluster 2022-04-28T21:26:59.765683+0000 mgr.y (mgr.24961) 3187 : cluster [DBG] pgmap v3512: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:01.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:00 smithi059 bash[61152]: cluster 2022-04-28T21:26:59.765683+0000 mgr.y (mgr.24961) 3187 : cluster [DBG] pgmap v3512: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:00 smithi139 bash[43301]: cluster 2022-04-28T21:26:59.765683+0000 mgr.y (mgr.24961) 3187 : cluster [DBG] pgmap v3512: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:01 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:01.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:02 smithi059 bash[61152]: cluster 2022-04-28T21:27:01.766905+0000 mgr.y (mgr.24961) 3188 : cluster [DBG] pgmap v3513: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:02 smithi059 bash[59745]: cluster 2022-04-28T21:27:01.766905+0000 mgr.y (mgr.24961) 3188 : cluster [DBG] pgmap v3513: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:02 smithi139 bash[43301]: cluster 2022-04-28T21:27:01.766905+0000 mgr.y (mgr.24961) 3188 : cluster [DBG] pgmap v3513: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:03.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:04 smithi059 bash[61152]: cluster 2022-04-28T21:27:03.767638+0000 mgr.y (mgr.24961) 3189 : cluster [DBG] pgmap v3514: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:05.106 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:04 smithi059 bash[59745]: cluster 2022-04-28T21:27:03.767638+0000 mgr.y (mgr.24961) 3189 : cluster [DBG] pgmap v3514: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:05.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:04 smithi139 bash[43301]: cluster 2022-04-28T21:27:03.767638+0000 mgr.y (mgr.24961) 3189 : cluster [DBG] pgmap v3514: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:06 smithi059 bash[61152]: cluster 2022-04-28T21:27:05.769045+0000 mgr.y (mgr.24961) 3190 : cluster [DBG] pgmap v3515: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:06 smithi059 bash[59745]: cluster 2022-04-28T21:27:05.769045+0000 mgr.y (mgr.24961) 3190 : cluster [DBG] pgmap v3515: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:06 smithi139 bash[43301]: cluster 2022-04-28T21:27:05.769045+0000 mgr.y (mgr.24961) 3190 : cluster [DBG] pgmap v3515: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:08 smithi059 bash[61152]: cluster 2022-04-28T21:27:07.770027+0000 mgr.y (mgr.24961) 3191 : cluster [DBG] pgmap v3516: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:08 smithi059 bash[59745]: cluster 2022-04-28T21:27:07.770027+0000 mgr.y (mgr.24961) 3191 : cluster [DBG] pgmap v3516: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:08 smithi139 bash[43301]: cluster 2022-04-28T21:27:07.770027+0000 mgr.y (mgr.24961) 3191 : cluster [DBG] pgmap v3516: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:10 smithi059 bash[61152]: cluster 2022-04-28T21:27:09.771066+0000 mgr.y (mgr.24961) 3192 : cluster [DBG] pgmap v3517: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:10 smithi059 bash[59745]: cluster 2022-04-28T21:27:09.771066+0000 mgr.y (mgr.24961) 3192 : cluster [DBG] pgmap v3517: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:10 smithi139 bash[43301]: cluster 2022-04-28T21:27:09.771066+0000 mgr.y (mgr.24961) 3192 : cluster [DBG] pgmap v3517: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:12.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:12 smithi059 bash[59745]: cluster 2022-04-28T21:27:11.772300+0000 mgr.y (mgr.24961) 3193 : cluster [DBG] pgmap v3518: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:12 smithi059 bash[61152]: cluster 2022-04-28T21:27:11.772300+0000 mgr.y (mgr.24961) 3193 : cluster [DBG] pgmap v3518: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:12 smithi139 bash[43301]: cluster 2022-04-28T21:27:11.772300+0000 mgr.y (mgr.24961) 3193 : cluster [DBG] pgmap v3518: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:13.893Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:14 smithi059 bash[61152]: cluster 2022-04-28T21:27:13.773017+0000 mgr.y (mgr.24961) 3194 : cluster [DBG] pgmap v3519: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:14 smithi059 bash[59745]: cluster 2022-04-28T21:27:13.773017+0000 mgr.y (mgr.24961) 3194 : cluster [DBG] pgmap v3519: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:14 smithi139 bash[43301]: cluster 2022-04-28T21:27:13.773017+0000 mgr.y (mgr.24961) 3194 : cluster [DBG] pgmap v3519: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:16 smithi139 bash[43301]: cluster 2022-04-28T21:27:15.774362+0000 mgr.y (mgr.24961) 3195 : cluster [DBG] pgmap v3520: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:16 smithi059 bash[61152]: cluster 2022-04-28T21:27:15.774362+0000 mgr.y (mgr.24961) 3195 : cluster [DBG] pgmap v3520: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:16 smithi059 bash[59745]: cluster 2022-04-28T21:27:15.774362+0000 mgr.y (mgr.24961) 3195 : cluster [DBG] pgmap v3520: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: cluster 2022-04-28T21:27:17.775386+0000 mgr.y (mgr.24961) 3196 : cluster [DBG] pgmap v3521: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: audit 2022-04-28T21:27:18.366079+0000 mon.a (mon.0) 2735 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:27:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: audit 2022-04-28T21:27:18.366620+0000 mon.a (mon.0) 2736 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:27:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: audit 2022-04-28T21:27:18.377086+0000 mon.a (mon.0) 2737 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:27:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: audit 2022-04-28T21:27:18.377743+0000 mon.a (mon.0) 2738 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:27:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: audit 2022-04-28T21:27:18.378455+0000 mon.a (mon.0) 2739 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:27:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:18 smithi139 bash[43301]: audit 2022-04-28T21:27:18.379123+0000 mon.a (mon.0) 2740 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:27:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: cluster 2022-04-28T21:27:17.775386+0000 mgr.y (mgr.24961) 3196 : cluster [DBG] pgmap v3521: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: audit 2022-04-28T21:27:18.366079+0000 mon.a (mon.0) 2735 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:27:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: audit 2022-04-28T21:27:18.366620+0000 mon.a (mon.0) 2736 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:27:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: audit 2022-04-28T21:27:18.377086+0000 mon.a (mon.0) 2737 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:27:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: audit 2022-04-28T21:27:18.377743+0000 mon.a (mon.0) 2738 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:27:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: audit 2022-04-28T21:27:18.378455+0000 mon.a (mon.0) 2739 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:27:19.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[61152]: audit 2022-04-28T21:27:18.379123+0000 mon.a (mon.0) 2740 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:27:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: cluster 2022-04-28T21:27:17.775386+0000 mgr.y (mgr.24961) 3196 : cluster [DBG] pgmap v3521: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: audit 2022-04-28T21:27:18.366079+0000 mon.a (mon.0) 2735 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:27:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: audit 2022-04-28T21:27:18.366620+0000 mon.a (mon.0) 2736 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:27:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: audit 2022-04-28T21:27:18.377086+0000 mon.a (mon.0) 2737 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:27:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: audit 2022-04-28T21:27:18.377743+0000 mon.a (mon.0) 2738 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:27:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: audit 2022-04-28T21:27:18.378455+0000 mon.a (mon.0) 2739 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:27:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:18 smithi059 bash[59745]: audit 2022-04-28T21:27:18.379123+0000 mon.a (mon.0) 2740 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:27:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:20 smithi059 bash[61152]: cluster 2022-04-28T21:27:19.776151+0000 mgr.y (mgr.24961) 3197 : cluster [DBG] pgmap v3522: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:20 smithi059 bash[59745]: cluster 2022-04-28T21:27:19.776151+0000 mgr.y (mgr.24961) 3197 : cluster [DBG] pgmap v3522: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:20 smithi139 bash[43301]: cluster 2022-04-28T21:27:19.776151+0000 mgr.y (mgr.24961) 3197 : cluster [DBG] pgmap v3522: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:21 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:21.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:22 smithi059 bash[61152]: cluster 2022-04-28T21:27:21.777371+0000 mgr.y (mgr.24961) 3198 : cluster [DBG] pgmap v3523: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:22 smithi059 bash[59745]: cluster 2022-04-28T21:27:21.777371+0000 mgr.y (mgr.24961) 3198 : cluster [DBG] pgmap v3523: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:22 smithi139 bash[43301]: cluster 2022-04-28T21:27:21.777371+0000 mgr.y (mgr.24961) 3198 : cluster [DBG] pgmap v3523: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:23.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:24 smithi059 bash[61152]: cluster 2022-04-28T21:27:23.778089+0000 mgr.y (mgr.24961) 3199 : cluster [DBG] pgmap v3524: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:24 smithi059 bash[59745]: cluster 2022-04-28T21:27:23.778089+0000 mgr.y (mgr.24961) 3199 : cluster [DBG] pgmap v3524: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:25.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:24 smithi139 bash[43301]: cluster 2022-04-28T21:27:23.778089+0000 mgr.y (mgr.24961) 3199 : cluster [DBG] pgmap v3524: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:27.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:26 smithi139 bash[43301]: cluster 2022-04-28T21:27:25.779510+0000 mgr.y (mgr.24961) 3200 : cluster [DBG] pgmap v3525: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:26 smithi059 bash[61152]: cluster 2022-04-28T21:27:25.779510+0000 mgr.y (mgr.24961) 3200 : cluster [DBG] pgmap v3525: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:26 smithi059 bash[59745]: cluster 2022-04-28T21:27:25.779510+0000 mgr.y (mgr.24961) 3200 : cluster [DBG] pgmap v3525: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:29.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:28 smithi139 bash[43301]: cluster 2022-04-28T21:27:27.780476+0000 mgr.y (mgr.24961) 3201 : cluster [DBG] pgmap v3526: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:28 smithi059 bash[61152]: cluster 2022-04-28T21:27:27.780476+0000 mgr.y (mgr.24961) 3201 : cluster [DBG] pgmap v3526: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:28 smithi059 bash[59745]: cluster 2022-04-28T21:27:27.780476+0000 mgr.y (mgr.24961) 3201 : cluster [DBG] pgmap v3526: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:30 smithi059 bash[61152]: cluster 2022-04-28T21:27:29.781524+0000 mgr.y (mgr.24961) 3202 : cluster [DBG] pgmap v3527: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:30 smithi059 bash[59745]: cluster 2022-04-28T21:27:29.781524+0000 mgr.y (mgr.24961) 3202 : cluster [DBG] pgmap v3527: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:30 smithi139 bash[43301]: cluster 2022-04-28T21:27:29.781524+0000 mgr.y (mgr.24961) 3202 : cluster [DBG] pgmap v3527: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:32.001Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:32 smithi059 bash[61152]: cluster 2022-04-28T21:27:31.782766+0000 mgr.y (mgr.24961) 3203 : cluster [DBG] pgmap v3528: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:32 smithi059 bash[59745]: cluster 2022-04-28T21:27:31.782766+0000 mgr.y (mgr.24961) 3203 : cluster [DBG] pgmap v3528: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:32 smithi139 bash[43301]: cluster 2022-04-28T21:27:31.782766+0000 mgr.y (mgr.24961) 3203 : cluster [DBG] pgmap v3528: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:33.895Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:35.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:34 smithi139 bash[43301]: cluster 2022-04-28T21:27:33.783602+0000 mgr.y (mgr.24961) 3204 : cluster [DBG] pgmap v3529: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:34 smithi059 bash[61152]: cluster 2022-04-28T21:27:33.783602+0000 mgr.y (mgr.24961) 3204 : cluster [DBG] pgmap v3529: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:34 smithi059 bash[59745]: cluster 2022-04-28T21:27:33.783602+0000 mgr.y (mgr.24961) 3204 : cluster [DBG] pgmap v3529: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:37.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:36 smithi139 bash[43301]: cluster 2022-04-28T21:27:35.785022+0000 mgr.y (mgr.24961) 3205 : cluster [DBG] pgmap v3530: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:36 smithi059 bash[61152]: cluster 2022-04-28T21:27:35.785022+0000 mgr.y (mgr.24961) 3205 : cluster [DBG] pgmap v3530: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:36 smithi059 bash[59745]: cluster 2022-04-28T21:27:35.785022+0000 mgr.y (mgr.24961) 3205 : cluster [DBG] pgmap v3530: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:39.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:38 smithi139 bash[43301]: cluster 2022-04-28T21:27:37.786022+0000 mgr.y (mgr.24961) 3206 : cluster [DBG] pgmap v3531: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:38 smithi059 bash[61152]: cluster 2022-04-28T21:27:37.786022+0000 mgr.y (mgr.24961) 3206 : cluster [DBG] pgmap v3531: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:38 smithi059 bash[59745]: cluster 2022-04-28T21:27:37.786022+0000 mgr.y (mgr.24961) 3206 : cluster [DBG] pgmap v3531: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:40 smithi059 bash[59745]: cluster 2022-04-28T21:27:39.786930+0000 mgr.y (mgr.24961) 3207 : cluster [DBG] pgmap v3532: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:41.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:40 smithi059 bash[61152]: cluster 2022-04-28T21:27:39.786930+0000 mgr.y (mgr.24961) 3207 : cluster [DBG] pgmap v3532: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:40 smithi139 bash[43301]: cluster 2022-04-28T21:27:39.786930+0000 mgr.y (mgr.24961) 3207 : cluster [DBG] pgmap v3532: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:42.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:42.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[59745]: audit 2022-04-28T21:27:40.773769+0000 mon.a (mon.0) 2741 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:27:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[59745]: audit 2022-04-28T21:27:40.780553+0000 mon.a (mon.0) 2742 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:27:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[59745]: audit 2022-04-28T21:27:40.782441+0000 mon.a (mon.0) 2743 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:27:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[59745]: cephadm 2022-04-28T21:27:40.783984+0000 mgr.y (mgr.24961) 3208 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:27:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[59745]: audit 2022-04-28T21:27:40.785301+0000 mon.a (mon.0) 2744 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:27:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[59745]: audit 2022-04-28T21:27:40.785970+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:27:42.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[61152]: audit 2022-04-28T21:27:40.773769+0000 mon.a (mon.0) 2741 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:27:42.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[61152]: audit 2022-04-28T21:27:40.780553+0000 mon.a (mon.0) 2742 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:27:42.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[61152]: audit 2022-04-28T21:27:40.782441+0000 mon.a (mon.0) 2743 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:27:42.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[61152]: cephadm 2022-04-28T21:27:40.783984+0000 mgr.y (mgr.24961) 3208 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:27:42.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[61152]: audit 2022-04-28T21:27:40.785301+0000 mon.a (mon.0) 2744 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:27:42.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:41 smithi059 bash[61152]: audit 2022-04-28T21:27:40.785970+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:27:42.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:41 smithi139 bash[43301]: audit 2022-04-28T21:27:40.773769+0000 mon.a (mon.0) 2741 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:27:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:41 smithi139 bash[43301]: audit 2022-04-28T21:27:40.780553+0000 mon.a (mon.0) 2742 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:27:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:41 smithi139 bash[43301]: audit 2022-04-28T21:27:40.782441+0000 mon.a (mon.0) 2743 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:27:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:41 smithi139 bash[43301]: cephadm 2022-04-28T21:27:40.783984+0000 mgr.y (mgr.24961) 3208 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:27:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:41 smithi139 bash[43301]: audit 2022-04-28T21:27:40.785301+0000 mon.a (mon.0) 2744 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:27:42.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:41 smithi139 bash[43301]: audit 2022-04-28T21:27:40.785970+0000 mon.a (mon.0) 2745 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:27:43.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:42 smithi059 bash[61152]: cluster 2022-04-28T21:27:41.788225+0000 mgr.y (mgr.24961) 3209 : cluster [DBG] pgmap v3533: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:43.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:42 smithi059 bash[59745]: cluster 2022-04-28T21:27:41.788225+0000 mgr.y (mgr.24961) 3209 : cluster [DBG] pgmap v3533: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:43.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:42 smithi139 bash[43301]: cluster 2022-04-28T21:27:41.788225+0000 mgr.y (mgr.24961) 3209 : cluster [DBG] pgmap v3533: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:43.894Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:45.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:44 smithi139 bash[43301]: cluster 2022-04-28T21:27:43.788998+0000 mgr.y (mgr.24961) 3210 : cluster [DBG] pgmap v3534: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:44 smithi059 bash[59745]: cluster 2022-04-28T21:27:43.788998+0000 mgr.y (mgr.24961) 3210 : cluster [DBG] pgmap v3534: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:44 smithi059 bash[61152]: cluster 2022-04-28T21:27:43.788998+0000 mgr.y (mgr.24961) 3210 : cluster [DBG] pgmap v3534: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:45 smithi059 bash[59745]: audit 2022-04-28T21:27:44.890566+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:45 smithi059 bash[59745]: audit 2022-04-28T21:27:45.118733+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:45 smithi059 bash[59745]: audit 2022-04-28T21:27:45.414072+0000 mon.a (mon.0) 2748 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:45 smithi059 bash[61152]: audit 2022-04-28T21:27:44.890566+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:45 smithi059 bash[61152]: audit 2022-04-28T21:27:45.118733+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:45 smithi059 bash[61152]: audit 2022-04-28T21:27:45.414072+0000 mon.a (mon.0) 2748 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:45 smithi139 bash[43301]: audit 2022-04-28T21:27:44.890566+0000 mon.a (mon.0) 2746 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:45 smithi139 bash[43301]: audit 2022-04-28T21:27:45.118733+0000 mon.a (mon.0) 2747 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:45 smithi139 bash[43301]: audit 2022-04-28T21:27:45.414072+0000 mon.a (mon.0) 2748 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:27:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:47 smithi059 bash[61152]: cluster 2022-04-28T21:27:45.790471+0000 mgr.y (mgr.24961) 3211 : cluster [DBG] pgmap v3535: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:47 smithi059 bash[59745]: cluster 2022-04-28T21:27:45.790471+0000 mgr.y (mgr.24961) 3211 : cluster [DBG] pgmap v3535: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:47 smithi139 bash[43301]: cluster 2022-04-28T21:27:45.790471+0000 mgr.y (mgr.24961) 3211 : cluster [DBG] pgmap v3535: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:49 smithi059 bash[61152]: cluster 2022-04-28T21:27:47.791456+0000 mgr.y (mgr.24961) 3212 : cluster [DBG] pgmap v3536: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:49 smithi059 bash[59745]: cluster 2022-04-28T21:27:47.791456+0000 mgr.y (mgr.24961) 3212 : cluster [DBG] pgmap v3536: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:49 smithi139 bash[43301]: cluster 2022-04-28T21:27:47.791456+0000 mgr.y (mgr.24961) 3212 : cluster [DBG] pgmap v3536: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:50 smithi059 bash[61152]: cluster 2022-04-28T21:27:49.792313+0000 mgr.y (mgr.24961) 3213 : cluster [DBG] pgmap v3537: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:50 smithi059 bash[59745]: cluster 2022-04-28T21:27:49.792313+0000 mgr.y (mgr.24961) 3213 : cluster [DBG] pgmap v3537: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:50 smithi139 bash[43301]: cluster 2022-04-28T21:27:49.792313+0000 mgr.y (mgr.24961) 3213 : cluster [DBG] pgmap v3537: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:52.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:52 smithi139 bash[43301]: cluster 2022-04-28T21:27:51.793525+0000 mgr.y (mgr.24961) 3214 : cluster [DBG] pgmap v3538: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:52 smithi059 bash[61152]: cluster 2022-04-28T21:27:51.793525+0000 mgr.y (mgr.24961) 3214 : cluster [DBG] pgmap v3538: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:52 smithi059 bash[59745]: cluster 2022-04-28T21:27:51.793525+0000 mgr.y (mgr.24961) 3214 : cluster [DBG] pgmap v3538: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:27:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:27:53.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:27:55.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:54 smithi139 bash[43301]: cluster 2022-04-28T21:27:53.794247+0000 mgr.y (mgr.24961) 3215 : cluster [DBG] pgmap v3539: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:54 smithi059 bash[61152]: cluster 2022-04-28T21:27:53.794247+0000 mgr.y (mgr.24961) 3215 : cluster [DBG] pgmap v3539: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:54 smithi059 bash[59745]: cluster 2022-04-28T21:27:53.794247+0000 mgr.y (mgr.24961) 3215 : cluster [DBG] pgmap v3539: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:57.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:56 smithi139 bash[43301]: cluster 2022-04-28T21:27:55.795940+0000 mgr.y (mgr.24961) 3216 : cluster [DBG] pgmap v3540: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:56 smithi059 bash[61152]: cluster 2022-04-28T21:27:55.795940+0000 mgr.y (mgr.24961) 3216 : cluster [DBG] pgmap v3540: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:56 smithi059 bash[59745]: cluster 2022-04-28T21:27:55.795940+0000 mgr.y (mgr.24961) 3216 : cluster [DBG] pgmap v3540: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:27:58 smithi059 bash[61152]: cluster 2022-04-28T21:27:57.796921+0000 mgr.y (mgr.24961) 3217 : cluster [DBG] pgmap v3541: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:59.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:27:58 smithi059 bash[59745]: cluster 2022-04-28T21:27:57.796921+0000 mgr.y (mgr.24961) 3217 : cluster [DBG] pgmap v3541: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:27:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:27:58 smithi139 bash[43301]: cluster 2022-04-28T21:27:57.796921+0000 mgr.y (mgr.24961) 3217 : cluster [DBG] pgmap v3541: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:00 smithi059 bash[59745]: cluster 2022-04-28T21:27:59.797874+0000 mgr.y (mgr.24961) 3218 : cluster [DBG] pgmap v3542: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:00 smithi059 bash[61152]: cluster 2022-04-28T21:27:59.797874+0000 mgr.y (mgr.24961) 3218 : cluster [DBG] pgmap v3542: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:00 smithi139 bash[43301]: cluster 2022-04-28T21:27:59.797874+0000 mgr.y (mgr.24961) 3218 : cluster [DBG] pgmap v3542: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:02.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:02 smithi139 bash[43301]: cluster 2022-04-28T21:28:01.799110+0000 mgr.y (mgr.24961) 3219 : cluster [DBG] pgmap v3543: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:02 smithi059 bash[61152]: cluster 2022-04-28T21:28:01.799110+0000 mgr.y (mgr.24961) 3219 : cluster [DBG] pgmap v3543: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:02 smithi059 bash[59745]: cluster 2022-04-28T21:28:01.799110+0000 mgr.y (mgr.24961) 3219 : cluster [DBG] pgmap v3543: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:03.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:05.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:04 smithi139 bash[43301]: cluster 2022-04-28T21:28:03.799861+0000 mgr.y (mgr.24961) 3220 : cluster [DBG] pgmap v3544: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:04 smithi059 bash[61152]: cluster 2022-04-28T21:28:03.799861+0000 mgr.y (mgr.24961) 3220 : cluster [DBG] pgmap v3544: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:04 smithi059 bash[59745]: cluster 2022-04-28T21:28:03.799861+0000 mgr.y (mgr.24961) 3220 : cluster [DBG] pgmap v3544: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:07.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:06 smithi139 bash[43301]: cluster 2022-04-28T21:28:05.801357+0000 mgr.y (mgr.24961) 3221 : cluster [DBG] pgmap v3545: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:06 smithi059 bash[61152]: cluster 2022-04-28T21:28:05.801357+0000 mgr.y (mgr.24961) 3221 : cluster [DBG] pgmap v3545: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:06 smithi059 bash[59745]: cluster 2022-04-28T21:28:05.801357+0000 mgr.y (mgr.24961) 3221 : cluster [DBG] pgmap v3545: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:08 smithi059 bash[61152]: cluster 2022-04-28T21:28:07.802451+0000 mgr.y (mgr.24961) 3222 : cluster [DBG] pgmap v3546: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:08 smithi059 bash[59745]: cluster 2022-04-28T21:28:07.802451+0000 mgr.y (mgr.24961) 3222 : cluster [DBG] pgmap v3546: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:08 smithi139 bash[43301]: cluster 2022-04-28T21:28:07.802451+0000 mgr.y (mgr.24961) 3222 : cluster [DBG] pgmap v3546: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:10 smithi059 bash[59745]: cluster 2022-04-28T21:28:09.803346+0000 mgr.y (mgr.24961) 3223 : cluster [DBG] pgmap v3547: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:11.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:10 smithi059 bash[61152]: cluster 2022-04-28T21:28:09.803346+0000 mgr.y (mgr.24961) 3223 : cluster [DBG] pgmap v3547: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:10 smithi139 bash[43301]: cluster 2022-04-28T21:28:09.803346+0000 mgr.y (mgr.24961) 3223 : cluster [DBG] pgmap v3547: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:12.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:12 smithi139 bash[43301]: cluster 2022-04-28T21:28:11.804558+0000 mgr.y (mgr.24961) 3224 : cluster [DBG] pgmap v3548: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:12 smithi059 bash[61152]: cluster 2022-04-28T21:28:11.804558+0000 mgr.y (mgr.24961) 3224 : cluster [DBG] pgmap v3548: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:12 smithi059 bash[59745]: cluster 2022-04-28T21:28:11.804558+0000 mgr.y (mgr.24961) 3224 : cluster [DBG] pgmap v3548: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:13.897Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:14 smithi059 bash[61152]: cluster 2022-04-28T21:28:13.805299+0000 mgr.y (mgr.24961) 3225 : cluster [DBG] pgmap v3549: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:14 smithi059 bash[59745]: cluster 2022-04-28T21:28:13.805299+0000 mgr.y (mgr.24961) 3225 : cluster [DBG] pgmap v3549: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:14 smithi139 bash[43301]: cluster 2022-04-28T21:28:13.805299+0000 mgr.y (mgr.24961) 3225 : cluster [DBG] pgmap v3549: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:16 smithi059 bash[61152]: cluster 2022-04-28T21:28:15.806691+0000 mgr.y (mgr.24961) 3226 : cluster [DBG] pgmap v3550: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:16 smithi059 bash[59745]: cluster 2022-04-28T21:28:15.806691+0000 mgr.y (mgr.24961) 3226 : cluster [DBG] pgmap v3550: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:16 smithi139 bash[43301]: cluster 2022-04-28T21:28:15.806691+0000 mgr.y (mgr.24961) 3226 : cluster [DBG] pgmap v3550: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: cluster 2022-04-28T21:28:17.807698+0000 mgr.y (mgr.24961) 3227 : cluster [DBG] pgmap v3551: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: audit 2022-04-28T21:28:18.379268+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:28:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: audit 2022-04-28T21:28:18.380065+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:28:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: audit 2022-04-28T21:28:18.392927+0000 mon.a (mon.0) 2751 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:28:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: audit 2022-04-28T21:28:18.393652+0000 mon.a (mon.0) 2752 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T21:28:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: audit 2022-04-28T21:28:18.394009+0000 mon.a (mon.0) 2753 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:28:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[59745]: audit 2022-04-28T21:28:18.394384+0000 mon.a (mon.0) 2754 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T21:28:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: cluster 2022-04-28T21:28:17.807698+0000 mgr.y (mgr.24961) 3227 : cluster [DBG] pgmap v3551: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: audit 2022-04-28T21:28:18.379268+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:28:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: audit 2022-04-28T21:28:18.380065+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:28:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: audit 2022-04-28T21:28:18.392927+0000 mon.a (mon.0) 2751 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:28:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: audit 2022-04-28T21:28:18.393652+0000 mon.a (mon.0) 2752 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T21:28:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: audit 2022-04-28T21:28:18.394009+0000 mon.a (mon.0) 2753 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:28:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:18 smithi059 bash[61152]: audit 2022-04-28T21:28:18.394384+0000 mon.a (mon.0) 2754 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T21:28:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: cluster 2022-04-28T21:28:17.807698+0000 mgr.y (mgr.24961) 3227 : cluster [DBG] pgmap v3551: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: audit 2022-04-28T21:28:18.379268+0000 mon.a (mon.0) 2749 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:28:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: audit 2022-04-28T21:28:18.380065+0000 mon.a (mon.0) 2750 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:28:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: audit 2022-04-28T21:28:18.392927+0000 mon.a (mon.0) 2751 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:28:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: audit 2022-04-28T21:28:18.393652+0000 mon.a (mon.0) 2752 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T21:28:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: audit 2022-04-28T21:28:18.394009+0000 mon.a (mon.0) 2753 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:28:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:18 smithi139 bash[43301]: audit 2022-04-28T21:28:18.394384+0000 mon.a (mon.0) 2754 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T21:28:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:20 smithi059 bash[61152]: cluster 2022-04-28T21:28:19.808593+0000 mgr.y (mgr.24961) 3228 : cluster [DBG] pgmap v3552: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:20 smithi059 bash[59745]: cluster 2022-04-28T21:28:19.808593+0000 mgr.y (mgr.24961) 3228 : cluster [DBG] pgmap v3552: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:20 smithi139 bash[43301]: cluster 2022-04-28T21:28:19.808593+0000 mgr.y (mgr.24961) 3228 : cluster [DBG] pgmap v3552: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:22.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:22 smithi059 bash[61152]: cluster 2022-04-28T21:28:21.809880+0000 mgr.y (mgr.24961) 3229 : cluster [DBG] pgmap v3553: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:22 smithi059 bash[59745]: cluster 2022-04-28T21:28:21.809880+0000 mgr.y (mgr.24961) 3229 : cluster [DBG] pgmap v3553: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:22 smithi139 bash[43301]: cluster 2022-04-28T21:28:21.809880+0000 mgr.y (mgr.24961) 3229 : cluster [DBG] pgmap v3553: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:23.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:24 smithi059 bash[61152]: cluster 2022-04-28T21:28:23.810686+0000 mgr.y (mgr.24961) 3230 : cluster [DBG] pgmap v3554: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:24 smithi059 bash[59745]: cluster 2022-04-28T21:28:23.810686+0000 mgr.y (mgr.24961) 3230 : cluster [DBG] pgmap v3554: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:24 smithi139 bash[43301]: cluster 2022-04-28T21:28:23.810686+0000 mgr.y (mgr.24961) 3230 : cluster [DBG] pgmap v3554: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:26 smithi059 bash[61152]: cluster 2022-04-28T21:28:25.812150+0000 mgr.y (mgr.24961) 3231 : cluster [DBG] pgmap v3555: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:26 smithi059 bash[59745]: cluster 2022-04-28T21:28:25.812150+0000 mgr.y (mgr.24961) 3231 : cluster [DBG] pgmap v3555: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:26 smithi139 bash[43301]: cluster 2022-04-28T21:28:25.812150+0000 mgr.y (mgr.24961) 3231 : cluster [DBG] pgmap v3555: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:28 smithi059 bash[61152]: cluster 2022-04-28T21:28:27.813097+0000 mgr.y (mgr.24961) 3232 : cluster [DBG] pgmap v3556: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:28 smithi059 bash[59745]: cluster 2022-04-28T21:28:27.813097+0000 mgr.y (mgr.24961) 3232 : cluster [DBG] pgmap v3556: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:28 smithi139 bash[43301]: cluster 2022-04-28T21:28:27.813097+0000 mgr.y (mgr.24961) 3232 : cluster [DBG] pgmap v3556: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:30 smithi059 bash[61152]: cluster 2022-04-28T21:28:29.814118+0000 mgr.y (mgr.24961) 3233 : cluster [DBG] pgmap v3557: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:30 smithi059 bash[59745]: cluster 2022-04-28T21:28:29.814118+0000 mgr.y (mgr.24961) 3233 : cluster [DBG] pgmap v3557: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:30 smithi139 bash[43301]: cluster 2022-04-28T21:28:29.814118+0000 mgr.y (mgr.24961) 3233 : cluster [DBG] pgmap v3557: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:32.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:32 smithi059 bash[61152]: cluster 2022-04-28T21:28:31.815355+0000 mgr.y (mgr.24961) 3234 : cluster [DBG] pgmap v3558: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:32 smithi059 bash[59745]: cluster 2022-04-28T21:28:31.815355+0000 mgr.y (mgr.24961) 3234 : cluster [DBG] pgmap v3558: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:32 smithi139 bash[43301]: cluster 2022-04-28T21:28:31.815355+0000 mgr.y (mgr.24961) 3234 : cluster [DBG] pgmap v3558: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:33.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:35.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:34 smithi059 bash[61152]: cluster 2022-04-28T21:28:33.816118+0000 mgr.y (mgr.24961) 3235 : cluster [DBG] pgmap v3559: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:34 smithi059 bash[59745]: cluster 2022-04-28T21:28:33.816118+0000 mgr.y (mgr.24961) 3235 : cluster [DBG] pgmap v3559: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:34 smithi139 bash[43301]: cluster 2022-04-28T21:28:33.816118+0000 mgr.y (mgr.24961) 3235 : cluster [DBG] pgmap v3559: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:36 smithi059 bash[59745]: cluster 2022-04-28T21:28:35.817511+0000 mgr.y (mgr.24961) 3236 : cluster [DBG] pgmap v3560: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:36 smithi059 bash[61152]: cluster 2022-04-28T21:28:35.817511+0000 mgr.y (mgr.24961) 3236 : cluster [DBG] pgmap v3560: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:36 smithi139 bash[43301]: cluster 2022-04-28T21:28:35.817511+0000 mgr.y (mgr.24961) 3236 : cluster [DBG] pgmap v3560: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:38 smithi059 bash[59745]: cluster 2022-04-28T21:28:37.818521+0000 mgr.y (mgr.24961) 3237 : cluster [DBG] pgmap v3561: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:38 smithi059 bash[61152]: cluster 2022-04-28T21:28:37.818521+0000 mgr.y (mgr.24961) 3237 : cluster [DBG] pgmap v3561: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:38 smithi139 bash[43301]: cluster 2022-04-28T21:28:37.818521+0000 mgr.y (mgr.24961) 3237 : cluster [DBG] pgmap v3561: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:41.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:40 smithi059 bash[61152]: cluster 2022-04-28T21:28:39.819721+0000 mgr.y (mgr.24961) 3238 : cluster [DBG] pgmap v3562: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:40 smithi059 bash[59745]: cluster 2022-04-28T21:28:39.819721+0000 mgr.y (mgr.24961) 3238 : cluster [DBG] pgmap v3562: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:40 smithi139 bash[43301]: cluster 2022-04-28T21:28:39.819721+0000 mgr.y (mgr.24961) 3238 : cluster [DBG] pgmap v3562: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:42.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:42 smithi059 bash[61152]: cluster 2022-04-28T21:28:41.820985+0000 mgr.y (mgr.24961) 3239 : cluster [DBG] pgmap v3563: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:42 smithi059 bash[59745]: cluster 2022-04-28T21:28:41.820985+0000 mgr.y (mgr.24961) 3239 : cluster [DBG] pgmap v3563: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:42 smithi139 bash[43301]: cluster 2022-04-28T21:28:41.820985+0000 mgr.y (mgr.24961) 3239 : cluster [DBG] pgmap v3563: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:43.898Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:44 smithi059 bash[61152]: cluster 2022-04-28T21:28:43.821727+0000 mgr.y (mgr.24961) 3240 : cluster [DBG] pgmap v3564: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:44 smithi059 bash[59745]: cluster 2022-04-28T21:28:43.821727+0000 mgr.y (mgr.24961) 3240 : cluster [DBG] pgmap v3564: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:44 smithi139 bash[43301]: cluster 2022-04-28T21:28:43.821727+0000 mgr.y (mgr.24961) 3240 : cluster [DBG] pgmap v3564: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[59745]: audit 2022-04-28T21:28:45.425031+0000 mon.a (mon.0) 2755 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:28:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[59745]: audit 2022-04-28T21:28:45.427926+0000 mon.a (mon.0) 2756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:28:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[59745]: audit 2022-04-28T21:28:45.429908+0000 mon.a (mon.0) 2757 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:28:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[59745]: cephadm 2022-04-28T21:28:45.431722+0000 mgr.y (mgr.24961) 3241 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:28:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[59745]: audit 2022-04-28T21:28:45.760107+0000 mon.a (mon.0) 2758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:28:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[61152]: audit 2022-04-28T21:28:45.425031+0000 mon.a (mon.0) 2755 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:28:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[61152]: audit 2022-04-28T21:28:45.427926+0000 mon.a (mon.0) 2756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:28:46.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[61152]: audit 2022-04-28T21:28:45.429908+0000 mon.a (mon.0) 2757 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:28:46.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[61152]: cephadm 2022-04-28T21:28:45.431722+0000 mgr.y (mgr.24961) 3241 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:28:46.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:45 smithi059 bash[61152]: audit 2022-04-28T21:28:45.760107+0000 mon.a (mon.0) 2758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:28:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:45 smithi139 bash[43301]: audit 2022-04-28T21:28:45.425031+0000 mon.a (mon.0) 2755 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:28:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:45 smithi139 bash[43301]: audit 2022-04-28T21:28:45.427926+0000 mon.a (mon.0) 2756 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:28:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:45 smithi139 bash[43301]: audit 2022-04-28T21:28:45.429908+0000 mon.a (mon.0) 2757 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:28:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:45 smithi139 bash[43301]: cephadm 2022-04-28T21:28:45.431722+0000 mgr.y (mgr.24961) 3241 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:28:46.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:45 smithi139 bash[43301]: audit 2022-04-28T21:28:45.760107+0000 mon.a (mon.0) 2758 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:28:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:46 smithi059 bash[61152]: cluster 2022-04-28T21:28:45.823159+0000 mgr.y (mgr.24961) 3242 : cluster [DBG] pgmap v3565: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:46 smithi059 bash[59745]: cluster 2022-04-28T21:28:45.823159+0000 mgr.y (mgr.24961) 3242 : cluster [DBG] pgmap v3565: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:46 smithi139 bash[43301]: cluster 2022-04-28T21:28:45.823159+0000 mgr.y (mgr.24961) 3242 : cluster [DBG] pgmap v3565: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:48 smithi059 bash[61152]: cluster 2022-04-28T21:28:47.824168+0000 mgr.y (mgr.24961) 3243 : cluster [DBG] pgmap v3566: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:48 smithi059 bash[59745]: cluster 2022-04-28T21:28:47.824168+0000 mgr.y (mgr.24961) 3243 : cluster [DBG] pgmap v3566: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:48 smithi139 bash[43301]: cluster 2022-04-28T21:28:47.824168+0000 mgr.y (mgr.24961) 3243 : cluster [DBG] pgmap v3566: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:51.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:50 smithi059 bash[61152]: cluster 2022-04-28T21:28:49.825317+0000 mgr.y (mgr.24961) 3244 : cluster [DBG] pgmap v3567: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:50 smithi059 bash[59745]: cluster 2022-04-28T21:28:49.825317+0000 mgr.y (mgr.24961) 3244 : cluster [DBG] pgmap v3567: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:50 smithi139 bash[43301]: cluster 2022-04-28T21:28:49.825317+0000 mgr.y (mgr.24961) 3244 : cluster [DBG] pgmap v3567: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:52.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:53.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:52 smithi059 bash[61152]: cluster 2022-04-28T21:28:51.826551+0000 mgr.y (mgr.24961) 3245 : cluster [DBG] pgmap v3568: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:52 smithi059 bash[59745]: cluster 2022-04-28T21:28:51.826551+0000 mgr.y (mgr.24961) 3245 : cluster [DBG] pgmap v3568: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:52 smithi139 bash[43301]: cluster 2022-04-28T21:28:51.826551+0000 mgr.y (mgr.24961) 3245 : cluster [DBG] pgmap v3568: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:28:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:28:53.899Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:28:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:54 smithi059 bash[61152]: cluster 2022-04-28T21:28:53.827287+0000 mgr.y (mgr.24961) 3246 : cluster [DBG] pgmap v3569: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:54 smithi059 bash[59745]: cluster 2022-04-28T21:28:53.827287+0000 mgr.y (mgr.24961) 3246 : cluster [DBG] pgmap v3569: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:54 smithi139 bash[43301]: cluster 2022-04-28T21:28:53.827287+0000 mgr.y (mgr.24961) 3246 : cluster [DBG] pgmap v3569: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:56 smithi059 bash[61152]: cluster 2022-04-28T21:28:55.828676+0000 mgr.y (mgr.24961) 3247 : cluster [DBG] pgmap v3570: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:56 smithi059 bash[59745]: cluster 2022-04-28T21:28:55.828676+0000 mgr.y (mgr.24961) 3247 : cluster [DBG] pgmap v3570: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:56 smithi139 bash[43301]: cluster 2022-04-28T21:28:55.828676+0000 mgr.y (mgr.24961) 3247 : cluster [DBG] pgmap v3570: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:28:58 smithi059 bash[59745]: cluster 2022-04-28T21:28:57.829684+0000 mgr.y (mgr.24961) 3248 : cluster [DBG] pgmap v3571: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:28:58 smithi059 bash[61152]: cluster 2022-04-28T21:28:57.829684+0000 mgr.y (mgr.24961) 3248 : cluster [DBG] pgmap v3571: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:28:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:28:58 smithi139 bash[43301]: cluster 2022-04-28T21:28:57.829684+0000 mgr.y (mgr.24961) 3248 : cluster [DBG] pgmap v3571: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:01.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:00 smithi059 bash[61152]: cluster 2022-04-28T21:28:59.830852+0000 mgr.y (mgr.24961) 3249 : cluster [DBG] pgmap v3572: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:00 smithi059 bash[59745]: cluster 2022-04-28T21:28:59.830852+0000 mgr.y (mgr.24961) 3249 : cluster [DBG] pgmap v3572: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:00 smithi139 bash[43301]: cluster 2022-04-28T21:28:59.830852+0000 mgr.y (mgr.24961) 3249 : cluster [DBG] pgmap v3572: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:02.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:02 smithi059 bash[61152]: cluster 2022-04-28T21:29:01.832060+0000 mgr.y (mgr.24961) 3250 : cluster [DBG] pgmap v3573: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:02 smithi059 bash[59745]: cluster 2022-04-28T21:29:01.832060+0000 mgr.y (mgr.24961) 3250 : cluster [DBG] pgmap v3573: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:02 smithi139 bash[43301]: cluster 2022-04-28T21:29:01.832060+0000 mgr.y (mgr.24961) 3250 : cluster [DBG] pgmap v3573: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:03.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:04 smithi059 bash[61152]: cluster 2022-04-28T21:29:03.832807+0000 mgr.y (mgr.24961) 3251 : cluster [DBG] pgmap v3574: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:04 smithi059 bash[59745]: cluster 2022-04-28T21:29:03.832807+0000 mgr.y (mgr.24961) 3251 : cluster [DBG] pgmap v3574: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:04 smithi139 bash[43301]: cluster 2022-04-28T21:29:03.832807+0000 mgr.y (mgr.24961) 3251 : cluster [DBG] pgmap v3574: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:06 smithi059 bash[61152]: cluster 2022-04-28T21:29:05.834306+0000 mgr.y (mgr.24961) 3252 : cluster [DBG] pgmap v3575: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:06 smithi059 bash[59745]: cluster 2022-04-28T21:29:05.834306+0000 mgr.y (mgr.24961) 3252 : cluster [DBG] pgmap v3575: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:06 smithi139 bash[43301]: cluster 2022-04-28T21:29:05.834306+0000 mgr.y (mgr.24961) 3252 : cluster [DBG] pgmap v3575: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:08 smithi059 bash[61152]: cluster 2022-04-28T21:29:07.835321+0000 mgr.y (mgr.24961) 3253 : cluster [DBG] pgmap v3576: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:08 smithi059 bash[59745]: cluster 2022-04-28T21:29:07.835321+0000 mgr.y (mgr.24961) 3253 : cluster [DBG] pgmap v3576: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:08 smithi139 bash[43301]: cluster 2022-04-28T21:29:07.835321+0000 mgr.y (mgr.24961) 3253 : cluster [DBG] pgmap v3576: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:10 smithi059 bash[61152]: cluster 2022-04-28T21:29:09.836498+0000 mgr.y (mgr.24961) 3254 : cluster [DBG] pgmap v3577: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:10 smithi059 bash[59745]: cluster 2022-04-28T21:29:09.836498+0000 mgr.y (mgr.24961) 3254 : cluster [DBG] pgmap v3577: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:10 smithi139 bash[43301]: cluster 2022-04-28T21:29:09.836498+0000 mgr.y (mgr.24961) 3254 : cluster [DBG] pgmap v3577: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:12.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:13.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:12 smithi059 bash[61152]: cluster 2022-04-28T21:29:11.837755+0000 mgr.y (mgr.24961) 3255 : cluster [DBG] pgmap v3578: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:12 smithi059 bash[59745]: cluster 2022-04-28T21:29:11.837755+0000 mgr.y (mgr.24961) 3255 : cluster [DBG] pgmap v3578: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:13.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:12 smithi139 bash[43301]: cluster 2022-04-28T21:29:11.837755+0000 mgr.y (mgr.24961) 3255 : cluster [DBG] pgmap v3578: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:14.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:13.902Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:15.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:14 smithi059 bash[59745]: cluster 2022-04-28T21:29:13.838485+0000 mgr.y (mgr.24961) 3256 : cluster [DBG] pgmap v3579: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:14 smithi059 bash[61152]: cluster 2022-04-28T21:29:13.838485+0000 mgr.y (mgr.24961) 3256 : cluster [DBG] pgmap v3579: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:14 smithi139 bash[43301]: cluster 2022-04-28T21:29:13.838485+0000 mgr.y (mgr.24961) 3256 : cluster [DBG] pgmap v3579: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:16 smithi059 bash[59745]: cluster 2022-04-28T21:29:15.840013+0000 mgr.y (mgr.24961) 3257 : cluster [DBG] pgmap v3580: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:16 smithi059 bash[61152]: cluster 2022-04-28T21:29:15.840013+0000 mgr.y (mgr.24961) 3257 : cluster [DBG] pgmap v3580: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:16 smithi139 bash[43301]: cluster 2022-04-28T21:29:15.840013+0000 mgr.y (mgr.24961) 3257 : cluster [DBG] pgmap v3580: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: cluster 2022-04-28T21:29:17.841097+0000 mgr.y (mgr.24961) 3258 : cluster [DBG] pgmap v3581: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: audit 2022-04-28T21:29:18.390403+0000 mon.a (mon.0) 2759 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:29:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: audit 2022-04-28T21:29:18.394032+0000 mon.a (mon.0) 2760 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:29:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: audit 2022-04-28T21:29:18.405302+0000 mon.a (mon.0) 2761 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:29:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: audit 2022-04-28T21:29:18.406284+0000 mon.a (mon.0) 2762 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:29:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: audit 2022-04-28T21:29:18.406535+0000 mon.a (mon.0) 2763 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T21:29:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[59745]: audit 2022-04-28T21:29:18.406808+0000 mon.a (mon.0) 2764 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:29:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: cluster 2022-04-28T21:29:17.841097+0000 mgr.y (mgr.24961) 3258 : cluster [DBG] pgmap v3581: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: audit 2022-04-28T21:29:18.390403+0000 mon.a (mon.0) 2759 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:29:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: audit 2022-04-28T21:29:18.394032+0000 mon.a (mon.0) 2760 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:29:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: audit 2022-04-28T21:29:18.405302+0000 mon.a (mon.0) 2761 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:29:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: audit 2022-04-28T21:29:18.406284+0000 mon.a (mon.0) 2762 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:29:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: audit 2022-04-28T21:29:18.406535+0000 mon.a (mon.0) 2763 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T21:29:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:18 smithi059 bash[61152]: audit 2022-04-28T21:29:18.406808+0000 mon.a (mon.0) 2764 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:29:19.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: cluster 2022-04-28T21:29:17.841097+0000 mgr.y (mgr.24961) 3258 : cluster [DBG] pgmap v3581: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: audit 2022-04-28T21:29:18.390403+0000 mon.a (mon.0) 2759 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:29:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: audit 2022-04-28T21:29:18.394032+0000 mon.a (mon.0) 2760 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:29:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: audit 2022-04-28T21:29:18.405302+0000 mon.a (mon.0) 2761 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:29:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: audit 2022-04-28T21:29:18.406284+0000 mon.a (mon.0) 2762 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:29:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: audit 2022-04-28T21:29:18.406535+0000 mon.a (mon.0) 2763 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T21:29:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:18 smithi139 bash[43301]: audit 2022-04-28T21:29:18.406808+0000 mon.a (mon.0) 2764 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:29:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:20 smithi059 bash[61152]: cluster 2022-04-28T21:29:19.842313+0000 mgr.y (mgr.24961) 3259 : cluster [DBG] pgmap v3582: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:20 smithi059 bash[59745]: cluster 2022-04-28T21:29:19.842313+0000 mgr.y (mgr.24961) 3259 : cluster [DBG] pgmap v3582: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:20 smithi139 bash[43301]: cluster 2022-04-28T21:29:19.842313+0000 mgr.y (mgr.24961) 3259 : cluster [DBG] pgmap v3582: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:22.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:23.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:22 smithi059 bash[61152]: cluster 2022-04-28T21:29:21.843631+0000 mgr.y (mgr.24961) 3260 : cluster [DBG] pgmap v3583: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:22 smithi059 bash[59745]: cluster 2022-04-28T21:29:21.843631+0000 mgr.y (mgr.24961) 3260 : cluster [DBG] pgmap v3583: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:22 smithi139 bash[43301]: cluster 2022-04-28T21:29:21.843631+0000 mgr.y (mgr.24961) 3260 : cluster [DBG] pgmap v3583: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:23.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:25.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:24 smithi059 bash[61152]: cluster 2022-04-28T21:29:23.844312+0000 mgr.y (mgr.24961) 3261 : cluster [DBG] pgmap v3584: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:24 smithi059 bash[59745]: cluster 2022-04-28T21:29:23.844312+0000 mgr.y (mgr.24961) 3261 : cluster [DBG] pgmap v3584: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:24 smithi139 bash[43301]: cluster 2022-04-28T21:29:23.844312+0000 mgr.y (mgr.24961) 3261 : cluster [DBG] pgmap v3584: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:26 smithi059 bash[61152]: cluster 2022-04-28T21:29:25.845700+0000 mgr.y (mgr.24961) 3262 : cluster [DBG] pgmap v3585: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:26 smithi059 bash[59745]: cluster 2022-04-28T21:29:25.845700+0000 mgr.y (mgr.24961) 3262 : cluster [DBG] pgmap v3585: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:26 smithi139 bash[43301]: cluster 2022-04-28T21:29:25.845700+0000 mgr.y (mgr.24961) 3262 : cluster [DBG] pgmap v3585: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:28 smithi059 bash[61152]: cluster 2022-04-28T21:29:27.846680+0000 mgr.y (mgr.24961) 3263 : cluster [DBG] pgmap v3586: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:28 smithi059 bash[59745]: cluster 2022-04-28T21:29:27.846680+0000 mgr.y (mgr.24961) 3263 : cluster [DBG] pgmap v3586: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:28 smithi139 bash[43301]: cluster 2022-04-28T21:29:27.846680+0000 mgr.y (mgr.24961) 3263 : cluster [DBG] pgmap v3586: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:31.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:30 smithi059 bash[61152]: cluster 2022-04-28T21:29:29.847590+0000 mgr.y (mgr.24961) 3264 : cluster [DBG] pgmap v3587: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:31.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:30 smithi059 bash[59745]: cluster 2022-04-28T21:29:29.847590+0000 mgr.y (mgr.24961) 3264 : cluster [DBG] pgmap v3587: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:30 smithi139 bash[43301]: cluster 2022-04-28T21:29:29.847590+0000 mgr.y (mgr.24961) 3264 : cluster [DBG] pgmap v3587: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:32.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:33.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:32 smithi059 bash[61152]: cluster 2022-04-28T21:29:31.848897+0000 mgr.y (mgr.24961) 3265 : cluster [DBG] pgmap v3588: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:32 smithi059 bash[59745]: cluster 2022-04-28T21:29:31.848897+0000 mgr.y (mgr.24961) 3265 : cluster [DBG] pgmap v3588: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:32 smithi139 bash[43301]: cluster 2022-04-28T21:29:31.848897+0000 mgr.y (mgr.24961) 3265 : cluster [DBG] pgmap v3588: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:33.901Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:34 smithi059 bash[61152]: cluster 2022-04-28T21:29:33.849733+0000 mgr.y (mgr.24961) 3266 : cluster [DBG] pgmap v3589: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:34 smithi059 bash[59745]: cluster 2022-04-28T21:29:33.849733+0000 mgr.y (mgr.24961) 3266 : cluster [DBG] pgmap v3589: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:34 smithi139 bash[43301]: cluster 2022-04-28T21:29:33.849733+0000 mgr.y (mgr.24961) 3266 : cluster [DBG] pgmap v3589: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:36 smithi059 bash[61152]: cluster 2022-04-28T21:29:35.851185+0000 mgr.y (mgr.24961) 3267 : cluster [DBG] pgmap v3590: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:36 smithi059 bash[59745]: cluster 2022-04-28T21:29:35.851185+0000 mgr.y (mgr.24961) 3267 : cluster [DBG] pgmap v3590: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:36 smithi139 bash[43301]: cluster 2022-04-28T21:29:35.851185+0000 mgr.y (mgr.24961) 3267 : cluster [DBG] pgmap v3590: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:39.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:38 smithi059 bash[61152]: cluster 2022-04-28T21:29:37.852176+0000 mgr.y (mgr.24961) 3268 : cluster [DBG] pgmap v3591: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:38 smithi059 bash[59745]: cluster 2022-04-28T21:29:37.852176+0000 mgr.y (mgr.24961) 3268 : cluster [DBG] pgmap v3591: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:38 smithi139 bash[43301]: cluster 2022-04-28T21:29:37.852176+0000 mgr.y (mgr.24961) 3268 : cluster [DBG] pgmap v3591: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:41.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:40 smithi059 bash[61152]: cluster 2022-04-28T21:29:39.853328+0000 mgr.y (mgr.24961) 3269 : cluster [DBG] pgmap v3592: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:41.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:40 smithi059 bash[59745]: cluster 2022-04-28T21:29:39.853328+0000 mgr.y (mgr.24961) 3269 : cluster [DBG] pgmap v3592: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:40 smithi139 bash[43301]: cluster 2022-04-28T21:29:39.853328+0000 mgr.y (mgr.24961) 3269 : cluster [DBG] pgmap v3592: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:42.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:43.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:42 smithi059 bash[61152]: cluster 2022-04-28T21:29:41.854556+0000 mgr.y (mgr.24961) 3270 : cluster [DBG] pgmap v3593: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:42 smithi059 bash[59745]: cluster 2022-04-28T21:29:41.854556+0000 mgr.y (mgr.24961) 3270 : cluster [DBG] pgmap v3593: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:42 smithi139 bash[43301]: cluster 2022-04-28T21:29:41.854556+0000 mgr.y (mgr.24961) 3270 : cluster [DBG] pgmap v3593: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:43.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:44 smithi059 bash[61152]: cluster 2022-04-28T21:29:43.855370+0000 mgr.y (mgr.24961) 3271 : cluster [DBG] pgmap v3594: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:44 smithi059 bash[59745]: cluster 2022-04-28T21:29:43.855370+0000 mgr.y (mgr.24961) 3271 : cluster [DBG] pgmap v3594: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:44 smithi139 bash[43301]: cluster 2022-04-28T21:29:43.855370+0000 mgr.y (mgr.24961) 3271 : cluster [DBG] pgmap v3594: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:46.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:45 smithi059 bash[59745]: audit 2022-04-28T21:29:45.767223+0000 mon.a (mon.0) 2765 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:29:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:45 smithi059 bash[59745]: audit 2022-04-28T21:29:45.770072+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:29:46.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:45 smithi059 bash[59745]: audit 2022-04-28T21:29:45.772074+0000 mon.a (mon.0) 2767 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:29:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:45 smithi059 bash[61152]: audit 2022-04-28T21:29:45.767223+0000 mon.a (mon.0) 2765 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:29:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:45 smithi059 bash[61152]: audit 2022-04-28T21:29:45.770072+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:29:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:45 smithi059 bash[61152]: audit 2022-04-28T21:29:45.772074+0000 mon.a (mon.0) 2767 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:29:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:45 smithi139 bash[43301]: audit 2022-04-28T21:29:45.767223+0000 mon.a (mon.0) 2765 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:29:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:45 smithi139 bash[43301]: audit 2022-04-28T21:29:45.770072+0000 mon.a (mon.0) 2766 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:29:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:45 smithi139 bash[43301]: audit 2022-04-28T21:29:45.772074+0000 mon.a (mon.0) 2767 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:29:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[59745]: cephadm 2022-04-28T21:29:45.773785+0000 mgr.y (mgr.24961) 3272 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:29:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[59745]: cluster 2022-04-28T21:29:45.856832+0000 mgr.y (mgr.24961) 3273 : cluster [DBG] pgmap v3595: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[59745]: audit 2022-04-28T21:29:46.077217+0000 mon.a (mon.0) 2768 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[59745]: audit 2022-04-28T21:29:46.083417+0000 mon.a (mon.0) 2769 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[59745]: audit 2022-04-28T21:29:46.386602+0000 mon.a (mon.0) 2770 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[61152]: cephadm 2022-04-28T21:29:45.773785+0000 mgr.y (mgr.24961) 3272 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:29:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[61152]: cluster 2022-04-28T21:29:45.856832+0000 mgr.y (mgr.24961) 3273 : cluster [DBG] pgmap v3595: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[61152]: audit 2022-04-28T21:29:46.077217+0000 mon.a (mon.0) 2768 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[61152]: audit 2022-04-28T21:29:46.083417+0000 mon.a (mon.0) 2769 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:47 smithi059 bash[61152]: audit 2022-04-28T21:29:46.386602+0000 mon.a (mon.0) 2770 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:47 smithi139 bash[43301]: cephadm 2022-04-28T21:29:45.773785+0000 mgr.y (mgr.24961) 3272 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:29:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:47 smithi139 bash[43301]: cluster 2022-04-28T21:29:45.856832+0000 mgr.y (mgr.24961) 3273 : cluster [DBG] pgmap v3595: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:47.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:47 smithi139 bash[43301]: audit 2022-04-28T21:29:46.077217+0000 mon.a (mon.0) 2768 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:47 smithi139 bash[43301]: audit 2022-04-28T21:29:46.083417+0000 mon.a (mon.0) 2769 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:47.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:47 smithi139 bash[43301]: audit 2022-04-28T21:29:46.386602+0000 mon.a (mon.0) 2770 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:29:49.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:49 smithi059 bash[61152]: cluster 2022-04-28T21:29:47.857811+0000 mgr.y (mgr.24961) 3274 : cluster [DBG] pgmap v3596: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:49 smithi059 bash[59745]: cluster 2022-04-28T21:29:47.857811+0000 mgr.y (mgr.24961) 3274 : cluster [DBG] pgmap v3596: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:49 smithi139 bash[43301]: cluster 2022-04-28T21:29:47.857811+0000 mgr.y (mgr.24961) 3274 : cluster [DBG] pgmap v3596: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:51.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:50 smithi059 bash[61152]: cluster 2022-04-28T21:29:49.858978+0000 mgr.y (mgr.24961) 3275 : cluster [DBG] pgmap v3597: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:50 smithi059 bash[59745]: cluster 2022-04-28T21:29:49.858978+0000 mgr.y (mgr.24961) 3275 : cluster [DBG] pgmap v3597: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:50 smithi139 bash[43301]: cluster 2022-04-28T21:29:49.858978+0000 mgr.y (mgr.24961) 3275 : cluster [DBG] pgmap v3597: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:52.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:53.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:52 smithi059 bash[59745]: cluster 2022-04-28T21:29:51.860267+0000 mgr.y (mgr.24961) 3276 : cluster [DBG] pgmap v3598: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:52 smithi059 bash[61152]: cluster 2022-04-28T21:29:51.860267+0000 mgr.y (mgr.24961) 3276 : cluster [DBG] pgmap v3598: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:52 smithi139 bash[43301]: cluster 2022-04-28T21:29:51.860267+0000 mgr.y (mgr.24961) 3276 : cluster [DBG] pgmap v3598: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:29:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:29:53.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:29:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:54 smithi059 bash[61152]: cluster 2022-04-28T21:29:53.860976+0000 mgr.y (mgr.24961) 3277 : cluster [DBG] pgmap v3599: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:54 smithi059 bash[59745]: cluster 2022-04-28T21:29:53.860976+0000 mgr.y (mgr.24961) 3277 : cluster [DBG] pgmap v3599: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:54 smithi139 bash[43301]: cluster 2022-04-28T21:29:53.860976+0000 mgr.y (mgr.24961) 3277 : cluster [DBG] pgmap v3599: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:56 smithi059 bash[61152]: cluster 2022-04-28T21:29:55.862368+0000 mgr.y (mgr.24961) 3278 : cluster [DBG] pgmap v3600: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:56 smithi059 bash[59745]: cluster 2022-04-28T21:29:55.862368+0000 mgr.y (mgr.24961) 3278 : cluster [DBG] pgmap v3600: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:56 smithi139 bash[43301]: cluster 2022-04-28T21:29:55.862368+0000 mgr.y (mgr.24961) 3278 : cluster [DBG] pgmap v3600: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:59.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:29:58 smithi059 bash[61152]: cluster 2022-04-28T21:29:57.863386+0000 mgr.y (mgr.24961) 3279 : cluster [DBG] pgmap v3601: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:29:58 smithi059 bash[59745]: cluster 2022-04-28T21:29:57.863386+0000 mgr.y (mgr.24961) 3279 : cluster [DBG] pgmap v3601: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:29:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:29:58 smithi139 bash[43301]: cluster 2022-04-28T21:29:57.863386+0000 mgr.y (mgr.24961) 3279 : cluster [DBG] pgmap v3601: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:00 smithi059 bash[61152]: cluster 2022-04-28T21:29:59.864539+0000 mgr.y (mgr.24961) 3280 : cluster [DBG] pgmap v3602: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:00 smithi059 bash[61152]: cluster 2022-04-28T21:30:00.000155+0000 mon.a (mon.0) 2771 : cluster [INF] overall HEALTH_OK 2022-04-28T21:30:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:00 smithi059 bash[59745]: cluster 2022-04-28T21:29:59.864539+0000 mgr.y (mgr.24961) 3280 : cluster [DBG] pgmap v3602: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:00 smithi059 bash[59745]: cluster 2022-04-28T21:30:00.000155+0000 mon.a (mon.0) 2771 : cluster [INF] overall HEALTH_OK 2022-04-28T21:30:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:00 smithi139 bash[43301]: cluster 2022-04-28T21:29:59.864539+0000 mgr.y (mgr.24961) 3280 : cluster [DBG] pgmap v3602: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:00 smithi139 bash[43301]: cluster 2022-04-28T21:30:00.000155+0000 mon.a (mon.0) 2771 : cluster [INF] overall HEALTH_OK 2022-04-28T21:30:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:02.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:03.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:02 smithi059 bash[61152]: cluster 2022-04-28T21:30:01.865706+0000 mgr.y (mgr.24961) 3281 : cluster [DBG] pgmap v3603: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:02 smithi059 bash[59745]: cluster 2022-04-28T21:30:01.865706+0000 mgr.y (mgr.24961) 3281 : cluster [DBG] pgmap v3603: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:03.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:02 smithi139 bash[43301]: cluster 2022-04-28T21:30:01.865706+0000 mgr.y (mgr.24961) 3281 : cluster [DBG] pgmap v3603: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:03.903Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:05.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:04 smithi059 bash[61152]: cluster 2022-04-28T21:30:03.866536+0000 mgr.y (mgr.24961) 3282 : cluster [DBG] pgmap v3604: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:04 smithi059 bash[59745]: cluster 2022-04-28T21:30:03.866536+0000 mgr.y (mgr.24961) 3282 : cluster [DBG] pgmap v3604: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:04 smithi139 bash[43301]: cluster 2022-04-28T21:30:03.866536+0000 mgr.y (mgr.24961) 3282 : cluster [DBG] pgmap v3604: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:06 smithi059 bash[61152]: cluster 2022-04-28T21:30:05.867996+0000 mgr.y (mgr.24961) 3283 : cluster [DBG] pgmap v3605: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:06 smithi059 bash[59745]: cluster 2022-04-28T21:30:05.867996+0000 mgr.y (mgr.24961) 3283 : cluster [DBG] pgmap v3605: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:06 smithi139 bash[43301]: cluster 2022-04-28T21:30:05.867996+0000 mgr.y (mgr.24961) 3283 : cluster [DBG] pgmap v3605: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:08 smithi059 bash[61152]: cluster 2022-04-28T21:30:07.868963+0000 mgr.y (mgr.24961) 3284 : cluster [DBG] pgmap v3606: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:08 smithi059 bash[59745]: cluster 2022-04-28T21:30:07.868963+0000 mgr.y (mgr.24961) 3284 : cluster [DBG] pgmap v3606: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:09.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:08 smithi139 bash[43301]: cluster 2022-04-28T21:30:07.868963+0000 mgr.y (mgr.24961) 3284 : cluster [DBG] pgmap v3606: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:10 smithi059 bash[61152]: cluster 2022-04-28T21:30:09.870055+0000 mgr.y (mgr.24961) 3285 : cluster [DBG] pgmap v3607: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:10 smithi059 bash[59745]: cluster 2022-04-28T21:30:09.870055+0000 mgr.y (mgr.24961) 3285 : cluster [DBG] pgmap v3607: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:10 smithi139 bash[43301]: cluster 2022-04-28T21:30:09.870055+0000 mgr.y (mgr.24961) 3285 : cluster [DBG] pgmap v3607: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:12.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:13.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:12 smithi059 bash[61152]: cluster 2022-04-28T21:30:11.871256+0000 mgr.y (mgr.24961) 3286 : cluster [DBG] pgmap v3608: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:12 smithi059 bash[59745]: cluster 2022-04-28T21:30:11.871256+0000 mgr.y (mgr.24961) 3286 : cluster [DBG] pgmap v3608: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:12 smithi139 bash[43301]: cluster 2022-04-28T21:30:11.871256+0000 mgr.y (mgr.24961) 3286 : cluster [DBG] pgmap v3608: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:13.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:14 smithi059 bash[61152]: cluster 2022-04-28T21:30:13.871993+0000 mgr.y (mgr.24961) 3287 : cluster [DBG] pgmap v3609: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:14 smithi059 bash[59745]: cluster 2022-04-28T21:30:13.871993+0000 mgr.y (mgr.24961) 3287 : cluster [DBG] pgmap v3609: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:14 smithi139 bash[43301]: cluster 2022-04-28T21:30:13.871993+0000 mgr.y (mgr.24961) 3287 : cluster [DBG] pgmap v3609: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:17.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:16 smithi059 bash[61152]: cluster 2022-04-28T21:30:15.873374+0000 mgr.y (mgr.24961) 3288 : cluster [DBG] pgmap v3610: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:16 smithi059 bash[59745]: cluster 2022-04-28T21:30:15.873374+0000 mgr.y (mgr.24961) 3288 : cluster [DBG] pgmap v3610: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:16 smithi139 bash[43301]: cluster 2022-04-28T21:30:15.873374+0000 mgr.y (mgr.24961) 3288 : cluster [DBG] pgmap v3610: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: cluster 2022-04-28T21:30:17.874372+0000 mgr.y (mgr.24961) 3289 : cluster [DBG] pgmap v3611: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: audit 2022-04-28T21:30:18.405800+0000 mon.a (mon.0) 2772 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:30:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: audit 2022-04-28T21:30:18.406250+0000 mon.a (mon.0) 2773 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:30:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: audit 2022-04-28T21:30:18.419777+0000 mon.a (mon.0) 2774 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:30:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: audit 2022-04-28T21:30:18.420288+0000 mon.a (mon.0) 2775 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T21:30:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: audit 2022-04-28T21:30:18.420668+0000 mon.a (mon.0) 2776 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:30:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[59745]: audit 2022-04-28T21:30:18.421062+0000 mon.a (mon.0) 2777 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:30:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: cluster 2022-04-28T21:30:17.874372+0000 mgr.y (mgr.24961) 3289 : cluster [DBG] pgmap v3611: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:19.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: audit 2022-04-28T21:30:18.405800+0000 mon.a (mon.0) 2772 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:30:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: audit 2022-04-28T21:30:18.406250+0000 mon.a (mon.0) 2773 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:30:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: audit 2022-04-28T21:30:18.419777+0000 mon.a (mon.0) 2774 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:30:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: audit 2022-04-28T21:30:18.420288+0000 mon.a (mon.0) 2775 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T21:30:19.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: audit 2022-04-28T21:30:18.420668+0000 mon.a (mon.0) 2776 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:30:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:18 smithi059 bash[61152]: audit 2022-04-28T21:30:18.421062+0000 mon.a (mon.0) 2777 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:30:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: cluster 2022-04-28T21:30:17.874372+0000 mgr.y (mgr.24961) 3289 : cluster [DBG] pgmap v3611: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: audit 2022-04-28T21:30:18.405800+0000 mon.a (mon.0) 2772 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:30:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: audit 2022-04-28T21:30:18.406250+0000 mon.a (mon.0) 2773 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:30:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: audit 2022-04-28T21:30:18.419777+0000 mon.a (mon.0) 2774 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:30:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: audit 2022-04-28T21:30:18.420288+0000 mon.a (mon.0) 2775 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2022-04-28T21:30:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: audit 2022-04-28T21:30:18.420668+0000 mon.a (mon.0) 2776 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:30:19.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:18 smithi139 bash[43301]: audit 2022-04-28T21:30:18.421062+0000 mon.a (mon.0) 2777 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:30:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:20 smithi139 bash[43301]: cluster 2022-04-28T21:30:19.875581+0000 mgr.y (mgr.24961) 3290 : cluster [DBG] pgmap v3612: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:21.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:20 smithi059 bash[61152]: cluster 2022-04-28T21:30:19.875581+0000 mgr.y (mgr.24961) 3290 : cluster [DBG] pgmap v3612: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:20 smithi059 bash[59745]: cluster 2022-04-28T21:30:19.875581+0000 mgr.y (mgr.24961) 3290 : cluster [DBG] pgmap v3612: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:22.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:23.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:22 smithi059 bash[61152]: cluster 2022-04-28T21:30:21.876757+0000 mgr.y (mgr.24961) 3291 : cluster [DBG] pgmap v3613: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:23.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:22 smithi059 bash[59745]: cluster 2022-04-28T21:30:21.876757+0000 mgr.y (mgr.24961) 3291 : cluster [DBG] pgmap v3613: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:22 smithi139 bash[43301]: cluster 2022-04-28T21:30:21.876757+0000 mgr.y (mgr.24961) 3291 : cluster [DBG] pgmap v3613: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:23.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:24 smithi059 bash[61152]: cluster 2022-04-28T21:30:23.877460+0000 mgr.y (mgr.24961) 3292 : cluster [DBG] pgmap v3614: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:24 smithi059 bash[59745]: cluster 2022-04-28T21:30:23.877460+0000 mgr.y (mgr.24961) 3292 : cluster [DBG] pgmap v3614: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:25.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:24 smithi139 bash[43301]: cluster 2022-04-28T21:30:23.877460+0000 mgr.y (mgr.24961) 3292 : cluster [DBG] pgmap v3614: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:27.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:26 smithi059 bash[61152]: cluster 2022-04-28T21:30:25.878867+0000 mgr.y (mgr.24961) 3293 : cluster [DBG] pgmap v3615: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:26 smithi059 bash[59745]: cluster 2022-04-28T21:30:25.878867+0000 mgr.y (mgr.24961) 3293 : cluster [DBG] pgmap v3615: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:26 smithi139 bash[43301]: cluster 2022-04-28T21:30:25.878867+0000 mgr.y (mgr.24961) 3293 : cluster [DBG] pgmap v3615: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:28 smithi059 bash[61152]: cluster 2022-04-28T21:30:27.879889+0000 mgr.y (mgr.24961) 3294 : cluster [DBG] pgmap v3616: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:28 smithi059 bash[59745]: cluster 2022-04-28T21:30:27.879889+0000 mgr.y (mgr.24961) 3294 : cluster [DBG] pgmap v3616: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:28 smithi139 bash[43301]: cluster 2022-04-28T21:30:27.879889+0000 mgr.y (mgr.24961) 3294 : cluster [DBG] pgmap v3616: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:31.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:30 smithi139 bash[43301]: cluster 2022-04-28T21:30:29.880979+0000 mgr.y (mgr.24961) 3295 : cluster [DBG] pgmap v3617: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:31.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:30 smithi059 bash[61152]: cluster 2022-04-28T21:30:29.880979+0000 mgr.y (mgr.24961) 3295 : cluster [DBG] pgmap v3617: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:30 smithi059 bash[59745]: cluster 2022-04-28T21:30:29.880979+0000 mgr.y (mgr.24961) 3295 : cluster [DBG] pgmap v3617: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:32.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:32 smithi059 bash[61152]: cluster 2022-04-28T21:30:31.882237+0000 mgr.y (mgr.24961) 3296 : cluster [DBG] pgmap v3618: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:33.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:32 smithi059 bash[59745]: cluster 2022-04-28T21:30:31.882237+0000 mgr.y (mgr.24961) 3296 : cluster [DBG] pgmap v3618: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:32 smithi139 bash[43301]: cluster 2022-04-28T21:30:31.882237+0000 mgr.y (mgr.24961) 3296 : cluster [DBG] pgmap v3618: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:33.904Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:35.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:34 smithi059 bash[61152]: cluster 2022-04-28T21:30:33.882938+0000 mgr.y (mgr.24961) 3297 : cluster [DBG] pgmap v3619: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:34 smithi059 bash[59745]: cluster 2022-04-28T21:30:33.882938+0000 mgr.y (mgr.24961) 3297 : cluster [DBG] pgmap v3619: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:34 smithi139 bash[43301]: cluster 2022-04-28T21:30:33.882938+0000 mgr.y (mgr.24961) 3297 : cluster [DBG] pgmap v3619: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:37.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:36 smithi059 bash[61152]: cluster 2022-04-28T21:30:35.884464+0000 mgr.y (mgr.24961) 3298 : cluster [DBG] pgmap v3620: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:36 smithi059 bash[59745]: cluster 2022-04-28T21:30:35.884464+0000 mgr.y (mgr.24961) 3298 : cluster [DBG] pgmap v3620: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:36 smithi139 bash[43301]: cluster 2022-04-28T21:30:35.884464+0000 mgr.y (mgr.24961) 3298 : cluster [DBG] pgmap v3620: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:39.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:38 smithi059 bash[61152]: cluster 2022-04-28T21:30:37.885569+0000 mgr.y (mgr.24961) 3299 : cluster [DBG] pgmap v3621: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:38 smithi059 bash[59745]: cluster 2022-04-28T21:30:37.885569+0000 mgr.y (mgr.24961) 3299 : cluster [DBG] pgmap v3621: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:38 smithi139 bash[43301]: cluster 2022-04-28T21:30:37.885569+0000 mgr.y (mgr.24961) 3299 : cluster [DBG] pgmap v3621: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:41.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:40 smithi139 bash[43301]: cluster 2022-04-28T21:30:39.886757+0000 mgr.y (mgr.24961) 3300 : cluster [DBG] pgmap v3622: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:41.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:40 smithi059 bash[61152]: cluster 2022-04-28T21:30:39.886757+0000 mgr.y (mgr.24961) 3300 : cluster [DBG] pgmap v3622: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:40 smithi059 bash[59745]: cluster 2022-04-28T21:30:39.886757+0000 mgr.y (mgr.24961) 3300 : cluster [DBG] pgmap v3622: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:42.011Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:43.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:42 smithi059 bash[61152]: cluster 2022-04-28T21:30:41.887990+0000 mgr.y (mgr.24961) 3301 : cluster [DBG] pgmap v3623: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:42 smithi059 bash[59745]: cluster 2022-04-28T21:30:41.887990+0000 mgr.y (mgr.24961) 3301 : cluster [DBG] pgmap v3623: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:42 smithi139 bash[43301]: cluster 2022-04-28T21:30:41.887990+0000 mgr.y (mgr.24961) 3301 : cluster [DBG] pgmap v3623: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:43.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:44 smithi059 bash[61152]: cluster 2022-04-28T21:30:43.888719+0000 mgr.y (mgr.24961) 3302 : cluster [DBG] pgmap v3624: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:44 smithi059 bash[59745]: cluster 2022-04-28T21:30:43.888719+0000 mgr.y (mgr.24961) 3302 : cluster [DBG] pgmap v3624: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:44 smithi139 bash[43301]: cluster 2022-04-28T21:30:43.888719+0000 mgr.y (mgr.24961) 3302 : cluster [DBG] pgmap v3624: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[59745]: cluster 2022-04-28T21:30:45.890145+0000 mgr.y (mgr.24961) 3303 : cluster [DBG] pgmap v3625: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[59745]: audit 2022-04-28T21:30:46.393766+0000 mon.a (mon.0) 2778 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:30:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[59745]: audit 2022-04-28T21:30:46.396810+0000 mon.a (mon.0) 2779 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:30:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[59745]: audit 2022-04-28T21:30:46.398860+0000 mon.a (mon.0) 2780 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:30:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[59745]: cephadm 2022-04-28T21:30:46.400642+0000 mgr.y (mgr.24961) 3304 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:30:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[59745]: audit 2022-04-28T21:30:46.707257+0000 mon.a (mon.0) 2781 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:30:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[61152]: cluster 2022-04-28T21:30:45.890145+0000 mgr.y (mgr.24961) 3303 : cluster [DBG] pgmap v3625: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[61152]: audit 2022-04-28T21:30:46.393766+0000 mon.a (mon.0) 2778 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:30:47.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[61152]: audit 2022-04-28T21:30:46.396810+0000 mon.a (mon.0) 2779 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:30:47.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[61152]: audit 2022-04-28T21:30:46.398860+0000 mon.a (mon.0) 2780 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:30:47.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[61152]: cephadm 2022-04-28T21:30:46.400642+0000 mgr.y (mgr.24961) 3304 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:30:47.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:46 smithi059 bash[61152]: audit 2022-04-28T21:30:46.707257+0000 mon.a (mon.0) 2781 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:30:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:46 smithi139 bash[43301]: cluster 2022-04-28T21:30:45.890145+0000 mgr.y (mgr.24961) 3303 : cluster [DBG] pgmap v3625: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:46 smithi139 bash[43301]: audit 2022-04-28T21:30:46.393766+0000 mon.a (mon.0) 2778 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:30:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:46 smithi139 bash[43301]: audit 2022-04-28T21:30:46.396810+0000 mon.a (mon.0) 2779 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:30:47.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:46 smithi139 bash[43301]: audit 2022-04-28T21:30:46.398860+0000 mon.a (mon.0) 2780 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:30:47.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:46 smithi139 bash[43301]: cephadm 2022-04-28T21:30:46.400642+0000 mgr.y (mgr.24961) 3304 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:30:47.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:46 smithi139 bash[43301]: audit 2022-04-28T21:30:46.707257+0000 mon.a (mon.0) 2781 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:30:49.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:48 smithi059 bash[61152]: cluster 2022-04-28T21:30:47.891129+0000 mgr.y (mgr.24961) 3305 : cluster [DBG] pgmap v3626: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:49.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:48 smithi059 bash[59745]: cluster 2022-04-28T21:30:47.891129+0000 mgr.y (mgr.24961) 3305 : cluster [DBG] pgmap v3626: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:49.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:48 smithi139 bash[43301]: cluster 2022-04-28T21:30:47.891129+0000 mgr.y (mgr.24961) 3305 : cluster [DBG] pgmap v3626: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:51.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:50 smithi139 bash[43301]: cluster 2022-04-28T21:30:49.892329+0000 mgr.y (mgr.24961) 3306 : cluster [DBG] pgmap v3627: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:50 smithi059 bash[61152]: cluster 2022-04-28T21:30:49.892329+0000 mgr.y (mgr.24961) 3306 : cluster [DBG] pgmap v3627: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:50 smithi059 bash[59745]: cluster 2022-04-28T21:30:49.892329+0000 mgr.y (mgr.24961) 3306 : cluster [DBG] pgmap v3627: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:52.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:53.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:52 smithi059 bash[61152]: cluster 2022-04-28T21:30:51.893544+0000 mgr.y (mgr.24961) 3307 : cluster [DBG] pgmap v3628: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:52 smithi059 bash[59745]: cluster 2022-04-28T21:30:51.893544+0000 mgr.y (mgr.24961) 3307 : cluster [DBG] pgmap v3628: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:53.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:52 smithi139 bash[43301]: cluster 2022-04-28T21:30:51.893544+0000 mgr.y (mgr.24961) 3307 : cluster [DBG] pgmap v3628: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:30:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:30:53.906Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:30:55.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:54 smithi059 bash[61152]: cluster 2022-04-28T21:30:53.895906+0000 mgr.y (mgr.24961) 3308 : cluster [DBG] pgmap v3629: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:54 smithi059 bash[59745]: cluster 2022-04-28T21:30:53.895906+0000 mgr.y (mgr.24961) 3308 : cluster [DBG] pgmap v3629: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:55.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:54 smithi139 bash[43301]: cluster 2022-04-28T21:30:53.895906+0000 mgr.y (mgr.24961) 3308 : cluster [DBG] pgmap v3629: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:57.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:56 smithi059 bash[61152]: cluster 2022-04-28T21:30:55.897270+0000 mgr.y (mgr.24961) 3309 : cluster [DBG] pgmap v3630: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:57.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:56 smithi059 bash[59745]: cluster 2022-04-28T21:30:55.897270+0000 mgr.y (mgr.24961) 3309 : cluster [DBG] pgmap v3630: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:57.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:56 smithi139 bash[43301]: cluster 2022-04-28T21:30:55.897270+0000 mgr.y (mgr.24961) 3309 : cluster [DBG] pgmap v3630: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:59.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:30:58 smithi059 bash[61152]: cluster 2022-04-28T21:30:57.898238+0000 mgr.y (mgr.24961) 3310 : cluster [DBG] pgmap v3631: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:30:58 smithi059 bash[59745]: cluster 2022-04-28T21:30:57.898238+0000 mgr.y (mgr.24961) 3310 : cluster [DBG] pgmap v3631: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:30:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:30:58 smithi139 bash[43301]: cluster 2022-04-28T21:30:57.898238+0000 mgr.y (mgr.24961) 3310 : cluster [DBG] pgmap v3631: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:00 smithi139 bash[43301]: cluster 2022-04-28T21:30:59.899341+0000 mgr.y (mgr.24961) 3311 : cluster [DBG] pgmap v3632: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:00 smithi059 bash[61152]: cluster 2022-04-28T21:30:59.899341+0000 mgr.y (mgr.24961) 3311 : cluster [DBG] pgmap v3632: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:00 smithi059 bash[59745]: cluster 2022-04-28T21:30:59.899341+0000 mgr.y (mgr.24961) 3311 : cluster [DBG] pgmap v3632: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:02.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:03.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:02 smithi059 bash[61152]: cluster 2022-04-28T21:31:01.900599+0000 mgr.y (mgr.24961) 3312 : cluster [DBG] pgmap v3633: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:02 smithi059 bash[59745]: cluster 2022-04-28T21:31:01.900599+0000 mgr.y (mgr.24961) 3312 : cluster [DBG] pgmap v3633: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:03.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:02 smithi139 bash[43301]: cluster 2022-04-28T21:31:01.900599+0000 mgr.y (mgr.24961) 3312 : cluster [DBG] pgmap v3633: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:03.907Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:05.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:04 smithi059 bash[61152]: cluster 2022-04-28T21:31:03.901440+0000 mgr.y (mgr.24961) 3313 : cluster [DBG] pgmap v3634: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:04 smithi059 bash[59745]: cluster 2022-04-28T21:31:03.901440+0000 mgr.y (mgr.24961) 3313 : cluster [DBG] pgmap v3634: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:05.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:04 smithi139 bash[43301]: cluster 2022-04-28T21:31:03.901440+0000 mgr.y (mgr.24961) 3313 : cluster [DBG] pgmap v3634: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:07.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:06 smithi059 bash[61152]: cluster 2022-04-28T21:31:05.902872+0000 mgr.y (mgr.24961) 3314 : cluster [DBG] pgmap v3635: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:07.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:06 smithi059 bash[59745]: cluster 2022-04-28T21:31:05.902872+0000 mgr.y (mgr.24961) 3314 : cluster [DBG] pgmap v3635: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:07.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:06 smithi139 bash[43301]: cluster 2022-04-28T21:31:05.902872+0000 mgr.y (mgr.24961) 3314 : cluster [DBG] pgmap v3635: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:09.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:08 smithi059 bash[61152]: cluster 2022-04-28T21:31:07.903856+0000 mgr.y (mgr.24961) 3315 : cluster [DBG] pgmap v3636: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:08 smithi059 bash[59745]: cluster 2022-04-28T21:31:07.903856+0000 mgr.y (mgr.24961) 3315 : cluster [DBG] pgmap v3636: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:09.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:08 smithi139 bash[43301]: cluster 2022-04-28T21:31:07.903856+0000 mgr.y (mgr.24961) 3315 : cluster [DBG] pgmap v3636: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:10 smithi139 bash[43301]: cluster 2022-04-28T21:31:09.905009+0000 mgr.y (mgr.24961) 3316 : cluster [DBG] pgmap v3637: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:11.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:10 smithi059 bash[61152]: cluster 2022-04-28T21:31:09.905009+0000 mgr.y (mgr.24961) 3316 : cluster [DBG] pgmap v3637: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:11.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:10 smithi059 bash[59745]: cluster 2022-04-28T21:31:09.905009+0000 mgr.y (mgr.24961) 3316 : cluster [DBG] pgmap v3637: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:12.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:12 smithi059 bash[61152]: cluster 2022-04-28T21:31:11.906263+0000 mgr.y (mgr.24961) 3317 : cluster [DBG] pgmap v3638: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:13.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:12 smithi059 bash[59745]: cluster 2022-04-28T21:31:11.906263+0000 mgr.y (mgr.24961) 3317 : cluster [DBG] pgmap v3638: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:13.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:12 smithi139 bash[43301]: cluster 2022-04-28T21:31:11.906263+0000 mgr.y (mgr.24961) 3317 : cluster [DBG] pgmap v3638: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:14.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:13.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:15.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:14 smithi059 bash[61152]: cluster 2022-04-28T21:31:13.906979+0000 mgr.y (mgr.24961) 3318 : cluster [DBG] pgmap v3639: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:15.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:14 smithi059 bash[59745]: cluster 2022-04-28T21:31:13.906979+0000 mgr.y (mgr.24961) 3318 : cluster [DBG] pgmap v3639: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:15.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:14 smithi139 bash[43301]: cluster 2022-04-28T21:31:13.906979+0000 mgr.y (mgr.24961) 3318 : cluster [DBG] pgmap v3639: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:17.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:16 smithi059 bash[61152]: cluster 2022-04-28T21:31:15.908438+0000 mgr.y (mgr.24961) 3319 : cluster [DBG] pgmap v3640: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:16 smithi059 bash[59745]: cluster 2022-04-28T21:31:15.908438+0000 mgr.y (mgr.24961) 3319 : cluster [DBG] pgmap v3640: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:16 smithi139 bash[43301]: cluster 2022-04-28T21:31:15.908438+0000 mgr.y (mgr.24961) 3319 : cluster [DBG] pgmap v3640: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: cluster 2022-04-28T21:31:17.909513+0000 mgr.y (mgr.24961) 3320 : cluster [DBG] pgmap v3641: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.885679+0000 mon.a (mon.0) 2782 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:31:19.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.886148+0000 mon.a (mon.0) 2783 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:31:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.886582+0000 mon.a (mon.0) 2784 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2022-04-28T21:31:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.887068+0000 mon.a (mon.0) 2785 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:31:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.887555+0000 mon.a (mon.0) 2786 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:31:19.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.888016+0000 mon.a (mon.0) 2787 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]: dispatch 2022-04-28T21:31:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.888479+0000 mon.a (mon.0) 2788 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2022-04-28T21:31:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.889168+0000 mon.a (mon.0) 2789 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:31:19.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[59745]: audit 2022-04-28T21:31:18.889867+0000 mon.a (mon.0) 2790 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:31:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: cluster 2022-04-28T21:31:17.909513+0000 mgr.y (mgr.24961) 3320 : cluster [DBG] pgmap v3641: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.885679+0000 mon.a (mon.0) 2782 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:31:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.886148+0000 mon.a (mon.0) 2783 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:31:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.886582+0000 mon.a (mon.0) 2784 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2022-04-28T21:31:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.887068+0000 mon.a (mon.0) 2785 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:31:19.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.887555+0000 mon.a (mon.0) 2786 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:31:19.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.888016+0000 mon.a (mon.0) 2787 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]: dispatch 2022-04-28T21:31:19.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.888479+0000 mon.a (mon.0) 2788 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2022-04-28T21:31:19.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.889168+0000 mon.a (mon.0) 2789 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:31:19.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:18 smithi059 bash[61152]: audit 2022-04-28T21:31:18.889867+0000 mon.a (mon.0) 2790 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:31:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: cluster 2022-04-28T21:31:17.909513+0000 mgr.y (mgr.24961) 3320 : cluster [DBG] pgmap v3641: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.885679+0000 mon.a (mon.0) 2782 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:31:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.886148+0000 mon.a (mon.0) 2783 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:31:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.886582+0000 mon.a (mon.0) 2784 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 0]}]: dispatch 2022-04-28T21:31:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.887068+0000 mon.a (mon.0) 2785 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:31:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.887555+0000 mon.a (mon.0) 2786 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:31:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.888016+0000 mon.a (mon.0) 2787 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 0]}]: dispatch 2022-04-28T21:31:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.888479+0000 mon.a (mon.0) 2788 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2022-04-28T21:31:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.889168+0000 mon.a (mon.0) 2789 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:31:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:18 smithi139 bash[43301]: audit 2022-04-28T21:31:18.889867+0000 mon.a (mon.0) 2790 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2022-04-28T21:31:20.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:20 smithi059 bash[59745]: cluster 2022-04-28T21:31:18.996672+0000 mon.a (mon.0) 2791 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2022-04-28T21:31:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:20 smithi059 bash[59745]: audit 2022-04-28T21:31:19.001824+0000 mon.a (mon.0) 2792 : audit [INF] from='client.? 172.21.15.59:0/141464087' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:20 smithi059 bash[61152]: cluster 2022-04-28T21:31:18.996672+0000 mon.a (mon.0) 2791 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2022-04-28T21:31:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:20 smithi059 bash[61152]: audit 2022-04-28T21:31:19.001824+0000 mon.a (mon.0) 2792 : audit [INF] from='client.? 172.21.15.59:0/141464087' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:20.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:20 smithi139 bash[43301]: cluster 2022-04-28T21:31:18.996672+0000 mon.a (mon.0) 2791 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2022-04-28T21:31:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:20 smithi139 bash[43301]: audit 2022-04-28T21:31:19.001824+0000 mon.a (mon.0) 2792 : audit [INF] from='client.? 172.21.15.59:0/141464087' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:21.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[61152]: cluster 2022-04-28T21:31:19.910831+0000 mgr.y (mgr.24961) 3321 : cluster [DBG] pgmap v3643: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:21.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[61152]: audit 2022-04-28T21:31:20.001568+0000 mon.a (mon.0) 2793 : audit [INF] from='client.? 172.21.15.59:0/141464087' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[61152]: cluster 2022-04-28T21:31:20.001902+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2022-04-28T21:31:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[61152]: cluster 2022-04-28T21:31:21.004994+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2022-04-28T21:31:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[59745]: cluster 2022-04-28T21:31:19.910831+0000 mgr.y (mgr.24961) 3321 : cluster [DBG] pgmap v3643: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[59745]: audit 2022-04-28T21:31:20.001568+0000 mon.a (mon.0) 2793 : audit [INF] from='client.? 172.21.15.59:0/141464087' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:21.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[59745]: cluster 2022-04-28T21:31:20.001902+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2022-04-28T21:31:21.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:21 smithi059 bash[59745]: cluster 2022-04-28T21:31:21.004994+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2022-04-28T21:31:21.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:21 smithi139 bash[43301]: cluster 2022-04-28T21:31:19.910831+0000 mgr.y (mgr.24961) 3321 : cluster [DBG] pgmap v3643: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:21 smithi139 bash[43301]: audit 2022-04-28T21:31:20.001568+0000 mon.a (mon.0) 2793 : audit [INF] from='client.? 172.21.15.59:0/141464087' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:21 smithi139 bash[43301]: cluster 2022-04-28T21:31:20.001902+0000 mon.a (mon.0) 2794 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2022-04-28T21:31:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:21 smithi139 bash[43301]: cluster 2022-04-28T21:31:21.004994+0000 mon.a (mon.0) 2795 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2022-04-28T21:31:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:22.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:23.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[61152]: cluster 2022-04-28T21:31:21.911747+0000 mgr.y (mgr.24961) 3322 : cluster [DBG] pgmap v3646: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[61152]: cluster 2022-04-28T21:31:22.024856+0000 mon.a (mon.0) 2796 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2022-04-28T21:31:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[61152]: audit 2022-04-28T21:31:22.028426+0000 mon.c (mon.1) 35 : audit [INF] from='client.? 172.21.15.59:0/2298690262' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[61152]: audit 2022-04-28T21:31:22.029434+0000 mon.a (mon.0) 2797 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[59745]: cluster 2022-04-28T21:31:21.911747+0000 mgr.y (mgr.24961) 3322 : cluster [DBG] pgmap v3646: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[59745]: cluster 2022-04-28T21:31:22.024856+0000 mon.a (mon.0) 2796 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2022-04-28T21:31:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[59745]: audit 2022-04-28T21:31:22.028426+0000 mon.c (mon.1) 35 : audit [INF] from='client.? 172.21.15.59:0/2298690262' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[59745]: audit 2022-04-28T21:31:22.029434+0000 mon.a (mon.0) 2797 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:23.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:23 smithi139 bash[43301]: cluster 2022-04-28T21:31:21.911747+0000 mgr.y (mgr.24961) 3322 : cluster [DBG] pgmap v3646: 137 pgs: 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:23 smithi139 bash[43301]: cluster 2022-04-28T21:31:22.024856+0000 mon.a (mon.0) 2796 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2022-04-28T21:31:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:23 smithi139 bash[43301]: audit 2022-04-28T21:31:22.028426+0000 mon.c (mon.1) 35 : audit [INF] from='client.? 172.21.15.59:0/2298690262' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:23 smithi139 bash[43301]: audit 2022-04-28T21:31:22.029434+0000 mon.a (mon.0) 2797 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:24.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:24 smithi059 bash[61152]: audit 2022-04-28T21:31:23.031734+0000 mon.a (mon.0) 2798 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:24 smithi059 bash[61152]: cluster 2022-04-28T21:31:23.032216+0000 mon.a (mon.0) 2799 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2022-04-28T21:31:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:24 smithi059 bash[61152]: cluster 2022-04-28T21:31:24.035715+0000 mon.a (mon.0) 2800 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2022-04-28T21:31:24.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:24 smithi059 bash[59745]: audit 2022-04-28T21:31:23.031734+0000 mon.a (mon.0) 2798 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:24 smithi059 bash[59745]: cluster 2022-04-28T21:31:23.032216+0000 mon.a (mon.0) 2799 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2022-04-28T21:31:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:24 smithi059 bash[59745]: cluster 2022-04-28T21:31:24.035715+0000 mon.a (mon.0) 2800 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2022-04-28T21:31:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:23.909Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:24.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:24 smithi139 bash[43301]: audit 2022-04-28T21:31:23.031734+0000 mon.a (mon.0) 2798 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:24.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:24 smithi139 bash[43301]: cluster 2022-04-28T21:31:23.032216+0000 mon.a (mon.0) 2799 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2022-04-28T21:31:24.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:24 smithi139 bash[43301]: cluster 2022-04-28T21:31:24.035715+0000 mon.a (mon.0) 2800 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2022-04-28T21:31:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:25 smithi059 bash[61152]: cluster 2022-04-28T21:31:23.912484+0000 mgr.y (mgr.24961) 3323 : cluster [DBG] pgmap v3649: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:25 smithi059 bash[59745]: cluster 2022-04-28T21:31:23.912484+0000 mgr.y (mgr.24961) 3323 : cluster [DBG] pgmap v3649: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:25.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:25 smithi139 bash[43301]: cluster 2022-04-28T21:31:23.912484+0000 mgr.y (mgr.24961) 3323 : cluster [DBG] pgmap v3649: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 728 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:26 smithi059 bash[61152]: cluster 2022-04-28T21:31:25.067267+0000 mon.a (mon.0) 2801 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2022-04-28T21:31:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:26 smithi059 bash[61152]: audit 2022-04-28T21:31:25.070543+0000 mon.c (mon.1) 36 : audit [INF] from='client.? 172.21.15.59:0/1569697303' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:26.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:26 smithi059 bash[61152]: audit 2022-04-28T21:31:25.071019+0000 mon.a (mon.0) 2802 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:26 smithi059 bash[59745]: cluster 2022-04-28T21:31:25.067267+0000 mon.a (mon.0) 2801 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2022-04-28T21:31:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:26 smithi059 bash[59745]: audit 2022-04-28T21:31:25.070543+0000 mon.c (mon.1) 36 : audit [INF] from='client.? 172.21.15.59:0/1569697303' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:26 smithi059 bash[59745]: audit 2022-04-28T21:31:25.071019+0000 mon.a (mon.0) 2802 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:26 smithi139 bash[43301]: cluster 2022-04-28T21:31:25.067267+0000 mon.a (mon.0) 2801 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2022-04-28T21:31:26.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:26 smithi139 bash[43301]: audit 2022-04-28T21:31:25.070543+0000 mon.c (mon.1) 36 : audit [INF] from='client.? 172.21.15.59:0/1569697303' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:26.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:26 smithi139 bash[43301]: audit 2022-04-28T21:31:25.071019+0000 mon.a (mon.0) 2802 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:27.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:27 smithi059 bash[59745]: cluster 2022-04-28T21:31:25.913501+0000 mgr.y (mgr.24961) 3324 : cluster [DBG] pgmap v3652: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:27 smithi059 bash[59745]: audit 2022-04-28T21:31:26.080277+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:27 smithi059 bash[59745]: cluster 2022-04-28T21:31:26.080518+0000 mon.a (mon.0) 2804 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2022-04-28T21:31:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:27 smithi059 bash[61152]: cluster 2022-04-28T21:31:25.913501+0000 mgr.y (mgr.24961) 3324 : cluster [DBG] pgmap v3652: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:27 smithi059 bash[61152]: audit 2022-04-28T21:31:26.080277+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:27 smithi059 bash[61152]: cluster 2022-04-28T21:31:26.080518+0000 mon.a (mon.0) 2804 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2022-04-28T21:31:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:27 smithi139 bash[43301]: cluster 2022-04-28T21:31:25.913501+0000 mgr.y (mgr.24961) 3324 : cluster [DBG] pgmap v3652: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:27 smithi139 bash[43301]: audit 2022-04-28T21:31:26.080277+0000 mon.a (mon.0) 2803 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:27 smithi139 bash[43301]: cluster 2022-04-28T21:31:26.080518+0000 mon.a (mon.0) 2804 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2022-04-28T21:31:28.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:28 smithi139 bash[43301]: cluster 2022-04-28T21:31:27.082859+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2022-04-28T21:31:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:28 smithi059 bash[61152]: cluster 2022-04-28T21:31:27.082859+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2022-04-28T21:31:28.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:28 smithi059 bash[59745]: cluster 2022-04-28T21:31:27.082859+0000 mon.a (mon.0) 2805 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2022-04-28T21:31:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[61152]: cluster 2022-04-28T21:31:27.914234+0000 mgr.y (mgr.24961) 3325 : cluster [DBG] pgmap v3655: 137 pgs: 137 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[61152]: cluster 2022-04-28T21:31:28.105112+0000 mon.a (mon.0) 2806 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2022-04-28T21:31:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[61152]: audit 2022-04-28T21:31:28.110598+0000 mon.b (mon.2) 13 : audit [INF] from='client.? 172.21.15.59:0/3656191801' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[61152]: audit 2022-04-28T21:31:28.114734+0000 mon.a (mon.0) 2807 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[59745]: cluster 2022-04-28T21:31:27.914234+0000 mgr.y (mgr.24961) 3325 : cluster [DBG] pgmap v3655: 137 pgs: 137 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:29.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[59745]: cluster 2022-04-28T21:31:28.105112+0000 mon.a (mon.0) 2806 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2022-04-28T21:31:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[59745]: audit 2022-04-28T21:31:28.110598+0000 mon.b (mon.2) 13 : audit [INF] from='client.? 172.21.15.59:0/3656191801' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:29 smithi059 bash[59745]: audit 2022-04-28T21:31:28.114734+0000 mon.a (mon.0) 2807 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:29 smithi139 bash[43301]: cluster 2022-04-28T21:31:27.914234+0000 mgr.y (mgr.24961) 3325 : cluster [DBG] pgmap v3655: 137 pgs: 137 active+clean; 24 KiB data, 734 MiB used, 715 GiB / 715 GiB avail 2022-04-28T21:31:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:29 smithi139 bash[43301]: cluster 2022-04-28T21:31:28.105112+0000 mon.a (mon.0) 2806 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2022-04-28T21:31:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:29 smithi139 bash[43301]: audit 2022-04-28T21:31:28.110598+0000 mon.b (mon.2) 13 : audit [INF] from='client.? 172.21.15.59:0/3656191801' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:29 smithi139 bash[43301]: audit 2022-04-28T21:31:28.114734+0000 mon.a (mon.0) 2807 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:30.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:30 smithi059 bash[61152]: audit 2022-04-28T21:31:29.113852+0000 mon.a (mon.0) 2808 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:30 smithi059 bash[61152]: cluster 2022-04-28T21:31:29.114165+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2022-04-28T21:31:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:30 smithi059 bash[59745]: audit 2022-04-28T21:31:29.113852+0000 mon.a (mon.0) 2808 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:30.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:30 smithi059 bash[59745]: cluster 2022-04-28T21:31:29.114165+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2022-04-28T21:31:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:30 smithi139 bash[43301]: audit 2022-04-28T21:31:29.113852+0000 mon.a (mon.0) 2808 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:30 smithi139 bash[43301]: cluster 2022-04-28T21:31:29.114165+0000 mon.a (mon.0) 2809 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2022-04-28T21:31:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:31 smithi059 bash[61152]: cluster 2022-04-28T21:31:29.915450+0000 mgr.y (mgr.24961) 3326 : cluster [DBG] pgmap v3658: 169 pgs: 4 creating+activating, 4 creating+peering, 22 unknown, 139 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:31.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:31 smithi059 bash[61152]: cluster 2022-04-28T21:31:30.134967+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2022-04-28T21:31:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:31 smithi059 bash[59745]: cluster 2022-04-28T21:31:29.915450+0000 mgr.y (mgr.24961) 3326 : cluster [DBG] pgmap v3658: 169 pgs: 4 creating+activating, 4 creating+peering, 22 unknown, 139 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:31 smithi059 bash[59745]: cluster 2022-04-28T21:31:30.134967+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2022-04-28T21:31:31.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:31 smithi139 bash[43301]: cluster 2022-04-28T21:31:29.915450+0000 mgr.y (mgr.24961) 3326 : cluster [DBG] pgmap v3658: 169 pgs: 4 creating+activating, 4 creating+peering, 22 unknown, 139 active+clean; 24 KiB data, 739 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:31.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:31 smithi139 bash[43301]: cluster 2022-04-28T21:31:30.134967+0000 mon.a (mon.0) 2810 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2022-04-28T21:31:32.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:32 smithi059 bash[61152]: cluster 2022-04-28T21:31:31.149090+0000 mon.a (mon.0) 2811 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2022-04-28T21:31:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:32 smithi059 bash[61152]: audit 2022-04-28T21:31:31.153897+0000 mon.a (mon.0) 2812 : audit [INF] from='client.? 172.21.15.59:0/2255032056' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:32.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:32 smithi059 bash[59745]: cluster 2022-04-28T21:31:31.149090+0000 mon.a (mon.0) 2811 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2022-04-28T21:31:32.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:32 smithi059 bash[59745]: audit 2022-04-28T21:31:31.153897+0000 mon.a (mon.0) 2812 : audit [INF] from='client.? 172.21.15.59:0/2255032056' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:32.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:32 smithi139 bash[43301]: cluster 2022-04-28T21:31:31.149090+0000 mon.a (mon.0) 2811 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2022-04-28T21:31:32.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:32 smithi139 bash[43301]: audit 2022-04-28T21:31:31.153897+0000 mon.a (mon.0) 2812 : audit [INF] from='client.? 172.21.15.59:0/2255032056' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[59745]: cluster 2022-04-28T21:31:31.916278+0000 mgr.y (mgr.24961) 3327 : cluster [DBG] pgmap v3661: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[59745]: audit 2022-04-28T21:31:32.154413+0000 mon.a (mon.0) 2813 : audit [INF] from='client.? 172.21.15.59:0/2255032056' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[59745]: cluster 2022-04-28T21:31:32.154809+0000 mon.a (mon.0) 2814 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2022-04-28T21:31:33.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[61152]: cluster 2022-04-28T21:31:31.916278+0000 mgr.y (mgr.24961) 3327 : cluster [DBG] pgmap v3661: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[61152]: audit 2022-04-28T21:31:32.154413+0000 mon.a (mon.0) 2813 : audit [INF] from='client.? 172.21.15.59:0/2255032056' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:33.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[61152]: cluster 2022-04-28T21:31:32.154809+0000 mon.a (mon.0) 2814 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2022-04-28T21:31:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:33 smithi139 bash[43301]: cluster 2022-04-28T21:31:31.916278+0000 mgr.y (mgr.24961) 3327 : cluster [DBG] pgmap v3661: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:33.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:33 smithi139 bash[43301]: audit 2022-04-28T21:31:32.154413+0000 mon.a (mon.0) 2813 : audit [INF] from='client.? 172.21.15.59:0/2255032056' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:33.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:33 smithi139 bash[43301]: cluster 2022-04-28T21:31:32.154809+0000 mon.a (mon.0) 2814 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2022-04-28T21:31:34.180 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:33.908Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:34.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:34 smithi059 bash[61152]: cluster 2022-04-28T21:31:33.173776+0000 mon.a (mon.0) 2815 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2022-04-28T21:31:34.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:34 smithi059 bash[59745]: cluster 2022-04-28T21:31:33.173776+0000 mon.a (mon.0) 2815 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2022-04-28T21:31:34.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:34 smithi139 bash[43301]: cluster 2022-04-28T21:31:33.173776+0000 mon.a (mon.0) 2815 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2022-04-28T21:31:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[59745]: cluster 2022-04-28T21:31:33.916991+0000 mgr.y (mgr.24961) 3328 : cluster [DBG] pgmap v3664: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[59745]: cluster 2022-04-28T21:31:34.184914+0000 mon.a (mon.0) 2816 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2022-04-28T21:31:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[59745]: audit 2022-04-28T21:31:34.186260+0000 mon.c (mon.1) 37 : audit [INF] from='client.? 172.21.15.59:0/1743253249' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[59745]: audit 2022-04-28T21:31:34.186824+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[61152]: cluster 2022-04-28T21:31:33.916991+0000 mgr.y (mgr.24961) 3328 : cluster [DBG] pgmap v3664: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[61152]: cluster 2022-04-28T21:31:34.184914+0000 mon.a (mon.0) 2816 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2022-04-28T21:31:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[61152]: audit 2022-04-28T21:31:34.186260+0000 mon.c (mon.1) 37 : audit [INF] from='client.? 172.21.15.59:0/1743253249' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:35.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:35 smithi059 bash[61152]: audit 2022-04-28T21:31:34.186824+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:35 smithi139 bash[43301]: cluster 2022-04-28T21:31:33.916991+0000 mgr.y (mgr.24961) 3328 : cluster [DBG] pgmap v3664: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:35.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:35 smithi139 bash[43301]: cluster 2022-04-28T21:31:34.184914+0000 mon.a (mon.0) 2816 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2022-04-28T21:31:35.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:35 smithi139 bash[43301]: audit 2022-04-28T21:31:34.186260+0000 mon.c (mon.1) 37 : audit [INF] from='client.? 172.21.15.59:0/1743253249' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:35.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:35 smithi139 bash[43301]: audit 2022-04-28T21:31:34.186824+0000 mon.a (mon.0) 2817 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:36.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:36 smithi059 bash[61152]: audit 2022-04-28T21:31:35.198648+0000 mon.a (mon.0) 2818 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:36 smithi059 bash[61152]: cluster 2022-04-28T21:31:35.204295+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2022-04-28T21:31:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:36 smithi059 bash[59745]: audit 2022-04-28T21:31:35.198648+0000 mon.a (mon.0) 2818 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:36.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:36 smithi059 bash[59745]: cluster 2022-04-28T21:31:35.204295+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2022-04-28T21:31:36.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:36 smithi139 bash[43301]: audit 2022-04-28T21:31:35.198648+0000 mon.a (mon.0) 2818 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:36.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:36 smithi139 bash[43301]: cluster 2022-04-28T21:31:35.204295+0000 mon.a (mon.0) 2819 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2022-04-28T21:31:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:37 smithi059 bash[61152]: cluster 2022-04-28T21:31:35.917764+0000 mgr.y (mgr.24961) 3329 : cluster [DBG] pgmap v3667: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:37 smithi059 bash[61152]: cluster 2022-04-28T21:31:36.203311+0000 mon.a (mon.0) 2820 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2022-04-28T21:31:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:37 smithi059 bash[59745]: cluster 2022-04-28T21:31:35.917764+0000 mgr.y (mgr.24961) 3329 : cluster [DBG] pgmap v3667: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:37 smithi059 bash[59745]: cluster 2022-04-28T21:31:36.203311+0000 mon.a (mon.0) 2820 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2022-04-28T21:31:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:37 smithi139 bash[43301]: cluster 2022-04-28T21:31:35.917764+0000 mgr.y (mgr.24961) 3329 : cluster [DBG] pgmap v3667: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:37.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:37 smithi139 bash[43301]: cluster 2022-04-28T21:31:36.203311+0000 mon.a (mon.0) 2820 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2022-04-28T21:31:38.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:38 smithi059 bash[61152]: cluster 2022-04-28T21:31:37.225856+0000 mon.a (mon.0) 2821 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2022-04-28T21:31:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:38 smithi059 bash[61152]: audit 2022-04-28T21:31:37.229476+0000 mon.c (mon.1) 38 : audit [INF] from='client.? 172.21.15.59:0/3998588740' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:38 smithi059 bash[61152]: audit 2022-04-28T21:31:37.233610+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:38 smithi059 bash[59745]: cluster 2022-04-28T21:31:37.225856+0000 mon.a (mon.0) 2821 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2022-04-28T21:31:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:38 smithi059 bash[59745]: audit 2022-04-28T21:31:37.229476+0000 mon.c (mon.1) 38 : audit [INF] from='client.? 172.21.15.59:0/3998588740' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:38 smithi059 bash[59745]: audit 2022-04-28T21:31:37.233610+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:38.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:38 smithi139 bash[43301]: cluster 2022-04-28T21:31:37.225856+0000 mon.a (mon.0) 2821 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2022-04-28T21:31:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:38 smithi139 bash[43301]: audit 2022-04-28T21:31:37.229476+0000 mon.c (mon.1) 38 : audit [INF] from='client.? 172.21.15.59:0/3998588740' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:38 smithi139 bash[43301]: audit 2022-04-28T21:31:37.233610+0000 mon.a (mon.0) 2822 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:39.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:39 smithi059 bash[61152]: cluster 2022-04-28T21:31:37.918515+0000 mgr.y (mgr.24961) 3330 : cluster [DBG] pgmap v3670: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:39 smithi059 bash[61152]: audit 2022-04-28T21:31:38.228524+0000 mon.a (mon.0) 2823 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:39 smithi059 bash[61152]: cluster 2022-04-28T21:31:38.228599+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2022-04-28T21:31:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:39 smithi059 bash[59745]: cluster 2022-04-28T21:31:37.918515+0000 mgr.y (mgr.24961) 3330 : cluster [DBG] pgmap v3670: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:39 smithi059 bash[59745]: audit 2022-04-28T21:31:38.228524+0000 mon.a (mon.0) 2823 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:39.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:39 smithi059 bash[59745]: cluster 2022-04-28T21:31:38.228599+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2022-04-28T21:31:39.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:39 smithi139 bash[43301]: cluster 2022-04-28T21:31:37.918515+0000 mgr.y (mgr.24961) 3330 : cluster [DBG] pgmap v3670: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:39 smithi139 bash[43301]: audit 2022-04-28T21:31:38.228524+0000 mon.a (mon.0) 2823 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-7","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:39 smithi139 bash[43301]: cluster 2022-04-28T21:31:38.228599+0000 mon.a (mon.0) 2824 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2022-04-28T21:31:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:40 smithi059 bash[61152]: cluster 2022-04-28T21:31:39.244976+0000 mon.a (mon.0) 2825 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2022-04-28T21:31:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:40 smithi059 bash[59745]: cluster 2022-04-28T21:31:39.244976+0000 mon.a (mon.0) 2825 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2022-04-28T21:31:40.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:40 smithi139 bash[43301]: cluster 2022-04-28T21:31:39.244976+0000 mon.a (mon.0) 2825 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2022-04-28T21:31:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:41 smithi059 bash[61152]: cluster 2022-04-28T21:31:39.919224+0000 mgr.y (mgr.24961) 3331 : cluster [DBG] pgmap v3673: 137 pgs: 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:41.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:41 smithi059 bash[61152]: cluster 2022-04-28T21:31:40.263317+0000 mon.a (mon.0) 2826 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2022-04-28T21:31:41.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:41 smithi059 bash[61152]: audit 2022-04-28T21:31:40.266397+0000 mon.a (mon.0) 2827 : audit [INF] from='client.? 172.21.15.59:0/216670807' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:41.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:41 smithi059 bash[59745]: cluster 2022-04-28T21:31:39.919224+0000 mgr.y (mgr.24961) 3331 : cluster [DBG] pgmap v3673: 137 pgs: 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:41.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:41 smithi059 bash[59745]: cluster 2022-04-28T21:31:40.263317+0000 mon.a (mon.0) 2826 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2022-04-28T21:31:41.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:41 smithi059 bash[59745]: audit 2022-04-28T21:31:40.266397+0000 mon.a (mon.0) 2827 : audit [INF] from='client.? 172.21.15.59:0/216670807' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:41 smithi139 bash[43301]: cluster 2022-04-28T21:31:39.919224+0000 mgr.y (mgr.24961) 3331 : cluster [DBG] pgmap v3673: 137 pgs: 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:41 smithi139 bash[43301]: cluster 2022-04-28T21:31:40.263317+0000 mon.a (mon.0) 2826 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2022-04-28T21:31:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:41 smithi139 bash[43301]: audit 2022-04-28T21:31:40.266397+0000 mon.a (mon.0) 2827 : audit [INF] from='client.? 172.21.15.59:0/216670807' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:42.287 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[59745]: audit 2022-04-28T21:31:41.269307+0000 mon.a (mon.0) 2828 : audit [INF] from='client.? 172.21.15.59:0/216670807' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-8","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:42.287 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[59745]: cluster 2022-04-28T21:31:41.269989+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2022-04-28T21:31:42.287 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[59745]: cluster 2022-04-28T21:31:42.273887+0000 mon.a (mon.0) 2830 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2022-04-28T21:31:42.287 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:42.016Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[61152]: audit 2022-04-28T21:31:41.269307+0000 mon.a (mon.0) 2828 : audit [INF] from='client.? 172.21.15.59:0/216670807' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-8","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:42.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[61152]: cluster 2022-04-28T21:31:41.269989+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2022-04-28T21:31:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:42 smithi059 bash[61152]: cluster 2022-04-28T21:31:42.273887+0000 mon.a (mon.0) 2830 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2022-04-28T21:31:42.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:42 smithi139 bash[43301]: audit 2022-04-28T21:31:41.269307+0000 mon.a (mon.0) 2828 : audit [INF] from='client.? 172.21.15.59:0/216670807' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-8","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:42.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:42 smithi139 bash[43301]: cluster 2022-04-28T21:31:41.269989+0000 mon.a (mon.0) 2829 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2022-04-28T21:31:42.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:42 smithi139 bash[43301]: cluster 2022-04-28T21:31:42.273887+0000 mon.a (mon.0) 2830 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2022-04-28T21:31:43.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:43 smithi059 bash[59745]: cluster 2022-04-28T21:31:41.920015+0000 mgr.y (mgr.24961) 3332 : cluster [DBG] pgmap v3676: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:43 smithi059 bash[61152]: cluster 2022-04-28T21:31:41.920015+0000 mgr.y (mgr.24961) 3332 : cluster [DBG] pgmap v3676: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:43.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:43 smithi139 bash[43301]: cluster 2022-04-28T21:31:41.920015+0000 mgr.y (mgr.24961) 3332 : cluster [DBG] pgmap v3676: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:44.307 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:43.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:44.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:44 smithi059 bash[61152]: cluster 2022-04-28T21:31:43.310540+0000 mon.a (mon.0) 2831 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2022-04-28T21:31:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:44 smithi059 bash[61152]: audit 2022-04-28T21:31:43.312656+0000 mon.a (mon.0) 2832 : audit [INF] from='client.? 172.21.15.59:0/1355358604' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:44.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:44 smithi059 bash[59745]: cluster 2022-04-28T21:31:43.310540+0000 mon.a (mon.0) 2831 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2022-04-28T21:31:44.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:44 smithi059 bash[59745]: audit 2022-04-28T21:31:43.312656+0000 mon.a (mon.0) 2832 : audit [INF] from='client.? 172.21.15.59:0/1355358604' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:44.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:44 smithi139 bash[43301]: cluster 2022-04-28T21:31:43.310540+0000 mon.a (mon.0) 2831 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2022-04-28T21:31:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:44 smithi139 bash[43301]: audit 2022-04-28T21:31:43.312656+0000 mon.a (mon.0) 2832 : audit [INF] from='client.? 172.21.15.59:0/1355358604' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:45.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:45 smithi059 bash[61152]: cluster 2022-04-28T21:31:43.920797+0000 mgr.y (mgr.24961) 3333 : cluster [DBG] pgmap v3679: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:45 smithi059 bash[61152]: audit 2022-04-28T21:31:44.321737+0000 mon.a (mon.0) 2833 : audit [INF] from='client.? 172.21.15.59:0/1355358604' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-9","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:45 smithi059 bash[61152]: cluster 2022-04-28T21:31:44.322113+0000 mon.a (mon.0) 2834 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2022-04-28T21:31:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:45 smithi059 bash[59745]: cluster 2022-04-28T21:31:43.920797+0000 mgr.y (mgr.24961) 3333 : cluster [DBG] pgmap v3679: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:45 smithi059 bash[59745]: audit 2022-04-28T21:31:44.321737+0000 mon.a (mon.0) 2833 : audit [INF] from='client.? 172.21.15.59:0/1355358604' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-9","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:45 smithi059 bash[59745]: cluster 2022-04-28T21:31:44.322113+0000 mon.a (mon.0) 2834 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2022-04-28T21:31:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:45 smithi139 bash[43301]: cluster 2022-04-28T21:31:43.920797+0000 mgr.y (mgr.24961) 3333 : cluster [DBG] pgmap v3679: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:45 smithi139 bash[43301]: audit 2022-04-28T21:31:44.321737+0000 mon.a (mon.0) 2833 : audit [INF] from='client.? 172.21.15.59:0/1355358604' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-9","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:45 smithi139 bash[43301]: cluster 2022-04-28T21:31:44.322113+0000 mon.a (mon.0) 2834 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2022-04-28T21:31:46.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:46 smithi059 bash[59745]: cluster 2022-04-28T21:31:45.322273+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2022-04-28T21:31:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:46 smithi059 bash[61152]: cluster 2022-04-28T21:31:45.322273+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2022-04-28T21:31:46.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:46 smithi139 bash[43301]: cluster 2022-04-28T21:31:45.322273+0000 mon.a (mon.0) 2835 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2022-04-28T21:31:47.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: cluster 2022-04-28T21:31:45.921461+0000 mgr.y (mgr.24961) 3334 : cluster [DBG] pgmap v3682: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:47.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: cluster 2022-04-28T21:31:46.344681+0000 mon.a (mon.0) 2836 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2022-04-28T21:31:47.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:46.351484+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.59:0/1782902407' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:47.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:46.355089+0000 mon.a (mon.0) 2837 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:47.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:46.713939+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:31:47.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:46.716806+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:31:47.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:46.718644+0000 mon.a (mon.0) 2840 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:31:47.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.008253+0000 mon.a (mon.0) 2841 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.011274+0000 mon.a (mon.0) 2842 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.012266+0000 mon.a (mon.0) 2843 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.012573+0000 mon.a (mon.0) 2844 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.013008+0000 mon.a (mon.0) 2845 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.013323+0000 mon.a (mon.0) 2846 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.013698+0000 mon.a (mon.0) 2847 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.014019+0000 mon.a (mon.0) 2848 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.014326+0000 mon.a (mon.0) 2849 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.014631+0000 mon.a (mon.0) 2850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.014969+0000 mon.a (mon.0) 2851 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.015281+0000 mon.a (mon.0) 2852 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.015592+0000 mon.a (mon.0) 2853 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.015914+0000 mon.a (mon.0) 2854 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.016231+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.016538+0000 mon.a (mon.0) 2856 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:31:47.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.016841+0000 mon.a (mon.0) 2857 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.017184+0000 mon.a (mon.0) 2858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.017463+0000 mon.a (mon.0) 2859 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:31:47.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.018076+0000 mon.a (mon.0) 2860 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:47 smithi139 bash[43301]: audit 2022-04-28T21:31:47.024209+0000 mon.a (mon.0) 2861 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:31:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: cluster 2022-04-28T21:31:45.921461+0000 mgr.y (mgr.24961) 3334 : cluster [DBG] pgmap v3682: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: cluster 2022-04-28T21:31:46.344681+0000 mon.a (mon.0) 2836 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2022-04-28T21:31:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:46.351484+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.59:0/1782902407' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:46.355089+0000 mon.a (mon.0) 2837 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:46.713939+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:31:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:46.716806+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:31:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:46.718644+0000 mon.a (mon.0) 2840 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:31:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.008253+0000 mon.a (mon.0) 2841 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.011274+0000 mon.a (mon.0) 2842 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.012266+0000 mon.a (mon.0) 2843 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.012573+0000 mon.a (mon.0) 2844 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.013008+0000 mon.a (mon.0) 2845 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.013323+0000 mon.a (mon.0) 2846 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.013698+0000 mon.a (mon.0) 2847 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.014019+0000 mon.a (mon.0) 2848 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.014326+0000 mon.a (mon.0) 2849 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: cluster 2022-04-28T21:31:45.921461+0000 mgr.y (mgr.24961) 3334 : cluster [DBG] pgmap v3682: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:47.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: cluster 2022-04-28T21:31:46.344681+0000 mon.a (mon.0) 2836 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2022-04-28T21:31:47.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:46.351484+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.59:0/1782902407' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:47.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:46.355089+0000 mon.a (mon.0) 2837 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:47.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:46.713939+0000 mon.a (mon.0) 2838 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:31:47.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:46.716806+0000 mon.a (mon.0) 2839 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:31:47.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:46.718644+0000 mon.a (mon.0) 2840 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:31:47.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.008253+0000 mon.a (mon.0) 2841 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.011274+0000 mon.a (mon.0) 2842 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.012266+0000 mon.a (mon.0) 2843 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.012573+0000 mon.a (mon.0) 2844 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.013008+0000 mon.a (mon.0) 2845 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.013323+0000 mon.a (mon.0) 2846 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.013698+0000 mon.a (mon.0) 2847 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.014019+0000 mon.a (mon.0) 2848 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.850 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.014326+0000 mon.a (mon.0) 2849 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.851 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.014631+0000 mon.a (mon.0) 2850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.851 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.014969+0000 mon.a (mon.0) 2851 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.852 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.015281+0000 mon.a (mon.0) 2852 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.852 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.015592+0000 mon.a (mon.0) 2853 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.852 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.015914+0000 mon.a (mon.0) 2854 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.853 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.016231+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.853 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.016538+0000 mon.a (mon.0) 2856 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:31:47.853 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.016841+0000 mon.a (mon.0) 2857 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.854 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.017184+0000 mon.a (mon.0) 2858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.854 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.017463+0000 mon.a (mon.0) 2859 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:31:47.854 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.018076+0000 mon.a (mon.0) 2860 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.855 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[59745]: audit 2022-04-28T21:31:47.024209+0000 mon.a (mon.0) 2861 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:31:47.855 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.014631+0000 mon.a (mon.0) 2850 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.014969+0000 mon.a (mon.0) 2851 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.015281+0000 mon.a (mon.0) 2852 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.015592+0000 mon.a (mon.0) 2853 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.015914+0000 mon.a (mon.0) 2854 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.016231+0000 mon.a (mon.0) 2855 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:31:47.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.016538+0000 mon.a (mon.0) 2856 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:31:47.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.016841+0000 mon.a (mon.0) 2857 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.017184+0000 mon.a (mon.0) 2858 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.017463+0000 mon.a (mon.0) 2859 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:31:47.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.018076+0000 mon.a (mon.0) 2860 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:31:47.860 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:47 smithi059 bash[61152]: audit 2022-04-28T21:31:47.024209+0000 mon.a (mon.0) 2861 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:31:48.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:48 smithi059 bash[61152]: cephadm 2022-04-28T21:31:46.720280+0000 mgr.y (mgr.24961) 3335 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:31:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:48 smithi059 bash[61152]: audit 2022-04-28T21:31:47.348594+0000 mon.a (mon.0) 2862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:48 smithi059 bash[61152]: cluster 2022-04-28T21:31:47.349078+0000 mon.a (mon.0) 2863 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2022-04-28T21:31:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:48 smithi059 bash[59745]: cephadm 2022-04-28T21:31:46.720280+0000 mgr.y (mgr.24961) 3335 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:31:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:48 smithi059 bash[59745]: audit 2022-04-28T21:31:47.348594+0000 mon.a (mon.0) 2862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:48 smithi059 bash[59745]: cluster 2022-04-28T21:31:47.349078+0000 mon.a (mon.0) 2863 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2022-04-28T21:31:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:48 smithi139 bash[43301]: cephadm 2022-04-28T21:31:46.720280+0000 mgr.y (mgr.24961) 3335 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:31:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:48 smithi139 bash[43301]: audit 2022-04-28T21:31:47.348594+0000 mon.a (mon.0) 2862 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-10","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:48 smithi139 bash[43301]: cluster 2022-04-28T21:31:47.349078+0000 mon.a (mon.0) 2863 : cluster [DBG] osdmap e957: 8 total, 8 up, 8 in 2022-04-28T21:31:49.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:49 smithi059 bash[61152]: cluster 2022-04-28T21:31:47.922169+0000 mgr.y (mgr.24961) 3336 : cluster [DBG] pgmap v3685: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:49 smithi059 bash[61152]: cluster 2022-04-28T21:31:48.361545+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2022-04-28T21:31:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:49 smithi059 bash[59745]: cluster 2022-04-28T21:31:47.922169+0000 mgr.y (mgr.24961) 3336 : cluster [DBG] pgmap v3685: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:49 smithi059 bash[59745]: cluster 2022-04-28T21:31:48.361545+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2022-04-28T21:31:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:49 smithi139 bash[43301]: cluster 2022-04-28T21:31:47.922169+0000 mgr.y (mgr.24961) 3336 : cluster [DBG] pgmap v3685: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:49 smithi139 bash[43301]: cluster 2022-04-28T21:31:48.361545+0000 mon.a (mon.0) 2864 : cluster [DBG] osdmap e958: 8 total, 8 up, 8 in 2022-04-28T21:31:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:50 smithi059 bash[59745]: cluster 2022-04-28T21:31:49.385686+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2022-04-28T21:31:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:50 smithi059 bash[59745]: audit 2022-04-28T21:31:49.389111+0000 mon.b (mon.2) 15 : audit [INF] from='client.? 172.21.15.59:0/4173102306' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:50 smithi059 bash[59745]: audit 2022-04-28T21:31:49.393853+0000 mon.a (mon.0) 2866 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:50 smithi059 bash[61152]: cluster 2022-04-28T21:31:49.385686+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2022-04-28T21:31:50.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:50 smithi059 bash[61152]: audit 2022-04-28T21:31:49.389111+0000 mon.b (mon.2) 15 : audit [INF] from='client.? 172.21.15.59:0/4173102306' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:50.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:50 smithi059 bash[61152]: audit 2022-04-28T21:31:49.393853+0000 mon.a (mon.0) 2866 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:50 smithi139 bash[43301]: cluster 2022-04-28T21:31:49.385686+0000 mon.a (mon.0) 2865 : cluster [DBG] osdmap e959: 8 total, 8 up, 8 in 2022-04-28T21:31:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:50 smithi139 bash[43301]: audit 2022-04-28T21:31:49.389111+0000 mon.b (mon.2) 15 : audit [INF] from='client.? 172.21.15.59:0/4173102306' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:50 smithi139 bash[43301]: audit 2022-04-28T21:31:49.393853+0000 mon.a (mon.0) 2866 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:51.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:51 smithi059 bash[59745]: cluster 2022-04-28T21:31:49.922945+0000 mgr.y (mgr.24961) 3337 : cluster [DBG] pgmap v3688: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:51 smithi059 bash[59745]: audit 2022-04-28T21:31:50.392682+0000 mon.a (mon.0) 2867 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:51 smithi059 bash[59745]: cluster 2022-04-28T21:31:50.392864+0000 mon.a (mon.0) 2868 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2022-04-28T21:31:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:51 smithi059 bash[61152]: cluster 2022-04-28T21:31:49.922945+0000 mgr.y (mgr.24961) 3337 : cluster [DBG] pgmap v3688: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:51.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:51 smithi059 bash[61152]: audit 2022-04-28T21:31:50.392682+0000 mon.a (mon.0) 2867 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:51.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:51 smithi059 bash[61152]: cluster 2022-04-28T21:31:50.392864+0000 mon.a (mon.0) 2868 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2022-04-28T21:31:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:51 smithi139 bash[43301]: cluster 2022-04-28T21:31:49.922945+0000 mgr.y (mgr.24961) 3337 : cluster [DBG] pgmap v3688: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:51 smithi139 bash[43301]: audit 2022-04-28T21:31:50.392682+0000 mon.a (mon.0) 2867 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-11","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:51 smithi139 bash[43301]: cluster 2022-04-28T21:31:50.392864+0000 mon.a (mon.0) 2868 : cluster [DBG] osdmap e960: 8 total, 8 up, 8 in 2022-04-28T21:31:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:52.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:52.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:52 smithi059 bash[59745]: cluster 2022-04-28T21:31:51.413211+0000 mon.a (mon.0) 2869 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2022-04-28T21:31:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:52 smithi059 bash[61152]: cluster 2022-04-28T21:31:51.413211+0000 mon.a (mon.0) 2869 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2022-04-28T21:31:52.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:52 smithi139 bash[43301]: cluster 2022-04-28T21:31:51.413211+0000 mon.a (mon.0) 2869 : cluster [DBG] osdmap e961: 8 total, 8 up, 8 in 2022-04-28T21:31:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[59745]: cluster 2022-04-28T21:31:51.923543+0000 mgr.y (mgr.24961) 3338 : cluster [DBG] pgmap v3691: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[59745]: cluster 2022-04-28T21:31:52.429039+0000 mon.a (mon.0) 2870 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2022-04-28T21:31:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[59745]: audit 2022-04-28T21:31:52.431872+0000 mon.c (mon.1) 39 : audit [INF] from='client.? 172.21.15.59:0/1565000078' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[59745]: audit 2022-04-28T21:31:52.433401+0000 mon.a (mon.0) 2871 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[61152]: cluster 2022-04-28T21:31:51.923543+0000 mgr.y (mgr.24961) 3338 : cluster [DBG] pgmap v3691: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[61152]: cluster 2022-04-28T21:31:52.429039+0000 mon.a (mon.0) 2870 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2022-04-28T21:31:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[61152]: audit 2022-04-28T21:31:52.431872+0000 mon.c (mon.1) 39 : audit [INF] from='client.? 172.21.15.59:0/1565000078' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:53.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[61152]: audit 2022-04-28T21:31:52.433401+0000 mon.a (mon.0) 2871 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:53 smithi139 bash[43301]: cluster 2022-04-28T21:31:51.923543+0000 mgr.y (mgr.24961) 3338 : cluster [DBG] pgmap v3691: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:53 smithi139 bash[43301]: cluster 2022-04-28T21:31:52.429039+0000 mon.a (mon.0) 2870 : cluster [DBG] osdmap e962: 8 total, 8 up, 8 in 2022-04-28T21:31:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:53 smithi139 bash[43301]: audit 2022-04-28T21:31:52.431872+0000 mon.c (mon.1) 39 : audit [INF] from='client.? 172.21.15.59:0/1565000078' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:53 smithi139 bash[43301]: audit 2022-04-28T21:31:52.433401+0000 mon.a (mon.0) 2871 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:31:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:31:53.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:31:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:54 smithi059 bash[59745]: audit 2022-04-28T21:31:53.429821+0000 mon.a (mon.0) 2872 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:54 smithi059 bash[59745]: cluster 2022-04-28T21:31:53.429991+0000 mon.a (mon.0) 2873 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2022-04-28T21:31:54.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:54 smithi059 bash[61152]: audit 2022-04-28T21:31:53.429821+0000 mon.a (mon.0) 2872 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:54.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:54 smithi059 bash[61152]: cluster 2022-04-28T21:31:53.429991+0000 mon.a (mon.0) 2873 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2022-04-28T21:31:54.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:54 smithi139 bash[43301]: audit 2022-04-28T21:31:53.429821+0000 mon.a (mon.0) 2872 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-12","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:54 smithi139 bash[43301]: cluster 2022-04-28T21:31:53.429991+0000 mon.a (mon.0) 2873 : cluster [DBG] osdmap e963: 8 total, 8 up, 8 in 2022-04-28T21:31:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:55 smithi059 bash[61152]: cluster 2022-04-28T21:31:53.924253+0000 mgr.y (mgr.24961) 3339 : cluster [DBG] pgmap v3694: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:55 smithi059 bash[61152]: cluster 2022-04-28T21:31:54.447247+0000 mon.a (mon.0) 2874 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2022-04-28T21:31:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:55 smithi059 bash[59745]: cluster 2022-04-28T21:31:53.924253+0000 mgr.y (mgr.24961) 3339 : cluster [DBG] pgmap v3694: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:55 smithi059 bash[59745]: cluster 2022-04-28T21:31:54.447247+0000 mon.a (mon.0) 2874 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2022-04-28T21:31:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:55 smithi139 bash[43301]: cluster 2022-04-28T21:31:53.924253+0000 mgr.y (mgr.24961) 3339 : cluster [DBG] pgmap v3694: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:55 smithi139 bash[43301]: cluster 2022-04-28T21:31:54.447247+0000 mon.a (mon.0) 2874 : cluster [DBG] osdmap e964: 8 total, 8 up, 8 in 2022-04-28T21:31:56.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:56 smithi059 bash[61152]: cluster 2022-04-28T21:31:55.466597+0000 mon.a (mon.0) 2875 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2022-04-28T21:31:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:56 smithi059 bash[61152]: audit 2022-04-28T21:31:55.468999+0000 mon.c (mon.1) 40 : audit [INF] from='client.? 172.21.15.59:0/1517223304' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:56 smithi059 bash[61152]: audit 2022-04-28T21:31:55.469548+0000 mon.a (mon.0) 2876 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:56 smithi059 bash[59745]: cluster 2022-04-28T21:31:55.466597+0000 mon.a (mon.0) 2875 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2022-04-28T21:31:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:56 smithi059 bash[59745]: audit 2022-04-28T21:31:55.468999+0000 mon.c (mon.1) 40 : audit [INF] from='client.? 172.21.15.59:0/1517223304' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:56 smithi059 bash[59745]: audit 2022-04-28T21:31:55.469548+0000 mon.a (mon.0) 2876 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:56 smithi139 bash[43301]: cluster 2022-04-28T21:31:55.466597+0000 mon.a (mon.0) 2875 : cluster [DBG] osdmap e965: 8 total, 8 up, 8 in 2022-04-28T21:31:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:56 smithi139 bash[43301]: audit 2022-04-28T21:31:55.468999+0000 mon.c (mon.1) 40 : audit [INF] from='client.? 172.21.15.59:0/1517223304' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:56 smithi139 bash[43301]: audit 2022-04-28T21:31:55.469548+0000 mon.a (mon.0) 2876 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:57.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:57 smithi059 bash[61152]: cluster 2022-04-28T21:31:55.925029+0000 mgr.y (mgr.24961) 3340 : cluster [DBG] pgmap v3697: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 762 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:57 smithi059 bash[61152]: audit 2022-04-28T21:31:56.476248+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:57 smithi059 bash[61152]: cluster 2022-04-28T21:31:56.476762+0000 mon.a (mon.0) 2878 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2022-04-28T21:31:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:57 smithi059 bash[59745]: cluster 2022-04-28T21:31:55.925029+0000 mgr.y (mgr.24961) 3340 : cluster [DBG] pgmap v3697: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 762 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:57 smithi059 bash[59745]: audit 2022-04-28T21:31:56.476248+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:57 smithi059 bash[59745]: cluster 2022-04-28T21:31:56.476762+0000 mon.a (mon.0) 2878 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2022-04-28T21:31:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:57 smithi139 bash[43301]: cluster 2022-04-28T21:31:55.925029+0000 mgr.y (mgr.24961) 3340 : cluster [DBG] pgmap v3697: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 762 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:57 smithi139 bash[43301]: audit 2022-04-28T21:31:56.476248+0000 mon.a (mon.0) 2877 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-13","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:31:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:57 smithi139 bash[43301]: cluster 2022-04-28T21:31:56.476762+0000 mon.a (mon.0) 2878 : cluster [DBG] osdmap e966: 8 total, 8 up, 8 in 2022-04-28T21:31:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:58 smithi059 bash[61152]: cluster 2022-04-28T21:31:57.491261+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2022-04-28T21:31:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:58 smithi059 bash[59745]: cluster 2022-04-28T21:31:57.491261+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2022-04-28T21:31:58.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:58 smithi139 bash[43301]: cluster 2022-04-28T21:31:57.491261+0000 mon.a (mon.0) 2879 : cluster [DBG] osdmap e967: 8 total, 8 up, 8 in 2022-04-28T21:31:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:59 smithi059 bash[61152]: cluster 2022-04-28T21:31:57.925665+0000 mgr.y (mgr.24961) 3341 : cluster [DBG] pgmap v3700: 137 pgs: 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:59 smithi059 bash[61152]: cluster 2022-04-28T21:31:58.511408+0000 mon.a (mon.0) 2880 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2022-04-28T21:31:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:31:59 smithi059 bash[61152]: audit 2022-04-28T21:31:58.514298+0000 mon.a (mon.0) 2881 : audit [INF] from='client.? 172.21.15.59:0/2027057911' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:59 smithi059 bash[59745]: cluster 2022-04-28T21:31:57.925665+0000 mgr.y (mgr.24961) 3341 : cluster [DBG] pgmap v3700: 137 pgs: 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:59 smithi059 bash[59745]: cluster 2022-04-28T21:31:58.511408+0000 mon.a (mon.0) 2880 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2022-04-28T21:31:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:31:59 smithi059 bash[59745]: audit 2022-04-28T21:31:58.514298+0000 mon.a (mon.0) 2881 : audit [INF] from='client.? 172.21.15.59:0/2027057911' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:31:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:59 smithi139 bash[43301]: cluster 2022-04-28T21:31:57.925665+0000 mgr.y (mgr.24961) 3341 : cluster [DBG] pgmap v3700: 137 pgs: 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:31:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:59 smithi139 bash[43301]: cluster 2022-04-28T21:31:58.511408+0000 mon.a (mon.0) 2880 : cluster [DBG] osdmap e968: 8 total, 8 up, 8 in 2022-04-28T21:31:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:31:59 smithi139 bash[43301]: audit 2022-04-28T21:31:58.514298+0000 mon.a (mon.0) 2881 : audit [INF] from='client.? 172.21.15.59:0/2027057911' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:00.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:00 smithi059 bash[61152]: audit 2022-04-28T21:31:59.521815+0000 mon.a (mon.0) 2882 : audit [INF] from='client.? 172.21.15.59:0/2027057911' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-14","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:00 smithi059 bash[61152]: cluster 2022-04-28T21:31:59.522024+0000 mon.a (mon.0) 2883 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2022-04-28T21:32:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:00 smithi059 bash[59745]: audit 2022-04-28T21:31:59.521815+0000 mon.a (mon.0) 2882 : audit [INF] from='client.? 172.21.15.59:0/2027057911' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-14","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:00 smithi059 bash[59745]: cluster 2022-04-28T21:31:59.522024+0000 mon.a (mon.0) 2883 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2022-04-28T21:32:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:00 smithi139 bash[43301]: audit 2022-04-28T21:31:59.521815+0000 mon.a (mon.0) 2882 : audit [INF] from='client.? 172.21.15.59:0/2027057911' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-14","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:00 smithi139 bash[43301]: cluster 2022-04-28T21:31:59.522024+0000 mon.a (mon.0) 2883 : cluster [DBG] osdmap e969: 8 total, 8 up, 8 in 2022-04-28T21:32:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:01 smithi059 bash[61152]: cluster 2022-04-28T21:31:59.926390+0000 mgr.y (mgr.24961) 3342 : cluster [DBG] pgmap v3703: 169 pgs: 14 creating+peering, 18 unknown, 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:01 smithi059 bash[61152]: cluster 2022-04-28T21:32:00.526078+0000 mon.a (mon.0) 2884 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2022-04-28T21:32:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:01 smithi059 bash[59745]: cluster 2022-04-28T21:31:59.926390+0000 mgr.y (mgr.24961) 3342 : cluster [DBG] pgmap v3703: 169 pgs: 14 creating+peering, 18 unknown, 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:01 smithi059 bash[59745]: cluster 2022-04-28T21:32:00.526078+0000 mon.a (mon.0) 2884 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2022-04-28T21:32:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:01 smithi139 bash[43301]: cluster 2022-04-28T21:31:59.926390+0000 mgr.y (mgr.24961) 3342 : cluster [DBG] pgmap v3703: 169 pgs: 14 creating+peering, 18 unknown, 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:01 smithi139 bash[43301]: cluster 2022-04-28T21:32:00.526078+0000 mon.a (mon.0) 2884 : cluster [DBG] osdmap e970: 8 total, 8 up, 8 in 2022-04-28T21:32:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:02.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:02.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[61152]: cluster 2022-04-28T21:32:01.547577+0000 mon.a (mon.0) 2885 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2022-04-28T21:32:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[61152]: audit 2022-04-28T21:32:01.549861+0000 mon.c (mon.1) 41 : audit [INF] from='client.? 172.21.15.59:0/1113950163' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[61152]: audit 2022-04-28T21:32:01.553444+0000 mon.a (mon.0) 2886 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[61152]: cluster 2022-04-28T21:32:01.927215+0000 mgr.y (mgr.24961) 3343 : cluster [DBG] pgmap v3706: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[59745]: cluster 2022-04-28T21:32:01.547577+0000 mon.a (mon.0) 2885 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2022-04-28T21:32:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[59745]: audit 2022-04-28T21:32:01.549861+0000 mon.c (mon.1) 41 : audit [INF] from='client.? 172.21.15.59:0/1113950163' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[59745]: audit 2022-04-28T21:32:01.553444+0000 mon.a (mon.0) 2886 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:02 smithi059 bash[59745]: cluster 2022-04-28T21:32:01.927215+0000 mgr.y (mgr.24961) 3343 : cluster [DBG] pgmap v3706: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:02 smithi139 bash[43301]: cluster 2022-04-28T21:32:01.547577+0000 mon.a (mon.0) 2885 : cluster [DBG] osdmap e971: 8 total, 8 up, 8 in 2022-04-28T21:32:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:02 smithi139 bash[43301]: audit 2022-04-28T21:32:01.549861+0000 mon.c (mon.1) 41 : audit [INF] from='client.? 172.21.15.59:0/1113950163' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:02 smithi139 bash[43301]: audit 2022-04-28T21:32:01.553444+0000 mon.a (mon.0) 2886 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:02 smithi139 bash[43301]: cluster 2022-04-28T21:32:01.927215+0000 mgr.y (mgr.24961) 3343 : cluster [DBG] pgmap v3706: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:03 smithi059 bash[61152]: audit 2022-04-28T21:32:02.555231+0000 mon.a (mon.0) 2887 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:03 smithi059 bash[61152]: cluster 2022-04-28T21:32:02.555715+0000 mon.a (mon.0) 2888 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2022-04-28T21:32:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:03 smithi059 bash[59745]: audit 2022-04-28T21:32:02.555231+0000 mon.a (mon.0) 2887 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:03 smithi059 bash[59745]: cluster 2022-04-28T21:32:02.555715+0000 mon.a (mon.0) 2888 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2022-04-28T21:32:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:03 smithi139 bash[43301]: audit 2022-04-28T21:32:02.555231+0000 mon.a (mon.0) 2887 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-15","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:03 smithi139 bash[43301]: cluster 2022-04-28T21:32:02.555715+0000 mon.a (mon.0) 2888 : cluster [DBG] osdmap e972: 8 total, 8 up, 8 in 2022-04-28T21:32:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:03.910Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:04 smithi139 bash[43301]: cluster 2022-04-28T21:32:03.578681+0000 mon.a (mon.0) 2889 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2022-04-28T21:32:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:04 smithi139 bash[43301]: cluster 2022-04-28T21:32:03.927909+0000 mgr.y (mgr.24961) 3344 : cluster [DBG] pgmap v3709: 137 pgs: 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:05.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:04 smithi059 bash[59745]: cluster 2022-04-28T21:32:03.578681+0000 mon.a (mon.0) 2889 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2022-04-28T21:32:05.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:04 smithi059 bash[59745]: cluster 2022-04-28T21:32:03.927909+0000 mgr.y (mgr.24961) 3344 : cluster [DBG] pgmap v3709: 137 pgs: 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:04 smithi059 bash[61152]: cluster 2022-04-28T21:32:03.578681+0000 mon.a (mon.0) 2889 : cluster [DBG] osdmap e973: 8 total, 8 up, 8 in 2022-04-28T21:32:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:04 smithi059 bash[61152]: cluster 2022-04-28T21:32:03.927909+0000 mgr.y (mgr.24961) 3344 : cluster [DBG] pgmap v3709: 137 pgs: 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:05 smithi139 bash[43301]: cluster 2022-04-28T21:32:04.595759+0000 mon.a (mon.0) 2890 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2022-04-28T21:32:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:05 smithi139 bash[43301]: audit 2022-04-28T21:32:04.599765+0000 mon.c (mon.1) 42 : audit [INF] from='client.? 172.21.15.59:0/2119069217' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:05 smithi139 bash[43301]: audit 2022-04-28T21:32:04.602833+0000 mon.a (mon.0) 2891 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:05 smithi059 bash[59745]: cluster 2022-04-28T21:32:04.595759+0000 mon.a (mon.0) 2890 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2022-04-28T21:32:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:05 smithi059 bash[59745]: audit 2022-04-28T21:32:04.599765+0000 mon.c (mon.1) 42 : audit [INF] from='client.? 172.21.15.59:0/2119069217' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:05 smithi059 bash[59745]: audit 2022-04-28T21:32:04.602833+0000 mon.a (mon.0) 2891 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:05 smithi059 bash[61152]: cluster 2022-04-28T21:32:04.595759+0000 mon.a (mon.0) 2890 : cluster [DBG] osdmap e974: 8 total, 8 up, 8 in 2022-04-28T21:32:06.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:05 smithi059 bash[61152]: audit 2022-04-28T21:32:04.599765+0000 mon.c (mon.1) 42 : audit [INF] from='client.? 172.21.15.59:0/2119069217' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:06.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:05 smithi059 bash[61152]: audit 2022-04-28T21:32:04.602833+0000 mon.a (mon.0) 2891 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:06 smithi139 bash[43301]: audit 2022-04-28T21:32:05.605621+0000 mon.a (mon.0) 2892 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:06 smithi139 bash[43301]: cluster 2022-04-28T21:32:05.605780+0000 mon.a (mon.0) 2893 : cluster [DBG] osdmap e975: 8 total, 8 up, 8 in 2022-04-28T21:32:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:06 smithi139 bash[43301]: cluster 2022-04-28T21:32:05.928690+0000 mgr.y (mgr.24961) 3345 : cluster [DBG] pgmap v3712: 169 pgs: 10 creating+peering, 22 unknown, 137 active+clean; 24 KiB data, 778 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:07.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:06 smithi059 bash[61152]: audit 2022-04-28T21:32:05.605621+0000 mon.a (mon.0) 2892 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:06 smithi059 bash[61152]: cluster 2022-04-28T21:32:05.605780+0000 mon.a (mon.0) 2893 : cluster [DBG] osdmap e975: 8 total, 8 up, 8 in 2022-04-28T21:32:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:06 smithi059 bash[61152]: cluster 2022-04-28T21:32:05.928690+0000 mgr.y (mgr.24961) 3345 : cluster [DBG] pgmap v3712: 169 pgs: 10 creating+peering, 22 unknown, 137 active+clean; 24 KiB data, 778 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:06 smithi059 bash[59745]: audit 2022-04-28T21:32:05.605621+0000 mon.a (mon.0) 2892 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-16","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:06 smithi059 bash[59745]: cluster 2022-04-28T21:32:05.605780+0000 mon.a (mon.0) 2893 : cluster [DBG] osdmap e975: 8 total, 8 up, 8 in 2022-04-28T21:32:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:06 smithi059 bash[59745]: cluster 2022-04-28T21:32:05.928690+0000 mgr.y (mgr.24961) 3345 : cluster [DBG] pgmap v3712: 169 pgs: 10 creating+peering, 22 unknown, 137 active+clean; 24 KiB data, 778 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:08.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:07 smithi059 bash[61152]: cluster 2022-04-28T21:32:06.611684+0000 mon.a (mon.0) 2894 : cluster [DBG] osdmap e976: 8 total, 8 up, 8 in 2022-04-28T21:32:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:07 smithi059 bash[59745]: cluster 2022-04-28T21:32:06.611684+0000 mon.a (mon.0) 2894 : cluster [DBG] osdmap e976: 8 total, 8 up, 8 in 2022-04-28T21:32:08.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:07 smithi139 bash[43301]: cluster 2022-04-28T21:32:06.611684+0000 mon.a (mon.0) 2894 : cluster [DBG] osdmap e976: 8 total, 8 up, 8 in 2022-04-28T21:32:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:08 smithi059 bash[61152]: cluster 2022-04-28T21:32:07.634591+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e977: 8 total, 8 up, 8 in 2022-04-28T21:32:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:08 smithi059 bash[61152]: audit 2022-04-28T21:32:07.636763+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.59:0/3282718013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:08 smithi059 bash[61152]: cluster 2022-04-28T21:32:07.929471+0000 mgr.y (mgr.24961) 3346 : cluster [DBG] pgmap v3715: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 783 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:08 smithi059 bash[59745]: cluster 2022-04-28T21:32:07.634591+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e977: 8 total, 8 up, 8 in 2022-04-28T21:32:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:08 smithi059 bash[59745]: audit 2022-04-28T21:32:07.636763+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.59:0/3282718013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:08 smithi059 bash[59745]: cluster 2022-04-28T21:32:07.929471+0000 mgr.y (mgr.24961) 3346 : cluster [DBG] pgmap v3715: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 783 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:08 smithi139 bash[43301]: cluster 2022-04-28T21:32:07.634591+0000 mon.a (mon.0) 2895 : cluster [DBG] osdmap e977: 8 total, 8 up, 8 in 2022-04-28T21:32:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:08 smithi139 bash[43301]: audit 2022-04-28T21:32:07.636763+0000 mon.a (mon.0) 2896 : audit [INF] from='client.? 172.21.15.59:0/3282718013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:08 smithi139 bash[43301]: cluster 2022-04-28T21:32:07.929471+0000 mgr.y (mgr.24961) 3346 : cluster [DBG] pgmap v3715: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 783 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:10.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:09 smithi059 bash[61152]: audit 2022-04-28T21:32:08.646965+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.59:0/3282718013' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-17","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:09 smithi059 bash[61152]: cluster 2022-04-28T21:32:08.647284+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e978: 8 total, 8 up, 8 in 2022-04-28T21:32:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:09 smithi059 bash[59745]: audit 2022-04-28T21:32:08.646965+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.59:0/3282718013' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-17","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:09 smithi059 bash[59745]: cluster 2022-04-28T21:32:08.647284+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e978: 8 total, 8 up, 8 in 2022-04-28T21:32:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:09 smithi139 bash[43301]: audit 2022-04-28T21:32:08.646965+0000 mon.a (mon.0) 2897 : audit [INF] from='client.? 172.21.15.59:0/3282718013' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-17","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:09 smithi139 bash[43301]: cluster 2022-04-28T21:32:08.647284+0000 mon.a (mon.0) 2898 : cluster [DBG] osdmap e978: 8 total, 8 up, 8 in 2022-04-28T21:32:11.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:10 smithi059 bash[59745]: cluster 2022-04-28T21:32:09.649529+0000 mon.a (mon.0) 2899 : cluster [DBG] osdmap e979: 8 total, 8 up, 8 in 2022-04-28T21:32:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:10 smithi059 bash[59745]: cluster 2022-04-28T21:32:09.930132+0000 mgr.y (mgr.24961) 3347 : cluster [DBG] pgmap v3718: 137 pgs: 137 active+clean; 24 KiB data, 784 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:10 smithi059 bash[61152]: cluster 2022-04-28T21:32:09.649529+0000 mon.a (mon.0) 2899 : cluster [DBG] osdmap e979: 8 total, 8 up, 8 in 2022-04-28T21:32:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:10 smithi059 bash[61152]: cluster 2022-04-28T21:32:09.930132+0000 mgr.y (mgr.24961) 3347 : cluster [DBG] pgmap v3718: 137 pgs: 137 active+clean; 24 KiB data, 784 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:10 smithi139 bash[43301]: cluster 2022-04-28T21:32:09.649529+0000 mon.a (mon.0) 2899 : cluster [DBG] osdmap e979: 8 total, 8 up, 8 in 2022-04-28T21:32:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:10 smithi139 bash[43301]: cluster 2022-04-28T21:32:09.930132+0000 mgr.y (mgr.24961) 3347 : cluster [DBG] pgmap v3718: 137 pgs: 137 active+clean; 24 KiB data, 784 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:12.019 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:11 smithi059 bash[61152]: cluster 2022-04-28T21:32:10.668365+0000 mon.a (mon.0) 2900 : cluster [DBG] osdmap e980: 8 total, 8 up, 8 in 2022-04-28T21:32:12.019 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:11 smithi059 bash[61152]: audit 2022-04-28T21:32:10.674007+0000 mon.c (mon.1) 43 : audit [INF] from='client.? 172.21.15.59:0/3773752405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:12.020 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:11 smithi059 bash[61152]: audit 2022-04-28T21:32:10.674619+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:12.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:11 smithi059 bash[59745]: cluster 2022-04-28T21:32:10.668365+0000 mon.a (mon.0) 2900 : cluster [DBG] osdmap e980: 8 total, 8 up, 8 in 2022-04-28T21:32:12.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:11 smithi059 bash[59745]: audit 2022-04-28T21:32:10.674007+0000 mon.c (mon.1) 43 : audit [INF] from='client.? 172.21.15.59:0/3773752405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:12.020 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:11 smithi059 bash[59745]: audit 2022-04-28T21:32:10.674619+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:11 smithi139 bash[43301]: cluster 2022-04-28T21:32:10.668365+0000 mon.a (mon.0) 2900 : cluster [DBG] osdmap e980: 8 total, 8 up, 8 in 2022-04-28T21:32:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:11 smithi139 bash[43301]: audit 2022-04-28T21:32:10.674007+0000 mon.c (mon.1) 43 : audit [INF] from='client.? 172.21.15.59:0/3773752405' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:11 smithi139 bash[43301]: audit 2022-04-28T21:32:10.674619+0000 mon.a (mon.0) 2901 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:12.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:13.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[61152]: audit 2022-04-28T21:32:11.678103+0000 mon.a (mon.0) 2902 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[61152]: cluster 2022-04-28T21:32:11.679162+0000 mon.a (mon.0) 2903 : cluster [DBG] osdmap e981: 8 total, 8 up, 8 in 2022-04-28T21:32:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[61152]: cluster 2022-04-28T21:32:11.930836+0000 mgr.y (mgr.24961) 3348 : cluster [DBG] pgmap v3721: 169 pgs: 13 creating+peering, 19 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[59745]: audit 2022-04-28T21:32:11.678103+0000 mon.a (mon.0) 2902 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[59745]: cluster 2022-04-28T21:32:11.679162+0000 mon.a (mon.0) 2903 : cluster [DBG] osdmap e981: 8 total, 8 up, 8 in 2022-04-28T21:32:13.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:12 smithi059 bash[59745]: cluster 2022-04-28T21:32:11.930836+0000 mgr.y (mgr.24961) 3348 : cluster [DBG] pgmap v3721: 169 pgs: 13 creating+peering, 19 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:13.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:12 smithi139 bash[43301]: audit 2022-04-28T21:32:11.678103+0000 mon.a (mon.0) 2902 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-18","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:12 smithi139 bash[43301]: cluster 2022-04-28T21:32:11.679162+0000 mon.a (mon.0) 2903 : cluster [DBG] osdmap e981: 8 total, 8 up, 8 in 2022-04-28T21:32:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:12 smithi139 bash[43301]: cluster 2022-04-28T21:32:11.930836+0000 mgr.y (mgr.24961) 3348 : cluster [DBG] pgmap v3721: 169 pgs: 13 creating+peering, 19 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:14.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:13 smithi059 bash[61152]: cluster 2022-04-28T21:32:12.697900+0000 mon.a (mon.0) 2904 : cluster [DBG] osdmap e982: 8 total, 8 up, 8 in 2022-04-28T21:32:14.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:13.912Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:14.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:13 smithi059 bash[59745]: cluster 2022-04-28T21:32:12.697900+0000 mon.a (mon.0) 2904 : cluster [DBG] osdmap e982: 8 total, 8 up, 8 in 2022-04-28T21:32:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:13 smithi139 bash[43301]: cluster 2022-04-28T21:32:12.697900+0000 mon.a (mon.0) 2904 : cluster [DBG] osdmap e982: 8 total, 8 up, 8 in 2022-04-28T21:32:15.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[61152]: cluster 2022-04-28T21:32:13.721168+0000 mon.a (mon.0) 2905 : cluster [DBG] osdmap e983: 8 total, 8 up, 8 in 2022-04-28T21:32:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[61152]: audit 2022-04-28T21:32:13.724799+0000 mon.b (mon.2) 16 : audit [INF] from='client.? 172.21.15.59:0/3689562189' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[61152]: audit 2022-04-28T21:32:13.728721+0000 mon.a (mon.0) 2906 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[61152]: cluster 2022-04-28T21:32:13.931627+0000 mgr.y (mgr.24961) 3349 : cluster [DBG] pgmap v3724: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[59745]: cluster 2022-04-28T21:32:13.721168+0000 mon.a (mon.0) 2905 : cluster [DBG] osdmap e983: 8 total, 8 up, 8 in 2022-04-28T21:32:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[59745]: audit 2022-04-28T21:32:13.724799+0000 mon.b (mon.2) 16 : audit [INF] from='client.? 172.21.15.59:0/3689562189' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[59745]: audit 2022-04-28T21:32:13.728721+0000 mon.a (mon.0) 2906 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:14 smithi059 bash[59745]: cluster 2022-04-28T21:32:13.931627+0000 mgr.y (mgr.24961) 3349 : cluster [DBG] pgmap v3724: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:14 smithi139 bash[43301]: cluster 2022-04-28T21:32:13.721168+0000 mon.a (mon.0) 2905 : cluster [DBG] osdmap e983: 8 total, 8 up, 8 in 2022-04-28T21:32:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:14 smithi139 bash[43301]: audit 2022-04-28T21:32:13.724799+0000 mon.b (mon.2) 16 : audit [INF] from='client.? 172.21.15.59:0/3689562189' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:14 smithi139 bash[43301]: audit 2022-04-28T21:32:13.728721+0000 mon.a (mon.0) 2906 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:14 smithi139 bash[43301]: cluster 2022-04-28T21:32:13.931627+0000 mgr.y (mgr.24961) 3349 : cluster [DBG] pgmap v3724: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 769 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:16.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:15 smithi059 bash[61152]: audit 2022-04-28T21:32:14.725289+0000 mon.a (mon.0) 2907 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:15 smithi059 bash[61152]: cluster 2022-04-28T21:32:14.725952+0000 mon.a (mon.0) 2908 : cluster [DBG] osdmap e984: 8 total, 8 up, 8 in 2022-04-28T21:32:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:15 smithi059 bash[59745]: audit 2022-04-28T21:32:14.725289+0000 mon.a (mon.0) 2907 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:15 smithi059 bash[59745]: cluster 2022-04-28T21:32:14.725952+0000 mon.a (mon.0) 2908 : cluster [DBG] osdmap e984: 8 total, 8 up, 8 in 2022-04-28T21:32:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:15 smithi139 bash[43301]: audit 2022-04-28T21:32:14.725289+0000 mon.a (mon.0) 2907 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-19","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:15 smithi139 bash[43301]: cluster 2022-04-28T21:32:14.725952+0000 mon.a (mon.0) 2908 : cluster [DBG] osdmap e984: 8 total, 8 up, 8 in 2022-04-28T21:32:17.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:16 smithi059 bash[61152]: cluster 2022-04-28T21:32:15.743524+0000 mon.a (mon.0) 2909 : cluster [DBG] osdmap e985: 8 total, 8 up, 8 in 2022-04-28T21:32:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:16 smithi059 bash[61152]: cluster 2022-04-28T21:32:15.932336+0000 mgr.y (mgr.24961) 3350 : cluster [DBG] pgmap v3727: 137 pgs: 137 active+clean; 24 KiB data, 775 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:17.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:16 smithi059 bash[59745]: cluster 2022-04-28T21:32:15.743524+0000 mon.a (mon.0) 2909 : cluster [DBG] osdmap e985: 8 total, 8 up, 8 in 2022-04-28T21:32:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:16 smithi059 bash[59745]: cluster 2022-04-28T21:32:15.932336+0000 mgr.y (mgr.24961) 3350 : cluster [DBG] pgmap v3727: 137 pgs: 137 active+clean; 24 KiB data, 775 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:16 smithi139 bash[43301]: cluster 2022-04-28T21:32:15.743524+0000 mon.a (mon.0) 2909 : cluster [DBG] osdmap e985: 8 total, 8 up, 8 in 2022-04-28T21:32:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:16 smithi139 bash[43301]: cluster 2022-04-28T21:32:15.932336+0000 mgr.y (mgr.24961) 3350 : cluster [DBG] pgmap v3727: 137 pgs: 137 active+clean; 24 KiB data, 775 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:17 smithi059 bash[61152]: cluster 2022-04-28T21:32:16.762183+0000 mon.a (mon.0) 2910 : cluster [DBG] osdmap e986: 8 total, 8 up, 8 in 2022-04-28T21:32:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:17 smithi059 bash[61152]: audit 2022-04-28T21:32:16.764355+0000 mon.c (mon.1) 44 : audit [INF] from='client.? 172.21.15.59:0/3213865588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:17 smithi059 bash[61152]: audit 2022-04-28T21:32:16.768041+0000 mon.a (mon.0) 2911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:17 smithi059 bash[59745]: cluster 2022-04-28T21:32:16.762183+0000 mon.a (mon.0) 2910 : cluster [DBG] osdmap e986: 8 total, 8 up, 8 in 2022-04-28T21:32:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:17 smithi059 bash[59745]: audit 2022-04-28T21:32:16.764355+0000 mon.c (mon.1) 44 : audit [INF] from='client.? 172.21.15.59:0/3213865588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:17 smithi059 bash[59745]: audit 2022-04-28T21:32:16.768041+0000 mon.a (mon.0) 2911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:18.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:17 smithi139 bash[43301]: cluster 2022-04-28T21:32:16.762183+0000 mon.a (mon.0) 2910 : cluster [DBG] osdmap e986: 8 total, 8 up, 8 in 2022-04-28T21:32:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:17 smithi139 bash[43301]: audit 2022-04-28T21:32:16.764355+0000 mon.c (mon.1) 44 : audit [INF] from='client.? 172.21.15.59:0/3213865588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:17 smithi139 bash[43301]: audit 2022-04-28T21:32:16.768041+0000 mon.a (mon.0) 2911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:19.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:18 smithi059 bash[59745]: audit 2022-04-28T21:32:17.767788+0000 mon.a (mon.0) 2912 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:18 smithi059 bash[59745]: cluster 2022-04-28T21:32:17.768115+0000 mon.a (mon.0) 2913 : cluster [DBG] osdmap e987: 8 total, 8 up, 8 in 2022-04-28T21:32:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:18 smithi059 bash[59745]: cluster 2022-04-28T21:32:17.933078+0000 mgr.y (mgr.24961) 3351 : cluster [DBG] pgmap v3730: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 775 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:18 smithi059 bash[61152]: audit 2022-04-28T21:32:17.767788+0000 mon.a (mon.0) 2912 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:18 smithi059 bash[61152]: cluster 2022-04-28T21:32:17.768115+0000 mon.a (mon.0) 2913 : cluster [DBG] osdmap e987: 8 total, 8 up, 8 in 2022-04-28T21:32:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:18 smithi059 bash[61152]: cluster 2022-04-28T21:32:17.933078+0000 mgr.y (mgr.24961) 3351 : cluster [DBG] pgmap v3730: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 775 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:19.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:18 smithi139 bash[43301]: audit 2022-04-28T21:32:17.767788+0000 mon.a (mon.0) 2912 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-20","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:18 smithi139 bash[43301]: cluster 2022-04-28T21:32:17.768115+0000 mon.a (mon.0) 2913 : cluster [DBG] osdmap e987: 8 total, 8 up, 8 in 2022-04-28T21:32:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:18 smithi139 bash[43301]: cluster 2022-04-28T21:32:17.933078+0000 mgr.y (mgr.24961) 3351 : cluster [DBG] pgmap v3730: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 775 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: cluster 2022-04-28T21:32:18.784261+0000 mon.a (mon.0) 2914 : cluster [DBG] osdmap e988: 8 total, 8 up, 8 in 2022-04-28T21:32:20.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: audit 2022-04-28T21:32:19.266983+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:32:20.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: audit 2022-04-28T21:32:19.267267+0000 mon.a (mon.0) 2916 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:32:20.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: audit 2022-04-28T21:32:19.267820+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:32:20.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: audit 2022-04-28T21:32:19.268278+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:32:20.133 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: audit 2022-04-28T21:32:19.268708+0000 mon.a (mon.0) 2919 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:32:20.134 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[61152]: audit 2022-04-28T21:32:19.269010+0000 mon.a (mon.0) 2920 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:32:20.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: cluster 2022-04-28T21:32:18.784261+0000 mon.a (mon.0) 2914 : cluster [DBG] osdmap e988: 8 total, 8 up, 8 in 2022-04-28T21:32:20.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: audit 2022-04-28T21:32:19.266983+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:32:20.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: audit 2022-04-28T21:32:19.267267+0000 mon.a (mon.0) 2916 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:32:20.135 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: audit 2022-04-28T21:32:19.267820+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:32:20.135 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: audit 2022-04-28T21:32:19.268278+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:32:20.135 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: audit 2022-04-28T21:32:19.268708+0000 mon.a (mon.0) 2919 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:32:20.135 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:19 smithi059 bash[59745]: audit 2022-04-28T21:32:19.269010+0000 mon.a (mon.0) 2920 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:32:20.136 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: cluster 2022-04-28T21:32:18.784261+0000 mon.a (mon.0) 2914 : cluster [DBG] osdmap e988: 8 total, 8 up, 8 in 2022-04-28T21:32:20.136 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: audit 2022-04-28T21:32:19.266983+0000 mon.a (mon.0) 2915 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:32:20.136 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: audit 2022-04-28T21:32:19.267267+0000 mon.a (mon.0) 2916 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:32:20.136 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: audit 2022-04-28T21:32:19.267820+0000 mon.a (mon.0) 2917 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:32:20.137 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: audit 2022-04-28T21:32:19.268278+0000 mon.a (mon.0) 2918 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:32:20.137 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: audit 2022-04-28T21:32:19.268708+0000 mon.a (mon.0) 2919 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:32:20.137 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:19 smithi139 bash[43301]: audit 2022-04-28T21:32:19.269010+0000 mon.a (mon.0) 2920 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:32:21.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[59745]: audit 2022-04-28T21:32:19.802431+0000 mon.b (mon.2) 17 : audit [INF] from='client.? 172.21.15.59:0/3463853916' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[59745]: cluster 2022-04-28T21:32:19.803789+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e989: 8 total, 8 up, 8 in 2022-04-28T21:32:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[59745]: audit 2022-04-28T21:32:19.810058+0000 mon.a (mon.0) 2922 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[59745]: cluster 2022-04-28T21:32:19.933838+0000 mgr.y (mgr.24961) 3352 : cluster [DBG] pgmap v3733: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[59745]: audit 2022-04-28T21:32:20.803909+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[59745]: cluster 2022-04-28T21:32:20.804305+0000 mon.a (mon.0) 2924 : cluster [DBG] osdmap e990: 8 total, 8 up, 8 in 2022-04-28T21:32:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[61152]: audit 2022-04-28T21:32:19.802431+0000 mon.b (mon.2) 17 : audit [INF] from='client.? 172.21.15.59:0/3463853916' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[61152]: cluster 2022-04-28T21:32:19.803789+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e989: 8 total, 8 up, 8 in 2022-04-28T21:32:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[61152]: audit 2022-04-28T21:32:19.810058+0000 mon.a (mon.0) 2922 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[61152]: cluster 2022-04-28T21:32:19.933838+0000 mgr.y (mgr.24961) 3352 : cluster [DBG] pgmap v3733: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[61152]: audit 2022-04-28T21:32:20.803909+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:20 smithi059 bash[61152]: cluster 2022-04-28T21:32:20.804305+0000 mon.a (mon.0) 2924 : cluster [DBG] osdmap e990: 8 total, 8 up, 8 in 2022-04-28T21:32:21.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:20 smithi139 bash[43301]: audit 2022-04-28T21:32:19.802431+0000 mon.b (mon.2) 17 : audit [INF] from='client.? 172.21.15.59:0/3463853916' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:20 smithi139 bash[43301]: cluster 2022-04-28T21:32:19.803789+0000 mon.a (mon.0) 2921 : cluster [DBG] osdmap e989: 8 total, 8 up, 8 in 2022-04-28T21:32:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:20 smithi139 bash[43301]: audit 2022-04-28T21:32:19.810058+0000 mon.a (mon.0) 2922 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:20 smithi139 bash[43301]: cluster 2022-04-28T21:32:19.933838+0000 mgr.y (mgr.24961) 3352 : cluster [DBG] pgmap v3733: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:20 smithi139 bash[43301]: audit 2022-04-28T21:32:20.803909+0000 mon.a (mon.0) 2923 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-21","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:20 smithi139 bash[43301]: cluster 2022-04-28T21:32:20.804305+0000 mon.a (mon.0) 2924 : cluster [DBG] osdmap e990: 8 total, 8 up, 8 in 2022-04-28T21:32:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:22.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:23.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:22 smithi059 bash[61152]: cluster 2022-04-28T21:32:21.824101+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e991: 8 total, 8 up, 8 in 2022-04-28T21:32:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:22 smithi059 bash[61152]: cluster 2022-04-28T21:32:21.934532+0000 mgr.y (mgr.24961) 3353 : cluster [DBG] pgmap v3736: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:22 smithi059 bash[59745]: cluster 2022-04-28T21:32:21.824101+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e991: 8 total, 8 up, 8 in 2022-04-28T21:32:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:22 smithi059 bash[59745]: cluster 2022-04-28T21:32:21.934532+0000 mgr.y (mgr.24961) 3353 : cluster [DBG] pgmap v3736: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:22 smithi139 bash[43301]: cluster 2022-04-28T21:32:21.824101+0000 mon.a (mon.0) 2925 : cluster [DBG] osdmap e991: 8 total, 8 up, 8 in 2022-04-28T21:32:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:22 smithi139 bash[43301]: cluster 2022-04-28T21:32:21.934532+0000 mgr.y (mgr.24961) 3353 : cluster [DBG] pgmap v3736: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:24.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[59745]: cluster 2022-04-28T21:32:22.831627+0000 mon.a (mon.0) 2926 : cluster [DBG] osdmap e992: 8 total, 8 up, 8 in 2022-04-28T21:32:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[59745]: audit 2022-04-28T21:32:22.839120+0000 mon.c (mon.1) 45 : audit [INF] from='client.? 172.21.15.59:0/1354744050' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:24.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[59745]: audit 2022-04-28T21:32:22.841687+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[61152]: cluster 2022-04-28T21:32:22.831627+0000 mon.a (mon.0) 2926 : cluster [DBG] osdmap e992: 8 total, 8 up, 8 in 2022-04-28T21:32:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[61152]: audit 2022-04-28T21:32:22.839120+0000 mon.c (mon.1) 45 : audit [INF] from='client.? 172.21.15.59:0/1354744050' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[61152]: audit 2022-04-28T21:32:22.841687+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:23.911Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:23 smithi139 bash[43301]: cluster 2022-04-28T21:32:22.831627+0000 mon.a (mon.0) 2926 : cluster [DBG] osdmap e992: 8 total, 8 up, 8 in 2022-04-28T21:32:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:23 smithi139 bash[43301]: audit 2022-04-28T21:32:22.839120+0000 mon.c (mon.1) 45 : audit [INF] from='client.? 172.21.15.59:0/1354744050' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:23 smithi139 bash[43301]: audit 2022-04-28T21:32:22.841687+0000 mon.a (mon.0) 2927 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:24 smithi139 bash[43301]: audit 2022-04-28T21:32:23.835535+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:24 smithi139 bash[43301]: cluster 2022-04-28T21:32:23.835695+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e993: 8 total, 8 up, 8 in 2022-04-28T21:32:25.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:24 smithi139 bash[43301]: cluster 2022-04-28T21:32:23.935284+0000 mgr.y (mgr.24961) 3354 : cluster [DBG] pgmap v3739: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:25.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:24 smithi059 bash[59745]: audit 2022-04-28T21:32:23.835535+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:24 smithi059 bash[59745]: cluster 2022-04-28T21:32:23.835695+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e993: 8 total, 8 up, 8 in 2022-04-28T21:32:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:24 smithi059 bash[59745]: cluster 2022-04-28T21:32:23.935284+0000 mgr.y (mgr.24961) 3354 : cluster [DBG] pgmap v3739: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:24 smithi059 bash[61152]: audit 2022-04-28T21:32:23.835535+0000 mon.a (mon.0) 2928 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-22","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:24 smithi059 bash[61152]: cluster 2022-04-28T21:32:23.835695+0000 mon.a (mon.0) 2929 : cluster [DBG] osdmap e993: 8 total, 8 up, 8 in 2022-04-28T21:32:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:24 smithi059 bash[61152]: cluster 2022-04-28T21:32:23.935284+0000 mgr.y (mgr.24961) 3354 : cluster [DBG] pgmap v3739: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:26.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:25 smithi059 bash[61152]: cluster 2022-04-28T21:32:24.859950+0000 mon.a (mon.0) 2930 : cluster [DBG] osdmap e994: 8 total, 8 up, 8 in 2022-04-28T21:32:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:25 smithi059 bash[59745]: cluster 2022-04-28T21:32:24.859950+0000 mon.a (mon.0) 2930 : cluster [DBG] osdmap e994: 8 total, 8 up, 8 in 2022-04-28T21:32:26.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:25 smithi139 bash[43301]: cluster 2022-04-28T21:32:24.859950+0000 mon.a (mon.0) 2930 : cluster [DBG] osdmap e994: 8 total, 8 up, 8 in 2022-04-28T21:32:27.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[61152]: cluster 2022-04-28T21:32:25.859015+0000 mon.a (mon.0) 2931 : cluster [DBG] osdmap e995: 8 total, 8 up, 8 in 2022-04-28T21:32:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[61152]: audit 2022-04-28T21:32:25.860212+0000 mon.a (mon.0) 2932 : audit [INF] from='client.? 172.21.15.59:0/1740009598' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[61152]: cluster 2022-04-28T21:32:25.936031+0000 mgr.y (mgr.24961) 3355 : cluster [DBG] pgmap v3742: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[61152]: audit 2022-04-28T21:32:26.864551+0000 mon.a (mon.0) 2933 : audit [INF] from='client.? 172.21.15.59:0/1740009598' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-23","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[61152]: cluster 2022-04-28T21:32:26.864844+0000 mon.a (mon.0) 2934 : cluster [DBG] osdmap e996: 8 total, 8 up, 8 in 2022-04-28T21:32:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[59745]: cluster 2022-04-28T21:32:25.859015+0000 mon.a (mon.0) 2931 : cluster [DBG] osdmap e995: 8 total, 8 up, 8 in 2022-04-28T21:32:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[59745]: audit 2022-04-28T21:32:25.860212+0000 mon.a (mon.0) 2932 : audit [INF] from='client.? 172.21.15.59:0/1740009598' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[59745]: cluster 2022-04-28T21:32:25.936031+0000 mgr.y (mgr.24961) 3355 : cluster [DBG] pgmap v3742: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[59745]: audit 2022-04-28T21:32:26.864551+0000 mon.a (mon.0) 2933 : audit [INF] from='client.? 172.21.15.59:0/1740009598' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-23","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:26 smithi059 bash[59745]: cluster 2022-04-28T21:32:26.864844+0000 mon.a (mon.0) 2934 : cluster [DBG] osdmap e996: 8 total, 8 up, 8 in 2022-04-28T21:32:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:26 smithi139 bash[43301]: cluster 2022-04-28T21:32:25.859015+0000 mon.a (mon.0) 2931 : cluster [DBG] osdmap e995: 8 total, 8 up, 8 in 2022-04-28T21:32:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:26 smithi139 bash[43301]: audit 2022-04-28T21:32:25.860212+0000 mon.a (mon.0) 2932 : audit [INF] from='client.? 172.21.15.59:0/1740009598' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:26 smithi139 bash[43301]: cluster 2022-04-28T21:32:25.936031+0000 mgr.y (mgr.24961) 3355 : cluster [DBG] pgmap v3742: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:26 smithi139 bash[43301]: audit 2022-04-28T21:32:26.864551+0000 mon.a (mon.0) 2933 : audit [INF] from='client.? 172.21.15.59:0/1740009598' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-23","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:26 smithi139 bash[43301]: cluster 2022-04-28T21:32:26.864844+0000 mon.a (mon.0) 2934 : cluster [DBG] osdmap e996: 8 total, 8 up, 8 in 2022-04-28T21:32:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:28 smithi059 bash[59745]: cluster 2022-04-28T21:32:27.882777+0000 mon.a (mon.0) 2935 : cluster [DBG] osdmap e997: 8 total, 8 up, 8 in 2022-04-28T21:32:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:28 smithi059 bash[59745]: cluster 2022-04-28T21:32:27.936628+0000 mgr.y (mgr.24961) 3356 : cluster [DBG] pgmap v3745: 137 pgs: 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:28 smithi059 bash[61152]: cluster 2022-04-28T21:32:27.882777+0000 mon.a (mon.0) 2935 : cluster [DBG] osdmap e997: 8 total, 8 up, 8 in 2022-04-28T21:32:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:28 smithi059 bash[61152]: cluster 2022-04-28T21:32:27.936628+0000 mgr.y (mgr.24961) 3356 : cluster [DBG] pgmap v3745: 137 pgs: 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:28 smithi139 bash[43301]: cluster 2022-04-28T21:32:27.882777+0000 mon.a (mon.0) 2935 : cluster [DBG] osdmap e997: 8 total, 8 up, 8 in 2022-04-28T21:32:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:28 smithi139 bash[43301]: cluster 2022-04-28T21:32:27.936628+0000 mgr.y (mgr.24961) 3356 : cluster [DBG] pgmap v3745: 137 pgs: 137 active+clean; 24 KiB data, 763 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:30.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:29 smithi059 bash[61152]: cluster 2022-04-28T21:32:28.895217+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e998: 8 total, 8 up, 8 in 2022-04-28T21:32:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:29 smithi059 bash[61152]: audit 2022-04-28T21:32:28.900952+0000 mon.c (mon.1) 46 : audit [INF] from='client.? 172.21.15.59:0/4104822064' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:30.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:29 smithi059 bash[61152]: audit 2022-04-28T21:32:28.901588+0000 mon.a (mon.0) 2937 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:29 smithi059 bash[59745]: cluster 2022-04-28T21:32:28.895217+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e998: 8 total, 8 up, 8 in 2022-04-28T21:32:30.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:29 smithi059 bash[59745]: audit 2022-04-28T21:32:28.900952+0000 mon.c (mon.1) 46 : audit [INF] from='client.? 172.21.15.59:0/4104822064' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:30.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:29 smithi059 bash[59745]: audit 2022-04-28T21:32:28.901588+0000 mon.a (mon.0) 2937 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:29 smithi139 bash[43301]: cluster 2022-04-28T21:32:28.895217+0000 mon.a (mon.0) 2936 : cluster [DBG] osdmap e998: 8 total, 8 up, 8 in 2022-04-28T21:32:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:29 smithi139 bash[43301]: audit 2022-04-28T21:32:28.900952+0000 mon.c (mon.1) 46 : audit [INF] from='client.? 172.21.15.59:0/4104822064' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:30.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:29 smithi139 bash[43301]: audit 2022-04-28T21:32:28.901588+0000 mon.a (mon.0) 2937 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:32:31.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[59745]: audit 2022-04-28T21:32:29.900036+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[59745]: cluster 2022-04-28T21:32:29.900806+0000 mon.a (mon.0) 2939 : cluster [DBG] osdmap e999: 8 total, 8 up, 8 in 2022-04-28T21:32:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[59745]: cluster 2022-04-28T21:32:29.937071+0000 mgr.y (mgr.24961) 3357 : cluster [DBG] pgmap v3748: 169 pgs: 13 creating+peering, 19 unknown, 137 active+clean; 24 KiB data, 768 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[59745]: cluster 2022-04-28T21:32:30.900922+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1000: 8 total, 8 up, 8 in 2022-04-28T21:32:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[61152]: audit 2022-04-28T21:32:29.900036+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[61152]: cluster 2022-04-28T21:32:29.900806+0000 mon.a (mon.0) 2939 : cluster [DBG] osdmap e999: 8 total, 8 up, 8 in 2022-04-28T21:32:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[61152]: cluster 2022-04-28T21:32:29.937071+0000 mgr.y (mgr.24961) 3357 : cluster [DBG] pgmap v3748: 169 pgs: 13 creating+peering, 19 unknown, 137 active+clean; 24 KiB data, 768 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:30 smithi059 bash[61152]: cluster 2022-04-28T21:32:30.900922+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1000: 8 total, 8 up, 8 in 2022-04-28T21:32:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:30 smithi139 bash[43301]: audit 2022-04-28T21:32:29.900036+0000 mon.a (mon.0) 2938 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83193-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:32:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:30 smithi139 bash[43301]: cluster 2022-04-28T21:32:29.900806+0000 mon.a (mon.0) 2939 : cluster [DBG] osdmap e999: 8 total, 8 up, 8 in 2022-04-28T21:32:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:30 smithi139 bash[43301]: cluster 2022-04-28T21:32:29.937071+0000 mgr.y (mgr.24961) 3357 : cluster [DBG] pgmap v3748: 169 pgs: 13 creating+peering, 19 unknown, 137 active+clean; 24 KiB data, 768 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:30 smithi139 bash[43301]: cluster 2022-04-28T21:32:30.900922+0000 mon.a (mon.0) 2940 : cluster [DBG] osdmap e1000: 8 total, 8 up, 8 in 2022-04-28T21:32:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:32.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[61152]: audit 2022-04-28T21:32:30.924587+0000 mon.c (mon.1) 47 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[61152]: audit 2022-04-28T21:32:30.932960+0000 mon.a (mon.0) 2941 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[61152]: audit 2022-04-28T21:32:30.933532+0000 mon.c (mon.1) 48 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[61152]: audit 2022-04-28T21:32:30.933818+0000 mon.a (mon.0) 2942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[61152]: audit 2022-04-28T21:32:30.934316+0000 mon.c (mon.1) 49 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[61152]: audit 2022-04-28T21:32:30.934607+0000 mon.a (mon.0) 2943 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[59745]: audit 2022-04-28T21:32:30.924587+0000 mon.c (mon.1) 47 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[59745]: audit 2022-04-28T21:32:30.932960+0000 mon.a (mon.0) 2941 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[59745]: audit 2022-04-28T21:32:30.933532+0000 mon.c (mon.1) 48 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[59745]: audit 2022-04-28T21:32:30.933818+0000 mon.a (mon.0) 2942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[59745]: audit 2022-04-28T21:32:30.934316+0000 mon.c (mon.1) 49 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:32.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:31 smithi059 bash[59745]: audit 2022-04-28T21:32:30.934607+0000 mon.a (mon.0) 2943 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:31 smithi139 bash[43301]: audit 2022-04-28T21:32:30.924587+0000 mon.c (mon.1) 47 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:31 smithi139 bash[43301]: audit 2022-04-28T21:32:30.932960+0000 mon.a (mon.0) 2941 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:31 smithi139 bash[43301]: audit 2022-04-28T21:32:30.933532+0000 mon.c (mon.1) 48 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:31 smithi139 bash[43301]: audit 2022-04-28T21:32:30.933818+0000 mon.a (mon.0) 2942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:31 smithi139 bash[43301]: audit 2022-04-28T21:32:30.934316+0000 mon.c (mon.1) 49 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:32.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:31 smithi139 bash[43301]: audit 2022-04-28T21:32:30.934607+0000 mon.a (mon.0) 2943 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[61152]: audit 2022-04-28T21:32:31.935033+0000 mon.a (mon.0) 2944 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[61152]: cluster 2022-04-28T21:32:31.935260+0000 mon.a (mon.0) 2945 : cluster [DBG] osdmap e1001: 8 total, 8 up, 8 in 2022-04-28T21:32:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[61152]: cluster 2022-04-28T21:32:31.937538+0000 mgr.y (mgr.24961) 3358 : cluster [DBG] pgmap v3751: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[61152]: audit 2022-04-28T21:32:31.938436+0000 mon.c (mon.1) 50 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[61152]: audit 2022-04-28T21:32:31.943445+0000 mon.a (mon.0) 2946 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[59745]: audit 2022-04-28T21:32:31.935033+0000 mon.a (mon.0) 2944 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[59745]: cluster 2022-04-28T21:32:31.935260+0000 mon.a (mon.0) 2945 : cluster [DBG] osdmap e1001: 8 total, 8 up, 8 in 2022-04-28T21:32:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[59745]: cluster 2022-04-28T21:32:31.937538+0000 mgr.y (mgr.24961) 3358 : cluster [DBG] pgmap v3751: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[59745]: audit 2022-04-28T21:32:31.938436+0000 mon.c (mon.1) 50 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[59745]: audit 2022-04-28T21:32:31.943445+0000 mon.a (mon.0) 2946 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:33 smithi139 bash[43301]: audit 2022-04-28T21:32:31.935033+0000 mon.a (mon.0) 2944 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-25", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:33 smithi139 bash[43301]: cluster 2022-04-28T21:32:31.935260+0000 mon.a (mon.0) 2945 : cluster [DBG] osdmap e1001: 8 total, 8 up, 8 in 2022-04-28T21:32:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:33 smithi139 bash[43301]: cluster 2022-04-28T21:32:31.937538+0000 mgr.y (mgr.24961) 3358 : cluster [DBG] pgmap v3751: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:33 smithi139 bash[43301]: audit 2022-04-28T21:32:31.938436+0000 mon.c (mon.1) 50 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:33 smithi139 bash[43301]: audit 2022-04-28T21:32:31.943445+0000 mon.a (mon.0) 2946 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:33.913Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:34.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:34 smithi059 bash[61152]: cluster 2022-04-28T21:32:33.407027+0000 mon.a (mon.0) 2947 : cluster [DBG] osdmap e1002: 8 total, 8 up, 8 in 2022-04-28T21:32:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:34 smithi059 bash[61152]: audit 2022-04-28T21:32:33.951285+0000 mon.a (mon.0) 2948 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:34 smithi059 bash[61152]: cluster 2022-04-28T21:32:33.951454+0000 mon.a (mon.0) 2949 : cluster [DBG] osdmap e1003: 8 total, 8 up, 8 in 2022-04-28T21:32:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:34 smithi059 bash[59745]: cluster 2022-04-28T21:32:33.407027+0000 mon.a (mon.0) 2947 : cluster [DBG] osdmap e1002: 8 total, 8 up, 8 in 2022-04-28T21:32:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:34 smithi059 bash[59745]: audit 2022-04-28T21:32:33.951285+0000 mon.a (mon.0) 2948 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:34 smithi059 bash[59745]: cluster 2022-04-28T21:32:33.951454+0000 mon.a (mon.0) 2949 : cluster [DBG] osdmap e1003: 8 total, 8 up, 8 in 2022-04-28T21:32:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:34 smithi139 bash[43301]: cluster 2022-04-28T21:32:33.407027+0000 mon.a (mon.0) 2947 : cluster [DBG] osdmap e1002: 8 total, 8 up, 8 in 2022-04-28T21:32:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:34 smithi139 bash[43301]: audit 2022-04-28T21:32:33.951285+0000 mon.a (mon.0) 2948 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-25", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:34 smithi139 bash[43301]: cluster 2022-04-28T21:32:33.951454+0000 mon.a (mon.0) 2949 : cluster [DBG] osdmap e1003: 8 total, 8 up, 8 in 2022-04-28T21:32:35.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:35 smithi059 bash[61152]: cluster 2022-04-28T21:32:33.938207+0000 mgr.y (mgr.24961) 3359 : cluster [DBG] pgmap v3753: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:35 smithi059 bash[61152]: cluster 2022-04-28T21:32:34.954024+0000 mon.a (mon.0) 2950 : cluster [DBG] osdmap e1004: 8 total, 8 up, 8 in 2022-04-28T21:32:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:35 smithi059 bash[59745]: cluster 2022-04-28T21:32:33.938207+0000 mgr.y (mgr.24961) 3359 : cluster [DBG] pgmap v3753: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:35 smithi059 bash[59745]: cluster 2022-04-28T21:32:34.954024+0000 mon.a (mon.0) 2950 : cluster [DBG] osdmap e1004: 8 total, 8 up, 8 in 2022-04-28T21:32:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:35 smithi139 bash[43301]: cluster 2022-04-28T21:32:33.938207+0000 mgr.y (mgr.24961) 3359 : cluster [DBG] pgmap v3753: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:35 smithi139 bash[43301]: cluster 2022-04-28T21:32:34.954024+0000 mon.a (mon.0) 2950 : cluster [DBG] osdmap e1004: 8 total, 8 up, 8 in 2022-04-28T21:32:37.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[61152]: cluster 2022-04-28T21:32:35.939398+0000 mgr.y (mgr.24961) 3360 : cluster [DBG] pgmap v3756: 145 pgs: 2 creating+activating, 4 creating+peering, 2 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[61152]: cluster 2022-04-28T21:32:35.962152+0000 mon.a (mon.0) 2951 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2022-04-28T21:32:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[61152]: audit 2022-04-28T21:32:35.963257+0000 mon.c (mon.1) 51 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[61152]: audit 2022-04-28T21:32:35.963835+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[59745]: cluster 2022-04-28T21:32:35.939398+0000 mgr.y (mgr.24961) 3360 : cluster [DBG] pgmap v3756: 145 pgs: 2 creating+activating, 4 creating+peering, 2 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[59745]: cluster 2022-04-28T21:32:35.962152+0000 mon.a (mon.0) 2951 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2022-04-28T21:32:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[59745]: audit 2022-04-28T21:32:35.963257+0000 mon.c (mon.1) 51 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:36 smithi059 bash[59745]: audit 2022-04-28T21:32:35.963835+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:37.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:36 smithi139 bash[43301]: cluster 2022-04-28T21:32:35.939398+0000 mgr.y (mgr.24961) 3360 : cluster [DBG] pgmap v3756: 145 pgs: 2 creating+activating, 4 creating+peering, 2 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:36 smithi139 bash[43301]: cluster 2022-04-28T21:32:35.962152+0000 mon.a (mon.0) 2951 : cluster [DBG] osdmap e1005: 8 total, 8 up, 8 in 2022-04-28T21:32:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:36 smithi139 bash[43301]: audit 2022-04-28T21:32:35.963257+0000 mon.c (mon.1) 51 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:37.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:36 smithi139 bash[43301]: audit 2022-04-28T21:32:35.963835+0000 mon.a (mon.0) 2952 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[59745]: audit 2022-04-28T21:32:36.972886+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[59745]: cluster 2022-04-28T21:32:36.973088+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2022-04-28T21:32:38.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[59745]: audit 2022-04-28T21:32:36.973865+0000 mon.c (mon.1) 52 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[59745]: audit 2022-04-28T21:32:36.974328+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[61152]: audit 2022-04-28T21:32:36.972886+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[61152]: cluster 2022-04-28T21:32:36.973088+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2022-04-28T21:32:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[61152]: audit 2022-04-28T21:32:36.973865+0000 mon.c (mon.1) 52 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:38.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:37 smithi059 bash[61152]: audit 2022-04-28T21:32:36.974328+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:38.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:37 smithi139 bash[43301]: audit 2022-04-28T21:32:36.972886+0000 mon.a (mon.0) 2953 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:37 smithi139 bash[43301]: cluster 2022-04-28T21:32:36.973088+0000 mon.a (mon.0) 2954 : cluster [DBG] osdmap e1006: 8 total, 8 up, 8 in 2022-04-28T21:32:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:37 smithi139 bash[43301]: audit 2022-04-28T21:32:36.973865+0000 mon.c (mon.1) 52 : audit [INF] from='client.? 172.21.15.59:0/1336417955' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:37 smithi139 bash[43301]: audit 2022-04-28T21:32:36.974328+0000 mon.a (mon.0) 2955 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]: dispatch 2022-04-28T21:32:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: cluster 2022-04-28T21:32:37.940183+0000 mgr.y (mgr.24961) 3361 : cluster [DBG] pgmap v3759: 137 pgs: 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.974811+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: cluster 2022-04-28T21:32:37.975045+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2022-04-28T21:32:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.989363+0000 mon.c (mon.1) 53 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.989715+0000 mon.a (mon.0) 2958 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.990536+0000 mon.c (mon.1) 54 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.990908+0000 mon.a (mon.0) 2959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.991526+0000 mon.c (mon.1) 55 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[61152]: audit 2022-04-28T21:32:37.991917+0000 mon.a (mon.0) 2960 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:39.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: cluster 2022-04-28T21:32:37.940183+0000 mgr.y (mgr.24961) 3361 : cluster [DBG] pgmap v3759: 137 pgs: 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:39.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.974811+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:39.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: cluster 2022-04-28T21:32:37.975045+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2022-04-28T21:32:39.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.989363+0000 mon.c (mon.1) 53 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.989715+0000 mon.a (mon.0) 2958 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.990536+0000 mon.c (mon.1) 54 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.990908+0000 mon.a (mon.0) 2959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.991526+0000 mon.c (mon.1) 55 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:39.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:38 smithi059 bash[59745]: audit 2022-04-28T21:32:37.991917+0000 mon.a (mon.0) 2960 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: cluster 2022-04-28T21:32:37.940183+0000 mgr.y (mgr.24961) 3361 : cluster [DBG] pgmap v3759: 137 pgs: 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.974811+0000 mon.a (mon.0) 2956 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-25"}]': finished 2022-04-28T21:32:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: cluster 2022-04-28T21:32:37.975045+0000 mon.a (mon.0) 2957 : cluster [DBG] osdmap e1007: 8 total, 8 up, 8 in 2022-04-28T21:32:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.989363+0000 mon.c (mon.1) 53 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.989715+0000 mon.a (mon.0) 2958 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.990536+0000 mon.c (mon.1) 54 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.990908+0000 mon.a (mon.0) 2959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.991526+0000 mon.c (mon.1) 55 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:38 smithi139 bash[43301]: audit 2022-04-28T21:32:37.991917+0000 mon.a (mon.0) 2960 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:40.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[61152]: audit 2022-04-28T21:32:38.994825+0000 mon.a (mon.0) 2961 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[61152]: cluster 2022-04-28T21:32:38.995005+0000 mon.a (mon.0) 2962 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2022-04-28T21:32:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[61152]: audit 2022-04-28T21:32:38.996084+0000 mon.c (mon.1) 56 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[61152]: audit 2022-04-28T21:32:38.996546+0000 mon.a (mon.0) 2963 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[59745]: audit 2022-04-28T21:32:38.994825+0000 mon.a (mon.0) 2961 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[59745]: cluster 2022-04-28T21:32:38.995005+0000 mon.a (mon.0) 2962 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2022-04-28T21:32:40.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[59745]: audit 2022-04-28T21:32:38.996084+0000 mon.c (mon.1) 56 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:40.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:39 smithi059 bash[59745]: audit 2022-04-28T21:32:38.996546+0000 mon.a (mon.0) 2963 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:39 smithi139 bash[43301]: audit 2022-04-28T21:32:38.994825+0000 mon.a (mon.0) 2961 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-26", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:39 smithi139 bash[43301]: cluster 2022-04-28T21:32:38.995005+0000 mon.a (mon.0) 2962 : cluster [DBG] osdmap e1008: 8 total, 8 up, 8 in 2022-04-28T21:32:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:39 smithi139 bash[43301]: audit 2022-04-28T21:32:38.996084+0000 mon.c (mon.1) 56 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:39 smithi139 bash[43301]: audit 2022-04-28T21:32:38.996546+0000 mon.a (mon.0) 2963 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:41.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:41 smithi059 bash[61152]: cluster 2022-04-28T21:32:39.941358+0000 mgr.y (mgr.24961) 3362 : cluster [DBG] pgmap v3762: 137 pgs: 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:41 smithi059 bash[61152]: cluster 2022-04-28T21:32:40.476678+0000 mon.a (mon.0) 2964 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2022-04-28T21:32:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:41 smithi059 bash[59745]: cluster 2022-04-28T21:32:39.941358+0000 mgr.y (mgr.24961) 3362 : cluster [DBG] pgmap v3762: 137 pgs: 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:41 smithi059 bash[59745]: cluster 2022-04-28T21:32:40.476678+0000 mon.a (mon.0) 2964 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2022-04-28T21:32:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:41 smithi139 bash[43301]: cluster 2022-04-28T21:32:39.941358+0000 mgr.y (mgr.24961) 3362 : cluster [DBG] pgmap v3762: 137 pgs: 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:41 smithi139 bash[43301]: cluster 2022-04-28T21:32:40.476678+0000 mon.a (mon.0) 2964 : cluster [DBG] osdmap e1009: 8 total, 8 up, 8 in 2022-04-28T21:32:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:42.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:42 smithi059 bash[61152]: audit 2022-04-28T21:32:41.017606+0000 mon.a (mon.0) 2965 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:42 smithi059 bash[61152]: cluster 2022-04-28T21:32:41.017738+0000 mon.a (mon.0) 2966 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2022-04-28T21:32:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:42 smithi059 bash[59745]: audit 2022-04-28T21:32:41.017606+0000 mon.a (mon.0) 2965 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:42 smithi059 bash[59745]: cluster 2022-04-28T21:32:41.017738+0000 mon.a (mon.0) 2966 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2022-04-28T21:32:42.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:42 smithi139 bash[43301]: audit 2022-04-28T21:32:41.017606+0000 mon.a (mon.0) 2965 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-26", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:42 smithi139 bash[43301]: cluster 2022-04-28T21:32:41.017738+0000 mon.a (mon.0) 2966 : cluster [DBG] osdmap e1010: 8 total, 8 up, 8 in 2022-04-28T21:32:43.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:43 smithi059 bash[61152]: cluster 2022-04-28T21:32:41.942312+0000 mgr.y (mgr.24961) 3363 : cluster [DBG] pgmap v3765: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:43 smithi059 bash[61152]: cluster 2022-04-28T21:32:42.028971+0000 mon.a (mon.0) 2967 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2022-04-28T21:32:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:43 smithi059 bash[59745]: cluster 2022-04-28T21:32:41.942312+0000 mgr.y (mgr.24961) 3363 : cluster [DBG] pgmap v3765: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:43.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:43 smithi059 bash[59745]: cluster 2022-04-28T21:32:42.028971+0000 mon.a (mon.0) 2967 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2022-04-28T21:32:43.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:43 smithi139 bash[43301]: cluster 2022-04-28T21:32:41.942312+0000 mgr.y (mgr.24961) 3363 : cluster [DBG] pgmap v3765: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:43 smithi139 bash[43301]: cluster 2022-04-28T21:32:42.028971+0000 mon.a (mon.0) 2967 : cluster [DBG] osdmap e1011: 8 total, 8 up, 8 in 2022-04-28T21:32:44.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:44 smithi059 bash[61152]: cluster 2022-04-28T21:32:43.041231+0000 mon.a (mon.0) 2968 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2022-04-28T21:32:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:44 smithi059 bash[61152]: audit 2022-04-28T21:32:43.043580+0000 mon.c (mon.1) 57 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:44 smithi059 bash[61152]: audit 2022-04-28T21:32:43.044169+0000 mon.a (mon.0) 2969 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:43.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:44 smithi059 bash[59745]: cluster 2022-04-28T21:32:43.041231+0000 mon.a (mon.0) 2968 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2022-04-28T21:32:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:44 smithi059 bash[59745]: audit 2022-04-28T21:32:43.043580+0000 mon.c (mon.1) 57 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:44 smithi059 bash[59745]: audit 2022-04-28T21:32:43.044169+0000 mon.a (mon.0) 2969 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:44 smithi139 bash[43301]: cluster 2022-04-28T21:32:43.041231+0000 mon.a (mon.0) 2968 : cluster [DBG] osdmap e1012: 8 total, 8 up, 8 in 2022-04-28T21:32:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:44 smithi139 bash[43301]: audit 2022-04-28T21:32:43.043580+0000 mon.c (mon.1) 57 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:44 smithi139 bash[43301]: audit 2022-04-28T21:32:43.044169+0000 mon.a (mon.0) 2969 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:45.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[59745]: cluster 2022-04-28T21:32:43.943158+0000 mgr.y (mgr.24961) 3364 : cluster [DBG] pgmap v3768: 137 pgs: 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[59745]: audit 2022-04-28T21:32:44.050501+0000 mon.a (mon.0) 2970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[59745]: cluster 2022-04-28T21:32:44.050610+0000 mon.a (mon.0) 2971 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2022-04-28T21:32:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[59745]: audit 2022-04-28T21:32:44.051852+0000 mon.c (mon.1) 58 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[59745]: audit 2022-04-28T21:32:44.052320+0000 mon.a (mon.0) 2972 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[61152]: cluster 2022-04-28T21:32:43.943158+0000 mgr.y (mgr.24961) 3364 : cluster [DBG] pgmap v3768: 137 pgs: 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[61152]: audit 2022-04-28T21:32:44.050501+0000 mon.a (mon.0) 2970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[61152]: cluster 2022-04-28T21:32:44.050610+0000 mon.a (mon.0) 2971 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2022-04-28T21:32:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[61152]: audit 2022-04-28T21:32:44.051852+0000 mon.c (mon.1) 58 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:45.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:45 smithi059 bash[61152]: audit 2022-04-28T21:32:44.052320+0000 mon.a (mon.0) 2972 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:45 smithi139 bash[43301]: cluster 2022-04-28T21:32:43.943158+0000 mgr.y (mgr.24961) 3364 : cluster [DBG] pgmap v3768: 137 pgs: 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:45 smithi139 bash[43301]: audit 2022-04-28T21:32:44.050501+0000 mon.a (mon.0) 2970 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:45 smithi139 bash[43301]: cluster 2022-04-28T21:32:44.050610+0000 mon.a (mon.0) 2971 : cluster [DBG] osdmap e1013: 8 total, 8 up, 8 in 2022-04-28T21:32:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:45 smithi139 bash[43301]: audit 2022-04-28T21:32:44.051852+0000 mon.c (mon.1) 58 : audit [INF] from='client.? 172.21.15.59:0/3933027399' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:45 smithi139 bash[43301]: audit 2022-04-28T21:32:44.052320+0000 mon.a (mon.0) 2972 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]: dispatch 2022-04-28T21:32:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.061041+0000 mon.a (mon.0) 2973 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: cluster 2022-04-28T21:32:45.061194+0000 mon.a (mon.0) 2974 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2022-04-28T21:32:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.073526+0000 mon.c (mon.1) 59 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.073995+0000 mon.a (mon.0) 2975 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.074702+0000 mon.c (mon.1) 60 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.075174+0000 mon.a (mon.0) 2976 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.075758+0000 mon.c (mon.1) 61 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[61152]: audit 2022-04-28T21:32:45.076188+0000 mon.a (mon.0) 2977 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.061041+0000 mon.a (mon.0) 2973 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: cluster 2022-04-28T21:32:45.061194+0000 mon.a (mon.0) 2974 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2022-04-28T21:32:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.073526+0000 mon.c (mon.1) 59 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.073995+0000 mon.a (mon.0) 2975 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.074702+0000 mon.c (mon.1) 60 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.075174+0000 mon.a (mon.0) 2976 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.075758+0000 mon.c (mon.1) 61 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:46.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:46 smithi059 bash[59745]: audit 2022-04-28T21:32:45.076188+0000 mon.a (mon.0) 2977 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.061041+0000 mon.a (mon.0) 2973 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-26"}]': finished 2022-04-28T21:32:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: cluster 2022-04-28T21:32:45.061194+0000 mon.a (mon.0) 2974 : cluster [DBG] osdmap e1014: 8 total, 8 up, 8 in 2022-04-28T21:32:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.073526+0000 mon.c (mon.1) 59 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.073995+0000 mon.a (mon.0) 2975 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.074702+0000 mon.c (mon.1) 60 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.075174+0000 mon.a (mon.0) 2976 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:46.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.075758+0000 mon.c (mon.1) 61 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:46.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:46 smithi139 bash[43301]: audit 2022-04-28T21:32:45.076188+0000 mon.a (mon.0) 2977 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: cluster 2022-04-28T21:32:45.944252+0000 mgr.y (mgr.24961) 3365 : cluster [DBG] pgmap v3771: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: audit 2022-04-28T21:32:46.065529+0000 mon.a (mon.0) 2978 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: cluster 2022-04-28T21:32:46.065719+0000 mon.a (mon.0) 2979 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2022-04-28T21:32:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: audit 2022-04-28T21:32:46.066751+0000 mon.c (mon.1) 62 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: audit 2022-04-28T21:32:46.067878+0000 mon.a (mon.0) 2980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: audit 2022-04-28T21:32:47.032709+0000 mon.a (mon.0) 2981 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:32:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: audit 2022-04-28T21:32:47.035576+0000 mon.a (mon.0) 2982 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:32:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[61152]: audit 2022-04-28T21:32:47.037535+0000 mon.a (mon.0) 2983 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:32:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: cluster 2022-04-28T21:32:45.944252+0000 mgr.y (mgr.24961) 3365 : cluster [DBG] pgmap v3771: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: audit 2022-04-28T21:32:46.065529+0000 mon.a (mon.0) 2978 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: cluster 2022-04-28T21:32:46.065719+0000 mon.a (mon.0) 2979 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2022-04-28T21:32:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: audit 2022-04-28T21:32:46.066751+0000 mon.c (mon.1) 62 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: audit 2022-04-28T21:32:46.067878+0000 mon.a (mon.0) 2980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: audit 2022-04-28T21:32:47.032709+0000 mon.a (mon.0) 2981 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:32:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: audit 2022-04-28T21:32:47.035576+0000 mon.a (mon.0) 2982 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:32:47.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:47 smithi059 bash[59745]: audit 2022-04-28T21:32:47.037535+0000 mon.a (mon.0) 2983 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:32:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: cluster 2022-04-28T21:32:45.944252+0000 mgr.y (mgr.24961) 3365 : cluster [DBG] pgmap v3771: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: audit 2022-04-28T21:32:46.065529+0000 mon.a (mon.0) 2978 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-27", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: cluster 2022-04-28T21:32:46.065719+0000 mon.a (mon.0) 2979 : cluster [DBG] osdmap e1015: 8 total, 8 up, 8 in 2022-04-28T21:32:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: audit 2022-04-28T21:32:46.066751+0000 mon.c (mon.1) 62 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: audit 2022-04-28T21:32:46.067878+0000 mon.a (mon.0) 2980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: audit 2022-04-28T21:32:47.032709+0000 mon.a (mon.0) 2981 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:32:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: audit 2022-04-28T21:32:47.035576+0000 mon.a (mon.0) 2982 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:32:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:47 smithi139 bash[43301]: audit 2022-04-28T21:32:47.037535+0000 mon.a (mon.0) 2983 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:32:48.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[61152]: cephadm 2022-04-28T21:32:47.039350+0000 mgr.y (mgr.24961) 3366 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:32:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[61152]: cluster 2022-04-28T21:32:47.530172+0000 mon.a (mon.0) 2984 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2022-04-28T21:32:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[61152]: audit 2022-04-28T21:32:47.545782+0000 mon.a (mon.0) 2985 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:32:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[61152]: audit 2022-04-28T21:32:48.072375+0000 mon.a (mon.0) 2986 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:48.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[61152]: cluster 2022-04-28T21:32:48.072547+0000 mon.a (mon.0) 2987 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2022-04-28T21:32:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[59745]: cephadm 2022-04-28T21:32:47.039350+0000 mgr.y (mgr.24961) 3366 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:32:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[59745]: cluster 2022-04-28T21:32:47.530172+0000 mon.a (mon.0) 2984 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2022-04-28T21:32:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[59745]: audit 2022-04-28T21:32:47.545782+0000 mon.a (mon.0) 2985 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:32:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[59745]: audit 2022-04-28T21:32:48.072375+0000 mon.a (mon.0) 2986 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:48.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:48 smithi059 bash[59745]: cluster 2022-04-28T21:32:48.072547+0000 mon.a (mon.0) 2987 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2022-04-28T21:32:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:48 smithi139 bash[43301]: cephadm 2022-04-28T21:32:47.039350+0000 mgr.y (mgr.24961) 3366 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:32:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:48 smithi139 bash[43301]: cluster 2022-04-28T21:32:47.530172+0000 mon.a (mon.0) 2984 : cluster [DBG] osdmap e1016: 8 total, 8 up, 8 in 2022-04-28T21:32:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:48 smithi139 bash[43301]: audit 2022-04-28T21:32:47.545782+0000 mon.a (mon.0) 2985 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:32:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:48 smithi139 bash[43301]: audit 2022-04-28T21:32:48.072375+0000 mon.a (mon.0) 2986 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-27", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:48 smithi139 bash[43301]: cluster 2022-04-28T21:32:48.072547+0000 mon.a (mon.0) 2987 : cluster [DBG] osdmap e1017: 8 total, 8 up, 8 in 2022-04-28T21:32:49.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:49 smithi059 bash[61152]: cluster 2022-04-28T21:32:47.944988+0000 mgr.y (mgr.24961) 3367 : cluster [DBG] pgmap v3774: 137 pgs: 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:49 smithi059 bash[61152]: cluster 2022-04-28T21:32:49.078070+0000 mon.a (mon.0) 2988 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2022-04-28T21:32:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:49 smithi059 bash[59745]: cluster 2022-04-28T21:32:47.944988+0000 mgr.y (mgr.24961) 3367 : cluster [DBG] pgmap v3774: 137 pgs: 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:49 smithi059 bash[59745]: cluster 2022-04-28T21:32:49.078070+0000 mon.a (mon.0) 2988 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2022-04-28T21:32:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:49 smithi139 bash[43301]: cluster 2022-04-28T21:32:47.944988+0000 mgr.y (mgr.24961) 3367 : cluster [DBG] pgmap v3774: 137 pgs: 137 active+clean; 24 KiB data, 760 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:49 smithi139 bash[43301]: cluster 2022-04-28T21:32:49.078070+0000 mon.a (mon.0) 2988 : cluster [DBG] osdmap e1018: 8 total, 8 up, 8 in 2022-04-28T21:32:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[61152]: cluster 2022-04-28T21:32:49.946104+0000 mgr.y (mgr.24961) 3368 : cluster [DBG] pgmap v3777: 145 pgs: 1 creating+activating, 1 creating+peering, 5 unknown, 138 active+clean; 25 KiB data, 755 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:32:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[61152]: cluster 2022-04-28T21:32:50.078815+0000 mon.a (mon.0) 2989 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2022-04-28T21:32:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[61152]: audit 2022-04-28T21:32:50.080723+0000 mon.c (mon.1) 63 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[61152]: audit 2022-04-28T21:32:50.081388+0000 mon.a (mon.0) 2990 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[61152]: cluster 2022-04-28T21:32:50.541653+0000 mon.a (mon.0) 2991 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2022-04-28T21:32:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[59745]: cluster 2022-04-28T21:32:49.946104+0000 mgr.y (mgr.24961) 3368 : cluster [DBG] pgmap v3777: 145 pgs: 1 creating+activating, 1 creating+peering, 5 unknown, 138 active+clean; 25 KiB data, 755 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:32:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[59745]: cluster 2022-04-28T21:32:50.078815+0000 mon.a (mon.0) 2989 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2022-04-28T21:32:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[59745]: audit 2022-04-28T21:32:50.080723+0000 mon.c (mon.1) 63 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[59745]: audit 2022-04-28T21:32:50.081388+0000 mon.a (mon.0) 2990 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:51.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:51 smithi059 bash[59745]: cluster 2022-04-28T21:32:50.541653+0000 mon.a (mon.0) 2991 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2022-04-28T21:32:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:51 smithi139 bash[43301]: cluster 2022-04-28T21:32:49.946104+0000 mgr.y (mgr.24961) 3368 : cluster [DBG] pgmap v3777: 145 pgs: 1 creating+activating, 1 creating+peering, 5 unknown, 138 active+clean; 25 KiB data, 755 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:32:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:51 smithi139 bash[43301]: cluster 2022-04-28T21:32:50.078815+0000 mon.a (mon.0) 2989 : cluster [DBG] osdmap e1019: 8 total, 8 up, 8 in 2022-04-28T21:32:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:51 smithi139 bash[43301]: audit 2022-04-28T21:32:50.080723+0000 mon.c (mon.1) 63 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:51 smithi139 bash[43301]: audit 2022-04-28T21:32:50.081388+0000 mon.a (mon.0) 2990 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:51 smithi139 bash[43301]: cluster 2022-04-28T21:32:50.541653+0000 mon.a (mon.0) 2991 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2022-04-28T21:32:52.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[61152]: audit 2022-04-28T21:32:51.083409+0000 mon.a (mon.0) 2992 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[61152]: cluster 2022-04-28T21:32:51.083541+0000 mon.a (mon.0) 2993 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2022-04-28T21:32:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[61152]: audit 2022-04-28T21:32:51.087076+0000 mon.c (mon.1) 64 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[61152]: audit 2022-04-28T21:32:51.094085+0000 mon.a (mon.0) 2994 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:52.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[59745]: audit 2022-04-28T21:32:51.083409+0000 mon.a (mon.0) 2992 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[59745]: cluster 2022-04-28T21:32:51.083541+0000 mon.a (mon.0) 2993 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2022-04-28T21:32:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[59745]: audit 2022-04-28T21:32:51.087076+0000 mon.c (mon.1) 64 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:52 smithi059 bash[59745]: audit 2022-04-28T21:32:51.094085+0000 mon.a (mon.0) 2994 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:52 smithi139 bash[43301]: audit 2022-04-28T21:32:51.083409+0000 mon.a (mon.0) 2992 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:52 smithi139 bash[43301]: cluster 2022-04-28T21:32:51.083541+0000 mon.a (mon.0) 2993 : cluster [DBG] osdmap e1020: 8 total, 8 up, 8 in 2022-04-28T21:32:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:52 smithi139 bash[43301]: audit 2022-04-28T21:32:51.087076+0000 mon.c (mon.1) 64 : audit [INF] from='client.? 172.21.15.59:0/4091864512' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:52 smithi139 bash[43301]: audit 2022-04-28T21:32:51.094085+0000 mon.a (mon.0) 2994 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]: dispatch 2022-04-28T21:32:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: cluster 2022-04-28T21:32:51.946950+0000 mgr.y (mgr.24961) 3369 : cluster [DBG] pgmap v3780: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: cluster 2022-04-28T21:32:52.091296+0000 mon.a (mon.0) 2995 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2022-04-28T21:32:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: cluster 2022-04-28T21:32:52.091358+0000 mon.a (mon.0) 2996 : cluster [INF] Cluster is now healthy 2022-04-28T21:32:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.111479+0000 mon.a (mon.0) 2997 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: cluster 2022-04-28T21:32:52.111580+0000 mon.a (mon.0) 2998 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2022-04-28T21:32:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.125663+0000 mon.c (mon.1) 65 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.126130+0000 mon.a (mon.0) 2999 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.126803+0000 mon.c (mon.1) 66 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.127192+0000 mon.a (mon.0) 3000 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.127922+0000 mon.c (mon.1) 67 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:53.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:53 smithi139 bash[43301]: audit 2022-04-28T21:32:52.128331+0000 mon.a (mon.0) 3001 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: cluster 2022-04-28T21:32:51.946950+0000 mgr.y (mgr.24961) 3369 : cluster [DBG] pgmap v3780: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: cluster 2022-04-28T21:32:52.091296+0000 mon.a (mon.0) 2995 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2022-04-28T21:32:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: cluster 2022-04-28T21:32:52.091358+0000 mon.a (mon.0) 2996 : cluster [INF] Cluster is now healthy 2022-04-28T21:32:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.111479+0000 mon.a (mon.0) 2997 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: cluster 2022-04-28T21:32:52.111580+0000 mon.a (mon.0) 2998 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2022-04-28T21:32:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.125663+0000 mon.c (mon.1) 65 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.126130+0000 mon.a (mon.0) 2999 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.126803+0000 mon.c (mon.1) 66 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.127192+0000 mon.a (mon.0) 3000 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.127922+0000 mon.c (mon.1) 67 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:53.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[61152]: audit 2022-04-28T21:32:52.128331+0000 mon.a (mon.0) 3001 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:53.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: cluster 2022-04-28T21:32:51.946950+0000 mgr.y (mgr.24961) 3369 : cluster [DBG] pgmap v3780: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: cluster 2022-04-28T21:32:52.091296+0000 mon.a (mon.0) 2995 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2022-04-28T21:32:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: cluster 2022-04-28T21:32:52.091358+0000 mon.a (mon.0) 2996 : cluster [INF] Cluster is now healthy 2022-04-28T21:32:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.111479+0000 mon.a (mon.0) 2997 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-27"}]': finished 2022-04-28T21:32:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: cluster 2022-04-28T21:32:52.111580+0000 mon.a (mon.0) 2998 : cluster [DBG] osdmap e1021: 8 total, 8 up, 8 in 2022-04-28T21:32:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.125663+0000 mon.c (mon.1) 65 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.126130+0000 mon.a (mon.0) 2999 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.126803+0000 mon.c (mon.1) 66 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.127192+0000 mon.a (mon.0) 3000 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.127922+0000 mon.c (mon.1) 67 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[59745]: audit 2022-04-28T21:32:52.128331+0000 mon.a (mon.0) 3001 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:32:54.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[61152]: audit 2022-04-28T21:32:53.109858+0000 mon.a (mon.0) 3002 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[61152]: cluster 2022-04-28T21:32:53.110000+0000 mon.a (mon.0) 3003 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2022-04-28T21:32:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[61152]: audit 2022-04-28T21:32:53.113273+0000 mon.c (mon.1) 68 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:54.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[61152]: audit 2022-04-28T21:32:53.113917+0000 mon.a (mon.0) 3004 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:32:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:32:53.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:32:54.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[59745]: audit 2022-04-28T21:32:53.109858+0000 mon.a (mon.0) 3002 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:54.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[59745]: cluster 2022-04-28T21:32:53.110000+0000 mon.a (mon.0) 3003 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2022-04-28T21:32:54.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[59745]: audit 2022-04-28T21:32:53.113273+0000 mon.c (mon.1) 68 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:54.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:54 smithi059 bash[59745]: audit 2022-04-28T21:32:53.113917+0000 mon.a (mon.0) 3004 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:54.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:54 smithi139 bash[43301]: audit 2022-04-28T21:32:53.109858+0000 mon.a (mon.0) 3002 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-28", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:32:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:54 smithi139 bash[43301]: cluster 2022-04-28T21:32:53.110000+0000 mon.a (mon.0) 3003 : cluster [DBG] osdmap e1022: 8 total, 8 up, 8 in 2022-04-28T21:32:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:54 smithi139 bash[43301]: audit 2022-04-28T21:32:53.113273+0000 mon.c (mon.1) 68 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:54 smithi139 bash[43301]: audit 2022-04-28T21:32:53.113917+0000 mon.a (mon.0) 3004 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:55 smithi059 bash[61152]: cluster 2022-04-28T21:32:53.947762+0000 mgr.y (mgr.24961) 3370 : cluster [DBG] pgmap v3783: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:55.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:55 smithi059 bash[61152]: cluster 2022-04-28T21:32:54.584075+0000 mon.a (mon.0) 3005 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2022-04-28T21:32:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:55 smithi059 bash[59745]: cluster 2022-04-28T21:32:53.947762+0000 mgr.y (mgr.24961) 3370 : cluster [DBG] pgmap v3783: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:55 smithi059 bash[59745]: cluster 2022-04-28T21:32:54.584075+0000 mon.a (mon.0) 3005 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2022-04-28T21:32:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:55 smithi139 bash[43301]: cluster 2022-04-28T21:32:53.947762+0000 mgr.y (mgr.24961) 3370 : cluster [DBG] pgmap v3783: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:55 smithi139 bash[43301]: cluster 2022-04-28T21:32:54.584075+0000 mon.a (mon.0) 3005 : cluster [DBG] osdmap e1023: 8 total, 8 up, 8 in 2022-04-28T21:32:56.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:56 smithi059 bash[61152]: audit 2022-04-28T21:32:55.136316+0000 mon.a (mon.0) 3006 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:32:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:56 smithi059 bash[61152]: cluster 2022-04-28T21:32:55.136432+0000 mon.a (mon.0) 3007 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2022-04-28T21:32:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:56 smithi059 bash[59745]: audit 2022-04-28T21:32:55.136316+0000 mon.a (mon.0) 3006 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:32:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:56 smithi059 bash[59745]: cluster 2022-04-28T21:32:55.136432+0000 mon.a (mon.0) 3007 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2022-04-28T21:32:56.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:56 smithi139 bash[43301]: audit 2022-04-28T21:32:55.136316+0000 mon.a (mon.0) 3006 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-28", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:32:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:56 smithi139 bash[43301]: cluster 2022-04-28T21:32:55.136432+0000 mon.a (mon.0) 3007 : cluster [DBG] osdmap e1024: 8 total, 8 up, 8 in 2022-04-28T21:32:57.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:57 smithi059 bash[59745]: cluster 2022-04-28T21:32:55.948524+0000 mgr.y (mgr.24961) 3371 : cluster [DBG] pgmap v3786: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:57 smithi059 bash[59745]: cluster 2022-04-28T21:32:56.139148+0000 mon.a (mon.0) 3008 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2022-04-28T21:32:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:57 smithi059 bash[61152]: cluster 2022-04-28T21:32:55.948524+0000 mgr.y (mgr.24961) 3371 : cluster [DBG] pgmap v3786: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:57 smithi059 bash[61152]: cluster 2022-04-28T21:32:56.139148+0000 mon.a (mon.0) 3008 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2022-04-28T21:32:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:57 smithi139 bash[43301]: cluster 2022-04-28T21:32:55.948524+0000 mgr.y (mgr.24961) 3371 : cluster [DBG] pgmap v3786: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:57.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:57 smithi139 bash[43301]: cluster 2022-04-28T21:32:56.139148+0000 mon.a (mon.0) 3008 : cluster [DBG] osdmap e1025: 8 total, 8 up, 8 in 2022-04-28T21:32:58.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:58 smithi059 bash[59745]: cluster 2022-04-28T21:32:57.158981+0000 mon.a (mon.0) 3009 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2022-04-28T21:32:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:58 smithi059 bash[59745]: audit 2022-04-28T21:32:57.161170+0000 mon.c (mon.1) 69 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:58.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:58 smithi059 bash[59745]: audit 2022-04-28T21:32:57.161735+0000 mon.a (mon.0) 3010 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:58 smithi059 bash[61152]: cluster 2022-04-28T21:32:57.158981+0000 mon.a (mon.0) 3009 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2022-04-28T21:32:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:58 smithi059 bash[61152]: audit 2022-04-28T21:32:57.161170+0000 mon.c (mon.1) 69 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:58 smithi059 bash[61152]: audit 2022-04-28T21:32:57.161735+0000 mon.a (mon.0) 3010 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:58 smithi139 bash[43301]: cluster 2022-04-28T21:32:57.158981+0000 mon.a (mon.0) 3009 : cluster [DBG] osdmap e1026: 8 total, 8 up, 8 in 2022-04-28T21:32:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:58 smithi139 bash[43301]: audit 2022-04-28T21:32:57.161170+0000 mon.c (mon.1) 69 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:58.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:58 smithi139 bash[43301]: audit 2022-04-28T21:32:57.161735+0000 mon.a (mon.0) 3010 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:59.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[59745]: cluster 2022-04-28T21:32:57.949262+0000 mgr.y (mgr.24961) 3372 : cluster [DBG] pgmap v3789: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[59745]: audit 2022-04-28T21:32:58.161924+0000 mon.a (mon.0) 3011 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:32:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[59745]: cluster 2022-04-28T21:32:58.162074+0000 mon.a (mon.0) 3012 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2022-04-28T21:32:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[59745]: audit 2022-04-28T21:32:58.162909+0000 mon.c (mon.1) 70 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[59745]: audit 2022-04-28T21:32:58.163512+0000 mon.a (mon.0) 3013 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[61152]: cluster 2022-04-28T21:32:57.949262+0000 mgr.y (mgr.24961) 3372 : cluster [DBG] pgmap v3789: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[61152]: audit 2022-04-28T21:32:58.161924+0000 mon.a (mon.0) 3011 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:32:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[61152]: cluster 2022-04-28T21:32:58.162074+0000 mon.a (mon.0) 3012 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2022-04-28T21:32:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[61152]: audit 2022-04-28T21:32:58.162909+0000 mon.c (mon.1) 70 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:32:59 smithi059 bash[61152]: audit 2022-04-28T21:32:58.163512+0000 mon.a (mon.0) 3013 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:59 smithi139 bash[43301]: cluster 2022-04-28T21:32:57.949262+0000 mgr.y (mgr.24961) 3372 : cluster [DBG] pgmap v3789: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:32:59.637 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:59 smithi139 bash[43301]: audit 2022-04-28T21:32:58.161924+0000 mon.a (mon.0) 3011 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:32:59.638 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:59 smithi139 bash[43301]: cluster 2022-04-28T21:32:58.162074+0000 mon.a (mon.0) 3012 : cluster [DBG] osdmap e1027: 8 total, 8 up, 8 in 2022-04-28T21:32:59.638 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:59 smithi139 bash[43301]: audit 2022-04-28T21:32:58.162909+0000 mon.c (mon.1) 70 : audit [INF] from='client.? 172.21.15.59:0/1511307587' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:32:59.638 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:32:59 smithi139 bash[43301]: audit 2022-04-28T21:32:58.163512+0000 mon.a (mon.0) 3013 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]: dispatch 2022-04-28T21:33:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[61152]: audit 2022-04-28T21:32:59.163479+0000 mon.a (mon.0) 3014 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:33:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[61152]: cluster 2022-04-28T21:32:59.163613+0000 mon.a (mon.0) 3015 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2022-04-28T21:33:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[61152]: audit 2022-04-28T21:32:59.185921+0000 mon.a (mon.0) 3016 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[61152]: audit 2022-04-28T21:32:59.187141+0000 mon.a (mon.0) 3017 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[61152]: audit 2022-04-28T21:32:59.187750+0000 mon.a (mon.0) 3018 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-29", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[59745]: audit 2022-04-28T21:32:59.163479+0000 mon.a (mon.0) 3014 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:33:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[59745]: cluster 2022-04-28T21:32:59.163613+0000 mon.a (mon.0) 3015 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2022-04-28T21:33:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[59745]: audit 2022-04-28T21:32:59.185921+0000 mon.a (mon.0) 3016 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[59745]: audit 2022-04-28T21:32:59.187141+0000 mon.a (mon.0) 3017 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:00 smithi059 bash[59745]: audit 2022-04-28T21:32:59.187750+0000 mon.a (mon.0) 3018 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-29", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:00.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:00 smithi139 bash[43301]: audit 2022-04-28T21:32:59.163479+0000 mon.a (mon.0) 3014 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-28"}]': finished 2022-04-28T21:33:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:00 smithi139 bash[43301]: cluster 2022-04-28T21:32:59.163613+0000 mon.a (mon.0) 3015 : cluster [DBG] osdmap e1028: 8 total, 8 up, 8 in 2022-04-28T21:33:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:00 smithi139 bash[43301]: audit 2022-04-28T21:32:59.185921+0000 mon.a (mon.0) 3016 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:00 smithi139 bash[43301]: audit 2022-04-28T21:32:59.187141+0000 mon.a (mon.0) 3017 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:00 smithi139 bash[43301]: audit 2022-04-28T21:32:59.187750+0000 mon.a (mon.0) 3018 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-29", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:02.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[59745]: cluster 2022-04-28T21:32:59.949995+0000 mgr.y (mgr.24961) 3373 : cluster [DBG] pgmap v3792: 137 pgs: 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:02.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[59745]: audit 2022-04-28T21:33:00.187938+0000 mon.a (mon.0) 3019 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-29", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:02.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[59745]: cluster 2022-04-28T21:33:00.188043+0000 mon.a (mon.0) 3020 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2022-04-28T21:33:02.023 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[59745]: audit 2022-04-28T21:33:00.189280+0000 mon.a (mon.0) 3021 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-29", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:02.024 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[61152]: cluster 2022-04-28T21:32:59.949995+0000 mgr.y (mgr.24961) 3373 : cluster [DBG] pgmap v3792: 137 pgs: 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:02.024 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[61152]: audit 2022-04-28T21:33:00.187938+0000 mon.a (mon.0) 3019 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-29", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:02.024 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[61152]: cluster 2022-04-28T21:33:00.188043+0000 mon.a (mon.0) 3020 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2022-04-28T21:33:02.024 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:01 smithi059 bash[61152]: audit 2022-04-28T21:33:00.189280+0000 mon.a (mon.0) 3021 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-29", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:01 smithi139 bash[43301]: cluster 2022-04-28T21:32:59.949995+0000 mgr.y (mgr.24961) 3373 : cluster [DBG] pgmap v3792: 137 pgs: 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:01 smithi139 bash[43301]: audit 2022-04-28T21:33:00.187938+0000 mon.a (mon.0) 3019 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-29", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:01 smithi139 bash[43301]: cluster 2022-04-28T21:33:00.188043+0000 mon.a (mon.0) 3020 : cluster [DBG] osdmap e1029: 8 total, 8 up, 8 in 2022-04-28T21:33:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:01 smithi139 bash[43301]: audit 2022-04-28T21:33:00.189280+0000 mon.a (mon.0) 3021 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-29", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:02.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:03.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[61152]: cluster 2022-04-28T21:33:01.654250+0000 mon.a (mon.0) 3022 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2022-04-28T21:33:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[61152]: cluster 2022-04-28T21:33:01.950738+0000 mgr.y (mgr.24961) 3374 : cluster [DBG] pgmap v3795: 137 pgs: 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[61152]: audit 2022-04-28T21:33:02.195628+0000 mon.a (mon.0) 3023 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-29", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[61152]: cluster 2022-04-28T21:33:02.195879+0000 mon.a (mon.0) 3024 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2022-04-28T21:33:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[59745]: cluster 2022-04-28T21:33:01.654250+0000 mon.a (mon.0) 3022 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2022-04-28T21:33:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[59745]: cluster 2022-04-28T21:33:01.950738+0000 mgr.y (mgr.24961) 3374 : cluster [DBG] pgmap v3795: 137 pgs: 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[59745]: audit 2022-04-28T21:33:02.195628+0000 mon.a (mon.0) 3023 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-29", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:02 smithi059 bash[59745]: cluster 2022-04-28T21:33:02.195879+0000 mon.a (mon.0) 3024 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2022-04-28T21:33:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:02 smithi139 bash[43301]: cluster 2022-04-28T21:33:01.654250+0000 mon.a (mon.0) 3022 : cluster [DBG] osdmap e1030: 8 total, 8 up, 8 in 2022-04-28T21:33:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:02 smithi139 bash[43301]: cluster 2022-04-28T21:33:01.950738+0000 mgr.y (mgr.24961) 3374 : cluster [DBG] pgmap v3795: 137 pgs: 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:02 smithi139 bash[43301]: audit 2022-04-28T21:33:02.195628+0000 mon.a (mon.0) 3023 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-29", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:02 smithi139 bash[43301]: cluster 2022-04-28T21:33:02.195879+0000 mon.a (mon.0) 3024 : cluster [DBG] osdmap e1031: 8 total, 8 up, 8 in 2022-04-28T21:33:04.202 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:03.914Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:04.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:04 smithi059 bash[61152]: cluster 2022-04-28T21:33:03.200143+0000 mon.a (mon.0) 3025 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2022-04-28T21:33:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:04 smithi059 bash[59745]: cluster 2022-04-28T21:33:03.200143+0000 mon.a (mon.0) 3025 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2022-04-28T21:33:04.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:04 smithi139 bash[43301]: cluster 2022-04-28T21:33:03.200143+0000 mon.a (mon.0) 3025 : cluster [DBG] osdmap e1032: 8 total, 8 up, 8 in 2022-04-28T21:33:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[59745]: cluster 2022-04-28T21:33:03.951517+0000 mgr.y (mgr.24961) 3375 : cluster [DBG] pgmap v3798: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[59745]: cluster 2022-04-28T21:33:04.197178+0000 mon.a (mon.0) 3026 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2022-04-28T21:33:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[59745]: audit 2022-04-28T21:33:04.198625+0000 mon.a (mon.0) 3027 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[59745]: audit 2022-04-28T21:33:05.201332+0000 mon.a (mon.0) 3028 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[59745]: cluster 2022-04-28T21:33:05.201501+0000 mon.a (mon.0) 3029 : cluster [DBG] osdmap e1034: 8 total, 8 up, 8 in 2022-04-28T21:33:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[61152]: cluster 2022-04-28T21:33:03.951517+0000 mgr.y (mgr.24961) 3375 : cluster [DBG] pgmap v3798: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[61152]: cluster 2022-04-28T21:33:04.197178+0000 mon.a (mon.0) 3026 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2022-04-28T21:33:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[61152]: audit 2022-04-28T21:33:04.198625+0000 mon.a (mon.0) 3027 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[61152]: audit 2022-04-28T21:33:05.201332+0000 mon.a (mon.0) 3028 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:05.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:05 smithi059 bash[61152]: cluster 2022-04-28T21:33:05.201501+0000 mon.a (mon.0) 3029 : cluster [DBG] osdmap e1034: 8 total, 8 up, 8 in 2022-04-28T21:33:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:05 smithi139 bash[43301]: cluster 2022-04-28T21:33:03.951517+0000 mgr.y (mgr.24961) 3375 : cluster [DBG] pgmap v3798: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 753 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:05 smithi139 bash[43301]: cluster 2022-04-28T21:33:04.197178+0000 mon.a (mon.0) 3026 : cluster [DBG] osdmap e1033: 8 total, 8 up, 8 in 2022-04-28T21:33:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:05 smithi139 bash[43301]: audit 2022-04-28T21:33:04.198625+0000 mon.a (mon.0) 3027 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:05 smithi139 bash[43301]: audit 2022-04-28T21:33:05.201332+0000 mon.a (mon.0) 3028 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:05 smithi139 bash[43301]: cluster 2022-04-28T21:33:05.201501+0000 mon.a (mon.0) 3029 : cluster [DBG] osdmap e1034: 8 total, 8 up, 8 in 2022-04-28T21:33:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:06 smithi059 bash[59745]: audit 2022-04-28T21:33:05.202644+0000 mon.a (mon.0) 3030 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:06 smithi059 bash[59745]: audit 2022-04-28T21:33:06.203777+0000 mon.a (mon.0) 3031 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:06 smithi059 bash[59745]: cluster 2022-04-28T21:33:06.203865+0000 mon.a (mon.0) 3032 : cluster [DBG] osdmap e1035: 8 total, 8 up, 8 in 2022-04-28T21:33:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:06 smithi059 bash[61152]: audit 2022-04-28T21:33:05.202644+0000 mon.a (mon.0) 3030 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:06 smithi059 bash[61152]: audit 2022-04-28T21:33:06.203777+0000 mon.a (mon.0) 3031 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:06 smithi059 bash[61152]: cluster 2022-04-28T21:33:06.203865+0000 mon.a (mon.0) 3032 : cluster [DBG] osdmap e1035: 8 total, 8 up, 8 in 2022-04-28T21:33:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:06 smithi139 bash[43301]: audit 2022-04-28T21:33:05.202644+0000 mon.a (mon.0) 3030 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]: dispatch 2022-04-28T21:33:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:06 smithi139 bash[43301]: audit 2022-04-28T21:33:06.203777+0000 mon.a (mon.0) 3031 : audit [INF] from='client.? 172.21.15.59:0/221343808' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-29"}]': finished 2022-04-28T21:33:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:06 smithi139 bash[43301]: cluster 2022-04-28T21:33:06.203865+0000 mon.a (mon.0) 3032 : cluster [DBG] osdmap e1035: 8 total, 8 up, 8 in 2022-04-28T21:33:07.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[61152]: cluster 2022-04-28T21:33:05.952261+0000 mgr.y (mgr.24961) 3376 : cluster [DBG] pgmap v3801: 137 pgs: 137 active+clean; 24 KiB data, 754 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[61152]: audit 2022-04-28T21:33:06.216503+0000 mon.a (mon.0) 3033 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[61152]: audit 2022-04-28T21:33:06.217287+0000 mon.a (mon.0) 3034 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[61152]: audit 2022-04-28T21:33:06.217807+0000 mon.a (mon.0) 3035 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-30", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[59745]: cluster 2022-04-28T21:33:05.952261+0000 mgr.y (mgr.24961) 3376 : cluster [DBG] pgmap v3801: 137 pgs: 137 active+clean; 24 KiB data, 754 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[59745]: audit 2022-04-28T21:33:06.216503+0000 mon.a (mon.0) 3033 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[59745]: audit 2022-04-28T21:33:06.217287+0000 mon.a (mon.0) 3034 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:07 smithi059 bash[59745]: audit 2022-04-28T21:33:06.217807+0000 mon.a (mon.0) 3035 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-30", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:07 smithi139 bash[43301]: cluster 2022-04-28T21:33:05.952261+0000 mgr.y (mgr.24961) 3376 : cluster [DBG] pgmap v3801: 137 pgs: 137 active+clean; 24 KiB data, 754 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:07 smithi139 bash[43301]: audit 2022-04-28T21:33:06.216503+0000 mon.a (mon.0) 3033 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:07 smithi139 bash[43301]: audit 2022-04-28T21:33:06.217287+0000 mon.a (mon.0) 3034 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:07 smithi139 bash[43301]: audit 2022-04-28T21:33:06.217807+0000 mon.a (mon.0) 3035 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-30", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:08 smithi059 bash[61152]: audit 2022-04-28T21:33:07.216101+0000 mon.a (mon.0) 3036 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-30", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:08 smithi059 bash[61152]: cluster 2022-04-28T21:33:07.216262+0000 mon.a (mon.0) 3037 : cluster [DBG] osdmap e1036: 8 total, 8 up, 8 in 2022-04-28T21:33:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:08 smithi059 bash[61152]: audit 2022-04-28T21:33:07.217865+0000 mon.a (mon.0) 3038 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-30", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:08 smithi059 bash[59745]: audit 2022-04-28T21:33:07.216101+0000 mon.a (mon.0) 3036 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-30", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:08 smithi059 bash[59745]: cluster 2022-04-28T21:33:07.216262+0000 mon.a (mon.0) 3037 : cluster [DBG] osdmap e1036: 8 total, 8 up, 8 in 2022-04-28T21:33:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:08 smithi059 bash[59745]: audit 2022-04-28T21:33:07.217865+0000 mon.a (mon.0) 3038 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-30", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:08 smithi139 bash[43301]: audit 2022-04-28T21:33:07.216101+0000 mon.a (mon.0) 3036 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-30", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:08 smithi139 bash[43301]: cluster 2022-04-28T21:33:07.216262+0000 mon.a (mon.0) 3037 : cluster [DBG] osdmap e1036: 8 total, 8 up, 8 in 2022-04-28T21:33:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:08 smithi139 bash[43301]: audit 2022-04-28T21:33:07.217865+0000 mon.a (mon.0) 3038 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-30", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[61152]: cluster 2022-04-28T21:33:07.953031+0000 mgr.y (mgr.24961) 3377 : cluster [DBG] pgmap v3804: 137 pgs: 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[61152]: cluster 2022-04-28T21:33:08.680860+0000 mon.a (mon.0) 3039 : cluster [DBG] osdmap e1037: 8 total, 8 up, 8 in 2022-04-28T21:33:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[61152]: audit 2022-04-28T21:33:09.228019+0000 mon.a (mon.0) 3040 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-30", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[61152]: cluster 2022-04-28T21:33:09.228194+0000 mon.a (mon.0) 3041 : cluster [DBG] osdmap e1038: 8 total, 8 up, 8 in 2022-04-28T21:33:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[59745]: cluster 2022-04-28T21:33:07.953031+0000 mgr.y (mgr.24961) 3377 : cluster [DBG] pgmap v3804: 137 pgs: 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[59745]: cluster 2022-04-28T21:33:08.680860+0000 mon.a (mon.0) 3039 : cluster [DBG] osdmap e1037: 8 total, 8 up, 8 in 2022-04-28T21:33:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[59745]: audit 2022-04-28T21:33:09.228019+0000 mon.a (mon.0) 3040 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-30", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:09 smithi059 bash[59745]: cluster 2022-04-28T21:33:09.228194+0000 mon.a (mon.0) 3041 : cluster [DBG] osdmap e1038: 8 total, 8 up, 8 in 2022-04-28T21:33:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:09 smithi139 bash[43301]: cluster 2022-04-28T21:33:07.953031+0000 mgr.y (mgr.24961) 3377 : cluster [DBG] pgmap v3804: 137 pgs: 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:09 smithi139 bash[43301]: cluster 2022-04-28T21:33:08.680860+0000 mon.a (mon.0) 3039 : cluster [DBG] osdmap e1037: 8 total, 8 up, 8 in 2022-04-28T21:33:10.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:09 smithi139 bash[43301]: audit 2022-04-28T21:33:09.228019+0000 mon.a (mon.0) 3040 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-30", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:10.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:09 smithi139 bash[43301]: cluster 2022-04-28T21:33:09.228194+0000 mon.a (mon.0) 3041 : cluster [DBG] osdmap e1038: 8 total, 8 up, 8 in 2022-04-28T21:33:11.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:11 smithi059 bash[61152]: cluster 2022-04-28T21:33:09.953783+0000 mgr.y (mgr.24961) 3378 : cluster [DBG] pgmap v3807: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:11 smithi059 bash[61152]: cluster 2022-04-28T21:33:10.227242+0000 mon.a (mon.0) 3042 : cluster [DBG] osdmap e1039: 8 total, 8 up, 8 in 2022-04-28T21:33:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:11 smithi059 bash[59745]: cluster 2022-04-28T21:33:09.953783+0000 mgr.y (mgr.24961) 3378 : cluster [DBG] pgmap v3807: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:11 smithi059 bash[59745]: cluster 2022-04-28T21:33:10.227242+0000 mon.a (mon.0) 3042 : cluster [DBG] osdmap e1039: 8 total, 8 up, 8 in 2022-04-28T21:33:11.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:11 smithi139 bash[43301]: cluster 2022-04-28T21:33:09.953783+0000 mgr.y (mgr.24961) 3378 : cluster [DBG] pgmap v3807: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 755 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:11 smithi139 bash[43301]: cluster 2022-04-28T21:33:10.227242+0000 mon.a (mon.0) 3042 : cluster [DBG] osdmap e1039: 8 total, 8 up, 8 in 2022-04-28T21:33:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:12 smithi059 bash[61152]: cluster 2022-04-28T21:33:11.228076+0000 mon.a (mon.0) 3043 : cluster [DBG] osdmap e1040: 8 total, 8 up, 8 in 2022-04-28T21:33:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:12 smithi059 bash[61152]: audit 2022-04-28T21:33:11.229835+0000 mon.a (mon.0) 3044 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:12.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:12 smithi059 bash[59745]: cluster 2022-04-28T21:33:11.228076+0000 mon.a (mon.0) 3043 : cluster [DBG] osdmap e1040: 8 total, 8 up, 8 in 2022-04-28T21:33:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:12 smithi059 bash[59745]: audit 2022-04-28T21:33:11.229835+0000 mon.a (mon.0) 3044 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:12.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:12 smithi139 bash[43301]: cluster 2022-04-28T21:33:11.228076+0000 mon.a (mon.0) 3043 : cluster [DBG] osdmap e1040: 8 total, 8 up, 8 in 2022-04-28T21:33:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:12 smithi139 bash[43301]: audit 2022-04-28T21:33:11.229835+0000 mon.a (mon.0) 3044 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:13.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[61152]: cluster 2022-04-28T21:33:11.954572+0000 mgr.y (mgr.24961) 3379 : cluster [DBG] pgmap v3810: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[61152]: audit 2022-04-28T21:33:12.230988+0000 mon.a (mon.0) 3045 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[61152]: cluster 2022-04-28T21:33:12.231161+0000 mon.a (mon.0) 3046 : cluster [DBG] osdmap e1041: 8 total, 8 up, 8 in 2022-04-28T21:33:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[61152]: audit 2022-04-28T21:33:12.232899+0000 mon.a (mon.0) 3047 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[61152]: audit 2022-04-28T21:33:13.232372+0000 mon.a (mon.0) 3048 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[61152]: cluster 2022-04-28T21:33:13.232462+0000 mon.a (mon.0) 3049 : cluster [DBG] osdmap e1042: 8 total, 8 up, 8 in 2022-04-28T21:33:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[59745]: cluster 2022-04-28T21:33:11.954572+0000 mgr.y (mgr.24961) 3379 : cluster [DBG] pgmap v3810: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[59745]: audit 2022-04-28T21:33:12.230988+0000 mon.a (mon.0) 3045 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[59745]: cluster 2022-04-28T21:33:12.231161+0000 mon.a (mon.0) 3046 : cluster [DBG] osdmap e1041: 8 total, 8 up, 8 in 2022-04-28T21:33:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[59745]: audit 2022-04-28T21:33:12.232899+0000 mon.a (mon.0) 3047 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[59745]: audit 2022-04-28T21:33:13.232372+0000 mon.a (mon.0) 3048 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[59745]: cluster 2022-04-28T21:33:13.232462+0000 mon.a (mon.0) 3049 : cluster [DBG] osdmap e1042: 8 total, 8 up, 8 in 2022-04-28T21:33:13.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:13 smithi139 bash[43301]: cluster 2022-04-28T21:33:11.954572+0000 mgr.y (mgr.24961) 3379 : cluster [DBG] pgmap v3810: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:13 smithi139 bash[43301]: audit 2022-04-28T21:33:12.230988+0000 mon.a (mon.0) 3045 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:13 smithi139 bash[43301]: cluster 2022-04-28T21:33:12.231161+0000 mon.a (mon.0) 3046 : cluster [DBG] osdmap e1041: 8 total, 8 up, 8 in 2022-04-28T21:33:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:13 smithi139 bash[43301]: audit 2022-04-28T21:33:12.232899+0000 mon.a (mon.0) 3047 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]: dispatch 2022-04-28T21:33:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:13 smithi139 bash[43301]: audit 2022-04-28T21:33:13.232372+0000 mon.a (mon.0) 3048 : audit [INF] from='client.? 172.21.15.59:0/4014958704' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-30"}]': finished 2022-04-28T21:33:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:13 smithi139 bash[43301]: cluster 2022-04-28T21:33:13.232462+0000 mon.a (mon.0) 3049 : cluster [DBG] osdmap e1042: 8 total, 8 up, 8 in 2022-04-28T21:33:14.257 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:13.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:14.257 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:14 smithi059 bash[59745]: audit 2022-04-28T21:33:13.249057+0000 mon.a (mon.0) 3050 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:14.257 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:14 smithi059 bash[59745]: audit 2022-04-28T21:33:13.250172+0000 mon.a (mon.0) 3051 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:14.258 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:14 smithi059 bash[59745]: audit 2022-04-28T21:33:13.250661+0000 mon.a (mon.0) 3052 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-31", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:14.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:14 smithi059 bash[61152]: audit 2022-04-28T21:33:13.249057+0000 mon.a (mon.0) 3050 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:14 smithi059 bash[61152]: audit 2022-04-28T21:33:13.250172+0000 mon.a (mon.0) 3051 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:14.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:14 smithi059 bash[61152]: audit 2022-04-28T21:33:13.250661+0000 mon.a (mon.0) 3052 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-31", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:14 smithi139 bash[43301]: audit 2022-04-28T21:33:13.249057+0000 mon.a (mon.0) 3050 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:14 smithi139 bash[43301]: audit 2022-04-28T21:33:13.250172+0000 mon.a (mon.0) 3051 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:14 smithi139 bash[43301]: audit 2022-04-28T21:33:13.250661+0000 mon.a (mon.0) 3052 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-31", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[61152]: cluster 2022-04-28T21:33:13.955263+0000 mgr.y (mgr.24961) 3380 : cluster [DBG] pgmap v3813: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[61152]: audit 2022-04-28T21:33:14.250629+0000 mon.a (mon.0) 3053 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-31", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[61152]: cluster 2022-04-28T21:33:14.250729+0000 mon.a (mon.0) 3054 : cluster [DBG] osdmap e1043: 8 total, 8 up, 8 in 2022-04-28T21:33:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[61152]: audit 2022-04-28T21:33:14.251801+0000 mon.a (mon.0) 3055 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-31", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[59745]: cluster 2022-04-28T21:33:13.955263+0000 mgr.y (mgr.24961) 3380 : cluster [DBG] pgmap v3813: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[59745]: audit 2022-04-28T21:33:14.250629+0000 mon.a (mon.0) 3053 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-31", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[59745]: cluster 2022-04-28T21:33:14.250729+0000 mon.a (mon.0) 3054 : cluster [DBG] osdmap e1043: 8 total, 8 up, 8 in 2022-04-28T21:33:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:15 smithi059 bash[59745]: audit 2022-04-28T21:33:14.251801+0000 mon.a (mon.0) 3055 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-31", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:16.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:15 smithi139 bash[43301]: cluster 2022-04-28T21:33:13.955263+0000 mgr.y (mgr.24961) 3380 : cluster [DBG] pgmap v3813: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:15 smithi139 bash[43301]: audit 2022-04-28T21:33:14.250629+0000 mon.a (mon.0) 3053 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-31", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:15 smithi139 bash[43301]: cluster 2022-04-28T21:33:14.250729+0000 mon.a (mon.0) 3054 : cluster [DBG] osdmap e1043: 8 total, 8 up, 8 in 2022-04-28T21:33:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:15 smithi139 bash[43301]: audit 2022-04-28T21:33:14.251801+0000 mon.a (mon.0) 3055 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-31", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:17.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[61152]: cluster 2022-04-28T21:33:15.720444+0000 mon.a (mon.0) 3056 : cluster [DBG] osdmap e1044: 8 total, 8 up, 8 in 2022-04-28T21:33:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[61152]: cluster 2022-04-28T21:33:15.956086+0000 mgr.y (mgr.24961) 3381 : cluster [DBG] pgmap v3816: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[61152]: audit 2022-04-28T21:33:16.255777+0000 mon.a (mon.0) 3057 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-31", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[61152]: cluster 2022-04-28T21:33:16.256005+0000 mon.a (mon.0) 3058 : cluster [DBG] osdmap e1045: 8 total, 8 up, 8 in 2022-04-28T21:33:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[59745]: cluster 2022-04-28T21:33:15.720444+0000 mon.a (mon.0) 3056 : cluster [DBG] osdmap e1044: 8 total, 8 up, 8 in 2022-04-28T21:33:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[59745]: cluster 2022-04-28T21:33:15.956086+0000 mgr.y (mgr.24961) 3381 : cluster [DBG] pgmap v3816: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[59745]: audit 2022-04-28T21:33:16.255777+0000 mon.a (mon.0) 3057 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-31", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:16 smithi059 bash[59745]: cluster 2022-04-28T21:33:16.256005+0000 mon.a (mon.0) 3058 : cluster [DBG] osdmap e1045: 8 total, 8 up, 8 in 2022-04-28T21:33:17.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:16 smithi139 bash[43301]: cluster 2022-04-28T21:33:15.720444+0000 mon.a (mon.0) 3056 : cluster [DBG] osdmap e1044: 8 total, 8 up, 8 in 2022-04-28T21:33:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:16 smithi139 bash[43301]: cluster 2022-04-28T21:33:15.956086+0000 mgr.y (mgr.24961) 3381 : cluster [DBG] pgmap v3816: 137 pgs: 137 active+clean; 24 KiB data, 756 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:16 smithi139 bash[43301]: audit 2022-04-28T21:33:16.255777+0000 mon.a (mon.0) 3057 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-31", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:16 smithi139 bash[43301]: cluster 2022-04-28T21:33:16.256005+0000 mon.a (mon.0) 3058 : cluster [DBG] osdmap e1045: 8 total, 8 up, 8 in 2022-04-28T21:33:18.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:18 smithi059 bash[59745]: cluster 2022-04-28T21:33:17.263094+0000 mon.a (mon.0) 3059 : cluster [DBG] osdmap e1046: 8 total, 8 up, 8 in 2022-04-28T21:33:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:18 smithi059 bash[61152]: cluster 2022-04-28T21:33:17.263094+0000 mon.a (mon.0) 3059 : cluster [DBG] osdmap e1046: 8 total, 8 up, 8 in 2022-04-28T21:33:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:18 smithi139 bash[43301]: cluster 2022-04-28T21:33:17.263094+0000 mon.a (mon.0) 3059 : cluster [DBG] osdmap e1046: 8 total, 8 up, 8 in 2022-04-28T21:33:19.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:19 smithi059 bash[59745]: cluster 2022-04-28T21:33:17.956871+0000 mgr.y (mgr.24961) 3382 : cluster [DBG] pgmap v3819: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:19 smithi059 bash[59745]: cluster 2022-04-28T21:33:18.262135+0000 mon.a (mon.0) 3060 : cluster [DBG] osdmap e1047: 8 total, 8 up, 8 in 2022-04-28T21:33:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:19 smithi059 bash[59745]: audit 2022-04-28T21:33:18.263759+0000 mon.a (mon.0) 3061 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:19 smithi059 bash[61152]: cluster 2022-04-28T21:33:17.956871+0000 mgr.y (mgr.24961) 3382 : cluster [DBG] pgmap v3819: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:19 smithi059 bash[61152]: cluster 2022-04-28T21:33:18.262135+0000 mon.a (mon.0) 3060 : cluster [DBG] osdmap e1047: 8 total, 8 up, 8 in 2022-04-28T21:33:19.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:19 smithi059 bash[61152]: audit 2022-04-28T21:33:18.263759+0000 mon.a (mon.0) 3061 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:19.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:19 smithi139 bash[43301]: cluster 2022-04-28T21:33:17.956871+0000 mgr.y (mgr.24961) 3382 : cluster [DBG] pgmap v3819: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:19.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:19 smithi139 bash[43301]: cluster 2022-04-28T21:33:18.262135+0000 mon.a (mon.0) 3060 : cluster [DBG] osdmap e1047: 8 total, 8 up, 8 in 2022-04-28T21:33:19.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:19 smithi139 bash[43301]: audit 2022-04-28T21:33:18.263759+0000 mon.a (mon.0) 3061 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.269115+0000 mon.a (mon.0) 3062 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: cluster 2022-04-28T21:33:19.269237+0000 mon.a (mon.0) 3063 : cluster [DBG] osdmap e1048: 8 total, 8 up, 8 in 2022-04-28T21:33:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.270396+0000 mon.a (mon.0) 3064 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:20.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.275893+0000 mon.a (mon.0) 3065 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:33:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.276339+0000 mon.a (mon.0) 3066 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:33:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.276671+0000 mon.a (mon.0) 3067 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:33:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.277068+0000 mon.a (mon.0) 3068 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:33:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.277421+0000 mon.a (mon.0) 3069 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:33:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:19.277717+0000 mon.a (mon.0) 3070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:33:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: audit 2022-04-28T21:33:20.267541+0000 mon.a (mon.0) 3071 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[59745]: cluster 2022-04-28T21:33:20.267634+0000 mon.a (mon.0) 3072 : cluster [DBG] osdmap e1049: 8 total, 8 up, 8 in 2022-04-28T21:33:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.269115+0000 mon.a (mon.0) 3062 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: cluster 2022-04-28T21:33:19.269237+0000 mon.a (mon.0) 3063 : cluster [DBG] osdmap e1048: 8 total, 8 up, 8 in 2022-04-28T21:33:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.270396+0000 mon.a (mon.0) 3064 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.275893+0000 mon.a (mon.0) 3065 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:33:20.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.276339+0000 mon.a (mon.0) 3066 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:33:20.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.276671+0000 mon.a (mon.0) 3067 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:33:20.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.277068+0000 mon.a (mon.0) 3068 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:33:20.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.277421+0000 mon.a (mon.0) 3069 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:33:20.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:19.277717+0000 mon.a (mon.0) 3070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:33:20.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: audit 2022-04-28T21:33:20.267541+0000 mon.a (mon.0) 3071 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:20.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:20 smithi059 bash[61152]: cluster 2022-04-28T21:33:20.267634+0000 mon.a (mon.0) 3072 : cluster [DBG] osdmap e1049: 8 total, 8 up, 8 in 2022-04-28T21:33:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.269115+0000 mon.a (mon.0) 3062 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: cluster 2022-04-28T21:33:19.269237+0000 mon.a (mon.0) 3063 : cluster [DBG] osdmap e1048: 8 total, 8 up, 8 in 2022-04-28T21:33:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.270396+0000 mon.a (mon.0) 3064 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]: dispatch 2022-04-28T21:33:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.275893+0000 mon.a (mon.0) 3065 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:33:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.276339+0000 mon.a (mon.0) 3066 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:33:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.276671+0000 mon.a (mon.0) 3067 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:33:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.277068+0000 mon.a (mon.0) 3068 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:33:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.277421+0000 mon.a (mon.0) 3069 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:33:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:19.277717+0000 mon.a (mon.0) 3070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.18", "id": [4, 0]}]: dispatch 2022-04-28T21:33:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: audit 2022-04-28T21:33:20.267541+0000 mon.a (mon.0) 3071 : audit [INF] from='client.? 172.21.15.59:0/3350446217' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-31"}]': finished 2022-04-28T21:33:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:20 smithi139 bash[43301]: cluster 2022-04-28T21:33:20.267634+0000 mon.a (mon.0) 3072 : cluster [DBG] osdmap e1049: 8 total, 8 up, 8 in 2022-04-28T21:33:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: cluster 2022-04-28T21:33:19.957586+0000 mgr.y (mgr.24961) 3383 : cluster [DBG] pgmap v3822: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:21.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: audit 2022-04-28T21:33:20.286297+0000 mon.c (mon.1) 71 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: audit 2022-04-28T21:33:20.286698+0000 mon.a (mon.0) 3073 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: audit 2022-04-28T21:33:20.287321+0000 mon.c (mon.1) 72 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: audit 2022-04-28T21:33:20.287721+0000 mon.a (mon.0) 3074 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: audit 2022-04-28T21:33:20.288256+0000 mon.c (mon.1) 73 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:21.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[59745]: audit 2022-04-28T21:33:20.288631+0000 mon.a (mon.0) 3075 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:21.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: cluster 2022-04-28T21:33:19.957586+0000 mgr.y (mgr.24961) 3383 : cluster [DBG] pgmap v3822: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:21.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: audit 2022-04-28T21:33:20.286297+0000 mon.c (mon.1) 71 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: audit 2022-04-28T21:33:20.286698+0000 mon.a (mon.0) 3073 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: audit 2022-04-28T21:33:20.287321+0000 mon.c (mon.1) 72 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: audit 2022-04-28T21:33:20.287721+0000 mon.a (mon.0) 3074 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: audit 2022-04-28T21:33:20.288256+0000 mon.c (mon.1) 73 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:21.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:21 smithi059 bash[61152]: audit 2022-04-28T21:33:20.288631+0000 mon.a (mon.0) 3075 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: cluster 2022-04-28T21:33:19.957586+0000 mgr.y (mgr.24961) 3383 : cluster [DBG] pgmap v3822: 137 pgs: 137 active+clean; 24 KiB data, 761 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: audit 2022-04-28T21:33:20.286297+0000 mon.c (mon.1) 71 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: audit 2022-04-28T21:33:20.286698+0000 mon.a (mon.0) 3073 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: audit 2022-04-28T21:33:20.287321+0000 mon.c (mon.1) 72 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: audit 2022-04-28T21:33:20.287721+0000 mon.a (mon.0) 3074 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: audit 2022-04-28T21:33:20.288256+0000 mon.c (mon.1) 73 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:21.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:21 smithi139 bash[43301]: audit 2022-04-28T21:33:20.288631+0000 mon.a (mon.0) 3075 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:22.296 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:22.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:22.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[61152]: audit 2022-04-28T21:33:21.295085+0000 mon.a (mon.0) 3076 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[61152]: cluster 2022-04-28T21:33:21.295207+0000 mon.a (mon.0) 3077 : cluster [DBG] osdmap e1050: 8 total, 8 up, 8 in 2022-04-28T21:33:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[61152]: audit 2022-04-28T21:33:21.296254+0000 mon.c (mon.1) 74 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[61152]: audit 2022-04-28T21:33:21.296856+0000 mon.a (mon.0) 3078 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:22.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[59745]: audit 2022-04-28T21:33:21.295085+0000 mon.a (mon.0) 3076 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[59745]: cluster 2022-04-28T21:33:21.295207+0000 mon.a (mon.0) 3077 : cluster [DBG] osdmap e1050: 8 total, 8 up, 8 in 2022-04-28T21:33:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[59745]: audit 2022-04-28T21:33:21.296254+0000 mon.c (mon.1) 74 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:22 smithi059 bash[59745]: audit 2022-04-28T21:33:21.296856+0000 mon.a (mon.0) 3078 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:22 smithi139 bash[43301]: audit 2022-04-28T21:33:21.295085+0000 mon.a (mon.0) 3076 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-32", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:22.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:22 smithi139 bash[43301]: cluster 2022-04-28T21:33:21.295207+0000 mon.a (mon.0) 3077 : cluster [DBG] osdmap e1050: 8 total, 8 up, 8 in 2022-04-28T21:33:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:22 smithi139 bash[43301]: audit 2022-04-28T21:33:21.296254+0000 mon.c (mon.1) 74 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:22 smithi139 bash[43301]: audit 2022-04-28T21:33:21.296856+0000 mon.a (mon.0) 3078 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:23.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:23 smithi059 bash[61152]: cluster 2022-04-28T21:33:21.958315+0000 mgr.y (mgr.24961) 3384 : cluster [DBG] pgmap v3825: 137 pgs: 137 active+clean; 24 KiB data, 742 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:23 smithi059 bash[61152]: cluster 2022-04-28T21:33:22.768346+0000 mon.a (mon.0) 3079 : cluster [DBG] osdmap e1051: 8 total, 8 up, 8 in 2022-04-28T21:33:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:23 smithi059 bash[59745]: cluster 2022-04-28T21:33:21.958315+0000 mgr.y (mgr.24961) 3384 : cluster [DBG] pgmap v3825: 137 pgs: 137 active+clean; 24 KiB data, 742 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:23 smithi059 bash[59745]: cluster 2022-04-28T21:33:22.768346+0000 mon.a (mon.0) 3079 : cluster [DBG] osdmap e1051: 8 total, 8 up, 8 in 2022-04-28T21:33:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:23 smithi139 bash[43301]: cluster 2022-04-28T21:33:21.958315+0000 mgr.y (mgr.24961) 3384 : cluster [DBG] pgmap v3825: 137 pgs: 137 active+clean; 24 KiB data, 742 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:23 smithi139 bash[43301]: cluster 2022-04-28T21:33:22.768346+0000 mon.a (mon.0) 3079 : cluster [DBG] osdmap e1051: 8 total, 8 up, 8 in 2022-04-28T21:33:24.322 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:23.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:24.322 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:24 smithi059 bash[59745]: audit 2022-04-28T21:33:23.314060+0000 mon.a (mon.0) 3080 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:24.322 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:24 smithi059 bash[59745]: cluster 2022-04-28T21:33:23.314203+0000 mon.a (mon.0) 3081 : cluster [DBG] osdmap e1052: 8 total, 8 up, 8 in 2022-04-28T21:33:24.323 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:24 smithi059 bash[59745]: cluster 2022-04-28T21:33:24.314782+0000 mon.a (mon.0) 3082 : cluster [DBG] osdmap e1053: 8 total, 8 up, 8 in 2022-04-28T21:33:24.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:24 smithi059 bash[61152]: audit 2022-04-28T21:33:23.314060+0000 mon.a (mon.0) 3080 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:24 smithi059 bash[61152]: cluster 2022-04-28T21:33:23.314203+0000 mon.a (mon.0) 3081 : cluster [DBG] osdmap e1052: 8 total, 8 up, 8 in 2022-04-28T21:33:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:24 smithi059 bash[61152]: cluster 2022-04-28T21:33:24.314782+0000 mon.a (mon.0) 3082 : cluster [DBG] osdmap e1053: 8 total, 8 up, 8 in 2022-04-28T21:33:24.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:24 smithi139 bash[43301]: audit 2022-04-28T21:33:23.314060+0000 mon.a (mon.0) 3080 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-32", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:24 smithi139 bash[43301]: cluster 2022-04-28T21:33:23.314203+0000 mon.a (mon.0) 3081 : cluster [DBG] osdmap e1052: 8 total, 8 up, 8 in 2022-04-28T21:33:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:24 smithi139 bash[43301]: cluster 2022-04-28T21:33:24.314782+0000 mon.a (mon.0) 3082 : cluster [DBG] osdmap e1053: 8 total, 8 up, 8 in 2022-04-28T21:33:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:25 smithi059 bash[61152]: cluster 2022-04-28T21:33:23.959087+0000 mgr.y (mgr.24961) 3385 : cluster [DBG] pgmap v3828: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 742 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:25 smithi059 bash[59745]: cluster 2022-04-28T21:33:23.959087+0000 mgr.y (mgr.24961) 3385 : cluster [DBG] pgmap v3828: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 742 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:25 smithi139 bash[43301]: cluster 2022-04-28T21:33:23.959087+0000 mgr.y (mgr.24961) 3385 : cluster [DBG] pgmap v3828: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 742 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:26.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[61152]: cluster 2022-04-28T21:33:25.332130+0000 mon.a (mon.0) 3083 : cluster [DBG] osdmap e1054: 8 total, 8 up, 8 in 2022-04-28T21:33:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[61152]: audit 2022-04-28T21:33:25.333597+0000 mon.c (mon.1) 75 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[61152]: audit 2022-04-28T21:33:25.334245+0000 mon.a (mon.0) 3084 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[61152]: audit 2022-04-28T21:33:26.333789+0000 mon.a (mon.0) 3085 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[61152]: cluster 2022-04-28T21:33:26.334034+0000 mon.a (mon.0) 3086 : cluster [DBG] osdmap e1055: 8 total, 8 up, 8 in 2022-04-28T21:33:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[59745]: cluster 2022-04-28T21:33:25.332130+0000 mon.a (mon.0) 3083 : cluster [DBG] osdmap e1054: 8 total, 8 up, 8 in 2022-04-28T21:33:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[59745]: audit 2022-04-28T21:33:25.333597+0000 mon.c (mon.1) 75 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[59745]: audit 2022-04-28T21:33:25.334245+0000 mon.a (mon.0) 3084 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[59745]: audit 2022-04-28T21:33:26.333789+0000 mon.a (mon.0) 3085 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:26 smithi059 bash[59745]: cluster 2022-04-28T21:33:26.334034+0000 mon.a (mon.0) 3086 : cluster [DBG] osdmap e1055: 8 total, 8 up, 8 in 2022-04-28T21:33:26.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:26 smithi139 bash[43301]: cluster 2022-04-28T21:33:25.332130+0000 mon.a (mon.0) 3083 : cluster [DBG] osdmap e1054: 8 total, 8 up, 8 in 2022-04-28T21:33:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:26 smithi139 bash[43301]: audit 2022-04-28T21:33:25.333597+0000 mon.c (mon.1) 75 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:26 smithi139 bash[43301]: audit 2022-04-28T21:33:25.334245+0000 mon.a (mon.0) 3084 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:26 smithi139 bash[43301]: audit 2022-04-28T21:33:26.333789+0000 mon.a (mon.0) 3085 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:26 smithi139 bash[43301]: cluster 2022-04-28T21:33:26.334034+0000 mon.a (mon.0) 3086 : cluster [DBG] osdmap e1055: 8 total, 8 up, 8 in 2022-04-28T21:33:27.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[59745]: cluster 2022-04-28T21:33:25.959855+0000 mgr.y (mgr.24961) 3386 : cluster [DBG] pgmap v3831: 137 pgs: 137 active+clean; 24 KiB data, 743 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[59745]: audit 2022-04-28T21:33:26.335024+0000 mon.c (mon.1) 76 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[59745]: audit 2022-04-28T21:33:26.335702+0000 mon.a (mon.0) 3087 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[59745]: audit 2022-04-28T21:33:27.335292+0000 mon.a (mon.0) 3088 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[59745]: cluster 2022-04-28T21:33:27.335469+0000 mon.a (mon.0) 3089 : cluster [DBG] osdmap e1056: 8 total, 8 up, 8 in 2022-04-28T21:33:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[61152]: cluster 2022-04-28T21:33:25.959855+0000 mgr.y (mgr.24961) 3386 : cluster [DBG] pgmap v3831: 137 pgs: 137 active+clean; 24 KiB data, 743 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[61152]: audit 2022-04-28T21:33:26.335024+0000 mon.c (mon.1) 76 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[61152]: audit 2022-04-28T21:33:26.335702+0000 mon.a (mon.0) 3087 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[61152]: audit 2022-04-28T21:33:27.335292+0000 mon.a (mon.0) 3088 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:27 smithi059 bash[61152]: cluster 2022-04-28T21:33:27.335469+0000 mon.a (mon.0) 3089 : cluster [DBG] osdmap e1056: 8 total, 8 up, 8 in 2022-04-28T21:33:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:27 smithi139 bash[43301]: cluster 2022-04-28T21:33:25.959855+0000 mgr.y (mgr.24961) 3386 : cluster [DBG] pgmap v3831: 137 pgs: 137 active+clean; 24 KiB data, 743 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:27 smithi139 bash[43301]: audit 2022-04-28T21:33:26.335024+0000 mon.c (mon.1) 76 : audit [INF] from='client.? 172.21.15.59:0/2836303141' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:27 smithi139 bash[43301]: audit 2022-04-28T21:33:26.335702+0000 mon.a (mon.0) 3087 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]: dispatch 2022-04-28T21:33:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:27 smithi139 bash[43301]: audit 2022-04-28T21:33:27.335292+0000 mon.a (mon.0) 3088 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-32"}]': finished 2022-04-28T21:33:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:27 smithi139 bash[43301]: cluster 2022-04-28T21:33:27.335469+0000 mon.a (mon.0) 3089 : cluster [DBG] osdmap e1056: 8 total, 8 up, 8 in 2022-04-28T21:33:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:28 smithi139 bash[43301]: audit 2022-04-28T21:33:27.349906+0000 mon.c (mon.1) 77 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:28 smithi139 bash[43301]: audit 2022-04-28T21:33:27.350327+0000 mon.a (mon.0) 3090 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:28 smithi139 bash[43301]: audit 2022-04-28T21:33:27.351274+0000 mon.c (mon.1) 78 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:28 smithi139 bash[43301]: audit 2022-04-28T21:33:27.351644+0000 mon.a (mon.0) 3091 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:28 smithi139 bash[43301]: audit 2022-04-28T21:33:27.352264+0000 mon.c (mon.1) 79 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:28 smithi139 bash[43301]: audit 2022-04-28T21:33:27.352627+0000 mon.a (mon.0) 3092 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[61152]: audit 2022-04-28T21:33:27.349906+0000 mon.c (mon.1) 77 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[61152]: audit 2022-04-28T21:33:27.350327+0000 mon.a (mon.0) 3090 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[61152]: audit 2022-04-28T21:33:27.351274+0000 mon.c (mon.1) 78 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[61152]: audit 2022-04-28T21:33:27.351644+0000 mon.a (mon.0) 3091 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[61152]: audit 2022-04-28T21:33:27.352264+0000 mon.c (mon.1) 79 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[61152]: audit 2022-04-28T21:33:27.352627+0000 mon.a (mon.0) 3092 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[59745]: audit 2022-04-28T21:33:27.349906+0000 mon.c (mon.1) 77 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[59745]: audit 2022-04-28T21:33:27.350327+0000 mon.a (mon.0) 3090 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[59745]: audit 2022-04-28T21:33:27.351274+0000 mon.c (mon.1) 78 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[59745]: audit 2022-04-28T21:33:27.351644+0000 mon.a (mon.0) 3091 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[59745]: audit 2022-04-28T21:33:27.352264+0000 mon.c (mon.1) 79 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:28 smithi059 bash[59745]: audit 2022-04-28T21:33:27.352627+0000 mon.a (mon.0) 3092 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:29 smithi139 bash[43301]: cluster 2022-04-28T21:33:27.960518+0000 mgr.y (mgr.24961) 3387 : cluster [DBG] pgmap v3834: 137 pgs: 137 active+clean; 24 KiB data, 743 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:29 smithi139 bash[43301]: audit 2022-04-28T21:33:28.354746+0000 mon.a (mon.0) 3093 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:29 smithi139 bash[43301]: cluster 2022-04-28T21:33:28.354873+0000 mon.a (mon.0) 3094 : cluster [DBG] osdmap e1057: 8 total, 8 up, 8 in 2022-04-28T21:33:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:29 smithi139 bash[43301]: audit 2022-04-28T21:33:28.355708+0000 mon.c (mon.1) 80 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:29 smithi139 bash[43301]: audit 2022-04-28T21:33:28.356363+0000 mon.a (mon.0) 3095 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:29.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[61152]: cluster 2022-04-28T21:33:27.960518+0000 mgr.y (mgr.24961) 3387 : cluster [DBG] pgmap v3834: 137 pgs: 137 active+clean; 24 KiB data, 743 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[61152]: audit 2022-04-28T21:33:28.354746+0000 mon.a (mon.0) 3093 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[61152]: cluster 2022-04-28T21:33:28.354873+0000 mon.a (mon.0) 3094 : cluster [DBG] osdmap e1057: 8 total, 8 up, 8 in 2022-04-28T21:33:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[61152]: audit 2022-04-28T21:33:28.355708+0000 mon.c (mon.1) 80 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[61152]: audit 2022-04-28T21:33:28.356363+0000 mon.a (mon.0) 3095 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[59745]: cluster 2022-04-28T21:33:27.960518+0000 mgr.y (mgr.24961) 3387 : cluster [DBG] pgmap v3834: 137 pgs: 137 active+clean; 24 KiB data, 743 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[59745]: audit 2022-04-28T21:33:28.354746+0000 mon.a (mon.0) 3093 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-33", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[59745]: cluster 2022-04-28T21:33:28.354873+0000 mon.a (mon.0) 3094 : cluster [DBG] osdmap e1057: 8 total, 8 up, 8 in 2022-04-28T21:33:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[59745]: audit 2022-04-28T21:33:28.355708+0000 mon.c (mon.1) 80 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:29 smithi059 bash[59745]: audit 2022-04-28T21:33:28.356363+0000 mon.a (mon.0) 3095 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:30.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:30 smithi139 bash[43301]: cluster 2022-04-28T21:33:29.822905+0000 mon.a (mon.0) 3096 : cluster [DBG] osdmap e1058: 8 total, 8 up, 8 in 2022-04-28T21:33:30.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:30 smithi059 bash[61152]: cluster 2022-04-28T21:33:29.822905+0000 mon.a (mon.0) 3096 : cluster [DBG] osdmap e1058: 8 total, 8 up, 8 in 2022-04-28T21:33:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:30 smithi059 bash[59745]: cluster 2022-04-28T21:33:29.822905+0000 mon.a (mon.0) 3096 : cluster [DBG] osdmap e1058: 8 total, 8 up, 8 in 2022-04-28T21:33:31.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:31 smithi059 bash[61152]: cluster 2022-04-28T21:33:29.961211+0000 mgr.y (mgr.24961) 3388 : cluster [DBG] pgmap v3837: 137 pgs: 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:31.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:31 smithi059 bash[61152]: audit 2022-04-28T21:33:30.380022+0000 mon.a (mon.0) 3097 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:31.927 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:31 smithi059 bash[61152]: cluster 2022-04-28T21:33:30.380307+0000 mon.a (mon.0) 3098 : cluster [DBG] osdmap e1059: 8 total, 8 up, 8 in 2022-04-28T21:33:31.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:31 smithi059 bash[59745]: cluster 2022-04-28T21:33:29.961211+0000 mgr.y (mgr.24961) 3388 : cluster [DBG] pgmap v3837: 137 pgs: 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:31.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:31 smithi059 bash[59745]: audit 2022-04-28T21:33:30.380022+0000 mon.a (mon.0) 3097 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:31.928 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:31 smithi059 bash[59745]: cluster 2022-04-28T21:33:30.380307+0000 mon.a (mon.0) 3098 : cluster [DBG] osdmap e1059: 8 total, 8 up, 8 in 2022-04-28T21:33:31.929 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:31 smithi139 bash[43301]: cluster 2022-04-28T21:33:29.961211+0000 mgr.y (mgr.24961) 3388 : cluster [DBG] pgmap v3837: 137 pgs: 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:31.929 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:31 smithi139 bash[43301]: audit 2022-04-28T21:33:30.380022+0000 mon.a (mon.0) 3097 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-33", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:31.929 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:31 smithi139 bash[43301]: cluster 2022-04-28T21:33:30.380307+0000 mon.a (mon.0) 3098 : cluster [DBG] osdmap e1059: 8 total, 8 up, 8 in 2022-04-28T21:33:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:32.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:32.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:32 smithi059 bash[61152]: cluster 2022-04-28T21:33:31.393623+0000 mon.a (mon.0) 3099 : cluster [DBG] osdmap e1060: 8 total, 8 up, 8 in 2022-04-28T21:33:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:32 smithi059 bash[59745]: cluster 2022-04-28T21:33:31.393623+0000 mon.a (mon.0) 3099 : cluster [DBG] osdmap e1060: 8 total, 8 up, 8 in 2022-04-28T21:33:32.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:32 smithi139 bash[43301]: cluster 2022-04-28T21:33:31.393623+0000 mon.a (mon.0) 3099 : cluster [DBG] osdmap e1060: 8 total, 8 up, 8 in 2022-04-28T21:33:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[61152]: cluster 2022-04-28T21:33:31.961915+0000 mgr.y (mgr.24961) 3389 : cluster [DBG] pgmap v3840: 145 pgs: 3 creating+peering, 5 unknown, 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[61152]: cluster 2022-04-28T21:33:32.396972+0000 mon.a (mon.0) 3100 : cluster [DBG] osdmap e1061: 8 total, 8 up, 8 in 2022-04-28T21:33:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[61152]: audit 2022-04-28T21:33:32.400036+0000 mon.c (mon.1) 81 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[61152]: audit 2022-04-28T21:33:32.405416+0000 mon.a (mon.0) 3101 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[59745]: cluster 2022-04-28T21:33:31.961915+0000 mgr.y (mgr.24961) 3389 : cluster [DBG] pgmap v3840: 145 pgs: 3 creating+peering, 5 unknown, 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[59745]: cluster 2022-04-28T21:33:32.396972+0000 mon.a (mon.0) 3100 : cluster [DBG] osdmap e1061: 8 total, 8 up, 8 in 2022-04-28T21:33:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[59745]: audit 2022-04-28T21:33:32.400036+0000 mon.c (mon.1) 81 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[59745]: audit 2022-04-28T21:33:32.405416+0000 mon.a (mon.0) 3101 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:33 smithi139 bash[43301]: cluster 2022-04-28T21:33:31.961915+0000 mgr.y (mgr.24961) 3389 : cluster [DBG] pgmap v3840: 145 pgs: 3 creating+peering, 5 unknown, 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:33 smithi139 bash[43301]: cluster 2022-04-28T21:33:32.396972+0000 mon.a (mon.0) 3100 : cluster [DBG] osdmap e1061: 8 total, 8 up, 8 in 2022-04-28T21:33:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:33 smithi139 bash[43301]: audit 2022-04-28T21:33:32.400036+0000 mon.c (mon.1) 81 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:33.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:33 smithi139 bash[43301]: audit 2022-04-28T21:33:32.405416+0000 mon.a (mon.0) 3101 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:33.916Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:34.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[59745]: audit 2022-04-28T21:33:33.409441+0000 mon.a (mon.0) 3102 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:34.866 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[59745]: cluster 2022-04-28T21:33:33.409550+0000 mon.a (mon.0) 3103 : cluster [DBG] osdmap e1062: 8 total, 8 up, 8 in 2022-04-28T21:33:34.866 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[59745]: audit 2022-04-28T21:33:33.412915+0000 mon.c (mon.1) 82 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:34.867 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[59745]: audit 2022-04-28T21:33:33.417745+0000 mon.a (mon.0) 3104 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:34.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[61152]: audit 2022-04-28T21:33:33.409441+0000 mon.a (mon.0) 3102 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:34.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[61152]: cluster 2022-04-28T21:33:33.409550+0000 mon.a (mon.0) 3103 : cluster [DBG] osdmap e1062: 8 total, 8 up, 8 in 2022-04-28T21:33:34.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[61152]: audit 2022-04-28T21:33:33.412915+0000 mon.c (mon.1) 82 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:34.868 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:34 smithi059 bash[61152]: audit 2022-04-28T21:33:33.417745+0000 mon.a (mon.0) 3104 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:34.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:34 smithi139 bash[43301]: audit 2022-04-28T21:33:33.409441+0000 mon.a (mon.0) 3102 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:34 smithi139 bash[43301]: cluster 2022-04-28T21:33:33.409550+0000 mon.a (mon.0) 3103 : cluster [DBG] osdmap e1062: 8 total, 8 up, 8 in 2022-04-28T21:33:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:34 smithi139 bash[43301]: audit 2022-04-28T21:33:33.412915+0000 mon.c (mon.1) 82 : audit [INF] from='client.? 172.21.15.59:0/1255672288' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:34 smithi139 bash[43301]: audit 2022-04-28T21:33:33.417745+0000 mon.a (mon.0) 3104 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]: dispatch 2022-04-28T21:33:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[59745]: cluster 2022-04-28T21:33:33.962616+0000 mgr.y (mgr.24961) 3390 : cluster [DBG] pgmap v3843: 137 pgs: 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[59745]: audit 2022-04-28T21:33:34.429157+0000 mon.a (mon.0) 3105 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[59745]: cluster 2022-04-28T21:33:34.429300+0000 mon.a (mon.0) 3106 : cluster [DBG] osdmap e1063: 8 total, 8 up, 8 in 2022-04-28T21:33:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[59745]: audit 2022-04-28T21:33:34.441479+0000 mon.a (mon.0) 3107 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[59745]: audit 2022-04-28T21:33:34.442950+0000 mon.a (mon.0) 3108 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[59745]: audit 2022-04-28T21:33:34.443398+0000 mon.a (mon.0) 3109 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-34", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:35.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[61152]: cluster 2022-04-28T21:33:33.962616+0000 mgr.y (mgr.24961) 3390 : cluster [DBG] pgmap v3843: 137 pgs: 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:35.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[61152]: audit 2022-04-28T21:33:34.429157+0000 mon.a (mon.0) 3105 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:35.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[61152]: cluster 2022-04-28T21:33:34.429300+0000 mon.a (mon.0) 3106 : cluster [DBG] osdmap e1063: 8 total, 8 up, 8 in 2022-04-28T21:33:35.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[61152]: audit 2022-04-28T21:33:34.441479+0000 mon.a (mon.0) 3107 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:35.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[61152]: audit 2022-04-28T21:33:34.442950+0000 mon.a (mon.0) 3108 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:35.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:35 smithi059 bash[61152]: audit 2022-04-28T21:33:34.443398+0000 mon.a (mon.0) 3109 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-34", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:35 smithi139 bash[43301]: cluster 2022-04-28T21:33:33.962616+0000 mgr.y (mgr.24961) 3390 : cluster [DBG] pgmap v3843: 137 pgs: 137 active+clean; 24 KiB data, 744 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:35 smithi139 bash[43301]: audit 2022-04-28T21:33:34.429157+0000 mon.a (mon.0) 3105 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-33"}]': finished 2022-04-28T21:33:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:35 smithi139 bash[43301]: cluster 2022-04-28T21:33:34.429300+0000 mon.a (mon.0) 3106 : cluster [DBG] osdmap e1063: 8 total, 8 up, 8 in 2022-04-28T21:33:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:35 smithi139 bash[43301]: audit 2022-04-28T21:33:34.441479+0000 mon.a (mon.0) 3107 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:35 smithi139 bash[43301]: audit 2022-04-28T21:33:34.442950+0000 mon.a (mon.0) 3108 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:35 smithi139 bash[43301]: audit 2022-04-28T21:33:34.443398+0000 mon.a (mon.0) 3109 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-34", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:37.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:36 smithi059 bash[61152]: audit 2022-04-28T21:33:35.441939+0000 mon.a (mon.0) 3110 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-34", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:36 smithi059 bash[61152]: cluster 2022-04-28T21:33:35.442072+0000 mon.a (mon.0) 3111 : cluster [DBG] osdmap e1064: 8 total, 8 up, 8 in 2022-04-28T21:33:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:36 smithi059 bash[61152]: audit 2022-04-28T21:33:35.442744+0000 mon.a (mon.0) 3112 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-34", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:36 smithi059 bash[59745]: audit 2022-04-28T21:33:35.441939+0000 mon.a (mon.0) 3110 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-34", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:36 smithi059 bash[59745]: cluster 2022-04-28T21:33:35.442072+0000 mon.a (mon.0) 3111 : cluster [DBG] osdmap e1064: 8 total, 8 up, 8 in 2022-04-28T21:33:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:36 smithi059 bash[59745]: audit 2022-04-28T21:33:35.442744+0000 mon.a (mon.0) 3112 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-34", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:36 smithi139 bash[43301]: audit 2022-04-28T21:33:35.441939+0000 mon.a (mon.0) 3110 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-34", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:37.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:36 smithi139 bash[43301]: cluster 2022-04-28T21:33:35.442072+0000 mon.a (mon.0) 3111 : cluster [DBG] osdmap e1064: 8 total, 8 up, 8 in 2022-04-28T21:33:37.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:36 smithi139 bash[43301]: audit 2022-04-28T21:33:35.442744+0000 mon.a (mon.0) 3112 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-34", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[61152]: cluster 2022-04-28T21:33:35.963355+0000 mgr.y (mgr.24961) 3391 : cluster [DBG] pgmap v3846: 137 pgs: 137 active+clean; 24 KiB data, 745 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[61152]: cluster 2022-04-28T21:33:36.905694+0000 mon.a (mon.0) 3113 : cluster [DBG] osdmap e1065: 8 total, 8 up, 8 in 2022-04-28T21:33:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[61152]: audit 2022-04-28T21:33:37.453951+0000 mon.a (mon.0) 3114 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-34", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[61152]: cluster 2022-04-28T21:33:37.454152+0000 mon.a (mon.0) 3115 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2022-04-28T21:33:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[59745]: cluster 2022-04-28T21:33:35.963355+0000 mgr.y (mgr.24961) 3391 : cluster [DBG] pgmap v3846: 137 pgs: 137 active+clean; 24 KiB data, 745 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[59745]: cluster 2022-04-28T21:33:36.905694+0000 mon.a (mon.0) 3113 : cluster [DBG] osdmap e1065: 8 total, 8 up, 8 in 2022-04-28T21:33:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[59745]: audit 2022-04-28T21:33:37.453951+0000 mon.a (mon.0) 3114 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-34", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:37 smithi059 bash[59745]: cluster 2022-04-28T21:33:37.454152+0000 mon.a (mon.0) 3115 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2022-04-28T21:33:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:37 smithi139 bash[43301]: cluster 2022-04-28T21:33:35.963355+0000 mgr.y (mgr.24961) 3391 : cluster [DBG] pgmap v3846: 137 pgs: 137 active+clean; 24 KiB data, 745 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:37 smithi139 bash[43301]: cluster 2022-04-28T21:33:36.905694+0000 mon.a (mon.0) 3113 : cluster [DBG] osdmap e1065: 8 total, 8 up, 8 in 2022-04-28T21:33:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:37 smithi139 bash[43301]: audit 2022-04-28T21:33:37.453951+0000 mon.a (mon.0) 3114 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-34", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:37 smithi139 bash[43301]: cluster 2022-04-28T21:33:37.454152+0000 mon.a (mon.0) 3115 : cluster [DBG] osdmap e1066: 8 total, 8 up, 8 in 2022-04-28T21:33:39.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:39 smithi059 bash[61152]: cluster 2022-04-28T21:33:37.963980+0000 mgr.y (mgr.24961) 3392 : cluster [DBG] pgmap v3849: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 745 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:39 smithi059 bash[61152]: cluster 2022-04-28T21:33:38.454146+0000 mon.a (mon.0) 3116 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2022-04-28T21:33:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:39 smithi059 bash[59745]: cluster 2022-04-28T21:33:37.963980+0000 mgr.y (mgr.24961) 3392 : cluster [DBG] pgmap v3849: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 745 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:39 smithi059 bash[59745]: cluster 2022-04-28T21:33:38.454146+0000 mon.a (mon.0) 3116 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2022-04-28T21:33:39.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:39 smithi139 bash[43301]: cluster 2022-04-28T21:33:37.963980+0000 mgr.y (mgr.24961) 3392 : cluster [DBG] pgmap v3849: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 745 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:39 smithi139 bash[43301]: cluster 2022-04-28T21:33:38.454146+0000 mon.a (mon.0) 3116 : cluster [DBG] osdmap e1067: 8 total, 8 up, 8 in 2022-04-28T21:33:40.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:40 smithi059 bash[61152]: cluster 2022-04-28T21:33:39.455391+0000 mon.a (mon.0) 3117 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2022-04-28T21:33:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:40 smithi059 bash[61152]: audit 2022-04-28T21:33:39.457070+0000 mon.a (mon.0) 3118 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:40 smithi059 bash[59745]: cluster 2022-04-28T21:33:39.455391+0000 mon.a (mon.0) 3117 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2022-04-28T21:33:40.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:40 smithi059 bash[59745]: audit 2022-04-28T21:33:39.457070+0000 mon.a (mon.0) 3118 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:40.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:40 smithi139 bash[43301]: cluster 2022-04-28T21:33:39.455391+0000 mon.a (mon.0) 3117 : cluster [DBG] osdmap e1068: 8 total, 8 up, 8 in 2022-04-28T21:33:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:40 smithi139 bash[43301]: audit 2022-04-28T21:33:39.457070+0000 mon.a (mon.0) 3118 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:41.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[59745]: cluster 2022-04-28T21:33:39.964656+0000 mgr.y (mgr.24961) 3393 : cluster [DBG] pgmap v3852: 137 pgs: 137 active+clean; 24 KiB data, 746 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[59745]: audit 2022-04-28T21:33:40.458286+0000 mon.a (mon.0) 3119 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[59745]: cluster 2022-04-28T21:33:40.458488+0000 mon.a (mon.0) 3120 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2022-04-28T21:33:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[59745]: audit 2022-04-28T21:33:40.459653+0000 mon.a (mon.0) 3121 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[61152]: cluster 2022-04-28T21:33:39.964656+0000 mgr.y (mgr.24961) 3393 : cluster [DBG] pgmap v3852: 137 pgs: 137 active+clean; 24 KiB data, 746 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[61152]: audit 2022-04-28T21:33:40.458286+0000 mon.a (mon.0) 3119 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[61152]: cluster 2022-04-28T21:33:40.458488+0000 mon.a (mon.0) 3120 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2022-04-28T21:33:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:41 smithi059 bash[61152]: audit 2022-04-28T21:33:40.459653+0000 mon.a (mon.0) 3121 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:41 smithi139 bash[43301]: cluster 2022-04-28T21:33:39.964656+0000 mgr.y (mgr.24961) 3393 : cluster [DBG] pgmap v3852: 137 pgs: 137 active+clean; 24 KiB data, 746 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:41 smithi139 bash[43301]: audit 2022-04-28T21:33:40.458286+0000 mon.a (mon.0) 3119 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:41 smithi139 bash[43301]: cluster 2022-04-28T21:33:40.458488+0000 mon.a (mon.0) 3120 : cluster [DBG] osdmap e1069: 8 total, 8 up, 8 in 2022-04-28T21:33:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:41 smithi139 bash[43301]: audit 2022-04-28T21:33:40.459653+0000 mon.a (mon.0) 3121 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]: dispatch 2022-04-28T21:33:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:42.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.465302+0000 mon.a (mon.0) 3122 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: cluster 2022-04-28T21:33:41.465510+0000 mon.a (mon.0) 3123 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2022-04-28T21:33:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.481584+0000 mon.c (mon.1) 83 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.481998+0000 mon.a (mon.0) 3124 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.482751+0000 mon.c (mon.1) 84 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.483075+0000 mon.a (mon.0) 3125 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.483618+0000 mon.c (mon.1) 85 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[61152]: audit 2022-04-28T21:33:41.483963+0000 mon.a (mon.0) 3126 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.465302+0000 mon.a (mon.0) 3122 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: cluster 2022-04-28T21:33:41.465510+0000 mon.a (mon.0) 3123 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2022-04-28T21:33:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.481584+0000 mon.c (mon.1) 83 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.481998+0000 mon.a (mon.0) 3124 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.482751+0000 mon.c (mon.1) 84 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.483075+0000 mon.a (mon.0) 3125 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.483618+0000 mon.c (mon.1) 85 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:42.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:42 smithi059 bash[59745]: audit 2022-04-28T21:33:41.483963+0000 mon.a (mon.0) 3126 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.465302+0000 mon.a (mon.0) 3122 : audit [INF] from='client.? 172.21.15.59:0/346724999' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-34"}]': finished 2022-04-28T21:33:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: cluster 2022-04-28T21:33:41.465510+0000 mon.a (mon.0) 3123 : cluster [DBG] osdmap e1070: 8 total, 8 up, 8 in 2022-04-28T21:33:42.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.481584+0000 mon.c (mon.1) 83 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.481998+0000 mon.a (mon.0) 3124 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.482751+0000 mon.c (mon.1) 84 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.483075+0000 mon.a (mon.0) 3125 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:42.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.483618+0000 mon.c (mon.1) 85 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:42.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:42 smithi139 bash[43301]: audit 2022-04-28T21:33:41.483963+0000 mon.a (mon.0) 3126 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:43.917Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[61152]: cluster 2022-04-28T21:33:41.965402+0000 mgr.y (mgr.24961) 3394 : cluster [DBG] pgmap v3855: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[61152]: audit 2022-04-28T21:33:42.487941+0000 mon.a (mon.0) 3127 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[61152]: cluster 2022-04-28T21:33:42.488017+0000 mon.a (mon.0) 3128 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2022-04-28T21:33:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[61152]: audit 2022-04-28T21:33:42.494463+0000 mon.c (mon.1) 86 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[61152]: audit 2022-04-28T21:33:42.494995+0000 mon.a (mon.0) 3129 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[59745]: cluster 2022-04-28T21:33:41.965402+0000 mgr.y (mgr.24961) 3394 : cluster [DBG] pgmap v3855: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[59745]: audit 2022-04-28T21:33:42.487941+0000 mon.a (mon.0) 3127 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[59745]: cluster 2022-04-28T21:33:42.488017+0000 mon.a (mon.0) 3128 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2022-04-28T21:33:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[59745]: audit 2022-04-28T21:33:42.494463+0000 mon.c (mon.1) 86 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:43 smithi059 bash[59745]: audit 2022-04-28T21:33:42.494995+0000 mon.a (mon.0) 3129 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:44.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:43 smithi139 bash[43301]: cluster 2022-04-28T21:33:41.965402+0000 mgr.y (mgr.24961) 3394 : cluster [DBG] pgmap v3855: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:43 smithi139 bash[43301]: audit 2022-04-28T21:33:42.487941+0000 mon.a (mon.0) 3127 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-35", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:43 smithi139 bash[43301]: cluster 2022-04-28T21:33:42.488017+0000 mon.a (mon.0) 3128 : cluster [DBG] osdmap e1071: 8 total, 8 up, 8 in 2022-04-28T21:33:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:43 smithi139 bash[43301]: audit 2022-04-28T21:33:42.494463+0000 mon.c (mon.1) 86 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:43 smithi139 bash[43301]: audit 2022-04-28T21:33:42.494995+0000 mon.a (mon.0) 3129 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:45.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[59745]: cluster 2022-04-28T21:33:43.961058+0000 mon.a (mon.0) 3130 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2022-04-28T21:33:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[59745]: cluster 2022-04-28T21:33:43.965900+0000 mgr.y (mgr.24961) 3395 : cluster [DBG] pgmap v3858: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[59745]: audit 2022-04-28T21:33:44.499786+0000 mon.a (mon.0) 3131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:45.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[59745]: cluster 2022-04-28T21:33:44.499894+0000 mon.a (mon.0) 3132 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2022-04-28T21:33:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[61152]: cluster 2022-04-28T21:33:43.961058+0000 mon.a (mon.0) 3130 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2022-04-28T21:33:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[61152]: cluster 2022-04-28T21:33:43.965900+0000 mgr.y (mgr.24961) 3395 : cluster [DBG] pgmap v3858: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[61152]: audit 2022-04-28T21:33:44.499786+0000 mon.a (mon.0) 3131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:44 smithi059 bash[61152]: cluster 2022-04-28T21:33:44.499894+0000 mon.a (mon.0) 3132 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2022-04-28T21:33:45.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:44 smithi139 bash[43301]: cluster 2022-04-28T21:33:43.961058+0000 mon.a (mon.0) 3130 : cluster [DBG] osdmap e1072: 8 total, 8 up, 8 in 2022-04-28T21:33:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:44 smithi139 bash[43301]: cluster 2022-04-28T21:33:43.965900+0000 mgr.y (mgr.24961) 3395 : cluster [DBG] pgmap v3858: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:44 smithi139 bash[43301]: audit 2022-04-28T21:33:44.499786+0000 mon.a (mon.0) 3131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-35", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:44 smithi139 bash[43301]: cluster 2022-04-28T21:33:44.499894+0000 mon.a (mon.0) 3132 : cluster [DBG] osdmap e1073: 8 total, 8 up, 8 in 2022-04-28T21:33:46.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:46 smithi059 bash[61152]: cluster 2022-04-28T21:33:45.500494+0000 mon.a (mon.0) 3133 : cluster [DBG] osdmap e1074: 8 total, 8 up, 8 in 2022-04-28T21:33:46.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:46 smithi059 bash[59745]: cluster 2022-04-28T21:33:45.500494+0000 mon.a (mon.0) 3133 : cluster [DBG] osdmap e1074: 8 total, 8 up, 8 in 2022-04-28T21:33:46.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:46 smithi139 bash[43301]: cluster 2022-04-28T21:33:45.500494+0000 mon.a (mon.0) 3133 : cluster [DBG] osdmap e1074: 8 total, 8 up, 8 in 2022-04-28T21:33:47.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[59745]: cluster 2022-04-28T21:33:45.966632+0000 mgr.y (mgr.24961) 3396 : cluster [DBG] pgmap v3861: 145 pgs: 2 creating+peering, 6 unknown, 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:47.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[59745]: cluster 2022-04-28T21:33:46.510298+0000 mon.a (mon.0) 3134 : cluster [DBG] osdmap e1075: 8 total, 8 up, 8 in 2022-04-28T21:33:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[59745]: audit 2022-04-28T21:33:46.513096+0000 mon.c (mon.1) 87 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[59745]: audit 2022-04-28T21:33:46.514966+0000 mon.a (mon.0) 3135 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[61152]: cluster 2022-04-28T21:33:45.966632+0000 mgr.y (mgr.24961) 3396 : cluster [DBG] pgmap v3861: 145 pgs: 2 creating+peering, 6 unknown, 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[61152]: cluster 2022-04-28T21:33:46.510298+0000 mon.a (mon.0) 3134 : cluster [DBG] osdmap e1075: 8 total, 8 up, 8 in 2022-04-28T21:33:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[61152]: audit 2022-04-28T21:33:46.513096+0000 mon.c (mon.1) 87 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:47 smithi059 bash[61152]: audit 2022-04-28T21:33:46.514966+0000 mon.a (mon.0) 3135 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:47 smithi139 bash[43301]: cluster 2022-04-28T21:33:45.966632+0000 mgr.y (mgr.24961) 3396 : cluster [DBG] pgmap v3861: 145 pgs: 2 creating+peering, 6 unknown, 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:47 smithi139 bash[43301]: cluster 2022-04-28T21:33:46.510298+0000 mon.a (mon.0) 3134 : cluster [DBG] osdmap e1075: 8 total, 8 up, 8 in 2022-04-28T21:33:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:47 smithi139 bash[43301]: audit 2022-04-28T21:33:46.513096+0000 mon.c (mon.1) 87 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:47 smithi139 bash[43301]: audit 2022-04-28T21:33:46.514966+0000 mon.a (mon.0) 3135 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.513266+0000 mon.a (mon.0) 3136 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: cluster 2022-04-28T21:33:47.513519+0000 mon.a (mon.0) 3137 : cluster [DBG] osdmap e1076: 8 total, 8 up, 8 in 2022-04-28T21:33:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.516572+0000 mon.c (mon.1) 88 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.522530+0000 mon.a (mon.0) 3138 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.555527+0000 mon.a (mon.0) 3139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:33:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.557917+0000 mon.a (mon.0) 3140 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:33:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.559571+0000 mon.a (mon.0) 3141 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:33:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[59745]: audit 2022-04-28T21:33:47.854188+0000 mon.a (mon.0) 3142 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:33:48.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.513266+0000 mon.a (mon.0) 3136 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:48.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: cluster 2022-04-28T21:33:47.513519+0000 mon.a (mon.0) 3137 : cluster [DBG] osdmap e1076: 8 total, 8 up, 8 in 2022-04-28T21:33:48.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.516572+0000 mon.c (mon.1) 88 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.522530+0000 mon.a (mon.0) 3138 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.555527+0000 mon.a (mon.0) 3139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:33:48.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.557917+0000 mon.a (mon.0) 3140 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:33:48.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.559571+0000 mon.a (mon.0) 3141 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:33:48.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:48 smithi059 bash[61152]: audit 2022-04-28T21:33:47.854188+0000 mon.a (mon.0) 3142 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:33:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.513266+0000 mon.a (mon.0) 3136 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: cluster 2022-04-28T21:33:47.513519+0000 mon.a (mon.0) 3137 : cluster [DBG] osdmap e1076: 8 total, 8 up, 8 in 2022-04-28T21:33:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.516572+0000 mon.c (mon.1) 88 : audit [INF] from='client.? 172.21.15.59:0/2625791764' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.522530+0000 mon.a (mon.0) 3138 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]: dispatch 2022-04-28T21:33:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.555527+0000 mon.a (mon.0) 3139 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:33:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.557917+0000 mon.a (mon.0) 3140 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:33:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.559571+0000 mon.a (mon.0) 3141 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:33:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:48 smithi139 bash[43301]: audit 2022-04-28T21:33:47.854188+0000 mon.a (mon.0) 3142 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:33:49.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: cephadm 2022-04-28T21:33:47.560915+0000 mgr.y (mgr.24961) 3397 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:33:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: cluster 2022-04-28T21:33:47.967325+0000 mgr.y (mgr.24961) 3398 : cluster [DBG] pgmap v3864: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: audit 2022-04-28T21:33:48.538968+0000 mon.a (mon.0) 3143 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: cluster 2022-04-28T21:33:48.539124+0000 mon.a (mon.0) 3144 : cluster [DBG] osdmap e1077: 8 total, 8 up, 8 in 2022-04-28T21:33:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: audit 2022-04-28T21:33:48.552092+0000 mon.a (mon.0) 3145 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:49.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: audit 2022-04-28T21:33:48.553295+0000 mon.a (mon.0) 3146 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:49.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[61152]: audit 2022-04-28T21:33:48.553732+0000 mon.a (mon.0) 3147 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: cephadm 2022-04-28T21:33:47.560915+0000 mgr.y (mgr.24961) 3397 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:33:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: cluster 2022-04-28T21:33:47.967325+0000 mgr.y (mgr.24961) 3398 : cluster [DBG] pgmap v3864: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: audit 2022-04-28T21:33:48.538968+0000 mon.a (mon.0) 3143 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: cluster 2022-04-28T21:33:48.539124+0000 mon.a (mon.0) 3144 : cluster [DBG] osdmap e1077: 8 total, 8 up, 8 in 2022-04-28T21:33:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: audit 2022-04-28T21:33:48.552092+0000 mon.a (mon.0) 3145 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: audit 2022-04-28T21:33:48.553295+0000 mon.a (mon.0) 3146 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:49 smithi059 bash[59745]: audit 2022-04-28T21:33:48.553732+0000 mon.a (mon.0) 3147 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: cephadm 2022-04-28T21:33:47.560915+0000 mgr.y (mgr.24961) 3397 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:33:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: cluster 2022-04-28T21:33:47.967325+0000 mgr.y (mgr.24961) 3398 : cluster [DBG] pgmap v3864: 137 pgs: 137 active+clean; 24 KiB data, 747 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: audit 2022-04-28T21:33:48.538968+0000 mon.a (mon.0) 3143 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-35"}]': finished 2022-04-28T21:33:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: cluster 2022-04-28T21:33:48.539124+0000 mon.a (mon.0) 3144 : cluster [DBG] osdmap e1077: 8 total, 8 up, 8 in 2022-04-28T21:33:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: audit 2022-04-28T21:33:48.552092+0000 mon.a (mon.0) 3145 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: audit 2022-04-28T21:33:48.553295+0000 mon.a (mon.0) 3146 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:49 smithi139 bash[43301]: audit 2022-04-28T21:33:48.553732+0000 mon.a (mon.0) 3147 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:51.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:51 smithi059 bash[61152]: audit 2022-04-28T21:33:49.540515+0000 mon.a (mon.0) 3148 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:51 smithi059 bash[61152]: cluster 2022-04-28T21:33:49.540668+0000 mon.a (mon.0) 3149 : cluster [DBG] osdmap e1078: 8 total, 8 up, 8 in 2022-04-28T21:33:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:51 smithi059 bash[61152]: audit 2022-04-28T21:33:49.541537+0000 mon.a (mon.0) 3150 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:51 smithi059 bash[59745]: audit 2022-04-28T21:33:49.540515+0000 mon.a (mon.0) 3148 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:51 smithi059 bash[59745]: cluster 2022-04-28T21:33:49.540668+0000 mon.a (mon.0) 3149 : cluster [DBG] osdmap e1078: 8 total, 8 up, 8 in 2022-04-28T21:33:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:51 smithi059 bash[59745]: audit 2022-04-28T21:33:49.541537+0000 mon.a (mon.0) 3150 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:51.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:51 smithi139 bash[43301]: audit 2022-04-28T21:33:49.540515+0000 mon.a (mon.0) 3148 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:51 smithi139 bash[43301]: cluster 2022-04-28T21:33:49.540668+0000 mon.a (mon.0) 3149 : cluster [DBG] osdmap e1078: 8 total, 8 up, 8 in 2022-04-28T21:33:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:51 smithi139 bash[43301]: audit 2022-04-28T21:33:49.541537+0000 mon.a (mon.0) 3150 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:52.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[61152]: cluster 2022-04-28T21:33:49.968056+0000 mgr.y (mgr.24961) 3399 : cluster [DBG] pgmap v3867: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[61152]: cluster 2022-04-28T21:33:51.000687+0000 mon.a (mon.0) 3151 : cluster [DBG] osdmap e1079: 8 total, 8 up, 8 in 2022-04-28T21:33:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[61152]: audit 2022-04-28T21:33:51.546119+0000 mon.a (mon.0) 3152 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[61152]: cluster 2022-04-28T21:33:51.546201+0000 mon.a (mon.0) 3153 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2022-04-28T21:33:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[59745]: cluster 2022-04-28T21:33:49.968056+0000 mgr.y (mgr.24961) 3399 : cluster [DBG] pgmap v3867: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[59745]: cluster 2022-04-28T21:33:51.000687+0000 mon.a (mon.0) 3151 : cluster [DBG] osdmap e1079: 8 total, 8 up, 8 in 2022-04-28T21:33:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[59745]: audit 2022-04-28T21:33:51.546119+0000 mon.a (mon.0) 3152 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:52 smithi059 bash[59745]: cluster 2022-04-28T21:33:51.546201+0000 mon.a (mon.0) 3153 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2022-04-28T21:33:52.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:52 smithi139 bash[43301]: cluster 2022-04-28T21:33:49.968056+0000 mgr.y (mgr.24961) 3399 : cluster [DBG] pgmap v3867: 137 pgs: 137 active+clean; 24 KiB data, 748 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:52 smithi139 bash[43301]: cluster 2022-04-28T21:33:51.000687+0000 mon.a (mon.0) 3151 : cluster [DBG] osdmap e1079: 8 total, 8 up, 8 in 2022-04-28T21:33:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:52 smithi139 bash[43301]: audit 2022-04-28T21:33:51.546119+0000 mon.a (mon.0) 3152 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:52 smithi139 bash[43301]: cluster 2022-04-28T21:33:51.546201+0000 mon.a (mon.0) 3153 : cluster [DBG] osdmap e1080: 8 total, 8 up, 8 in 2022-04-28T21:33:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:53 smithi059 bash[61152]: cluster 2022-04-28T21:33:51.968812+0000 mgr.y (mgr.24961) 3400 : cluster [DBG] pgmap v3870: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:53 smithi059 bash[61152]: cluster 2022-04-28T21:33:52.555917+0000 mon.a (mon.0) 3154 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2022-04-28T21:33:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:53 smithi059 bash[59745]: cluster 2022-04-28T21:33:51.968812+0000 mgr.y (mgr.24961) 3400 : cluster [DBG] pgmap v3870: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:53 smithi059 bash[59745]: cluster 2022-04-28T21:33:52.555917+0000 mon.a (mon.0) 3154 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2022-04-28T21:33:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:53 smithi139 bash[43301]: cluster 2022-04-28T21:33:51.968812+0000 mgr.y (mgr.24961) 3400 : cluster [DBG] pgmap v3870: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:53.889 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:53 smithi139 bash[43301]: cluster 2022-04-28T21:33:52.555917+0000 mon.a (mon.0) 3154 : cluster [DBG] osdmap e1081: 8 total, 8 up, 8 in 2022-04-28T21:33:54.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:33:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:33:53.918Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:33:54.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:54 smithi059 bash[61152]: cluster 2022-04-28T21:33:53.558425+0000 mon.a (mon.0) 3155 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2022-04-28T21:33:54.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:54 smithi059 bash[61152]: audit 2022-04-28T21:33:53.560310+0000 mon.a (mon.0) 3156 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:54 smithi059 bash[61152]: cluster 2022-04-28T21:33:53.969514+0000 mgr.y (mgr.24961) 3401 : cluster [DBG] pgmap v3873: 137 pgs: 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:54 smithi059 bash[59745]: cluster 2022-04-28T21:33:53.558425+0000 mon.a (mon.0) 3155 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2022-04-28T21:33:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:54 smithi059 bash[59745]: audit 2022-04-28T21:33:53.560310+0000 mon.a (mon.0) 3156 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:54 smithi059 bash[59745]: cluster 2022-04-28T21:33:53.969514+0000 mgr.y (mgr.24961) 3401 : cluster [DBG] pgmap v3873: 137 pgs: 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:54 smithi139 bash[43301]: cluster 2022-04-28T21:33:53.558425+0000 mon.a (mon.0) 3155 : cluster [DBG] osdmap e1082: 8 total, 8 up, 8 in 2022-04-28T21:33:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:54 smithi139 bash[43301]: audit 2022-04-28T21:33:53.560310+0000 mon.a (mon.0) 3156 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:54 smithi139 bash[43301]: cluster 2022-04-28T21:33:53.969514+0000 mgr.y (mgr.24961) 3401 : cluster [DBG] pgmap v3873: 137 pgs: 137 active+clean; 24 KiB data, 749 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:55.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[59745]: audit 2022-04-28T21:33:54.562846+0000 mon.a (mon.0) 3157 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[59745]: cluster 2022-04-28T21:33:54.563406+0000 mon.a (mon.0) 3158 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2022-04-28T21:33:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[59745]: audit 2022-04-28T21:33:54.564885+0000 mon.a (mon.0) 3159 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[59745]: audit 2022-04-28T21:33:55.564227+0000 mon.a (mon.0) 3160 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[59745]: cluster 2022-04-28T21:33:55.564390+0000 mon.a (mon.0) 3161 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2022-04-28T21:33:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[61152]: audit 2022-04-28T21:33:54.562846+0000 mon.a (mon.0) 3157 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[61152]: cluster 2022-04-28T21:33:54.563406+0000 mon.a (mon.0) 3158 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2022-04-28T21:33:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[61152]: audit 2022-04-28T21:33:54.564885+0000 mon.a (mon.0) 3159 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[61152]: audit 2022-04-28T21:33:55.564227+0000 mon.a (mon.0) 3160 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:55 smithi059 bash[61152]: cluster 2022-04-28T21:33:55.564390+0000 mon.a (mon.0) 3161 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2022-04-28T21:33:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:55 smithi139 bash[43301]: audit 2022-04-28T21:33:54.562846+0000 mon.a (mon.0) 3157 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:55 smithi139 bash[43301]: cluster 2022-04-28T21:33:54.563406+0000 mon.a (mon.0) 3158 : cluster [DBG] osdmap e1083: 8 total, 8 up, 8 in 2022-04-28T21:33:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:55 smithi139 bash[43301]: audit 2022-04-28T21:33:54.564885+0000 mon.a (mon.0) 3159 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]: dispatch 2022-04-28T21:33:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:55 smithi139 bash[43301]: audit 2022-04-28T21:33:55.564227+0000 mon.a (mon.0) 3160 : audit [INF] from='client.? 172.21.15.59:0/2248442083' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-36"}]': finished 2022-04-28T21:33:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:55 smithi139 bash[43301]: cluster 2022-04-28T21:33:55.564390+0000 mon.a (mon.0) 3161 : cluster [DBG] osdmap e1084: 8 total, 8 up, 8 in 2022-04-28T21:33:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: audit 2022-04-28T21:33:55.582850+0000 mon.c (mon.1) 89 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: audit 2022-04-28T21:33:55.583299+0000 mon.a (mon.0) 3162 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: audit 2022-04-28T21:33:55.584038+0000 mon.c (mon.1) 90 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: audit 2022-04-28T21:33:55.584483+0000 mon.a (mon.0) 3163 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: audit 2022-04-28T21:33:55.585164+0000 mon.c (mon.1) 91 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:56.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: audit 2022-04-28T21:33:55.585658+0000 mon.a (mon.0) 3164 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:56.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:56 smithi139 bash[43301]: cluster 2022-04-28T21:33:55.970316+0000 mgr.y (mgr.24961) 3402 : cluster [DBG] pgmap v3876: 137 pgs: 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:57.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: audit 2022-04-28T21:33:55.582850+0000 mon.c (mon.1) 89 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: audit 2022-04-28T21:33:55.583299+0000 mon.a (mon.0) 3162 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: audit 2022-04-28T21:33:55.584038+0000 mon.c (mon.1) 90 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: audit 2022-04-28T21:33:55.584483+0000 mon.a (mon.0) 3163 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: audit 2022-04-28T21:33:55.585164+0000 mon.c (mon.1) 91 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: audit 2022-04-28T21:33:55.585658+0000 mon.a (mon.0) 3164 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[59745]: cluster 2022-04-28T21:33:55.970316+0000 mgr.y (mgr.24961) 3402 : cluster [DBG] pgmap v3876: 137 pgs: 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: audit 2022-04-28T21:33:55.582850+0000 mon.c (mon.1) 89 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: audit 2022-04-28T21:33:55.583299+0000 mon.a (mon.0) 3162 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: audit 2022-04-28T21:33:55.584038+0000 mon.c (mon.1) 90 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: audit 2022-04-28T21:33:55.584483+0000 mon.a (mon.0) 3163 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:57.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: audit 2022-04-28T21:33:55.585164+0000 mon.c (mon.1) 91 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:57.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: audit 2022-04-28T21:33:55.585658+0000 mon.a (mon.0) 3164 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:33:57.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:56 smithi059 bash[61152]: cluster 2022-04-28T21:33:55.970316+0000 mgr.y (mgr.24961) 3402 : cluster [DBG] pgmap v3876: 137 pgs: 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:58.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[61152]: audit 2022-04-28T21:33:56.587587+0000 mon.a (mon.0) 3165 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[61152]: cluster 2022-04-28T21:33:56.587747+0000 mon.a (mon.0) 3166 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2022-04-28T21:33:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[61152]: audit 2022-04-28T21:33:56.591875+0000 mon.c (mon.1) 92 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[61152]: audit 2022-04-28T21:33:56.596221+0000 mon.a (mon.0) 3167 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[59745]: audit 2022-04-28T21:33:56.587587+0000 mon.a (mon.0) 3165 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[59745]: cluster 2022-04-28T21:33:56.587747+0000 mon.a (mon.0) 3166 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2022-04-28T21:33:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[59745]: audit 2022-04-28T21:33:56.591875+0000 mon.c (mon.1) 92 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:58.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:58 smithi059 bash[59745]: audit 2022-04-28T21:33:56.596221+0000 mon.a (mon.0) 3167 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:58.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:58 smithi139 bash[43301]: audit 2022-04-28T21:33:56.587587+0000 mon.a (mon.0) 3165 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-37", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:33:58.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:58 smithi139 bash[43301]: cluster 2022-04-28T21:33:56.587747+0000 mon.a (mon.0) 3166 : cluster [DBG] osdmap e1085: 8 total, 8 up, 8 in 2022-04-28T21:33:58.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:58 smithi139 bash[43301]: audit 2022-04-28T21:33:56.591875+0000 mon.c (mon.1) 92 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:58.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:58 smithi139 bash[43301]: audit 2022-04-28T21:33:56.596221+0000 mon.a (mon.0) 3167 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:33:59.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[61152]: cluster 2022-04-28T21:33:57.971018+0000 mgr.y (mgr.24961) 3403 : cluster [DBG] pgmap v3879: 137 pgs: 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[61152]: cluster 2022-04-28T21:33:58.059734+0000 mon.a (mon.0) 3168 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2022-04-28T21:33:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[61152]: audit 2022-04-28T21:33:58.611255+0000 mon.a (mon.0) 3169 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:33:59.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[61152]: cluster 2022-04-28T21:33:58.611555+0000 mon.a (mon.0) 3170 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2022-04-28T21:33:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[59745]: cluster 2022-04-28T21:33:57.971018+0000 mgr.y (mgr.24961) 3403 : cluster [DBG] pgmap v3879: 137 pgs: 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[59745]: cluster 2022-04-28T21:33:58.059734+0000 mon.a (mon.0) 3168 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2022-04-28T21:33:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[59745]: audit 2022-04-28T21:33:58.611255+0000 mon.a (mon.0) 3169 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:33:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:33:59 smithi059 bash[59745]: cluster 2022-04-28T21:33:58.611555+0000 mon.a (mon.0) 3170 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2022-04-28T21:33:59.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:59 smithi139 bash[43301]: cluster 2022-04-28T21:33:57.971018+0000 mgr.y (mgr.24961) 3403 : cluster [DBG] pgmap v3879: 137 pgs: 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:33:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:59 smithi139 bash[43301]: cluster 2022-04-28T21:33:58.059734+0000 mon.a (mon.0) 3168 : cluster [DBG] osdmap e1086: 8 total, 8 up, 8 in 2022-04-28T21:33:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:59 smithi139 bash[43301]: audit 2022-04-28T21:33:58.611255+0000 mon.a (mon.0) 3169 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-37", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:33:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:33:59 smithi139 bash[43301]: cluster 2022-04-28T21:33:58.611555+0000 mon.a (mon.0) 3170 : cluster [DBG] osdmap e1087: 8 total, 8 up, 8 in 2022-04-28T21:34:01.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:00 smithi059 bash[61152]: cluster 2022-04-28T21:33:59.613979+0000 mon.a (mon.0) 3171 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2022-04-28T21:34:01.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:00 smithi059 bash[61152]: cluster 2022-04-28T21:33:59.971804+0000 mgr.y (mgr.24961) 3404 : cluster [DBG] pgmap v3882: 145 pgs: 3 creating+peering, 5 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:00 smithi059 bash[59745]: cluster 2022-04-28T21:33:59.613979+0000 mon.a (mon.0) 3171 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2022-04-28T21:34:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:00 smithi059 bash[59745]: cluster 2022-04-28T21:33:59.971804+0000 mgr.y (mgr.24961) 3404 : cluster [DBG] pgmap v3882: 145 pgs: 3 creating+peering, 5 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:01.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:00 smithi139 bash[43301]: cluster 2022-04-28T21:33:59.613979+0000 mon.a (mon.0) 3171 : cluster [DBG] osdmap e1088: 8 total, 8 up, 8 in 2022-04-28T21:34:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:00 smithi139 bash[43301]: cluster 2022-04-28T21:33:59.971804+0000 mgr.y (mgr.24961) 3404 : cluster [DBG] pgmap v3882: 145 pgs: 3 creating+peering, 5 unknown, 137 active+clean; 24 KiB data, 750 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:02.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:01 smithi059 bash[61152]: cluster 2022-04-28T21:34:00.617553+0000 mon.a (mon.0) 3172 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2022-04-28T21:34:02.026 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:01 smithi059 bash[61152]: audit 2022-04-28T21:34:00.619747+0000 mon.c (mon.1) 93 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:02.027 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:01 smithi059 bash[61152]: audit 2022-04-28T21:34:00.631158+0000 mon.a (mon.0) 3173 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:02.027 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:02.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:02.027 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:01 smithi059 bash[59745]: cluster 2022-04-28T21:34:00.617553+0000 mon.a (mon.0) 3172 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2022-04-28T21:34:02.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:01 smithi059 bash[59745]: audit 2022-04-28T21:34:00.619747+0000 mon.c (mon.1) 93 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:02.028 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:01 smithi059 bash[59745]: audit 2022-04-28T21:34:00.631158+0000 mon.a (mon.0) 3173 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:01 smithi139 bash[43301]: cluster 2022-04-28T21:34:00.617553+0000 mon.a (mon.0) 3172 : cluster [DBG] osdmap e1089: 8 total, 8 up, 8 in 2022-04-28T21:34:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:01 smithi139 bash[43301]: audit 2022-04-28T21:34:00.619747+0000 mon.c (mon.1) 93 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:01 smithi139 bash[43301]: audit 2022-04-28T21:34:00.631158+0000 mon.a (mon.0) 3173 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[59745]: audit 2022-04-28T21:34:01.640667+0000 mon.a (mon.0) 3174 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:34:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[59745]: cluster 2022-04-28T21:34:01.640777+0000 mon.a (mon.0) 3175 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2022-04-28T21:34:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[59745]: audit 2022-04-28T21:34:01.643195+0000 mon.c (mon.1) 94 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[59745]: audit 2022-04-28T21:34:01.643594+0000 mon.a (mon.0) 3176 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[59745]: cluster 2022-04-28T21:34:01.972620+0000 mgr.y (mgr.24961) 3405 : cluster [DBG] pgmap v3885: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:03.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[61152]: audit 2022-04-28T21:34:01.640667+0000 mon.a (mon.0) 3174 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:34:03.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[61152]: cluster 2022-04-28T21:34:01.640777+0000 mon.a (mon.0) 3175 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2022-04-28T21:34:03.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[61152]: audit 2022-04-28T21:34:01.643195+0000 mon.c (mon.1) 94 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[61152]: audit 2022-04-28T21:34:01.643594+0000 mon.a (mon.0) 3176 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:02 smithi059 bash[61152]: cluster 2022-04-28T21:34:01.972620+0000 mgr.y (mgr.24961) 3405 : cluster [DBG] pgmap v3885: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:03.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:02 smithi139 bash[43301]: audit 2022-04-28T21:34:01.640667+0000 mon.a (mon.0) 3174 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:34:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:02 smithi139 bash[43301]: cluster 2022-04-28T21:34:01.640777+0000 mon.a (mon.0) 3175 : cluster [DBG] osdmap e1090: 8 total, 8 up, 8 in 2022-04-28T21:34:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:02 smithi139 bash[43301]: audit 2022-04-28T21:34:01.643195+0000 mon.c (mon.1) 94 : audit [INF] from='client.? 172.21.15.59:0/1269088836' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:02 smithi139 bash[43301]: audit 2022-04-28T21:34:01.643594+0000 mon.a (mon.0) 3176 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]: dispatch 2022-04-28T21:34:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:02 smithi139 bash[43301]: cluster 2022-04-28T21:34:01.972620+0000 mgr.y (mgr.24961) 3405 : cluster [DBG] pgmap v3885: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:03.919 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[61152]: audit 2022-04-28T21:34:02.645455+0000 mon.a (mon.0) 3177 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:34:03.920 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[61152]: cluster 2022-04-28T21:34:02.645617+0000 mon.a (mon.0) 3178 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2022-04-28T21:34:03.920 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[61152]: audit 2022-04-28T21:34:02.657414+0000 mon.a (mon.0) 3179 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:03.920 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[61152]: audit 2022-04-28T21:34:02.658703+0000 mon.a (mon.0) 3180 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:03.920 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[61152]: audit 2022-04-28T21:34:02.659127+0000 mon.a (mon.0) 3181 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-38", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:03.920 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[59745]: audit 2022-04-28T21:34:02.645455+0000 mon.a (mon.0) 3177 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:34:03.921 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[59745]: cluster 2022-04-28T21:34:02.645617+0000 mon.a (mon.0) 3178 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2022-04-28T21:34:03.921 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[59745]: audit 2022-04-28T21:34:02.657414+0000 mon.a (mon.0) 3179 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:03.921 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[59745]: audit 2022-04-28T21:34:02.658703+0000 mon.a (mon.0) 3180 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:03.921 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[59745]: audit 2022-04-28T21:34:02.659127+0000 mon.a (mon.0) 3181 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-38", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:04.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:03 smithi139 bash[43301]: audit 2022-04-28T21:34:02.645455+0000 mon.a (mon.0) 3177 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-37"}]': finished 2022-04-28T21:34:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:03 smithi139 bash[43301]: cluster 2022-04-28T21:34:02.645617+0000 mon.a (mon.0) 3178 : cluster [DBG] osdmap e1091: 8 total, 8 up, 8 in 2022-04-28T21:34:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:03 smithi139 bash[43301]: audit 2022-04-28T21:34:02.657414+0000 mon.a (mon.0) 3179 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:03 smithi139 bash[43301]: audit 2022-04-28T21:34:02.658703+0000 mon.a (mon.0) 3180 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:03 smithi139 bash[43301]: audit 2022-04-28T21:34:02.659127+0000 mon.a (mon.0) 3181 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-38", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:03.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:05.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[61152]: audit 2022-04-28T21:34:03.659963+0000 mon.a (mon.0) 3182 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-38", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:05.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[61152]: cluster 2022-04-28T21:34:03.660187+0000 mon.a (mon.0) 3183 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2022-04-28T21:34:05.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[61152]: audit 2022-04-28T21:34:03.661257+0000 mon.a (mon.0) 3184 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-38", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:05.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[61152]: cluster 2022-04-28T21:34:03.973426+0000 mgr.y (mgr.24961) 3406 : cluster [DBG] pgmap v3888: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[59745]: audit 2022-04-28T21:34:03.659963+0000 mon.a (mon.0) 3182 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-38", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[59745]: cluster 2022-04-28T21:34:03.660187+0000 mon.a (mon.0) 3183 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2022-04-28T21:34:05.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[59745]: audit 2022-04-28T21:34:03.661257+0000 mon.a (mon.0) 3184 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-38", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:05.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:05 smithi059 bash[59745]: cluster 2022-04-28T21:34:03.973426+0000 mgr.y (mgr.24961) 3406 : cluster [DBG] pgmap v3888: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:05 smithi139 bash[43301]: audit 2022-04-28T21:34:03.659963+0000 mon.a (mon.0) 3182 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-38", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:05 smithi139 bash[43301]: cluster 2022-04-28T21:34:03.660187+0000 mon.a (mon.0) 3183 : cluster [DBG] osdmap e1092: 8 total, 8 up, 8 in 2022-04-28T21:34:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:05 smithi139 bash[43301]: audit 2022-04-28T21:34:03.661257+0000 mon.a (mon.0) 3184 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-38", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:05 smithi139 bash[43301]: cluster 2022-04-28T21:34:03.973426+0000 mgr.y (mgr.24961) 3406 : cluster [DBG] pgmap v3888: 137 pgs: 137 active+clean; 24 KiB data, 751 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:06.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:06 smithi059 bash[61152]: cluster 2022-04-28T21:34:05.131025+0000 mon.a (mon.0) 3185 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2022-04-28T21:34:06.139 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:06 smithi059 bash[59745]: cluster 2022-04-28T21:34:05.131025+0000 mon.a (mon.0) 3185 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2022-04-28T21:34:06.140 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:06 smithi059 bash[59745]: audit 2022-04-28T21:34:05.664716+0000 mon.a (mon.0) 3186 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-38", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:06.140 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:06 smithi059 bash[59745]: cluster 2022-04-28T21:34:05.665078+0000 mon.a (mon.0) 3187 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2022-04-28T21:34:06.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:06 smithi059 bash[61152]: audit 2022-04-28T21:34:05.664716+0000 mon.a (mon.0) 3186 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-38", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:06 smithi059 bash[61152]: cluster 2022-04-28T21:34:05.665078+0000 mon.a (mon.0) 3187 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2022-04-28T21:34:06.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:06 smithi139 bash[43301]: cluster 2022-04-28T21:34:05.131025+0000 mon.a (mon.0) 3185 : cluster [DBG] osdmap e1093: 8 total, 8 up, 8 in 2022-04-28T21:34:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:06 smithi139 bash[43301]: audit 2022-04-28T21:34:05.664716+0000 mon.a (mon.0) 3186 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-38", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:06 smithi139 bash[43301]: cluster 2022-04-28T21:34:05.665078+0000 mon.a (mon.0) 3187 : cluster [DBG] osdmap e1094: 8 total, 8 up, 8 in 2022-04-28T21:34:07.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:07 smithi059 bash[61152]: cluster 2022-04-28T21:34:05.974272+0000 mgr.y (mgr.24961) 3407 : cluster [DBG] pgmap v3891: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 752 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:07 smithi059 bash[61152]: cluster 2022-04-28T21:34:06.675112+0000 mon.a (mon.0) 3188 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2022-04-28T21:34:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:07 smithi059 bash[59745]: cluster 2022-04-28T21:34:05.974272+0000 mgr.y (mgr.24961) 3407 : cluster [DBG] pgmap v3891: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 752 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:07 smithi059 bash[59745]: cluster 2022-04-28T21:34:06.675112+0000 mon.a (mon.0) 3188 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2022-04-28T21:34:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:07 smithi139 bash[43301]: cluster 2022-04-28T21:34:05.974272+0000 mgr.y (mgr.24961) 3407 : cluster [DBG] pgmap v3891: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 752 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:07 smithi139 bash[43301]: cluster 2022-04-28T21:34:06.675112+0000 mon.a (mon.0) 3188 : cluster [DBG] osdmap e1095: 8 total, 8 up, 8 in 2022-04-28T21:34:09.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:08 smithi059 bash[59745]: cluster 2022-04-28T21:34:07.678704+0000 mon.a (mon.0) 3189 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2022-04-28T21:34:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:08 smithi059 bash[59745]: audit 2022-04-28T21:34:07.681019+0000 mon.a (mon.0) 3190 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:08 smithi059 bash[59745]: cluster 2022-04-28T21:34:07.975052+0000 mgr.y (mgr.24961) 3408 : cluster [DBG] pgmap v3894: 137 pgs: 137 active+clean; 24 KiB data, 752 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:08 smithi059 bash[61152]: cluster 2022-04-28T21:34:07.678704+0000 mon.a (mon.0) 3189 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2022-04-28T21:34:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:08 smithi059 bash[61152]: audit 2022-04-28T21:34:07.681019+0000 mon.a (mon.0) 3190 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:08 smithi059 bash[61152]: cluster 2022-04-28T21:34:07.975052+0000 mgr.y (mgr.24961) 3408 : cluster [DBG] pgmap v3894: 137 pgs: 137 active+clean; 24 KiB data, 752 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:09.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:08 smithi139 bash[43301]: cluster 2022-04-28T21:34:07.678704+0000 mon.a (mon.0) 3189 : cluster [DBG] osdmap e1096: 8 total, 8 up, 8 in 2022-04-28T21:34:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:08 smithi139 bash[43301]: audit 2022-04-28T21:34:07.681019+0000 mon.a (mon.0) 3190 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:08 smithi139 bash[43301]: cluster 2022-04-28T21:34:07.975052+0000 mgr.y (mgr.24961) 3408 : cluster [DBG] pgmap v3894: 137 pgs: 137 active+clean; 24 KiB data, 752 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:10.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:09 smithi059 bash[59745]: audit 2022-04-28T21:34:08.679578+0000 mon.a (mon.0) 3191 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:09 smithi059 bash[59745]: cluster 2022-04-28T21:34:08.679668+0000 mon.a (mon.0) 3192 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2022-04-28T21:34:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:09 smithi059 bash[59745]: audit 2022-04-28T21:34:08.682232+0000 mon.a (mon.0) 3193 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:09 smithi059 bash[61152]: audit 2022-04-28T21:34:08.679578+0000 mon.a (mon.0) 3191 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:09 smithi059 bash[61152]: cluster 2022-04-28T21:34:08.679668+0000 mon.a (mon.0) 3192 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2022-04-28T21:34:10.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:09 smithi059 bash[61152]: audit 2022-04-28T21:34:08.682232+0000 mon.a (mon.0) 3193 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:09 smithi139 bash[43301]: audit 2022-04-28T21:34:08.679578+0000 mon.a (mon.0) 3191 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:09 smithi139 bash[43301]: cluster 2022-04-28T21:34:08.679668+0000 mon.a (mon.0) 3192 : cluster [DBG] osdmap e1097: 8 total, 8 up, 8 in 2022-04-28T21:34:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:09 smithi139 bash[43301]: audit 2022-04-28T21:34:08.682232+0000 mon.a (mon.0) 3193 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]: dispatch 2022-04-28T21:34:11.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[61152]: audit 2022-04-28T21:34:09.683308+0000 mon.a (mon.0) 3194 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[61152]: cluster 2022-04-28T21:34:09.683420+0000 mon.a (mon.0) 3195 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2022-04-28T21:34:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[61152]: audit 2022-04-28T21:34:09.698815+0000 mon.a (mon.0) 3196 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[61152]: audit 2022-04-28T21:34:09.699776+0000 mon.a (mon.0) 3197 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[61152]: audit 2022-04-28T21:34:09.700171+0000 mon.a (mon.0) 3198 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-39", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:11.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[61152]: cluster 2022-04-28T21:34:09.975829+0000 mgr.y (mgr.24961) 3409 : cluster [DBG] pgmap v3897: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[59745]: audit 2022-04-28T21:34:09.683308+0000 mon.a (mon.0) 3194 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[59745]: cluster 2022-04-28T21:34:09.683420+0000 mon.a (mon.0) 3195 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2022-04-28T21:34:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[59745]: audit 2022-04-28T21:34:09.698815+0000 mon.a (mon.0) 3196 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:11.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[59745]: audit 2022-04-28T21:34:09.699776+0000 mon.a (mon.0) 3197 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:11.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[59745]: audit 2022-04-28T21:34:09.700171+0000 mon.a (mon.0) 3198 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-39", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:11.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:10 smithi059 bash[59745]: cluster 2022-04-28T21:34:09.975829+0000 mgr.y (mgr.24961) 3409 : cluster [DBG] pgmap v3897: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:10 smithi139 bash[43301]: audit 2022-04-28T21:34:09.683308+0000 mon.a (mon.0) 3194 : audit [INF] from='client.? 172.21.15.59:0/3958230461' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-38"}]': finished 2022-04-28T21:34:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:10 smithi139 bash[43301]: cluster 2022-04-28T21:34:09.683420+0000 mon.a (mon.0) 3195 : cluster [DBG] osdmap e1098: 8 total, 8 up, 8 in 2022-04-28T21:34:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:10 smithi139 bash[43301]: audit 2022-04-28T21:34:09.698815+0000 mon.a (mon.0) 3196 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:10 smithi139 bash[43301]: audit 2022-04-28T21:34:09.699776+0000 mon.a (mon.0) 3197 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:10 smithi139 bash[43301]: audit 2022-04-28T21:34:09.700171+0000 mon.a (mon.0) 3198 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-39", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:10 smithi139 bash[43301]: cluster 2022-04-28T21:34:09.975829+0000 mgr.y (mgr.24961) 3409 : cluster [DBG] pgmap v3897: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:12.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[61152]: audit 2022-04-28T21:34:10.705302+0000 mon.a (mon.0) 3199 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-39", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[61152]: cluster 2022-04-28T21:34:10.705430+0000 mon.a (mon.0) 3200 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2022-04-28T21:34:12.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[61152]: audit 2022-04-28T21:34:10.706032+0000 mon.a (mon.0) 3201 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-39", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:12.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[59745]: audit 2022-04-28T21:34:10.705302+0000 mon.a (mon.0) 3199 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-39", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[59745]: cluster 2022-04-28T21:34:10.705430+0000 mon.a (mon.0) 3200 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2022-04-28T21:34:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:12 smithi059 bash[59745]: audit 2022-04-28T21:34:10.706032+0000 mon.a (mon.0) 3201 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-39", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:12 smithi139 bash[43301]: audit 2022-04-28T21:34:10.705302+0000 mon.a (mon.0) 3199 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-39", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:12 smithi139 bash[43301]: cluster 2022-04-28T21:34:10.705430+0000 mon.a (mon.0) 3200 : cluster [DBG] osdmap e1099: 8 total, 8 up, 8 in 2022-04-28T21:34:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:12 smithi139 bash[43301]: audit 2022-04-28T21:34:10.706032+0000 mon.a (mon.0) 3201 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-39", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:13.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[61152]: cluster 2022-04-28T21:34:11.976526+0000 mgr.y (mgr.24961) 3410 : cluster [DBG] pgmap v3900: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[61152]: cluster 2022-04-28T21:34:12.171438+0000 mon.a (mon.0) 3202 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2022-04-28T21:34:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[61152]: audit 2022-04-28T21:34:12.718522+0000 mon.a (mon.0) 3203 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-39", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[61152]: cluster 2022-04-28T21:34:12.718703+0000 mon.a (mon.0) 3204 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2022-04-28T21:34:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[59745]: cluster 2022-04-28T21:34:11.976526+0000 mgr.y (mgr.24961) 3410 : cluster [DBG] pgmap v3900: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[59745]: cluster 2022-04-28T21:34:12.171438+0000 mon.a (mon.0) 3202 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2022-04-28T21:34:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[59745]: audit 2022-04-28T21:34:12.718522+0000 mon.a (mon.0) 3203 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-39", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[59745]: cluster 2022-04-28T21:34:12.718703+0000 mon.a (mon.0) 3204 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2022-04-28T21:34:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:13 smithi139 bash[43301]: cluster 2022-04-28T21:34:11.976526+0000 mgr.y (mgr.24961) 3410 : cluster [DBG] pgmap v3900: 137 pgs: 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:13 smithi139 bash[43301]: cluster 2022-04-28T21:34:12.171438+0000 mon.a (mon.0) 3202 : cluster [DBG] osdmap e1100: 8 total, 8 up, 8 in 2022-04-28T21:34:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:13 smithi139 bash[43301]: audit 2022-04-28T21:34:12.718522+0000 mon.a (mon.0) 3203 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-39", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:13 smithi139 bash[43301]: cluster 2022-04-28T21:34:12.718703+0000 mon.a (mon.0) 3204 : cluster [DBG] osdmap e1101: 8 total, 8 up, 8 in 2022-04-28T21:34:14.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:13.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:15.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:14 smithi059 bash[61152]: cluster 2022-04-28T21:34:13.722350+0000 mon.a (mon.0) 3205 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2022-04-28T21:34:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:14 smithi059 bash[61152]: cluster 2022-04-28T21:34:13.977291+0000 mgr.y (mgr.24961) 3411 : cluster [DBG] pgmap v3903: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:14 smithi059 bash[59745]: cluster 2022-04-28T21:34:13.722350+0000 mon.a (mon.0) 3205 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2022-04-28T21:34:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:14 smithi059 bash[59745]: cluster 2022-04-28T21:34:13.977291+0000 mgr.y (mgr.24961) 3411 : cluster [DBG] pgmap v3903: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:14 smithi139 bash[43301]: cluster 2022-04-28T21:34:13.722350+0000 mon.a (mon.0) 3205 : cluster [DBG] osdmap e1102: 8 total, 8 up, 8 in 2022-04-28T21:34:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:14 smithi139 bash[43301]: cluster 2022-04-28T21:34:13.977291+0000 mgr.y (mgr.24961) 3411 : cluster [DBG] pgmap v3903: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 757 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:16.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[61152]: cluster 2022-04-28T21:34:14.725251+0000 mon.a (mon.0) 3206 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2022-04-28T21:34:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[61152]: audit 2022-04-28T21:34:14.727964+0000 mon.a (mon.0) 3207 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[61152]: audit 2022-04-28T21:34:15.727923+0000 mon.a (mon.0) 3208 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[61152]: cluster 2022-04-28T21:34:15.728074+0000 mon.a (mon.0) 3209 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2022-04-28T21:34:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[61152]: audit 2022-04-28T21:34:15.728804+0000 mon.a (mon.0) 3210 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[59745]: cluster 2022-04-28T21:34:14.725251+0000 mon.a (mon.0) 3206 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2022-04-28T21:34:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[59745]: audit 2022-04-28T21:34:14.727964+0000 mon.a (mon.0) 3207 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[59745]: audit 2022-04-28T21:34:15.727923+0000 mon.a (mon.0) 3208 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[59745]: cluster 2022-04-28T21:34:15.728074+0000 mon.a (mon.0) 3209 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2022-04-28T21:34:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:15 smithi059 bash[59745]: audit 2022-04-28T21:34:15.728804+0000 mon.a (mon.0) 3210 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:15 smithi139 bash[43301]: cluster 2022-04-28T21:34:14.725251+0000 mon.a (mon.0) 3206 : cluster [DBG] osdmap e1103: 8 total, 8 up, 8 in 2022-04-28T21:34:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:15 smithi139 bash[43301]: audit 2022-04-28T21:34:14.727964+0000 mon.a (mon.0) 3207 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:15 smithi139 bash[43301]: audit 2022-04-28T21:34:15.727923+0000 mon.a (mon.0) 3208 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:15 smithi139 bash[43301]: cluster 2022-04-28T21:34:15.728074+0000 mon.a (mon.0) 3209 : cluster [DBG] osdmap e1104: 8 total, 8 up, 8 in 2022-04-28T21:34:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:15 smithi139 bash[43301]: audit 2022-04-28T21:34:15.728804+0000 mon.a (mon.0) 3210 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]: dispatch 2022-04-28T21:34:17.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[59745]: cluster 2022-04-28T21:34:15.978011+0000 mgr.y (mgr.24961) 3412 : cluster [DBG] pgmap v3906: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[59745]: audit 2022-04-28T21:34:16.730029+0000 mon.a (mon.0) 3211 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[59745]: cluster 2022-04-28T21:34:16.730166+0000 mon.a (mon.0) 3212 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2022-04-28T21:34:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[59745]: audit 2022-04-28T21:34:16.742809+0000 mon.a (mon.0) 3213 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:17.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[59745]: audit 2022-04-28T21:34:16.743684+0000 mon.a (mon.0) 3214 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[59745]: audit 2022-04-28T21:34:16.744216+0000 mon.a (mon.0) 3215 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-40", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[61152]: cluster 2022-04-28T21:34:15.978011+0000 mgr.y (mgr.24961) 3412 : cluster [DBG] pgmap v3906: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[61152]: audit 2022-04-28T21:34:16.730029+0000 mon.a (mon.0) 3211 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[61152]: cluster 2022-04-28T21:34:16.730166+0000 mon.a (mon.0) 3212 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2022-04-28T21:34:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[61152]: audit 2022-04-28T21:34:16.742809+0000 mon.a (mon.0) 3213 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:17.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[61152]: audit 2022-04-28T21:34:16.743684+0000 mon.a (mon.0) 3214 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:17.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:17 smithi059 bash[61152]: audit 2022-04-28T21:34:16.744216+0000 mon.a (mon.0) 3215 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-40", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:17 smithi139 bash[43301]: cluster 2022-04-28T21:34:15.978011+0000 mgr.y (mgr.24961) 3412 : cluster [DBG] pgmap v3906: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:17 smithi139 bash[43301]: audit 2022-04-28T21:34:16.730029+0000 mon.a (mon.0) 3211 : audit [INF] from='client.? 172.21.15.59:0/2197627060' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-39"}]': finished 2022-04-28T21:34:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:17 smithi139 bash[43301]: cluster 2022-04-28T21:34:16.730166+0000 mon.a (mon.0) 3212 : cluster [DBG] osdmap e1105: 8 total, 8 up, 8 in 2022-04-28T21:34:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:17 smithi139 bash[43301]: audit 2022-04-28T21:34:16.742809+0000 mon.a (mon.0) 3213 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:17 smithi139 bash[43301]: audit 2022-04-28T21:34:16.743684+0000 mon.a (mon.0) 3214 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:17 smithi139 bash[43301]: audit 2022-04-28T21:34:16.744216+0000 mon.a (mon.0) 3215 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-40", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:34:19.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[61152]: audit 2022-04-28T21:34:17.741564+0000 mon.a (mon.0) 3216 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-40", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[61152]: cluster 2022-04-28T21:34:17.741670+0000 mon.a (mon.0) 3217 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2022-04-28T21:34:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[61152]: audit 2022-04-28T21:34:17.745134+0000 mon.a (mon.0) 3218 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-40", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[61152]: cluster 2022-04-28T21:34:17.978914+0000 mgr.y (mgr.24961) 3413 : cluster [DBG] pgmap v3909: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[59745]: audit 2022-04-28T21:34:17.741564+0000 mon.a (mon.0) 3216 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-40", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[59745]: cluster 2022-04-28T21:34:17.741670+0000 mon.a (mon.0) 3217 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2022-04-28T21:34:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[59745]: audit 2022-04-28T21:34:17.745134+0000 mon.a (mon.0) 3218 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-40", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:19 smithi059 bash[59745]: cluster 2022-04-28T21:34:17.978914+0000 mgr.y (mgr.24961) 3413 : cluster [DBG] pgmap v3909: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:19.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:19 smithi139 bash[43301]: audit 2022-04-28T21:34:17.741564+0000 mon.a (mon.0) 3216 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83193-40", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:34:19.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:19 smithi139 bash[43301]: cluster 2022-04-28T21:34:17.741670+0000 mon.a (mon.0) 3217 : cluster [DBG] osdmap e1106: 8 total, 8 up, 8 in 2022-04-28T21:34:19.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:19 smithi139 bash[43301]: audit 2022-04-28T21:34:17.745134+0000 mon.a (mon.0) 3218 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-40", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:19.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:19 smithi139 bash[43301]: cluster 2022-04-28T21:34:17.978914+0000 mgr.y (mgr.24961) 3413 : cluster [DBG] pgmap v3909: 137 pgs: 137 active+clean; 24 KiB data, 758 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: cluster 2022-04-28T21:34:19.209387+0000 mon.a (mon.0) 3219 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2022-04-28T21:34:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.284735+0000 mon.a (mon.0) 3220 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:34:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.285146+0000 mon.a (mon.0) 3221 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:34:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.291299+0000 mon.a (mon.0) 3222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:34:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.294225+0000 mon.a (mon.0) 3223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T21:34:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.294952+0000 mon.a (mon.0) 3224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:34:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.295572+0000 mon.a (mon.0) 3225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:34:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: audit 2022-04-28T21:34:19.749477+0000 mon.a (mon.0) 3226 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-40", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[61152]: cluster 2022-04-28T21:34:19.749651+0000 mon.a (mon.0) 3227 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2022-04-28T21:34:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: cluster 2022-04-28T21:34:19.209387+0000 mon.a (mon.0) 3219 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2022-04-28T21:34:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.284735+0000 mon.a (mon.0) 3220 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:34:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.285146+0000 mon.a (mon.0) 3221 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:34:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.291299+0000 mon.a (mon.0) 3222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:34:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.294225+0000 mon.a (mon.0) 3223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T21:34:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.294952+0000 mon.a (mon.0) 3224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:34:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.295572+0000 mon.a (mon.0) 3225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:34:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: audit 2022-04-28T21:34:19.749477+0000 mon.a (mon.0) 3226 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-40", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:20 smithi059 bash[59745]: cluster 2022-04-28T21:34:19.749651+0000 mon.a (mon.0) 3227 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2022-04-28T21:34:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: cluster 2022-04-28T21:34:19.209387+0000 mon.a (mon.0) 3219 : cluster [DBG] osdmap e1107: 8 total, 8 up, 8 in 2022-04-28T21:34:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.284735+0000 mon.a (mon.0) 3220 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:34:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.285146+0000 mon.a (mon.0) 3221 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:34:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.291299+0000 mon.a (mon.0) 3222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T21:34:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.294225+0000 mon.a (mon.0) 3223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2022-04-28T21:34:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.294952+0000 mon.a (mon.0) 3224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:34:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.295572+0000 mon.a (mon.0) 3225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:34:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: audit 2022-04-28T21:34:19.749477+0000 mon.a (mon.0) 3226 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83193-40", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:20 smithi139 bash[43301]: cluster 2022-04-28T21:34:19.749651+0000 mon.a (mon.0) 3227 : cluster [DBG] osdmap e1108: 8 total, 8 up, 8 in 2022-04-28T21:34:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:21 smithi059 bash[61152]: cluster 2022-04-28T21:34:19.979758+0000 mgr.y (mgr.24961) 3414 : cluster [DBG] pgmap v3912: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:21.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:21 smithi059 bash[61152]: cluster 2022-04-28T21:34:20.756688+0000 mon.a (mon.0) 3228 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2022-04-28T21:34:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:21 smithi059 bash[59745]: cluster 2022-04-28T21:34:19.979758+0000 mgr.y (mgr.24961) 3414 : cluster [DBG] pgmap v3912: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:21 smithi059 bash[59745]: cluster 2022-04-28T21:34:20.756688+0000 mon.a (mon.0) 3228 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2022-04-28T21:34:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:21 smithi139 bash[43301]: cluster 2022-04-28T21:34:19.979758+0000 mgr.y (mgr.24961) 3414 : cluster [DBG] pgmap v3912: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 759 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:21 smithi139 bash[43301]: cluster 2022-04-28T21:34:20.756688+0000 mon.a (mon.0) 3228 : cluster [DBG] osdmap e1109: 8 total, 8 up, 8 in 2022-04-28T21:34:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:22.028Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:23.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[61152]: cluster 2022-04-28T21:34:21.770464+0000 mon.a (mon.0) 3229 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2022-04-28T21:34:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[61152]: audit 2022-04-28T21:34:21.771738+0000 mon.a (mon.0) 3230 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:23.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[61152]: cluster 2022-04-28T21:34:21.980517+0000 mgr.y (mgr.24961) 3415 : cluster [DBG] pgmap v3915: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[59745]: cluster 2022-04-28T21:34:21.770464+0000 mon.a (mon.0) 3229 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2022-04-28T21:34:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[59745]: audit 2022-04-28T21:34:21.771738+0000 mon.a (mon.0) 3230 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:23.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:22 smithi059 bash[59745]: cluster 2022-04-28T21:34:21.980517+0000 mgr.y (mgr.24961) 3415 : cluster [DBG] pgmap v3915: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:23.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:22 smithi139 bash[43301]: cluster 2022-04-28T21:34:21.770464+0000 mon.a (mon.0) 3229 : cluster [DBG] osdmap e1110: 8 total, 8 up, 8 in 2022-04-28T21:34:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:22 smithi139 bash[43301]: audit 2022-04-28T21:34:21.771738+0000 mon.a (mon.0) 3230 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:22 smithi139 bash[43301]: cluster 2022-04-28T21:34:21.980517+0000 mgr.y (mgr.24961) 3415 : cluster [DBG] pgmap v3915: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:24.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:23.919Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[61152]: audit 2022-04-28T21:34:22.772329+0000 mon.a (mon.0) 3231 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[61152]: cluster 2022-04-28T21:34:22.772465+0000 mon.a (mon.0) 3232 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2022-04-28T21:34:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[61152]: audit 2022-04-28T21:34:22.773503+0000 mon.a (mon.0) 3233 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[61152]: audit 2022-04-28T21:34:23.774258+0000 mon.a (mon.0) 3234 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[61152]: cluster 2022-04-28T21:34:23.774476+0000 mon.a (mon.0) 3235 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2022-04-28T21:34:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[59745]: audit 2022-04-28T21:34:22.772329+0000 mon.a (mon.0) 3231 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[59745]: cluster 2022-04-28T21:34:22.772465+0000 mon.a (mon.0) 3232 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2022-04-28T21:34:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[59745]: audit 2022-04-28T21:34:22.773503+0000 mon.a (mon.0) 3233 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[59745]: audit 2022-04-28T21:34:23.774258+0000 mon.a (mon.0) 3234 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:23 smithi059 bash[59745]: cluster 2022-04-28T21:34:23.774476+0000 mon.a (mon.0) 3235 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2022-04-28T21:34:24.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:23 smithi139 bash[43301]: audit 2022-04-28T21:34:22.772329+0000 mon.a (mon.0) 3231 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:23 smithi139 bash[43301]: cluster 2022-04-28T21:34:22.772465+0000 mon.a (mon.0) 3232 : cluster [DBG] osdmap e1111: 8 total, 8 up, 8 in 2022-04-28T21:34:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:23 smithi139 bash[43301]: audit 2022-04-28T21:34:22.773503+0000 mon.a (mon.0) 3233 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]: dispatch 2022-04-28T21:34:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:23 smithi139 bash[43301]: audit 2022-04-28T21:34:23.774258+0000 mon.a (mon.0) 3234 : audit [INF] from='client.? 172.21.15.59:0/1341602548' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83193-40"}]': finished 2022-04-28T21:34:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:23 smithi139 bash[43301]: cluster 2022-04-28T21:34:23.774476+0000 mon.a (mon.0) 3235 : cluster [DBG] osdmap e1112: 8 total, 8 up, 8 in 2022-04-28T21:34:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:24 smithi059 bash[59745]: cluster 2022-04-28T21:34:23.981254+0000 mgr.y (mgr.24961) 3416 : cluster [DBG] pgmap v3918: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:24 smithi059 bash[61152]: cluster 2022-04-28T21:34:23.981254+0000 mgr.y (mgr.24961) 3416 : cluster [DBG] pgmap v3918: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:24 smithi139 bash[43301]: cluster 2022-04-28T21:34:23.981254+0000 mgr.y (mgr.24961) 3416 : cluster [DBG] pgmap v3918: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:27.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:27 smithi059 bash[61152]: cluster 2022-04-28T21:34:25.982795+0000 mgr.y (mgr.24961) 3417 : cluster [DBG] pgmap v3919: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:27 smithi059 bash[59745]: cluster 2022-04-28T21:34:25.982795+0000 mgr.y (mgr.24961) 3417 : cluster [DBG] pgmap v3919: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:27.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:27 smithi139 bash[43301]: cluster 2022-04-28T21:34:25.982795+0000 mgr.y (mgr.24961) 3417 : cluster [DBG] pgmap v3919: 137 pgs: 137 active+clean; 24 KiB data, 764 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:29.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:29 smithi059 bash[61152]: cluster 2022-04-28T21:34:27.983864+0000 mgr.y (mgr.24961) 3418 : cluster [DBG] pgmap v3920: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:29 smithi059 bash[59745]: cluster 2022-04-28T21:34:27.983864+0000 mgr.y (mgr.24961) 3418 : cluster [DBG] pgmap v3920: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:29 smithi139 bash[43301]: cluster 2022-04-28T21:34:27.983864+0000 mgr.y (mgr.24961) 3418 : cluster [DBG] pgmap v3920: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:31.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:31 smithi059 bash[61152]: cluster 2022-04-28T21:34:29.985042+0000 mgr.y (mgr.24961) 3419 : cluster [DBG] pgmap v3921: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:31 smithi059 bash[59745]: cluster 2022-04-28T21:34:29.985042+0000 mgr.y (mgr.24961) 3419 : cluster [DBG] pgmap v3921: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:31.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:31 smithi139 bash[43301]: cluster 2022-04-28T21:34:29.985042+0000 mgr.y (mgr.24961) 3419 : cluster [DBG] pgmap v3921: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:32.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:33.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:33 smithi059 bash[61152]: cluster 2022-04-28T21:34:31.986298+0000 mgr.y (mgr.24961) 3420 : cluster [DBG] pgmap v3922: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:33 smithi059 bash[59745]: cluster 2022-04-28T21:34:31.986298+0000 mgr.y (mgr.24961) 3420 : cluster [DBG] pgmap v3922: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:33.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:33 smithi139 bash[43301]: cluster 2022-04-28T21:34:31.986298+0000 mgr.y (mgr.24961) 3420 : cluster [DBG] pgmap v3922: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:33.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:35.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:35 smithi059 bash[61152]: cluster 2022-04-28T21:34:33.986816+0000 mgr.y (mgr.24961) 3421 : cluster [DBG] pgmap v3923: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:35 smithi059 bash[59745]: cluster 2022-04-28T21:34:33.986816+0000 mgr.y (mgr.24961) 3421 : cluster [DBG] pgmap v3923: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:35.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:35 smithi139 bash[43301]: cluster 2022-04-28T21:34:33.986816+0000 mgr.y (mgr.24961) 3421 : cluster [DBG] pgmap v3923: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:37.340 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:37 smithi059 bash[61152]: cluster 2022-04-28T21:34:35.988219+0000 mgr.y (mgr.24961) 3422 : cluster [DBG] pgmap v3924: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:37.387 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:37 smithi059 bash[59745]: cluster 2022-04-28T21:34:35.988219+0000 mgr.y (mgr.24961) 3422 : cluster [DBG] pgmap v3924: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:37.387 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:37 smithi139 bash[43301]: cluster 2022-04-28T21:34:35.988219+0000 mgr.y (mgr.24961) 3422 : cluster [DBG] pgmap v3924: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:39.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:39 smithi059 bash[59745]: cluster 2022-04-28T21:34:37.989190+0000 mgr.y (mgr.24961) 3423 : cluster [DBG] pgmap v3925: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:39 smithi059 bash[61152]: cluster 2022-04-28T21:34:37.989190+0000 mgr.y (mgr.24961) 3423 : cluster [DBG] pgmap v3925: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:39.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:39 smithi139 bash[43301]: cluster 2022-04-28T21:34:37.989190+0000 mgr.y (mgr.24961) 3423 : cluster [DBG] pgmap v3925: 137 pgs: 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:40.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:40 smithi139 bash[43301]: cluster 2022-04-28T21:34:39.088321+0000 mon.a (mon.0) 3236 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2022-04-28T21:34:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:40 smithi139 bash[43301]: audit 2022-04-28T21:34:39.093624+0000 mon.c (mon.1) 95 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:34:40.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:40 smithi139 bash[43301]: audit 2022-04-28T21:34:39.094720+0000 mon.a (mon.0) 3237 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:34:40.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:40 smithi059 bash[59745]: cluster 2022-04-28T21:34:39.088321+0000 mon.a (mon.0) 3236 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2022-04-28T21:34:40.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:40 smithi059 bash[59745]: audit 2022-04-28T21:34:39.093624+0000 mon.c (mon.1) 95 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:34:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:40 smithi059 bash[59745]: audit 2022-04-28T21:34:39.094720+0000 mon.a (mon.0) 3237 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:34:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:40 smithi059 bash[61152]: cluster 2022-04-28T21:34:39.088321+0000 mon.a (mon.0) 3236 : cluster [DBG] osdmap e1113: 8 total, 8 up, 8 in 2022-04-28T21:34:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:40 smithi059 bash[61152]: audit 2022-04-28T21:34:39.093624+0000 mon.c (mon.1) 95 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:34:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:40 smithi059 bash[61152]: audit 2022-04-28T21:34:39.094720+0000 mon.a (mon.0) 3237 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:34:41.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:41 smithi139 bash[43301]: cluster 2022-04-28T21:34:39.990275+0000 mgr.y (mgr.24961) 3424 : cluster [DBG] pgmap v3927: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:41 smithi139 bash[43301]: audit 2022-04-28T21:34:40.102444+0000 mon.a (mon.0) 3238 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:34:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:41 smithi139 bash[43301]: cluster 2022-04-28T21:34:40.102732+0000 mon.a (mon.0) 3239 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2022-04-28T21:34:41.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:41 smithi059 bash[61152]: cluster 2022-04-28T21:34:39.990275+0000 mgr.y (mgr.24961) 3424 : cluster [DBG] pgmap v3927: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:41.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:41 smithi059 bash[61152]: audit 2022-04-28T21:34:40.102444+0000 mon.a (mon.0) 3238 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:34:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:41 smithi059 bash[61152]: cluster 2022-04-28T21:34:40.102732+0000 mon.a (mon.0) 3239 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2022-04-28T21:34:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:41 smithi059 bash[59745]: cluster 2022-04-28T21:34:39.990275+0000 mgr.y (mgr.24961) 3424 : cluster [DBG] pgmap v3927: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 765 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:34:41.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:41 smithi059 bash[59745]: audit 2022-04-28T21:34:40.102444+0000 mon.a (mon.0) 3238 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:34:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:41 smithi059 bash[59745]: cluster 2022-04-28T21:34:40.102732+0000 mon.a (mon.0) 3239 : cluster [DBG] osdmap e1114: 8 total, 8 up, 8 in 2022-04-28T21:34:42.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:42 smithi059 bash[59745]: cluster 2022-04-28T21:34:41.101569+0000 mon.a (mon.0) 3240 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2022-04-28T21:34:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:42 smithi059 bash[61152]: cluster 2022-04-28T21:34:41.101569+0000 mon.a (mon.0) 3240 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2022-04-28T21:34:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:42.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:42.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:42 smithi139 bash[43301]: cluster 2022-04-28T21:34:41.101569+0000 mon.a (mon.0) 3240 : cluster [DBG] osdmap e1115: 8 total, 8 up, 8 in 2022-04-28T21:34:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: cluster 2022-04-28T21:34:41.990942+0000 mgr.y (mgr.24961) 3425 : cluster [DBG] pgmap v3930: 169 pgs: 5 creating+activating, 13 creating+peering, 3 unknown, 148 active+clean; 24 KiB data, 774 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 236 op/s 2022-04-28T21:34:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.214388+0000 mon.c (mon.1) 96 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:34:43.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.214709+0000 mgr.y (mgr.24961) 3426 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:34:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.215437+0000 mon.c (mon.1) 97 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:34:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.215673+0000 mgr.y (mgr.24961) 3427 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:34:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.216294+0000 mon.c (mon.1) 98 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:34:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.216527+0000 mgr.y (mgr.24961) 3428 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:34:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.217097+0000 mon.c (mon.1) 99 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:34:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.217324+0000 mgr.y (mgr.24961) 3429 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:34:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.217882+0000 mon.c (mon.1) 100 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:34:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.218107+0000 mgr.y (mgr.24961) 3430 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:34:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.218717+0000 mon.c (mon.1) 101 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:34:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.218915+0000 mgr.y (mgr.24961) 3431 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:34:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.219512+0000 mon.c (mon.1) 102 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:34:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.219724+0000 mgr.y (mgr.24961) 3432 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:34:43.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.220283+0000 mon.c (mon.1) 103 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:34:43.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.220469+0000 mgr.y (mgr.24961) 3433 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:34:43.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.220999+0000 mon.c (mon.1) 104 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:34:43.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: cluster 2022-04-28T21:34:41.990942+0000 mgr.y (mgr.24961) 3425 : cluster [DBG] pgmap v3930: 169 pgs: 5 creating+activating, 13 creating+peering, 3 unknown, 148 active+clean; 24 KiB data, 774 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 236 op/s 2022-04-28T21:34:43.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.214388+0000 mon.c (mon.1) 96 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:34:43.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.214709+0000 mgr.y (mgr.24961) 3426 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:34:43.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.215437+0000 mon.c (mon.1) 97 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:34:43.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.215673+0000 mgr.y (mgr.24961) 3427 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:34:43.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.216294+0000 mon.c (mon.1) 98 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:34:43.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.216527+0000 mgr.y (mgr.24961) 3428 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:34:43.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.217097+0000 mon.c (mon.1) 99 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:34:43.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.217324+0000 mgr.y (mgr.24961) 3429 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:34:43.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.217882+0000 mon.c (mon.1) 100 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:34:43.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.218107+0000 mgr.y (mgr.24961) 3430 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:34:43.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.218717+0000 mon.c (mon.1) 101 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:34:43.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.218915+0000 mgr.y (mgr.24961) 3431 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:34:43.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.219512+0000 mon.c (mon.1) 102 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:34:43.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.219724+0000 mgr.y (mgr.24961) 3432 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:34:43.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.220283+0000 mon.c (mon.1) 103 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:34:43.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.220469+0000 mgr.y (mgr.24961) 3433 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:34:43.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.220999+0000 mon.c (mon.1) 104 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:34:43.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.221194+0000 mgr.y (mgr.24961) 3434 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:34:43.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.221681+0000 mon.c (mon.1) 105 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:34:43.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[61152]: audit 2022-04-28T21:34:42.221890+0000 mgr.y (mgr.24961) 3435 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:34:43.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.221194+0000 mgr.y (mgr.24961) 3434 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:34:43.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.221681+0000 mon.c (mon.1) 105 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:34:43.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[59745]: audit 2022-04-28T21:34:42.221890+0000 mgr.y (mgr.24961) 3435 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:34:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: cluster 2022-04-28T21:34:41.990942+0000 mgr.y (mgr.24961) 3425 : cluster [DBG] pgmap v3930: 169 pgs: 5 creating+activating, 13 creating+peering, 3 unknown, 148 active+clean; 24 KiB data, 774 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 236 op/s 2022-04-28T21:34:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.214388+0000 mon.c (mon.1) 96 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:34:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.214709+0000 mgr.y (mgr.24961) 3426 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:34:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.215437+0000 mon.c (mon.1) 97 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:34:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.215673+0000 mgr.y (mgr.24961) 3427 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:34:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.216294+0000 mon.c (mon.1) 98 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:34:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.216527+0000 mgr.y (mgr.24961) 3428 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:34:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.217097+0000 mon.c (mon.1) 99 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:34:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.217324+0000 mgr.y (mgr.24961) 3429 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:34:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.217882+0000 mon.c (mon.1) 100 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:34:43.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.218107+0000 mgr.y (mgr.24961) 3430 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:34:43.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.218717+0000 mon.c (mon.1) 101 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:34:43.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.218915+0000 mgr.y (mgr.24961) 3431 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:34:43.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.219512+0000 mon.c (mon.1) 102 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:34:43.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.219724+0000 mgr.y (mgr.24961) 3432 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:34:43.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.220283+0000 mon.c (mon.1) 103 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:34:43.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.220469+0000 mgr.y (mgr.24961) 3433 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:34:43.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.220999+0000 mon.c (mon.1) 104 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:34:43.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.221194+0000 mgr.y (mgr.24961) 3434 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:34:43.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.221681+0000 mon.c (mon.1) 105 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:34:43.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:43 smithi139 bash[43301]: audit 2022-04-28T21:34:42.221890+0000 mgr.y (mgr.24961) 3435 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:34:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:43.921Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:44 smithi059 bash[61152]: cluster 2022-04-28T21:34:43.114648+0000 osd.3 (osd.3) 1 : cluster [DBG] 76.1 deep-scrub ok 2022-04-28T21:34:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:44 smithi059 bash[61152]: cluster 2022-04-28T21:34:43.205918+0000 osd.1 (osd.1) 1 : cluster [DBG] 76.5 deep-scrub ok 2022-04-28T21:34:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:44 smithi059 bash[61152]: cluster 2022-04-28T21:34:43.680612+0000 osd.2 (osd.2) 1 : cluster [DBG] 76.0 deep-scrub ok 2022-04-28T21:34:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:44 smithi059 bash[59745]: cluster 2022-04-28T21:34:43.114648+0000 osd.3 (osd.3) 1 : cluster [DBG] 76.1 deep-scrub ok 2022-04-28T21:34:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:44 smithi059 bash[59745]: cluster 2022-04-28T21:34:43.205918+0000 osd.1 (osd.1) 1 : cluster [DBG] 76.5 deep-scrub ok 2022-04-28T21:34:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:44 smithi059 bash[59745]: cluster 2022-04-28T21:34:43.680612+0000 osd.2 (osd.2) 1 : cluster [DBG] 76.0 deep-scrub ok 2022-04-28T21:34:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:44 smithi139 bash[43301]: cluster 2022-04-28T21:34:43.114648+0000 osd.3 (osd.3) 1 : cluster [DBG] 76.1 deep-scrub ok 2022-04-28T21:34:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:44 smithi139 bash[43301]: cluster 2022-04-28T21:34:43.205918+0000 osd.1 (osd.1) 1 : cluster [DBG] 76.5 deep-scrub ok 2022-04-28T21:34:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:44 smithi139 bash[43301]: cluster 2022-04-28T21:34:43.680612+0000 osd.2 (osd.2) 1 : cluster [DBG] 76.0 deep-scrub ok 2022-04-28T21:34:45.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[61152]: cluster 2022-04-28T21:34:43.477529+0000 osd.0 (osd.0) 1 : cluster [DBG] 76.4 deep-scrub ok 2022-04-28T21:34:45.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[61152]: cluster 2022-04-28T21:34:43.991712+0000 mgr.y (mgr.24961) 3436 : cluster [DBG] pgmap v3931: 169 pgs: 5 creating+activating, 13 creating+peering, 3 unknown, 148 active+clean; 24 KiB data, 774 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 236 op/s 2022-04-28T21:34:45.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[61152]: cluster 2022-04-28T21:34:44.065222+0000 osd.3 (osd.3) 2 : cluster [DBG] 76.8 deep-scrub ok 2022-04-28T21:34:45.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[61152]: cluster 2022-04-28T21:34:44.250110+0000 osd.1 (osd.1) 2 : cluster [DBG] 76.6 deep-scrub ok 2022-04-28T21:34:45.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[59745]: cluster 2022-04-28T21:34:43.477529+0000 osd.0 (osd.0) 1 : cluster [DBG] 76.4 deep-scrub ok 2022-04-28T21:34:45.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[59745]: cluster 2022-04-28T21:34:43.991712+0000 mgr.y (mgr.24961) 3436 : cluster [DBG] pgmap v3931: 169 pgs: 5 creating+activating, 13 creating+peering, 3 unknown, 148 active+clean; 24 KiB data, 774 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 236 op/s 2022-04-28T21:34:45.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[59745]: cluster 2022-04-28T21:34:44.065222+0000 osd.3 (osd.3) 2 : cluster [DBG] 76.8 deep-scrub ok 2022-04-28T21:34:45.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:45 smithi059 bash[59745]: cluster 2022-04-28T21:34:44.250110+0000 osd.1 (osd.1) 2 : cluster [DBG] 76.6 deep-scrub ok 2022-04-28T21:34:45.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:45 smithi139 bash[43301]: cluster 2022-04-28T21:34:43.477529+0000 osd.0 (osd.0) 1 : cluster [DBG] 76.4 deep-scrub ok 2022-04-28T21:34:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:45 smithi139 bash[43301]: cluster 2022-04-28T21:34:43.991712+0000 mgr.y (mgr.24961) 3436 : cluster [DBG] pgmap v3931: 169 pgs: 5 creating+activating, 13 creating+peering, 3 unknown, 148 active+clean; 24 KiB data, 774 MiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 236 op/s 2022-04-28T21:34:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:45 smithi139 bash[43301]: cluster 2022-04-28T21:34:44.065222+0000 osd.3 (osd.3) 2 : cluster [DBG] 76.8 deep-scrub ok 2022-04-28T21:34:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:45 smithi139 bash[43301]: cluster 2022-04-28T21:34:44.250110+0000 osd.1 (osd.1) 2 : cluster [DBG] 76.6 deep-scrub ok 2022-04-28T21:34:47.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:47 smithi059 bash[61152]: cluster 2022-04-28T21:34:45.993392+0000 mgr.y (mgr.24961) 3437 : cluster [DBG] pgmap v3932: 169 pgs: 169 active+clean; 96 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 15 MiB/s wr, 210 op/s 2022-04-28T21:34:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:47 smithi059 bash[61152]: cluster 2022-04-28T21:34:46.655646+0000 osd.2 (osd.2) 2 : cluster [DBG] 76.2 deep-scrub ok 2022-04-28T21:34:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:47 smithi059 bash[59745]: cluster 2022-04-28T21:34:45.993392+0000 mgr.y (mgr.24961) 3437 : cluster [DBG] pgmap v3932: 169 pgs: 169 active+clean; 96 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 15 MiB/s wr, 210 op/s 2022-04-28T21:34:47.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:47 smithi059 bash[59745]: cluster 2022-04-28T21:34:46.655646+0000 osd.2 (osd.2) 2 : cluster [DBG] 76.2 deep-scrub ok 2022-04-28T21:34:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:47 smithi139 bash[43301]: cluster 2022-04-28T21:34:45.993392+0000 mgr.y (mgr.24961) 3437 : cluster [DBG] pgmap v3932: 169 pgs: 169 active+clean; 96 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 15 MiB/s wr, 210 op/s 2022-04-28T21:34:47.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:47 smithi139 bash[43301]: cluster 2022-04-28T21:34:46.655646+0000 osd.2 (osd.2) 2 : cluster [DBG] 76.2 deep-scrub ok 2022-04-28T21:34:48.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[61152]: cluster 2022-04-28T21:34:47.079805+0000 osd.3 (osd.3) 3 : cluster [DBG] 76.9 deep-scrub ok 2022-04-28T21:34:49.029 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[61152]: audit 2022-04-28T21:34:47.861136+0000 mon.a (mon.0) 3241 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:34:49.029 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[61152]: audit 2022-04-28T21:34:47.864414+0000 mon.a (mon.0) 3242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:34:49.030 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[61152]: audit 2022-04-28T21:34:47.866484+0000 mon.a (mon.0) 3243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:34:49.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[59745]: cluster 2022-04-28T21:34:47.079805+0000 osd.3 (osd.3) 3 : cluster [DBG] 76.9 deep-scrub ok 2022-04-28T21:34:49.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[59745]: audit 2022-04-28T21:34:47.861136+0000 mon.a (mon.0) 3241 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:34:49.030 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[59745]: audit 2022-04-28T21:34:47.864414+0000 mon.a (mon.0) 3242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:34:49.031 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:48 smithi059 bash[59745]: audit 2022-04-28T21:34:47.866484+0000 mon.a (mon.0) 3243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:34:49.031 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:48 smithi139 bash[43301]: cluster 2022-04-28T21:34:47.079805+0000 osd.3 (osd.3) 3 : cluster [DBG] 76.9 deep-scrub ok 2022-04-28T21:34:49.031 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:48 smithi139 bash[43301]: audit 2022-04-28T21:34:47.861136+0000 mon.a (mon.0) 3241 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:34:49.032 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:48 smithi139 bash[43301]: audit 2022-04-28T21:34:47.864414+0000 mon.a (mon.0) 3242 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:34:49.032 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:48 smithi139 bash[43301]: audit 2022-04-28T21:34:47.866484+0000 mon.a (mon.0) 3243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:34:49.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:49 smithi059 bash[61152]: cephadm 2022-04-28T21:34:47.868274+0000 mgr.y (mgr.24961) 3438 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:34:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:49 smithi059 bash[61152]: cluster 2022-04-28T21:34:47.994160+0000 mgr.y (mgr.24961) 3439 : cluster [DBG] pgmap v3933: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 19 MiB/s wr, 256 op/s 2022-04-28T21:34:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:49 smithi059 bash[61152]: audit 2022-04-28T21:34:48.167891+0000 mon.a (mon.0) 3244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:34:49.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:49 smithi059 bash[59745]: cephadm 2022-04-28T21:34:47.868274+0000 mgr.y (mgr.24961) 3438 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:34:49.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:49 smithi059 bash[59745]: cluster 2022-04-28T21:34:47.994160+0000 mgr.y (mgr.24961) 3439 : cluster [DBG] pgmap v3933: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 19 MiB/s wr, 256 op/s 2022-04-28T21:34:49.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:49 smithi059 bash[59745]: audit 2022-04-28T21:34:48.167891+0000 mon.a (mon.0) 3244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:34:49.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:49 smithi139 bash[43301]: cephadm 2022-04-28T21:34:47.868274+0000 mgr.y (mgr.24961) 3438 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:34:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:49 smithi139 bash[43301]: cluster 2022-04-28T21:34:47.994160+0000 mgr.y (mgr.24961) 3439 : cluster [DBG] pgmap v3933: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 19 MiB/s wr, 256 op/s 2022-04-28T21:34:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:49 smithi139 bash[43301]: audit 2022-04-28T21:34:48.167891+0000 mon.a (mon.0) 3244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:34:51.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:51 smithi059 bash[61152]: cluster 2022-04-28T21:34:49.995072+0000 mgr.y (mgr.24961) 3440 : cluster [DBG] pgmap v3934: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 15 MiB/s wr, 207 op/s 2022-04-28T21:34:51.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:51 smithi059 bash[59745]: cluster 2022-04-28T21:34:49.995072+0000 mgr.y (mgr.24961) 3440 : cluster [DBG] pgmap v3934: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 15 MiB/s wr, 207 op/s 2022-04-28T21:34:51.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:51 smithi139 bash[43301]: cluster 2022-04-28T21:34:49.995072+0000 mgr.y (mgr.24961) 3440 : cluster [DBG] pgmap v3934: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 15 MiB/s wr, 207 op/s 2022-04-28T21:34:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:52.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:53.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:53 smithi059 bash[61152]: cluster 2022-04-28T21:34:51.996508+0000 mgr.y (mgr.24961) 3441 : cluster [DBG] pgmap v3935: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 14 MiB/s wr, 58 op/s 2022-04-28T21:34:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:53 smithi059 bash[59745]: cluster 2022-04-28T21:34:51.996508+0000 mgr.y (mgr.24961) 3441 : cluster [DBG] pgmap v3935: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 14 MiB/s wr, 58 op/s 2022-04-28T21:34:53.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:53 smithi139 bash[43301]: cluster 2022-04-28T21:34:51.996508+0000 mgr.y (mgr.24961) 3441 : cluster [DBG] pgmap v3935: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 14 MiB/s wr, 58 op/s 2022-04-28T21:34:54.191 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:34:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:34:53.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:34:54.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:54 smithi059 bash[61152]: cluster 2022-04-28T21:34:53.376955+0000 osd.4 (osd.4) 1 : cluster [DBG] 76.3 deep-scrub ok 2022-04-28T21:34:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:54 smithi059 bash[59745]: cluster 2022-04-28T21:34:53.376955+0000 osd.4 (osd.4) 1 : cluster [DBG] 76.3 deep-scrub ok 2022-04-28T21:34:54.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:54 smithi139 bash[43301]: cluster 2022-04-28T21:34:53.376955+0000 osd.4 (osd.4) 1 : cluster [DBG] 76.3 deep-scrub ok 2022-04-28T21:34:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:55 smithi059 bash[61152]: cluster 2022-04-28T21:34:53.997266+0000 mgr.y (mgr.24961) 3442 : cluster [DBG] pgmap v3936: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 12 MiB/s wr, 53 op/s 2022-04-28T21:34:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:55 smithi059 bash[59745]: cluster 2022-04-28T21:34:53.997266+0000 mgr.y (mgr.24961) 3442 : cluster [DBG] pgmap v3936: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 12 MiB/s wr, 53 op/s 2022-04-28T21:34:55.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:55 smithi139 bash[43301]: cluster 2022-04-28T21:34:53.997266+0000 mgr.y (mgr.24961) 3442 : cluster [DBG] pgmap v3936: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 12 MiB/s wr, 53 op/s 2022-04-28T21:34:57.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:57 smithi059 bash[61152]: cluster 2022-04-28T21:34:55.998867+0000 mgr.y (mgr.24961) 3443 : cluster [DBG] pgmap v3937: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 12 MiB/s wr, 53 op/s 2022-04-28T21:34:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:57 smithi059 bash[61152]: cluster 2022-04-28T21:34:56.316883+0000 osd.4 (osd.4) 2 : cluster [DBG] 76.7 deep-scrub ok 2022-04-28T21:34:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:57 smithi059 bash[59745]: cluster 2022-04-28T21:34:55.998867+0000 mgr.y (mgr.24961) 3443 : cluster [DBG] pgmap v3937: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 12 MiB/s wr, 53 op/s 2022-04-28T21:34:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:57 smithi059 bash[59745]: cluster 2022-04-28T21:34:56.316883+0000 osd.4 (osd.4) 2 : cluster [DBG] 76.7 deep-scrub ok 2022-04-28T21:34:57.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:57 smithi139 bash[43301]: cluster 2022-04-28T21:34:55.998867+0000 mgr.y (mgr.24961) 3443 : cluster [DBG] pgmap v3937: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 12 MiB/s wr, 53 op/s 2022-04-28T21:34:57.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:57 smithi139 bash[43301]: cluster 2022-04-28T21:34:56.316883+0000 osd.4 (osd.4) 2 : cluster [DBG] 76.7 deep-scrub ok 2022-04-28T21:34:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:34:59 smithi059 bash[61152]: cluster 2022-04-28T21:34:57.999610+0000 mgr.y (mgr.24961) 3444 : cluster [DBG] pgmap v3938: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.0 MiB/s wr, 49 op/s 2022-04-28T21:34:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:34:59 smithi059 bash[59745]: cluster 2022-04-28T21:34:57.999610+0000 mgr.y (mgr.24961) 3444 : cluster [DBG] pgmap v3938: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.0 MiB/s wr, 49 op/s 2022-04-28T21:34:59.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:34:59 smithi139 bash[43301]: cluster 2022-04-28T21:34:57.999610+0000 mgr.y (mgr.24961) 3444 : cluster [DBG] pgmap v3938: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.0 MiB/s wr, 49 op/s 2022-04-28T21:35:01.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:01 smithi059 bash[59745]: cluster 2022-04-28T21:35:00.000867+0000 mgr.y (mgr.24961) 3445 : cluster [DBG] pgmap v3939: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:01 smithi059 bash[61152]: cluster 2022-04-28T21:35:00.000867+0000 mgr.y (mgr.24961) 3445 : cluster [DBG] pgmap v3939: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:01.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:01 smithi139 bash[43301]: cluster 2022-04-28T21:35:00.000867+0000 mgr.y (mgr.24961) 3445 : cluster [DBG] pgmap v3939: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:02.031Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:03.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:03 smithi059 bash[61152]: cluster 2022-04-28T21:35:02.002232+0000 mgr.y (mgr.24961) 3446 : cluster [DBG] pgmap v3940: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:03.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:03 smithi059 bash[59745]: cluster 2022-04-28T21:35:02.002232+0000 mgr.y (mgr.24961) 3446 : cluster [DBG] pgmap v3940: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:03.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:03 smithi139 bash[43301]: cluster 2022-04-28T21:35:02.002232+0000 mgr.y (mgr.24961) 3446 : cluster [DBG] pgmap v3940: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:03.923Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:05.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:05 smithi059 bash[61152]: cluster 2022-04-28T21:35:04.002977+0000 mgr.y (mgr.24961) 3447 : cluster [DBG] pgmap v3941: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:05.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:05 smithi059 bash[59745]: cluster 2022-04-28T21:35:04.002977+0000 mgr.y (mgr.24961) 3447 : cluster [DBG] pgmap v3941: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:05.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:05 smithi139 bash[43301]: cluster 2022-04-28T21:35:04.002977+0000 mgr.y (mgr.24961) 3447 : cluster [DBG] pgmap v3941: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:07.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:07 smithi059 bash[61152]: cluster 2022-04-28T21:35:06.004296+0000 mgr.y (mgr.24961) 3448 : cluster [DBG] pgmap v3942: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:07.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:07 smithi059 bash[59745]: cluster 2022-04-28T21:35:06.004296+0000 mgr.y (mgr.24961) 3448 : cluster [DBG] pgmap v3942: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:07.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:07 smithi139 bash[43301]: cluster 2022-04-28T21:35:06.004296+0000 mgr.y (mgr.24961) 3448 : cluster [DBG] pgmap v3942: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:09.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:09 smithi059 bash[61152]: cluster 2022-04-28T21:35:08.005308+0000 mgr.y (mgr.24961) 3449 : cluster [DBG] pgmap v3943: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:09.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:09 smithi059 bash[59745]: cluster 2022-04-28T21:35:08.005308+0000 mgr.y (mgr.24961) 3449 : cluster [DBG] pgmap v3943: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:09.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:09 smithi139 bash[43301]: cluster 2022-04-28T21:35:08.005308+0000 mgr.y (mgr.24961) 3449 : cluster [DBG] pgmap v3943: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:11.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:11 smithi059 bash[61152]: cluster 2022-04-28T21:35:10.006622+0000 mgr.y (mgr.24961) 3450 : cluster [DBG] pgmap v3944: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:11.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:11 smithi059 bash[59745]: cluster 2022-04-28T21:35:10.006622+0000 mgr.y (mgr.24961) 3450 : cluster [DBG] pgmap v3944: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:11 smithi139 bash[43301]: cluster 2022-04-28T21:35:10.006622+0000 mgr.y (mgr.24961) 3450 : cluster [DBG] pgmap v3944: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:12.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: cluster 2022-04-28T21:35:12.007999+0000 mgr.y (mgr.24961) 3451 : cluster [DBG] pgmap v3945: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.484863+0000 mon.c (mon.1) 106 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:35:13.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.485190+0000 mgr.y (mgr.24961) 3452 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:35:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.485756+0000 mon.c (mon.1) 107 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:35:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.485961+0000 mgr.y (mgr.24961) 3453 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:35:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.486648+0000 mon.c (mon.1) 108 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:35:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.486816+0000 mgr.y (mgr.24961) 3454 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:35:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.487311+0000 mon.c (mon.1) 109 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:35:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.487526+0000 mgr.y (mgr.24961) 3455 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:35:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.487976+0000 mon.c (mon.1) 110 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:35:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.488157+0000 mgr.y (mgr.24961) 3456 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:35:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.488688+0000 mon.c (mon.1) 111 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:35:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.488874+0000 mgr.y (mgr.24961) 3457 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:35:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.489462+0000 mon.c (mon.1) 112 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:35:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.489655+0000 mgr.y (mgr.24961) 3458 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:35:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.490205+0000 mon.c (mon.1) 113 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:35:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.490399+0000 mgr.y (mgr.24961) 3459 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:35:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.490923+0000 mon.c (mon.1) 114 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:35:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.491114+0000 mgr.y (mgr.24961) 3460 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:35:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.491768+0000 mon.c (mon.1) 115 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:35:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: audit 2022-04-28T21:35:12.491963+0000 mgr.y (mgr.24961) 3461 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:35:13.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[59745]: cluster 2022-04-28T21:35:12.580879+0000 osd.2 (osd.2) 3 : cluster [DBG] 76.0 deep-scrub ok 2022-04-28T21:35:13.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: cluster 2022-04-28T21:35:12.007999+0000 mgr.y (mgr.24961) 3451 : cluster [DBG] pgmap v3945: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:13.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.484863+0000 mon.c (mon.1) 106 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:35:13.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.485190+0000 mgr.y (mgr.24961) 3452 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:35:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.485756+0000 mon.c (mon.1) 107 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:35:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.485961+0000 mgr.y (mgr.24961) 3453 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:35:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.486648+0000 mon.c (mon.1) 108 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:35:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.486816+0000 mgr.y (mgr.24961) 3454 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:35:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.487311+0000 mon.c (mon.1) 109 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:35:13.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.487526+0000 mgr.y (mgr.24961) 3455 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:35:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.487976+0000 mon.c (mon.1) 110 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:35:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.488157+0000 mgr.y (mgr.24961) 3456 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:35:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.488688+0000 mon.c (mon.1) 111 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:35:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.488874+0000 mgr.y (mgr.24961) 3457 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:35:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.489462+0000 mon.c (mon.1) 112 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:35:13.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.489655+0000 mgr.y (mgr.24961) 3458 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:35:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.490205+0000 mon.c (mon.1) 113 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:35:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.490399+0000 mgr.y (mgr.24961) 3459 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:35:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.490923+0000 mon.c (mon.1) 114 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:35:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.491114+0000 mgr.y (mgr.24961) 3460 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:35:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.491768+0000 mon.c (mon.1) 115 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:35:13.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: audit 2022-04-28T21:35:12.491963+0000 mgr.y (mgr.24961) 3461 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:35:13.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[61152]: cluster 2022-04-28T21:35:12.580879+0000 osd.2 (osd.2) 3 : cluster [DBG] 76.0 deep-scrub ok 2022-04-28T21:35:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: cluster 2022-04-28T21:35:12.007999+0000 mgr.y (mgr.24961) 3451 : cluster [DBG] pgmap v3945: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.484863+0000 mon.c (mon.1) 106 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:35:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.485190+0000 mgr.y (mgr.24961) 3452 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.0"}]: dispatch 2022-04-28T21:35:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.485756+0000 mon.c (mon.1) 107 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:35:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.485961+0000 mgr.y (mgr.24961) 3453 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.1"}]: dispatch 2022-04-28T21:35:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.486648+0000 mon.c (mon.1) 108 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:35:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.486816+0000 mgr.y (mgr.24961) 3454 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.2"}]: dispatch 2022-04-28T21:35:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.487311+0000 mon.c (mon.1) 109 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:35:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.487526+0000 mgr.y (mgr.24961) 3455 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.3"}]: dispatch 2022-04-28T21:35:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.487976+0000 mon.c (mon.1) 110 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:35:13.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.488157+0000 mgr.y (mgr.24961) 3456 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.4"}]: dispatch 2022-04-28T21:35:13.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.488688+0000 mon.c (mon.1) 111 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:35:13.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.488874+0000 mgr.y (mgr.24961) 3457 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.5"}]: dispatch 2022-04-28T21:35:13.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.489462+0000 mon.c (mon.1) 112 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:35:13.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.489655+0000 mgr.y (mgr.24961) 3458 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.6"}]: dispatch 2022-04-28T21:35:13.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.490205+0000 mon.c (mon.1) 113 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:35:13.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.490399+0000 mgr.y (mgr.24961) 3459 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.7"}]: dispatch 2022-04-28T21:35:13.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.490923+0000 mon.c (mon.1) 114 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:35:13.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.491114+0000 mgr.y (mgr.24961) 3460 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.8"}]: dispatch 2022-04-28T21:35:13.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.491768+0000 mon.c (mon.1) 115 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:35:13.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: audit 2022-04-28T21:35:12.491963+0000 mgr.y (mgr.24961) 3461 : audit [DBG] from='mon.1 -' entity='mon.' cmd=[{"prefix": "pg deep-scrub", "pgid": "76.9"}]: dispatch 2022-04-28T21:35:13.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:13 smithi139 bash[43301]: cluster 2022-04-28T21:35:12.580879+0000 osd.2 (osd.2) 3 : cluster [DBG] 76.0 deep-scrub ok 2022-04-28T21:35:14.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:13.922Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:15.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:15 smithi059 bash[59745]: cluster 2022-04-28T21:35:13.577388+0000 osd.0 (osd.0) 2 : cluster [DBG] 76.4 deep-scrub ok 2022-04-28T21:35:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:15 smithi059 bash[59745]: cluster 2022-04-28T21:35:14.008753+0000 mgr.y (mgr.24961) 3462 : cluster [DBG] pgmap v3946: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:15.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:15 smithi059 bash[59745]: cluster 2022-04-28T21:35:14.970396+0000 osd.1 (osd.1) 3 : cluster [DBG] 76.5 deep-scrub ok 2022-04-28T21:35:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:15 smithi059 bash[61152]: cluster 2022-04-28T21:35:13.577388+0000 osd.0 (osd.0) 2 : cluster [DBG] 76.4 deep-scrub ok 2022-04-28T21:35:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:15 smithi059 bash[61152]: cluster 2022-04-28T21:35:14.008753+0000 mgr.y (mgr.24961) 3462 : cluster [DBG] pgmap v3946: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:15.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:15 smithi059 bash[61152]: cluster 2022-04-28T21:35:14.970396+0000 osd.1 (osd.1) 3 : cluster [DBG] 76.5 deep-scrub ok 2022-04-28T21:35:15.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:15 smithi139 bash[43301]: cluster 2022-04-28T21:35:13.577388+0000 osd.0 (osd.0) 2 : cluster [DBG] 76.4 deep-scrub ok 2022-04-28T21:35:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:15 smithi139 bash[43301]: cluster 2022-04-28T21:35:14.008753+0000 mgr.y (mgr.24961) 3462 : cluster [DBG] pgmap v3946: 169 pgs: 169 active+clean; 144 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:15 smithi139 bash[43301]: cluster 2022-04-28T21:35:14.970396+0000 osd.1 (osd.1) 3 : cluster [DBG] 76.5 deep-scrub ok 2022-04-28T21:35:16.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:16 smithi059 bash[61152]: cluster 2022-04-28T21:35:15.601542+0000 osd.2 (osd.2) 4 : cluster [DBG] 76.2 deep-scrub ok 2022-04-28T21:35:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:16 smithi059 bash[61152]: cluster 2022-04-28T21:35:15.937504+0000 osd.1 (osd.1) 4 : cluster [DBG] 76.6 deep-scrub ok 2022-04-28T21:35:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:16 smithi059 bash[59745]: cluster 2022-04-28T21:35:15.601542+0000 osd.2 (osd.2) 4 : cluster [DBG] 76.2 deep-scrub ok 2022-04-28T21:35:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:16 smithi059 bash[59745]: cluster 2022-04-28T21:35:15.937504+0000 osd.1 (osd.1) 4 : cluster [DBG] 76.6 deep-scrub ok 2022-04-28T21:35:16.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:16 smithi139 bash[43301]: cluster 2022-04-28T21:35:15.601542+0000 osd.2 (osd.2) 4 : cluster [DBG] 76.2 deep-scrub ok 2022-04-28T21:35:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:16 smithi139 bash[43301]: cluster 2022-04-28T21:35:15.937504+0000 osd.1 (osd.1) 4 : cluster [DBG] 76.6 deep-scrub ok 2022-04-28T21:35:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:17 smithi059 bash[61152]: cluster 2022-04-28T21:35:16.010447+0000 mgr.y (mgr.24961) 3463 : cluster [DBG] pgmap v3947: 169 pgs: 169 active+clean; 192 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s wr, 0 op/s 2022-04-28T21:35:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:17 smithi059 bash[61152]: cluster 2022-04-28T21:35:16.218961+0000 osd.3 (osd.3) 4 : cluster [DBG] 76.1 deep-scrub ok 2022-04-28T21:35:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:17 smithi059 bash[59745]: cluster 2022-04-28T21:35:16.010447+0000 mgr.y (mgr.24961) 3463 : cluster [DBG] pgmap v3947: 169 pgs: 169 active+clean; 192 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s wr, 0 op/s 2022-04-28T21:35:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:17 smithi059 bash[59745]: cluster 2022-04-28T21:35:16.218961+0000 osd.3 (osd.3) 4 : cluster [DBG] 76.1 deep-scrub ok 2022-04-28T21:35:17.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:17 smithi139 bash[43301]: cluster 2022-04-28T21:35:16.010447+0000 mgr.y (mgr.24961) 3463 : cluster [DBG] pgmap v3947: 169 pgs: 169 active+clean; 192 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s wr, 0 op/s 2022-04-28T21:35:17.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:17 smithi139 bash[43301]: cluster 2022-04-28T21:35:16.218961+0000 osd.3 (osd.3) 4 : cluster [DBG] 76.1 deep-scrub ok 2022-04-28T21:35:18.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:18 smithi059 bash[61152]: cluster 2022-04-28T21:35:17.186283+0000 osd.3 (osd.3) 5 : cluster [DBG] 76.8 deep-scrub ok 2022-04-28T21:35:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:18 smithi059 bash[61152]: cluster 2022-04-28T21:35:17.424963+0000 osd.4 (osd.4) 3 : cluster [DBG] 76.3 deep-scrub ok 2022-04-28T21:35:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:18 smithi059 bash[59745]: cluster 2022-04-28T21:35:17.186283+0000 osd.3 (osd.3) 5 : cluster [DBG] 76.8 deep-scrub ok 2022-04-28T21:35:18.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:18 smithi059 bash[59745]: cluster 2022-04-28T21:35:17.424963+0000 osd.4 (osd.4) 3 : cluster [DBG] 76.3 deep-scrub ok 2022-04-28T21:35:18.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:18 smithi139 bash[43301]: cluster 2022-04-28T21:35:17.186283+0000 osd.3 (osd.3) 5 : cluster [DBG] 76.8 deep-scrub ok 2022-04-28T21:35:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:18 smithi139 bash[43301]: cluster 2022-04-28T21:35:17.424963+0000 osd.4 (osd.4) 3 : cluster [DBG] 76.3 deep-scrub ok 2022-04-28T21:35:19.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:19 smithi059 bash[61152]: cluster 2022-04-28T21:35:18.011623+0000 mgr.y (mgr.24961) 3464 : cluster [DBG] pgmap v3948: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:19.728 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:19 smithi059 bash[61152]: cluster 2022-04-28T21:35:18.417254+0000 osd.4 (osd.4) 4 : cluster [DBG] 76.7 deep-scrub ok 2022-04-28T21:35:19.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:19 smithi059 bash[59745]: cluster 2022-04-28T21:35:18.011623+0000 mgr.y (mgr.24961) 3464 : cluster [DBG] pgmap v3948: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:19.728 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:19 smithi059 bash[59745]: cluster 2022-04-28T21:35:18.417254+0000 osd.4 (osd.4) 4 : cluster [DBG] 76.7 deep-scrub ok 2022-04-28T21:35:19.729 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:19 smithi139 bash[43301]: cluster 2022-04-28T21:35:18.011623+0000 mgr.y (mgr.24961) 3464 : cluster [DBG] pgmap v3948: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:19.729 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:19 smithi139 bash[43301]: cluster 2022-04-28T21:35:18.417254+0000 osd.4 (osd.4) 4 : cluster [DBG] 76.7 deep-scrub ok 2022-04-28T21:35:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: cluster 2022-04-28T21:35:19.188889+0000 osd.3 (osd.3) 6 : cluster [DBG] 76.9 deep-scrub ok 2022-04-28T21:35:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: audit 2022-04-28T21:35:19.294225+0000 mon.a (mon.0) 3245 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:35:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: audit 2022-04-28T21:35:19.294610+0000 mon.a (mon.0) 3246 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:35:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: audit 2022-04-28T21:35:19.305165+0000 mon.a (mon.0) 3247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:35:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: audit 2022-04-28T21:35:19.305765+0000 mon.a (mon.0) 3248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:35:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: audit 2022-04-28T21:35:19.306316+0000 mon.a (mon.0) 3249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:35:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[61152]: audit 2022-04-28T21:35:19.306800+0000 mon.a (mon.0) 3250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:35:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: cluster 2022-04-28T21:35:19.188889+0000 osd.3 (osd.3) 6 : cluster [DBG] 76.9 deep-scrub ok 2022-04-28T21:35:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: audit 2022-04-28T21:35:19.294225+0000 mon.a (mon.0) 3245 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:35:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: audit 2022-04-28T21:35:19.294610+0000 mon.a (mon.0) 3246 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:35:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: audit 2022-04-28T21:35:19.305165+0000 mon.a (mon.0) 3247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:35:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: audit 2022-04-28T21:35:19.305765+0000 mon.a (mon.0) 3248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:35:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: audit 2022-04-28T21:35:19.306316+0000 mon.a (mon.0) 3249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:35:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:20 smithi059 bash[59745]: audit 2022-04-28T21:35:19.306800+0000 mon.a (mon.0) 3250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:35:20.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: cluster 2022-04-28T21:35:19.188889+0000 osd.3 (osd.3) 6 : cluster [DBG] 76.9 deep-scrub ok 2022-04-28T21:35:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: audit 2022-04-28T21:35:19.294225+0000 mon.a (mon.0) 3245 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:35:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: audit 2022-04-28T21:35:19.294610+0000 mon.a (mon.0) 3246 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:35:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: audit 2022-04-28T21:35:19.305165+0000 mon.a (mon.0) 3247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:35:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: audit 2022-04-28T21:35:19.305765+0000 mon.a (mon.0) 3248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:35:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: audit 2022-04-28T21:35:19.306316+0000 mon.a (mon.0) 3249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:35:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:20 smithi139 bash[43301]: audit 2022-04-28T21:35:19.306800+0000 mon.a (mon.0) 3250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:35:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:21 smithi059 bash[61152]: cluster 2022-04-28T21:35:20.012933+0000 mgr.y (mgr.24961) 3465 : cluster [DBG] pgmap v3949: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:21.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:21 smithi059 bash[59745]: cluster 2022-04-28T21:35:20.012933+0000 mgr.y (mgr.24961) 3465 : cluster [DBG] pgmap v3949: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:21 smithi139 bash[43301]: cluster 2022-04-28T21:35:20.012933+0000 mgr.y (mgr.24961) 3465 : cluster [DBG] pgmap v3949: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:22.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:23 smithi059 bash[61152]: cluster 2022-04-28T21:35:22.014231+0000 mgr.y (mgr.24961) 3466 : cluster [DBG] pgmap v3950: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:23 smithi059 bash[59745]: cluster 2022-04-28T21:35:22.014231+0000 mgr.y (mgr.24961) 3466 : cluster [DBG] pgmap v3950: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:23 smithi139 bash[43301]: cluster 2022-04-28T21:35:22.014231+0000 mgr.y (mgr.24961) 3466 : cluster [DBG] pgmap v3950: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:23.925Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:25 smithi059 bash[61152]: cluster 2022-04-28T21:35:24.014944+0000 mgr.y (mgr.24961) 3467 : cluster [DBG] pgmap v3951: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:25 smithi059 bash[59745]: cluster 2022-04-28T21:35:24.014944+0000 mgr.y (mgr.24961) 3467 : cluster [DBG] pgmap v3951: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:25 smithi139 bash[43301]: cluster 2022-04-28T21:35:24.014944+0000 mgr.y (mgr.24961) 3467 : cluster [DBG] pgmap v3951: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:27.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:27 smithi059 bash[61152]: cluster 2022-04-28T21:35:26.016641+0000 mgr.y (mgr.24961) 3468 : cluster [DBG] pgmap v3952: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:27 smithi059 bash[59745]: cluster 2022-04-28T21:35:26.016641+0000 mgr.y (mgr.24961) 3468 : cluster [DBG] pgmap v3952: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:27 smithi139 bash[43301]: cluster 2022-04-28T21:35:26.016641+0000 mgr.y (mgr.24961) 3468 : cluster [DBG] pgmap v3952: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s wr, 0 op/s 2022-04-28T21:35:29.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:29 smithi059 bash[61152]: cluster 2022-04-28T21:35:28.017806+0000 mgr.y (mgr.24961) 3469 : cluster [DBG] pgmap v3953: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s wr, 0 op/s 2022-04-28T21:35:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:29 smithi059 bash[59745]: cluster 2022-04-28T21:35:28.017806+0000 mgr.y (mgr.24961) 3469 : cluster [DBG] pgmap v3953: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s wr, 0 op/s 2022-04-28T21:35:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:29 smithi139 bash[43301]: cluster 2022-04-28T21:35:28.017806+0000 mgr.y (mgr.24961) 3469 : cluster [DBG] pgmap v3953: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s wr, 0 op/s 2022-04-28T21:35:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:31 smithi059 bash[61152]: cluster 2022-04-28T21:35:30.019110+0000 mgr.y (mgr.24961) 3470 : cluster [DBG] pgmap v3954: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:31 smithi059 bash[59745]: cluster 2022-04-28T21:35:30.019110+0000 mgr.y (mgr.24961) 3470 : cluster [DBG] pgmap v3954: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:31 smithi139 bash[43301]: cluster 2022-04-28T21:35:30.019110+0000 mgr.y (mgr.24961) 3470 : cluster [DBG] pgmap v3954: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:32.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:33.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:33 smithi059 bash[61152]: cluster 2022-04-28T21:35:32.020542+0000 mgr.y (mgr.24961) 3471 : cluster [DBG] pgmap v3955: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:33.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:33 smithi059 bash[59745]: cluster 2022-04-28T21:35:32.020542+0000 mgr.y (mgr.24961) 3471 : cluster [DBG] pgmap v3955: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:33 smithi139 bash[43301]: cluster 2022-04-28T21:35:32.020542+0000 mgr.y (mgr.24961) 3471 : cluster [DBG] pgmap v3955: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:33.926Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:35.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:35 smithi059 bash[61152]: cluster 2022-04-28T21:35:34.021279+0000 mgr.y (mgr.24961) 3472 : cluster [DBG] pgmap v3956: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:35 smithi059 bash[59745]: cluster 2022-04-28T21:35:34.021279+0000 mgr.y (mgr.24961) 3472 : cluster [DBG] pgmap v3956: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:35 smithi139 bash[43301]: cluster 2022-04-28T21:35:34.021279+0000 mgr.y (mgr.24961) 3472 : cluster [DBG] pgmap v3956: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:37 smithi059 bash[61152]: cluster 2022-04-28T21:35:36.022929+0000 mgr.y (mgr.24961) 3473 : cluster [DBG] pgmap v3957: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:37 smithi059 bash[59745]: cluster 2022-04-28T21:35:36.022929+0000 mgr.y (mgr.24961) 3473 : cluster [DBG] pgmap v3957: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:37 smithi139 bash[43301]: cluster 2022-04-28T21:35:36.022929+0000 mgr.y (mgr.24961) 3473 : cluster [DBG] pgmap v3957: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:39 smithi139 bash[43301]: cluster 2022-04-28T21:35:38.023959+0000 mgr.y (mgr.24961) 3474 : cluster [DBG] pgmap v3958: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:39.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:39 smithi059 bash[61152]: cluster 2022-04-28T21:35:38.023959+0000 mgr.y (mgr.24961) 3474 : cluster [DBG] pgmap v3958: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:39 smithi059 bash[59745]: cluster 2022-04-28T21:35:38.023959+0000 mgr.y (mgr.24961) 3474 : cluster [DBG] pgmap v3958: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:41.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:41 smithi139 bash[43301]: cluster 2022-04-28T21:35:40.025241+0000 mgr.y (mgr.24961) 3475 : cluster [DBG] pgmap v3959: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:41.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:41 smithi059 bash[61152]: cluster 2022-04-28T21:35:40.025241+0000 mgr.y (mgr.24961) 3475 : cluster [DBG] pgmap v3959: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:41.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:41 smithi059 bash[59745]: cluster 2022-04-28T21:35:40.025241+0000 mgr.y (mgr.24961) 3475 : cluster [DBG] pgmap v3959: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:42.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:43 smithi139 bash[43301]: cluster 2022-04-28T21:35:42.027005+0000 mgr.y (mgr.24961) 3476 : cluster [DBG] pgmap v3960: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:43.638 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:43 smithi139 bash[43301]: audit 2022-04-28T21:35:42.504279+0000 mon.c (mon.1) 116 : audit [DBG] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2022-04-28T21:35:43.638 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:43 smithi139 bash[43301]: audit 2022-04-28T21:35:42.504915+0000 mon.c (mon.1) 117 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:43.638 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:43 smithi139 bash[43301]: audit 2022-04-28T21:35:42.505259+0000 mon.a (mon.0) 3251 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:43.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[61152]: cluster 2022-04-28T21:35:42.027005+0000 mgr.y (mgr.24961) 3476 : cluster [DBG] pgmap v3960: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[61152]: audit 2022-04-28T21:35:42.504279+0000 mon.c (mon.1) 116 : audit [DBG] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2022-04-28T21:35:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[61152]: audit 2022-04-28T21:35:42.504915+0000 mon.c (mon.1) 117 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[61152]: audit 2022-04-28T21:35:42.505259+0000 mon.a (mon.0) 3251 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[59745]: cluster 2022-04-28T21:35:42.027005+0000 mgr.y (mgr.24961) 3476 : cluster [DBG] pgmap v3960: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[59745]: audit 2022-04-28T21:35:42.504279+0000 mon.c (mon.1) 116 : audit [DBG] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2022-04-28T21:35:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[59745]: audit 2022-04-28T21:35:42.504915+0000 mon.c (mon.1) 117 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[59745]: audit 2022-04-28T21:35:42.505259+0000 mon.a (mon.0) 3251 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:43.927Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:44.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[61152]: audit 2022-04-28T21:35:43.358179+0000 mon.a (mon.0) 3252 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[61152]: cluster 2022-04-28T21:35:43.358414+0000 mon.a (mon.0) 3253 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2022-04-28T21:35:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[61152]: audit 2022-04-28T21:35:43.364604+0000 mon.c (mon.1) 118 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2"}]: dispatch 2022-04-28T21:35:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[61152]: audit 2022-04-28T21:35:43.364999+0000 mon.c (mon.1) 119 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[61152]: audit 2022-04-28T21:35:43.365301+0000 mon.a (mon.0) 3254 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[59745]: audit 2022-04-28T21:35:43.358179+0000 mon.a (mon.0) 3252 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[59745]: cluster 2022-04-28T21:35:43.358414+0000 mon.a (mon.0) 3253 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2022-04-28T21:35:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[59745]: audit 2022-04-28T21:35:43.364604+0000 mon.c (mon.1) 118 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2"}]: dispatch 2022-04-28T21:35:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[59745]: audit 2022-04-28T21:35:43.364999+0000 mon.c (mon.1) 119 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:44 smithi059 bash[59745]: audit 2022-04-28T21:35:43.365301+0000 mon.a (mon.0) 3254 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:44.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:44 smithi139 bash[43301]: audit 2022-04-28T21:35:43.358179+0000 mon.a (mon.0) 3252 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app1","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:44 smithi139 bash[43301]: cluster 2022-04-28T21:35:43.358414+0000 mon.a (mon.0) 3253 : cluster [DBG] osdmap e1116: 8 total, 8 up, 8 in 2022-04-28T21:35:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:44 smithi139 bash[43301]: audit 2022-04-28T21:35:43.364604+0000 mon.c (mon.1) 118 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2"}]: dispatch 2022-04-28T21:35:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:44 smithi139 bash[43301]: audit 2022-04-28T21:35:43.364999+0000 mon.c (mon.1) 119 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:44 smithi139 bash[43301]: audit 2022-04-28T21:35:43.365301+0000 mon.a (mon.0) 3254 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[61152]: cluster 2022-04-28T21:35:44.027799+0000 mgr.y (mgr.24961) 3477 : cluster [DBG] pgmap v3962: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[61152]: audit 2022-04-28T21:35:44.368593+0000 mon.a (mon.0) 3255 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[61152]: cluster 2022-04-28T21:35:44.368733+0000 mon.a (mon.0) 3256 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2022-04-28T21:35:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[61152]: audit 2022-04-28T21:35:44.379126+0000 mon.c (mon.1) 120 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"dne","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[61152]: audit 2022-04-28T21:35:44.379621+0000 mon.c (mon.1) 121 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[61152]: audit 2022-04-28T21:35:44.380040+0000 mon.a (mon.0) 3257 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[59745]: cluster 2022-04-28T21:35:44.027799+0000 mgr.y (mgr.24961) 3477 : cluster [DBG] pgmap v3962: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[59745]: audit 2022-04-28T21:35:44.368593+0000 mon.a (mon.0) 3255 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:45.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[59745]: cluster 2022-04-28T21:35:44.368733+0000 mon.a (mon.0) 3256 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2022-04-28T21:35:45.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[59745]: audit 2022-04-28T21:35:44.379126+0000 mon.c (mon.1) 120 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"dne","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[59745]: audit 2022-04-28T21:35:44.379621+0000 mon.c (mon.1) 121 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:45 smithi059 bash[59745]: audit 2022-04-28T21:35:44.380040+0000 mon.a (mon.0) 3257 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:45 smithi139 bash[43301]: cluster 2022-04-28T21:35:44.027799+0000 mgr.y (mgr.24961) 3477 : cluster [DBG] pgmap v3962: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:45 smithi139 bash[43301]: audit 2022-04-28T21:35:44.368593+0000 mon.a (mon.0) 3255 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-1","app": "app2","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:45 smithi139 bash[43301]: cluster 2022-04-28T21:35:44.368733+0000 mon.a (mon.0) 3256 : cluster [DBG] osdmap e1117: 8 total, 8 up, 8 in 2022-04-28T21:35:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:45 smithi139 bash[43301]: audit 2022-04-28T21:35:44.379126+0000 mon.c (mon.1) 120 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"dne","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:45 smithi139 bash[43301]: audit 2022-04-28T21:35:44.379621+0000 mon.c (mon.1) 121 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:45.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:45 smithi139 bash[43301]: audit 2022-04-28T21:35:44.380040+0000 mon.a (mon.0) 3257 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]: dispatch 2022-04-28T21:35:46.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[59745]: audit 2022-04-28T21:35:45.387571+0000 mon.a (mon.0) 3258 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]': finished 2022-04-28T21:35:46.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[59745]: cluster 2022-04-28T21:35:45.387667+0000 mon.a (mon.0) 3259 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2022-04-28T21:35:46.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[59745]: audit 2022-04-28T21:35:45.389823+0000 mon.c (mon.1) 122 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]: dispatch 2022-04-28T21:35:46.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[59745]: audit 2022-04-28T21:35:45.391879+0000 mon.a (mon.0) 3260 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]: dispatch 2022-04-28T21:35:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[61152]: audit 2022-04-28T21:35:45.387571+0000 mon.a (mon.0) 3258 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]': finished 2022-04-28T21:35:46.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[61152]: cluster 2022-04-28T21:35:45.387667+0000 mon.a (mon.0) 3259 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2022-04-28T21:35:46.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[61152]: audit 2022-04-28T21:35:45.389823+0000 mon.c (mon.1) 122 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]: dispatch 2022-04-28T21:35:46.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:46 smithi059 bash[61152]: audit 2022-04-28T21:35:45.391879+0000 mon.a (mon.0) 3260 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]: dispatch 2022-04-28T21:35:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:46 smithi139 bash[43301]: audit 2022-04-28T21:35:45.387571+0000 mon.a (mon.0) 3258 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1","value":"value1"}]': finished 2022-04-28T21:35:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:46 smithi139 bash[43301]: cluster 2022-04-28T21:35:45.387667+0000 mon.a (mon.0) 3259 : cluster [DBG] osdmap e1118: 8 total, 8 up, 8 in 2022-04-28T21:35:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:46 smithi139 bash[43301]: audit 2022-04-28T21:35:45.389823+0000 mon.c (mon.1) 122 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]: dispatch 2022-04-28T21:35:46.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:46 smithi139 bash[43301]: audit 2022-04-28T21:35:45.391879+0000 mon.a (mon.0) 3260 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]: dispatch 2022-04-28T21:35:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[61152]: cluster 2022-04-28T21:35:46.028709+0000 mgr.y (mgr.24961) 3478 : cluster [DBG] pgmap v3965: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 682 B/s wr, 0 op/s 2022-04-28T21:35:47.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[61152]: audit 2022-04-28T21:35:46.402048+0000 mon.a (mon.0) 3261 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]': finished 2022-04-28T21:35:47.856 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[61152]: cluster 2022-04-28T21:35:46.402179+0000 mon.a (mon.0) 3262 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2022-04-28T21:35:47.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[61152]: audit 2022-04-28T21:35:46.404942+0000 mon.c (mon.1) 123 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]: dispatch 2022-04-28T21:35:47.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[61152]: audit 2022-04-28T21:35:46.405427+0000 mon.a (mon.0) 3263 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]: dispatch 2022-04-28T21:35:47.859 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[59745]: cluster 2022-04-28T21:35:46.028709+0000 mgr.y (mgr.24961) 3478 : cluster [DBG] pgmap v3965: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 682 B/s wr, 0 op/s 2022-04-28T21:35:47.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[59745]: audit 2022-04-28T21:35:46.402048+0000 mon.a (mon.0) 3261 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]': finished 2022-04-28T21:35:47.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[59745]: cluster 2022-04-28T21:35:46.402179+0000 mon.a (mon.0) 3262 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2022-04-28T21:35:47.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[59745]: audit 2022-04-28T21:35:46.404942+0000 mon.c (mon.1) 123 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]: dispatch 2022-04-28T21:35:47.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:47 smithi059 bash[59745]: audit 2022-04-28T21:35:46.405427+0000 mon.a (mon.0) 3263 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]: dispatch 2022-04-28T21:35:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:47 smithi139 bash[43301]: cluster 2022-04-28T21:35:46.028709+0000 mgr.y (mgr.24961) 3478 : cluster [DBG] pgmap v3965: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 682 B/s wr, 0 op/s 2022-04-28T21:35:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:47 smithi139 bash[43301]: audit 2022-04-28T21:35:46.402048+0000 mon.a (mon.0) 3261 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application set","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key2","value":"value2"}]': finished 2022-04-28T21:35:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:47 smithi139 bash[43301]: cluster 2022-04-28T21:35:46.402179+0000 mon.a (mon.0) 3262 : cluster [DBG] osdmap e1119: 8 total, 8 up, 8 in 2022-04-28T21:35:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:47 smithi139 bash[43301]: audit 2022-04-28T21:35:46.404942+0000 mon.c (mon.1) 123 : audit [INF] from='client.? 172.21.15.59:0/25340382' entity='client.admin' cmd=[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]: dispatch 2022-04-28T21:35:47.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:47 smithi139 bash[43301]: audit 2022-04-28T21:35:46.405427+0000 mon.a (mon.0) 3263 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]: dispatch 2022-04-28T21:35:48.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[61152]: audit 2022-04-28T21:35:47.403828+0000 mon.a (mon.0) 3264 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]': finished 2022-04-28T21:35:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[61152]: cluster 2022-04-28T21:35:47.403984+0000 mon.a (mon.0) 3265 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2022-04-28T21:35:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[61152]: audit 2022-04-28T21:35:48.175735+0000 mon.a (mon.0) 3266 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:35:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[61152]: audit 2022-04-28T21:35:48.177937+0000 mon.a (mon.0) 3267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:35:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[61152]: audit 2022-04-28T21:35:48.179682+0000 mon.a (mon.0) 3268 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:35:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[59745]: audit 2022-04-28T21:35:47.403828+0000 mon.a (mon.0) 3264 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]': finished 2022-04-28T21:35:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[59745]: cluster 2022-04-28T21:35:47.403984+0000 mon.a (mon.0) 3265 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2022-04-28T21:35:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[59745]: audit 2022-04-28T21:35:48.175735+0000 mon.a (mon.0) 3266 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:35:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[59745]: audit 2022-04-28T21:35:48.177937+0000 mon.a (mon.0) 3267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:35:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:48 smithi059 bash[59745]: audit 2022-04-28T21:35:48.179682+0000 mon.a (mon.0) 3268 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:35:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:48 smithi139 bash[43301]: audit 2022-04-28T21:35:47.403828+0000 mon.a (mon.0) 3264 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix":"osd pool application rm","pool":"test-rados-api-smithi059-83986-1","app":"app1","key":"key1"}]': finished 2022-04-28T21:35:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:48 smithi139 bash[43301]: cluster 2022-04-28T21:35:47.403984+0000 mon.a (mon.0) 3265 : cluster [DBG] osdmap e1120: 8 total, 8 up, 8 in 2022-04-28T21:35:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:48 smithi139 bash[43301]: audit 2022-04-28T21:35:48.175735+0000 mon.a (mon.0) 3266 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:35:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:48 smithi139 bash[43301]: audit 2022-04-28T21:35:48.177937+0000 mon.a (mon.0) 3267 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:35:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:48 smithi139 bash[43301]: audit 2022-04-28T21:35:48.179682+0000 mon.a (mon.0) 3268 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:35:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: cluster 2022-04-28T21:35:48.029458+0000 mgr.y (mgr.24961) 3479 : cluster [DBG] pgmap v3968: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T21:35:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: cephadm 2022-04-28T21:35:48.181216+0000 mgr.y (mgr.24961) 3480 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:35:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: cluster 2022-04-28T21:35:48.416751+0000 mon.a (mon.0) 3269 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2022-04-28T21:35:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: audit 2022-04-28T21:35:48.438036+0000 mon.a (mon.0) 3270 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: audit 2022-04-28T21:35:48.438442+0000 mon.a (mon.0) 3271 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: audit 2022-04-28T21:35:48.438801+0000 mon.a (mon.0) 3272 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-24", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:35:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[59745]: audit 2022-04-28T21:35:48.477985+0000 mon.a (mon.0) 3273 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:35:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: cluster 2022-04-28T21:35:48.029458+0000 mgr.y (mgr.24961) 3479 : cluster [DBG] pgmap v3968: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T21:35:49.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: cephadm 2022-04-28T21:35:48.181216+0000 mgr.y (mgr.24961) 3480 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:35:49.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: cluster 2022-04-28T21:35:48.416751+0000 mon.a (mon.0) 3269 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2022-04-28T21:35:49.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: audit 2022-04-28T21:35:48.438036+0000 mon.a (mon.0) 3270 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:49.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: audit 2022-04-28T21:35:48.438442+0000 mon.a (mon.0) 3271 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:49.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: audit 2022-04-28T21:35:48.438801+0000 mon.a (mon.0) 3272 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-24", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:35:49.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:49 smithi059 bash[61152]: audit 2022-04-28T21:35:48.477985+0000 mon.a (mon.0) 3273 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:35:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: cluster 2022-04-28T21:35:48.029458+0000 mgr.y (mgr.24961) 3479 : cluster [DBG] pgmap v3968: 169 pgs: 169 active+clean; 216 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1023 B/s wr, 1 op/s 2022-04-28T21:35:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: cephadm 2022-04-28T21:35:48.181216+0000 mgr.y (mgr.24961) 3480 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:35:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: cluster 2022-04-28T21:35:48.416751+0000 mon.a (mon.0) 3269 : cluster [DBG] osdmap e1121: 8 total, 8 up, 8 in 2022-04-28T21:35:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: audit 2022-04-28T21:35:48.438036+0000 mon.a (mon.0) 3270 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: audit 2022-04-28T21:35:48.438442+0000 mon.a (mon.0) 3271 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: audit 2022-04-28T21:35:48.438801+0000 mon.a (mon.0) 3272 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-24", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:35:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:49 smithi139 bash[43301]: audit 2022-04-28T21:35:48.477985+0000 mon.a (mon.0) 3273 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:35:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:50 smithi059 bash[61152]: audit 2022-04-28T21:35:49.425895+0000 mon.a (mon.0) 3274 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-24", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:35:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:50 smithi059 bash[61152]: cluster 2022-04-28T21:35:49.425992+0000 mon.a (mon.0) 3275 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2022-04-28T21:35:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:50 smithi059 bash[61152]: audit 2022-04-28T21:35:49.431886+0000 mon.a (mon.0) 3276 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-24", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:50 smithi059 bash[59745]: audit 2022-04-28T21:35:49.425895+0000 mon.a (mon.0) 3274 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-24", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:35:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:50 smithi059 bash[59745]: cluster 2022-04-28T21:35:49.425992+0000 mon.a (mon.0) 3275 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2022-04-28T21:35:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:50 smithi059 bash[59745]: audit 2022-04-28T21:35:49.431886+0000 mon.a (mon.0) 3276 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-24", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:50 smithi139 bash[43301]: audit 2022-04-28T21:35:49.425895+0000 mon.a (mon.0) 3274 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-24", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:35:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:50 smithi139 bash[43301]: cluster 2022-04-28T21:35:49.425992+0000 mon.a (mon.0) 3275 : cluster [DBG] osdmap e1122: 8 total, 8 up, 8 in 2022-04-28T21:35:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:50 smithi139 bash[43301]: audit 2022-04-28T21:35:49.431886+0000 mon.a (mon.0) 3276 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-24", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:52.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[61152]: cluster 2022-04-28T21:35:50.030298+0000 mgr.y (mgr.24961) 3481 : cluster [DBG] pgmap v3971: 137 pgs: 137 active+clean; 24 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[61152]: cluster 2022-04-28T21:35:50.894457+0000 mon.a (mon.0) 3277 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2022-04-28T21:35:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[61152]: audit 2022-04-28T21:35:51.434674+0000 mon.a (mon.0) 3278 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-24", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:52.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[61152]: cluster 2022-04-28T21:35:51.434911+0000 mon.a (mon.0) 3279 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2022-04-28T21:35:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[59745]: cluster 2022-04-28T21:35:50.030298+0000 mgr.y (mgr.24961) 3481 : cluster [DBG] pgmap v3971: 137 pgs: 137 active+clean; 24 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[59745]: cluster 2022-04-28T21:35:50.894457+0000 mon.a (mon.0) 3277 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2022-04-28T21:35:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[59745]: audit 2022-04-28T21:35:51.434674+0000 mon.a (mon.0) 3278 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-24", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:52.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:51 smithi059 bash[59745]: cluster 2022-04-28T21:35:51.434911+0000 mon.a (mon.0) 3279 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2022-04-28T21:35:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:51 smithi139 bash[43301]: cluster 2022-04-28T21:35:50.030298+0000 mgr.y (mgr.24961) 3481 : cluster [DBG] pgmap v3971: 137 pgs: 137 active+clean; 24 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:51 smithi139 bash[43301]: cluster 2022-04-28T21:35:50.894457+0000 mon.a (mon.0) 3277 : cluster [DBG] osdmap e1123: 8 total, 8 up, 8 in 2022-04-28T21:35:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:51 smithi139 bash[43301]: audit 2022-04-28T21:35:51.434674+0000 mon.a (mon.0) 3278 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-24", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:51 smithi139 bash[43301]: cluster 2022-04-28T21:35:51.434911+0000 mon.a (mon.0) 3279 : cluster [DBG] osdmap e1124: 8 total, 8 up, 8 in 2022-04-28T21:35:53.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[61152]: cluster 2022-04-28T21:35:52.031017+0000 mgr.y (mgr.24961) 3482 : cluster [DBG] pgmap v3974: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 820 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[61152]: cluster 2022-04-28T21:35:52.438680+0000 mon.a (mon.0) 3280 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2022-04-28T21:35:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[61152]: audit 2022-04-28T21:35:52.441036+0000 mon.a (mon.0) 3281 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[59745]: cluster 2022-04-28T21:35:52.031017+0000 mgr.y (mgr.24961) 3482 : cluster [DBG] pgmap v3974: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 820 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[59745]: cluster 2022-04-28T21:35:52.438680+0000 mon.a (mon.0) 3280 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2022-04-28T21:35:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[59745]: audit 2022-04-28T21:35:52.441036+0000 mon.a (mon.0) 3281 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:53 smithi139 bash[43301]: cluster 2022-04-28T21:35:52.031017+0000 mgr.y (mgr.24961) 3482 : cluster [DBG] pgmap v3974: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 820 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:53 smithi139 bash[43301]: cluster 2022-04-28T21:35:52.438680+0000 mon.a (mon.0) 3280 : cluster [DBG] osdmap e1125: 8 total, 8 up, 8 in 2022-04-28T21:35:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:53 smithi139 bash[43301]: audit 2022-04-28T21:35:52.441036+0000 mon.a (mon.0) 3281 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:54.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:35:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:35:53.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:35:54.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:54 smithi059 bash[59745]: audit 2022-04-28T21:35:53.439606+0000 mon.a (mon.0) 3282 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:54 smithi059 bash[59745]: cluster 2022-04-28T21:35:53.440221+0000 mon.a (mon.0) 3283 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2022-04-28T21:35:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:54 smithi059 bash[59745]: audit 2022-04-28T21:35:53.449566+0000 mon.a (mon.0) 3284 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:54 smithi059 bash[61152]: audit 2022-04-28T21:35:53.439606+0000 mon.a (mon.0) 3282 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:54 smithi059 bash[61152]: cluster 2022-04-28T21:35:53.440221+0000 mon.a (mon.0) 3283 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2022-04-28T21:35:54.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:54 smithi059 bash[61152]: audit 2022-04-28T21:35:53.449566+0000 mon.a (mon.0) 3284 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:54.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:54 smithi139 bash[43301]: audit 2022-04-28T21:35:53.439606+0000 mon.a (mon.0) 3282 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:54 smithi139 bash[43301]: cluster 2022-04-28T21:35:53.440221+0000 mon.a (mon.0) 3283 : cluster [DBG] osdmap e1126: 8 total, 8 up, 8 in 2022-04-28T21:35:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:54 smithi139 bash[43301]: audit 2022-04-28T21:35:53.449566+0000 mon.a (mon.0) 3284 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:35:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:55 smithi059 bash[61152]: cluster 2022-04-28T21:35:54.031785+0000 mgr.y (mgr.24961) 3483 : cluster [DBG] pgmap v3977: 177 pgs: 40 unknown, 137 active+clean; 24 KiB data, 820 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:55 smithi059 bash[61152]: audit 2022-04-28T21:35:54.452024+0000 mon.a (mon.0) 3285 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-25","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:55 smithi059 bash[61152]: cluster 2022-04-28T21:35:54.452129+0000 mon.a (mon.0) 3286 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2022-04-28T21:35:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:55 smithi059 bash[59745]: cluster 2022-04-28T21:35:54.031785+0000 mgr.y (mgr.24961) 3483 : cluster [DBG] pgmap v3977: 177 pgs: 40 unknown, 137 active+clean; 24 KiB data, 820 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:55 smithi059 bash[59745]: audit 2022-04-28T21:35:54.452024+0000 mon.a (mon.0) 3285 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-25","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:55.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:55 smithi059 bash[59745]: cluster 2022-04-28T21:35:54.452129+0000 mon.a (mon.0) 3286 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2022-04-28T21:35:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:55 smithi139 bash[43301]: cluster 2022-04-28T21:35:54.031785+0000 mgr.y (mgr.24961) 3483 : cluster [DBG] pgmap v3977: 177 pgs: 40 unknown, 137 active+clean; 24 KiB data, 820 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:55 smithi139 bash[43301]: audit 2022-04-28T21:35:54.452024+0000 mon.a (mon.0) 3285 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-25","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:35:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:55 smithi139 bash[43301]: cluster 2022-04-28T21:35:54.452129+0000 mon.a (mon.0) 3286 : cluster [DBG] osdmap e1127: 8 total, 8 up, 8 in 2022-04-28T21:35:56.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:56 smithi059 bash[61152]: cluster 2022-04-28T21:35:55.471324+0000 mon.a (mon.0) 3287 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2022-04-28T21:35:56.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:56 smithi059 bash[59745]: cluster 2022-04-28T21:35:55.471324+0000 mon.a (mon.0) 3287 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2022-04-28T21:35:56.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:56 smithi139 bash[43301]: cluster 2022-04-28T21:35:55.471324+0000 mon.a (mon.0) 3287 : cluster [DBG] osdmap e1128: 8 total, 8 up, 8 in 2022-04-28T21:35:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:57 smithi059 bash[61152]: cluster 2022-04-28T21:35:56.041870+0000 mgr.y (mgr.24961) 3484 : cluster [DBG] pgmap v3980: 145 pgs: 145 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:57 smithi059 bash[61152]: cluster 2022-04-28T21:35:56.482158+0000 mon.a (mon.0) 3288 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2022-04-28T21:35:57.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:57 smithi059 bash[61152]: audit 2022-04-28T21:35:56.484463+0000 mon.a (mon.0) 3289 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:57 smithi059 bash[59745]: cluster 2022-04-28T21:35:56.041870+0000 mgr.y (mgr.24961) 3484 : cluster [DBG] pgmap v3980: 145 pgs: 145 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:57.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:57 smithi059 bash[59745]: cluster 2022-04-28T21:35:56.482158+0000 mon.a (mon.0) 3288 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2022-04-28T21:35:57.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:57 smithi059 bash[59745]: audit 2022-04-28T21:35:56.484463+0000 mon.a (mon.0) 3289 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:57 smithi139 bash[43301]: cluster 2022-04-28T21:35:56.041870+0000 mgr.y (mgr.24961) 3484 : cluster [DBG] pgmap v3980: 145 pgs: 145 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:57 smithi139 bash[43301]: cluster 2022-04-28T21:35:56.482158+0000 mon.a (mon.0) 3288 : cluster [DBG] osdmap e1129: 8 total, 8 up, 8 in 2022-04-28T21:35:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:57 smithi139 bash[43301]: audit 2022-04-28T21:35:56.484463+0000 mon.a (mon.0) 3289 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:58.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:58 smithi059 bash[61152]: audit 2022-04-28T21:35:57.482434+0000 mon.a (mon.0) 3290 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:58 smithi059 bash[61152]: cluster 2022-04-28T21:35:57.482739+0000 mon.a (mon.0) 3291 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2022-04-28T21:35:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:58 smithi059 bash[61152]: audit 2022-04-28T21:35:57.485213+0000 mon.a (mon.0) 3292 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:58 smithi059 bash[59745]: audit 2022-04-28T21:35:57.482434+0000 mon.a (mon.0) 3290 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:58.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:58 smithi059 bash[59745]: cluster 2022-04-28T21:35:57.482739+0000 mon.a (mon.0) 3291 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2022-04-28T21:35:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:58 smithi059 bash[59745]: audit 2022-04-28T21:35:57.485213+0000 mon.a (mon.0) 3292 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:58 smithi139 bash[43301]: audit 2022-04-28T21:35:57.482434+0000 mon.a (mon.0) 3290 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:58 smithi139 bash[43301]: cluster 2022-04-28T21:35:57.482739+0000 mon.a (mon.0) 3291 : cluster [DBG] osdmap e1130: 8 total, 8 up, 8 in 2022-04-28T21:35:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:58 smithi139 bash[43301]: audit 2022-04-28T21:35:57.485213+0000 mon.a (mon.0) 3292 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]: dispatch 2022-04-28T21:35:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:59 smithi059 bash[61152]: cluster 2022-04-28T21:35:58.042600+0000 mgr.y (mgr.24961) 3485 : cluster [DBG] pgmap v3983: 137 pgs: 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:59 smithi059 bash[61152]: audit 2022-04-28T21:35:58.488565+0000 mon.a (mon.0) 3293 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:35:59 smithi059 bash[61152]: cluster 2022-04-28T21:35:58.488746+0000 mon.a (mon.0) 3294 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2022-04-28T21:35:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:59 smithi059 bash[59745]: cluster 2022-04-28T21:35:58.042600+0000 mgr.y (mgr.24961) 3485 : cluster [DBG] pgmap v3983: 137 pgs: 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:59 smithi059 bash[59745]: audit 2022-04-28T21:35:58.488565+0000 mon.a (mon.0) 3293 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:35:59 smithi059 bash[59745]: cluster 2022-04-28T21:35:58.488746+0000 mon.a (mon.0) 3294 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2022-04-28T21:35:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:59 smithi139 bash[43301]: cluster 2022-04-28T21:35:58.042600+0000 mgr.y (mgr.24961) 3485 : cluster [DBG] pgmap v3983: 137 pgs: 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:35:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:59 smithi139 bash[43301]: audit 2022-04-28T21:35:58.488565+0000 mon.a (mon.0) 3293 : audit [INF] from='client.? 172.21.15.59:0/2587725970' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-24"}]': finished 2022-04-28T21:35:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:35:59 smithi139 bash[43301]: cluster 2022-04-28T21:35:58.488746+0000 mon.a (mon.0) 3294 : cluster [DBG] osdmap e1131: 8 total, 8 up, 8 in 2022-04-28T21:36:00.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:00 smithi059 bash[61152]: cluster 2022-04-28T21:35:59.513544+0000 mon.a (mon.0) 3295 : cluster [DBG] osdmap e1132: 8 total, 8 up, 8 in 2022-04-28T21:36:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:00 smithi059 bash[61152]: audit 2022-04-28T21:35:59.518137+0000 mon.a (mon.0) 3296 : audit [INF] from='client.? 172.21.15.59:0/3792111943' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:00 smithi059 bash[59745]: cluster 2022-04-28T21:35:59.513544+0000 mon.a (mon.0) 3295 : cluster [DBG] osdmap e1132: 8 total, 8 up, 8 in 2022-04-28T21:36:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:00 smithi059 bash[59745]: audit 2022-04-28T21:35:59.518137+0000 mon.a (mon.0) 3296 : audit [INF] from='client.? 172.21.15.59:0/3792111943' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:00.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:00 smithi139 bash[43301]: cluster 2022-04-28T21:35:59.513544+0000 mon.a (mon.0) 3295 : cluster [DBG] osdmap e1132: 8 total, 8 up, 8 in 2022-04-28T21:36:00.909 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:00 smithi139 bash[43301]: audit 2022-04-28T21:35:59.518137+0000 mon.a (mon.0) 3296 : audit [INF] from='client.? 172.21.15.59:0/3792111943' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:01.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:01 smithi059 bash[59745]: cluster 2022-04-28T21:36:00.043439+0000 mgr.y (mgr.24961) 3486 : cluster [DBG] pgmap v3986: 169 pgs: 3 creating+peering, 29 unknown, 137 active+clean; 24 KiB data, 822 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:01 smithi059 bash[59745]: audit 2022-04-28T21:36:00.525656+0000 mon.a (mon.0) 3297 : audit [INF] from='client.? 172.21.15.59:0/3792111943' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-27","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:01 smithi059 bash[59745]: cluster 2022-04-28T21:36:00.526226+0000 mon.a (mon.0) 3298 : cluster [DBG] osdmap e1133: 8 total, 8 up, 8 in 2022-04-28T21:36:01.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:01 smithi059 bash[61152]: cluster 2022-04-28T21:36:00.043439+0000 mgr.y (mgr.24961) 3486 : cluster [DBG] pgmap v3986: 169 pgs: 3 creating+peering, 29 unknown, 137 active+clean; 24 KiB data, 822 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:01.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:01 smithi059 bash[61152]: audit 2022-04-28T21:36:00.525656+0000 mon.a (mon.0) 3297 : audit [INF] from='client.? 172.21.15.59:0/3792111943' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-27","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:01.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:01 smithi059 bash[61152]: cluster 2022-04-28T21:36:00.526226+0000 mon.a (mon.0) 3298 : cluster [DBG] osdmap e1133: 8 total, 8 up, 8 in 2022-04-28T21:36:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:01 smithi139 bash[43301]: cluster 2022-04-28T21:36:00.043439+0000 mgr.y (mgr.24961) 3486 : cluster [DBG] pgmap v3986: 169 pgs: 3 creating+peering, 29 unknown, 137 active+clean; 24 KiB data, 822 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:01 smithi139 bash[43301]: audit 2022-04-28T21:36:00.525656+0000 mon.a (mon.0) 3297 : audit [INF] from='client.? 172.21.15.59:0/3792111943' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-27","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:01 smithi139 bash[43301]: cluster 2022-04-28T21:36:00.526226+0000 mon.a (mon.0) 3298 : cluster [DBG] osdmap e1133: 8 total, 8 up, 8 in 2022-04-28T21:36:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:02.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:02.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:02 smithi059 bash[61152]: cluster 2022-04-28T21:36:01.526596+0000 mon.a (mon.0) 3299 : cluster [DBG] osdmap e1134: 8 total, 8 up, 8 in 2022-04-28T21:36:02.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:02 smithi059 bash[59745]: cluster 2022-04-28T21:36:01.526596+0000 mon.a (mon.0) 3299 : cluster [DBG] osdmap e1134: 8 total, 8 up, 8 in 2022-04-28T21:36:02.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:02 smithi139 bash[43301]: cluster 2022-04-28T21:36:01.526596+0000 mon.a (mon.0) 3299 : cluster [DBG] osdmap e1134: 8 total, 8 up, 8 in 2022-04-28T21:36:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[61152]: cluster 2022-04-28T21:36:02.044196+0000 mgr.y (mgr.24961) 3487 : cluster [DBG] pgmap v3989: 137 pgs: 137 active+clean; 24 KiB data, 823 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[61152]: cluster 2022-04-28T21:36:02.551921+0000 mon.a (mon.0) 3300 : cluster [DBG] osdmap e1135: 8 total, 8 up, 8 in 2022-04-28T21:36:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[61152]: audit 2022-04-28T21:36:02.556725+0000 mon.c (mon.1) 124 : audit [INF] from='client.? 172.21.15.59:0/4233977533' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[61152]: audit 2022-04-28T21:36:02.560687+0000 mon.a (mon.0) 3301 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[59745]: cluster 2022-04-28T21:36:02.044196+0000 mgr.y (mgr.24961) 3487 : cluster [DBG] pgmap v3989: 137 pgs: 137 active+clean; 24 KiB data, 823 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[59745]: cluster 2022-04-28T21:36:02.551921+0000 mon.a (mon.0) 3300 : cluster [DBG] osdmap e1135: 8 total, 8 up, 8 in 2022-04-28T21:36:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[59745]: audit 2022-04-28T21:36:02.556725+0000 mon.c (mon.1) 124 : audit [INF] from='client.? 172.21.15.59:0/4233977533' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[59745]: audit 2022-04-28T21:36:02.560687+0000 mon.a (mon.0) 3301 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:03 smithi139 bash[43301]: cluster 2022-04-28T21:36:02.044196+0000 mgr.y (mgr.24961) 3487 : cluster [DBG] pgmap v3989: 137 pgs: 137 active+clean; 24 KiB data, 823 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:03 smithi139 bash[43301]: cluster 2022-04-28T21:36:02.551921+0000 mon.a (mon.0) 3300 : cluster [DBG] osdmap e1135: 8 total, 8 up, 8 in 2022-04-28T21:36:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:03 smithi139 bash[43301]: audit 2022-04-28T21:36:02.556725+0000 mon.c (mon.1) 124 : audit [INF] from='client.? 172.21.15.59:0/4233977533' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:03 smithi139 bash[43301]: audit 2022-04-28T21:36:02.560687+0000 mon.a (mon.0) 3301 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:03.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:04.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:04 smithi059 bash[61152]: audit 2022-04-28T21:36:03.558473+0000 mon.a (mon.0) 3302 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:04.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:04 smithi059 bash[61152]: cluster 2022-04-28T21:36:03.560096+0000 mon.a (mon.0) 3303 : cluster [DBG] osdmap e1136: 8 total, 8 up, 8 in 2022-04-28T21:36:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:04 smithi059 bash[59745]: audit 2022-04-28T21:36:03.558473+0000 mon.a (mon.0) 3302 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:04.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:04 smithi059 bash[59745]: cluster 2022-04-28T21:36:03.560096+0000 mon.a (mon.0) 3303 : cluster [DBG] osdmap e1136: 8 total, 8 up, 8 in 2022-04-28T21:36:04.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:04 smithi139 bash[43301]: audit 2022-04-28T21:36:03.558473+0000 mon.a (mon.0) 3302 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-30","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:04 smithi139 bash[43301]: cluster 2022-04-28T21:36:03.560096+0000 mon.a (mon.0) 3303 : cluster [DBG] osdmap e1136: 8 total, 8 up, 8 in 2022-04-28T21:36:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:05 smithi059 bash[61152]: cluster 2022-04-28T21:36:04.045118+0000 mgr.y (mgr.24961) 3488 : cluster [DBG] pgmap v3992: 169 pgs: 29 unknown, 140 active+clean; 24 KiB data, 823 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:05 smithi059 bash[61152]: cluster 2022-04-28T21:36:04.583957+0000 mon.a (mon.0) 3304 : cluster [DBG] osdmap e1137: 8 total, 8 up, 8 in 2022-04-28T21:36:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:05 smithi059 bash[59745]: cluster 2022-04-28T21:36:04.045118+0000 mgr.y (mgr.24961) 3488 : cluster [DBG] pgmap v3992: 169 pgs: 29 unknown, 140 active+clean; 24 KiB data, 823 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:05 smithi059 bash[59745]: cluster 2022-04-28T21:36:04.583957+0000 mon.a (mon.0) 3304 : cluster [DBG] osdmap e1137: 8 total, 8 up, 8 in 2022-04-28T21:36:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:05 smithi139 bash[43301]: cluster 2022-04-28T21:36:04.045118+0000 mgr.y (mgr.24961) 3488 : cluster [DBG] pgmap v3992: 169 pgs: 29 unknown, 140 active+clean; 24 KiB data, 823 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:05 smithi139 bash[43301]: cluster 2022-04-28T21:36:04.583957+0000 mon.a (mon.0) 3304 : cluster [DBG] osdmap e1137: 8 total, 8 up, 8 in 2022-04-28T21:36:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:06 smithi139 bash[43301]: audit 2022-04-28T21:36:05.596792+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.59:0/2733118565' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:06 smithi139 bash[43301]: cluster 2022-04-28T21:36:05.597257+0000 mon.a (mon.0) 3305 : cluster [DBG] osdmap e1138: 8 total, 8 up, 8 in 2022-04-28T21:36:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:06 smithi139 bash[43301]: audit 2022-04-28T21:36:05.601199+0000 mon.a (mon.0) 3306 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:06 smithi139 bash[43301]: cluster 2022-04-28T21:36:06.046212+0000 mgr.y (mgr.24961) 3489 : cluster [DBG] pgmap v3995: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[61152]: audit 2022-04-28T21:36:05.596792+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.59:0/2733118565' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[61152]: cluster 2022-04-28T21:36:05.597257+0000 mon.a (mon.0) 3305 : cluster [DBG] osdmap e1138: 8 total, 8 up, 8 in 2022-04-28T21:36:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[61152]: audit 2022-04-28T21:36:05.601199+0000 mon.a (mon.0) 3306 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:07.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[61152]: cluster 2022-04-28T21:36:06.046212+0000 mgr.y (mgr.24961) 3489 : cluster [DBG] pgmap v3995: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[59745]: audit 2022-04-28T21:36:05.596792+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.59:0/2733118565' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[59745]: cluster 2022-04-28T21:36:05.597257+0000 mon.a (mon.0) 3305 : cluster [DBG] osdmap e1138: 8 total, 8 up, 8 in 2022-04-28T21:36:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[59745]: audit 2022-04-28T21:36:05.601199+0000 mon.a (mon.0) 3306 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:07.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:06 smithi059 bash[59745]: cluster 2022-04-28T21:36:06.046212+0000 mgr.y (mgr.24961) 3489 : cluster [DBG] pgmap v3995: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:07 smithi139 bash[43301]: audit 2022-04-28T21:36:06.602338+0000 mon.a (mon.0) 3307 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:07 smithi139 bash[43301]: cluster 2022-04-28T21:36:06.602591+0000 mon.a (mon.0) 3308 : cluster [DBG] osdmap e1139: 8 total, 8 up, 8 in 2022-04-28T21:36:08.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:07 smithi059 bash[61152]: audit 2022-04-28T21:36:06.602338+0000 mon.a (mon.0) 3307 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:07 smithi059 bash[61152]: cluster 2022-04-28T21:36:06.602591+0000 mon.a (mon.0) 3308 : cluster [DBG] osdmap e1139: 8 total, 8 up, 8 in 2022-04-28T21:36:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:07 smithi059 bash[59745]: audit 2022-04-28T21:36:06.602338+0000 mon.a (mon.0) 3307 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-83986-33","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:07 smithi059 bash[59745]: cluster 2022-04-28T21:36:06.602591+0000 mon.a (mon.0) 3308 : cluster [DBG] osdmap e1139: 8 total, 8 up, 8 in 2022-04-28T21:36:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: cluster 2022-04-28T21:36:07.626145+0000 mon.a (mon.0) 3309 : cluster [DBG] osdmap e1140: 8 total, 8 up, 8 in 2022-04-28T21:36:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: audit 2022-04-28T21:36:07.643342+0000 mon.c (mon.1) 125 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: audit 2022-04-28T21:36:07.647871+0000 mon.a (mon.0) 3310 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: audit 2022-04-28T21:36:07.648700+0000 mon.c (mon.1) 126 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: audit 2022-04-28T21:36:07.649139+0000 mon.a (mon.0) 3311 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: audit 2022-04-28T21:36:07.649679+0000 mon.c (mon.1) 127 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: audit 2022-04-28T21:36:07.650046+0000 mon.a (mon.0) 3312 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[61152]: cluster 2022-04-28T21:36:08.046920+0000 mgr.y (mgr.24961) 3490 : cluster [DBG] pgmap v3998: 137 pgs: 137 active+clean; 24 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: cluster 2022-04-28T21:36:07.626145+0000 mon.a (mon.0) 3309 : cluster [DBG] osdmap e1140: 8 total, 8 up, 8 in 2022-04-28T21:36:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: audit 2022-04-28T21:36:07.643342+0000 mon.c (mon.1) 125 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: audit 2022-04-28T21:36:07.647871+0000 mon.a (mon.0) 3310 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: audit 2022-04-28T21:36:07.648700+0000 mon.c (mon.1) 126 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: audit 2022-04-28T21:36:07.649139+0000 mon.a (mon.0) 3311 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: audit 2022-04-28T21:36:07.649679+0000 mon.c (mon.1) 127 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:09.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: audit 2022-04-28T21:36:07.650046+0000 mon.a (mon.0) 3312 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:09.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:08 smithi059 bash[59745]: cluster 2022-04-28T21:36:08.046920+0000 mgr.y (mgr.24961) 3490 : cluster [DBG] pgmap v3998: 137 pgs: 137 active+clean; 24 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: cluster 2022-04-28T21:36:07.626145+0000 mon.a (mon.0) 3309 : cluster [DBG] osdmap e1140: 8 total, 8 up, 8 in 2022-04-28T21:36:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: audit 2022-04-28T21:36:07.643342+0000 mon.c (mon.1) 125 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: audit 2022-04-28T21:36:07.647871+0000 mon.a (mon.0) 3310 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: audit 2022-04-28T21:36:07.648700+0000 mon.c (mon.1) 126 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: audit 2022-04-28T21:36:07.649139+0000 mon.a (mon.0) 3311 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:09.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: audit 2022-04-28T21:36:07.649679+0000 mon.c (mon.1) 127 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:09.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: audit 2022-04-28T21:36:07.650046+0000 mon.a (mon.0) 3312 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:09.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:08 smithi139 bash[43301]: cluster 2022-04-28T21:36:08.046920+0000 mgr.y (mgr.24961) 3490 : cluster [DBG] pgmap v3998: 137 pgs: 137 active+clean; 24 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:10.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[61152]: audit 2022-04-28T21:36:08.638275+0000 mon.a (mon.0) 3313 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:36:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[61152]: cluster 2022-04-28T21:36:08.638447+0000 mon.a (mon.0) 3314 : cluster [DBG] osdmap e1141: 8 total, 8 up, 8 in 2022-04-28T21:36:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[61152]: audit 2022-04-28T21:36:08.642952+0000 mon.c (mon.1) 128 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[61152]: audit 2022-04-28T21:36:08.643549+0000 mon.a (mon.0) 3315 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[59745]: audit 2022-04-28T21:36:08.638275+0000 mon.a (mon.0) 3313 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:36:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[59745]: cluster 2022-04-28T21:36:08.638447+0000 mon.a (mon.0) 3314 : cluster [DBG] osdmap e1141: 8 total, 8 up, 8 in 2022-04-28T21:36:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[59745]: audit 2022-04-28T21:36:08.642952+0000 mon.c (mon.1) 128 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:09 smithi059 bash[59745]: audit 2022-04-28T21:36:08.643549+0000 mon.a (mon.0) 3315 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:09 smithi139 bash[43301]: audit 2022-04-28T21:36:08.638275+0000 mon.a (mon.0) 3313 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-83986-36", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:36:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:09 smithi139 bash[43301]: cluster 2022-04-28T21:36:08.638447+0000 mon.a (mon.0) 3314 : cluster [DBG] osdmap e1141: 8 total, 8 up, 8 in 2022-04-28T21:36:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:09 smithi139 bash[43301]: audit 2022-04-28T21:36:08.642952+0000 mon.c (mon.1) 128 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:09 smithi139 bash[43301]: audit 2022-04-28T21:36:08.643549+0000 mon.a (mon.0) 3315 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:11.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:10 smithi059 bash[61152]: cluster 2022-04-28T21:36:10.047680+0000 mgr.y (mgr.24961) 3491 : cluster [DBG] pgmap v4001: 137 pgs: 137 active+clean; 24 KiB data, 810 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:10 smithi059 bash[61152]: cluster 2022-04-28T21:36:10.118563+0000 mon.a (mon.0) 3316 : cluster [DBG] osdmap e1142: 8 total, 8 up, 8 in 2022-04-28T21:36:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:10 smithi059 bash[59745]: cluster 2022-04-28T21:36:10.047680+0000 mgr.y (mgr.24961) 3491 : cluster [DBG] pgmap v4001: 137 pgs: 137 active+clean; 24 KiB data, 810 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:11.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:10 smithi059 bash[59745]: cluster 2022-04-28T21:36:10.118563+0000 mon.a (mon.0) 3316 : cluster [DBG] osdmap e1142: 8 total, 8 up, 8 in 2022-04-28T21:36:11.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:10 smithi139 bash[43301]: cluster 2022-04-28T21:36:10.047680+0000 mgr.y (mgr.24961) 3491 : cluster [DBG] pgmap v4001: 137 pgs: 137 active+clean; 24 KiB data, 810 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:10 smithi139 bash[43301]: cluster 2022-04-28T21:36:10.118563+0000 mon.a (mon.0) 3316 : cluster [DBG] osdmap e1142: 8 total, 8 up, 8 in 2022-04-28T21:36:12.035 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:11 smithi059 bash[61152]: audit 2022-04-28T21:36:10.663993+0000 mon.a (mon.0) 3317 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:12.035 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:11 smithi059 bash[61152]: cluster 2022-04-28T21:36:10.664831+0000 mon.a (mon.0) 3318 : cluster [DBG] osdmap e1143: 8 total, 8 up, 8 in 2022-04-28T21:36:12.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:11 smithi059 bash[59745]: audit 2022-04-28T21:36:10.663993+0000 mon.a (mon.0) 3317 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:12.036 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:11 smithi059 bash[59745]: cluster 2022-04-28T21:36:10.664831+0000 mon.a (mon.0) 3318 : cluster [DBG] osdmap e1143: 8 total, 8 up, 8 in 2022-04-28T21:36:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:11 smithi139 bash[43301]: audit 2022-04-28T21:36:10.663993+0000 mon.a (mon.0) 3317 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-83986-36", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:11 smithi139 bash[43301]: cluster 2022-04-28T21:36:10.664831+0000 mon.a (mon.0) 3318 : cluster [DBG] osdmap e1143: 8 total, 8 up, 8 in 2022-04-28T21:36:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:12.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:12 smithi059 bash[61152]: cluster 2022-04-28T21:36:11.670293+0000 mon.a (mon.0) 3319 : cluster [DBG] osdmap e1144: 8 total, 8 up, 8 in 2022-04-28T21:36:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:12 smithi059 bash[61152]: cluster 2022-04-28T21:36:12.048415+0000 mgr.y (mgr.24961) 3492 : cluster [DBG] pgmap v4004: 145 pgs: 5 creating+peering, 3 unknown, 137 active+clean; 24 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:13.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:12 smithi059 bash[59745]: cluster 2022-04-28T21:36:11.670293+0000 mon.a (mon.0) 3319 : cluster [DBG] osdmap e1144: 8 total, 8 up, 8 in 2022-04-28T21:36:13.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:12 smithi059 bash[59745]: cluster 2022-04-28T21:36:12.048415+0000 mgr.y (mgr.24961) 3492 : cluster [DBG] pgmap v4004: 145 pgs: 5 creating+peering, 3 unknown, 137 active+clean; 24 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:12 smithi139 bash[43301]: cluster 2022-04-28T21:36:11.670293+0000 mon.a (mon.0) 3319 : cluster [DBG] osdmap e1144: 8 total, 8 up, 8 in 2022-04-28T21:36:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:12 smithi139 bash[43301]: cluster 2022-04-28T21:36:12.048415+0000 mgr.y (mgr.24961) 3492 : cluster [DBG] pgmap v4004: 145 pgs: 5 creating+peering, 3 unknown, 137 active+clean; 24 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:14.090 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:13.928Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[61152]: cluster 2022-04-28T21:36:12.680071+0000 mon.a (mon.0) 3320 : cluster [DBG] osdmap e1145: 8 total, 8 up, 8 in 2022-04-28T21:36:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[61152]: audit 2022-04-28T21:36:12.681442+0000 mon.c (mon.1) 129 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[61152]: audit 2022-04-28T21:36:12.683863+0000 mon.a (mon.0) 3321 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[61152]: audit 2022-04-28T21:36:13.683516+0000 mon.a (mon.0) 3322 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:14.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[61152]: cluster 2022-04-28T21:36:13.683732+0000 mon.a (mon.0) 3323 : cluster [DBG] osdmap e1146: 8 total, 8 up, 8 in 2022-04-28T21:36:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[59745]: cluster 2022-04-28T21:36:12.680071+0000 mon.a (mon.0) 3320 : cluster [DBG] osdmap e1145: 8 total, 8 up, 8 in 2022-04-28T21:36:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[59745]: audit 2022-04-28T21:36:12.681442+0000 mon.c (mon.1) 129 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[59745]: audit 2022-04-28T21:36:12.683863+0000 mon.a (mon.0) 3321 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[59745]: audit 2022-04-28T21:36:13.683516+0000 mon.a (mon.0) 3322 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:13 smithi059 bash[59745]: cluster 2022-04-28T21:36:13.683732+0000 mon.a (mon.0) 3323 : cluster [DBG] osdmap e1146: 8 total, 8 up, 8 in 2022-04-28T21:36:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:13 smithi139 bash[43301]: cluster 2022-04-28T21:36:12.680071+0000 mon.a (mon.0) 3320 : cluster [DBG] osdmap e1145: 8 total, 8 up, 8 in 2022-04-28T21:36:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:13 smithi139 bash[43301]: audit 2022-04-28T21:36:12.681442+0000 mon.c (mon.1) 129 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:13 smithi139 bash[43301]: audit 2022-04-28T21:36:12.683863+0000 mon.a (mon.0) 3321 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:13 smithi139 bash[43301]: audit 2022-04-28T21:36:13.683516+0000 mon.a (mon.0) 3322 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:13 smithi139 bash[43301]: cluster 2022-04-28T21:36:13.683732+0000 mon.a (mon.0) 3323 : cluster [DBG] osdmap e1146: 8 total, 8 up, 8 in 2022-04-28T21:36:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[61152]: audit 2022-04-28T21:36:13.684476+0000 mon.c (mon.1) 130 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[61152]: audit 2022-04-28T21:36:13.685555+0000 mon.a (mon.0) 3324 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[61152]: cluster 2022-04-28T21:36:14.049322+0000 mgr.y (mgr.24961) 3493 : cluster [DBG] pgmap v4007: 137 pgs: 137 active+clean; 24 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[61152]: audit 2022-04-28T21:36:14.684948+0000 mon.a (mon.0) 3325 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:15.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[61152]: cluster 2022-04-28T21:36:14.685165+0000 mon.a (mon.0) 3326 : cluster [DBG] osdmap e1147: 8 total, 8 up, 8 in 2022-04-28T21:36:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[59745]: audit 2022-04-28T21:36:13.684476+0000 mon.c (mon.1) 130 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[59745]: audit 2022-04-28T21:36:13.685555+0000 mon.a (mon.0) 3324 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:15.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[59745]: cluster 2022-04-28T21:36:14.049322+0000 mgr.y (mgr.24961) 3493 : cluster [DBG] pgmap v4007: 137 pgs: 137 active+clean; 24 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:15.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[59745]: audit 2022-04-28T21:36:14.684948+0000 mon.a (mon.0) 3325 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:15.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:14 smithi059 bash[59745]: cluster 2022-04-28T21:36:14.685165+0000 mon.a (mon.0) 3326 : cluster [DBG] osdmap e1147: 8 total, 8 up, 8 in 2022-04-28T21:36:15.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:14 smithi139 bash[43301]: audit 2022-04-28T21:36:13.684476+0000 mon.c (mon.1) 130 : audit [INF] from='client.? 172.21.15.59:0/3901640198' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:14 smithi139 bash[43301]: audit 2022-04-28T21:36:13.685555+0000 mon.a (mon.0) 3324 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]: dispatch 2022-04-28T21:36:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:14 smithi139 bash[43301]: cluster 2022-04-28T21:36:14.049322+0000 mgr.y (mgr.24961) 3493 : cluster [DBG] pgmap v4007: 137 pgs: 137 active+clean; 24 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:14 smithi139 bash[43301]: audit 2022-04-28T21:36:14.684948+0000 mon.a (mon.0) 3325 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-83986-36"}]': finished 2022-04-28T21:36:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:14 smithi139 bash[43301]: cluster 2022-04-28T21:36:14.685165+0000 mon.a (mon.0) 3326 : cluster [DBG] osdmap e1147: 8 total, 8 up, 8 in 2022-04-28T21:36:17.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:17 smithi139 bash[43301]: cluster 2022-04-28T21:36:16.050501+0000 mgr.y (mgr.24961) 3494 : cluster [DBG] pgmap v4009: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:17 smithi059 bash[61152]: cluster 2022-04-28T21:36:16.050501+0000 mgr.y (mgr.24961) 3494 : cluster [DBG] pgmap v4009: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:17 smithi059 bash[59745]: cluster 2022-04-28T21:36:16.050501+0000 mgr.y (mgr.24961) 3494 : cluster [DBG] pgmap v4009: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:19.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:19 smithi139 bash[43301]: cluster 2022-04-28T21:36:18.051228+0000 mgr.y (mgr.24961) 3495 : cluster [DBG] pgmap v4010: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:19.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:19 smithi059 bash[61152]: cluster 2022-04-28T21:36:18.051228+0000 mgr.y (mgr.24961) 3495 : cluster [DBG] pgmap v4010: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:19.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:19 smithi059 bash[59745]: cluster 2022-04-28T21:36:18.051228+0000 mgr.y (mgr.24961) 3495 : cluster [DBG] pgmap v4010: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[61152]: audit 2022-04-28T21:36:19.303909+0000 mon.a (mon.0) 3327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:36:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[61152]: audit 2022-04-28T21:36:19.304469+0000 mon.a (mon.0) 3328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:36:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[61152]: audit 2022-04-28T21:36:19.316012+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:36:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[61152]: audit 2022-04-28T21:36:19.316451+0000 mon.a (mon.0) 3330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:36:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[61152]: audit 2022-04-28T21:36:19.316846+0000 mon.a (mon.0) 3331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:36:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[61152]: audit 2022-04-28T21:36:19.317193+0000 mon.a (mon.0) 3332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:36:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[59745]: audit 2022-04-28T21:36:19.303909+0000 mon.a (mon.0) 3327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:36:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[59745]: audit 2022-04-28T21:36:19.304469+0000 mon.a (mon.0) 3328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:36:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[59745]: audit 2022-04-28T21:36:19.316012+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:36:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[59745]: audit 2022-04-28T21:36:19.316451+0000 mon.a (mon.0) 3330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:36:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[59745]: audit 2022-04-28T21:36:19.316846+0000 mon.a (mon.0) 3331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:36:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:20 smithi059 bash[59745]: audit 2022-04-28T21:36:19.317193+0000 mon.a (mon.0) 3332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:36:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:20 smithi139 bash[43301]: audit 2022-04-28T21:36:19.303909+0000 mon.a (mon.0) 3327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:36:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:20 smithi139 bash[43301]: audit 2022-04-28T21:36:19.304469+0000 mon.a (mon.0) 3328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:36:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:20 smithi139 bash[43301]: audit 2022-04-28T21:36:19.316012+0000 mon.a (mon.0) 3329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:36:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:20 smithi139 bash[43301]: audit 2022-04-28T21:36:19.316451+0000 mon.a (mon.0) 3330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:36:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:20 smithi139 bash[43301]: audit 2022-04-28T21:36:19.316846+0000 mon.a (mon.0) 3331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:36:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:20 smithi139 bash[43301]: audit 2022-04-28T21:36:19.317193+0000 mon.a (mon.0) 3332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:36:21.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:21 smithi059 bash[61152]: cluster 2022-04-28T21:36:20.052433+0000 mgr.y (mgr.24961) 3496 : cluster [DBG] pgmap v4011: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:21.607 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:21 smithi059 bash[59745]: cluster 2022-04-28T21:36:20.052433+0000 mgr.y (mgr.24961) 3496 : cluster [DBG] pgmap v4011: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:21.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:21 smithi139 bash[43301]: cluster 2022-04-28T21:36:20.052433+0000 mgr.y (mgr.24961) 3496 : cluster [DBG] pgmap v4011: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:22.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:23.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:23 smithi059 bash[61152]: cluster 2022-04-28T21:36:22.053644+0000 mgr.y (mgr.24961) 3497 : cluster [DBG] pgmap v4012: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:23.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:23 smithi059 bash[59745]: cluster 2022-04-28T21:36:22.053644+0000 mgr.y (mgr.24961) 3497 : cluster [DBG] pgmap v4012: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:23.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:23 smithi139 bash[43301]: cluster 2022-04-28T21:36:22.053644+0000 mgr.y (mgr.24961) 3497 : cluster [DBG] pgmap v4012: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:23.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:25.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:25 smithi059 bash[61152]: cluster 2022-04-28T21:36:24.054535+0000 mgr.y (mgr.24961) 3498 : cluster [DBG] pgmap v4013: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:25 smithi059 bash[59745]: cluster 2022-04-28T21:36:24.054535+0000 mgr.y (mgr.24961) 3498 : cluster [DBG] pgmap v4013: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:25.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:25 smithi139 bash[43301]: cluster 2022-04-28T21:36:24.054535+0000 mgr.y (mgr.24961) 3498 : cluster [DBG] pgmap v4013: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:27.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:27 smithi059 bash[61152]: cluster 2022-04-28T21:36:26.056154+0000 mgr.y (mgr.24961) 3499 : cluster [DBG] pgmap v4014: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:27.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:27 smithi059 bash[59745]: cluster 2022-04-28T21:36:26.056154+0000 mgr.y (mgr.24961) 3499 : cluster [DBG] pgmap v4014: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:27.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:27 smithi139 bash[43301]: cluster 2022-04-28T21:36:26.056154+0000 mgr.y (mgr.24961) 3499 : cluster [DBG] pgmap v4014: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:29.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:29 smithi059 bash[61152]: cluster 2022-04-28T21:36:28.056893+0000 mgr.y (mgr.24961) 3500 : cluster [DBG] pgmap v4015: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:29.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:29 smithi059 bash[59745]: cluster 2022-04-28T21:36:28.056893+0000 mgr.y (mgr.24961) 3500 : cluster [DBG] pgmap v4015: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:29.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:29 smithi139 bash[43301]: cluster 2022-04-28T21:36:28.056893+0000 mgr.y (mgr.24961) 3500 : cluster [DBG] pgmap v4015: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:31.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:31 smithi059 bash[61152]: cluster 2022-04-28T21:36:30.058085+0000 mgr.y (mgr.24961) 3501 : cluster [DBG] pgmap v4016: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:31.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:31 smithi059 bash[59745]: cluster 2022-04-28T21:36:30.058085+0000 mgr.y (mgr.24961) 3501 : cluster [DBG] pgmap v4016: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:31.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:31 smithi139 bash[43301]: cluster 2022-04-28T21:36:30.058085+0000 mgr.y (mgr.24961) 3501 : cluster [DBG] pgmap v4016: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:32.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:33.590 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:33 smithi059 bash[59745]: cluster 2022-04-28T21:36:32.059243+0000 mgr.y (mgr.24961) 3502 : cluster [DBG] pgmap v4017: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:33.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:33 smithi059 bash[61152]: cluster 2022-04-28T21:36:32.059243+0000 mgr.y (mgr.24961) 3502 : cluster [DBG] pgmap v4017: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:33.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:33 smithi139 bash[43301]: cluster 2022-04-28T21:36:32.059243+0000 mgr.y (mgr.24961) 3502 : cluster [DBG] pgmap v4017: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:33.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:35 smithi059 bash[61152]: cluster 2022-04-28T21:36:34.060144+0000 mgr.y (mgr.24961) 3503 : cluster [DBG] pgmap v4018: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:35 smithi059 bash[59745]: cluster 2022-04-28T21:36:34.060144+0000 mgr.y (mgr.24961) 3503 : cluster [DBG] pgmap v4018: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:35.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:35 smithi139 bash[43301]: cluster 2022-04-28T21:36:34.060144+0000 mgr.y (mgr.24961) 3503 : cluster [DBG] pgmap v4018: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:37.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:37 smithi059 bash[61152]: cluster 2022-04-28T21:36:36.061597+0000 mgr.y (mgr.24961) 3504 : cluster [DBG] pgmap v4019: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:37.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:37 smithi059 bash[59745]: cluster 2022-04-28T21:36:36.061597+0000 mgr.y (mgr.24961) 3504 : cluster [DBG] pgmap v4019: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:37.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:37 smithi139 bash[43301]: cluster 2022-04-28T21:36:36.061597+0000 mgr.y (mgr.24961) 3504 : cluster [DBG] pgmap v4019: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:39.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:39 smithi059 bash[61152]: cluster 2022-04-28T21:36:38.062320+0000 mgr.y (mgr.24961) 3505 : cluster [DBG] pgmap v4020: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:39.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:39 smithi059 bash[59745]: cluster 2022-04-28T21:36:38.062320+0000 mgr.y (mgr.24961) 3505 : cluster [DBG] pgmap v4020: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:39 smithi139 bash[43301]: cluster 2022-04-28T21:36:38.062320+0000 mgr.y (mgr.24961) 3505 : cluster [DBG] pgmap v4020: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:41.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:41 smithi059 bash[59745]: cluster 2022-04-28T21:36:40.063546+0000 mgr.y (mgr.24961) 3506 : cluster [DBG] pgmap v4021: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:41 smithi059 bash[61152]: cluster 2022-04-28T21:36:40.063546+0000 mgr.y (mgr.24961) 3506 : cluster [DBG] pgmap v4021: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:41.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:41 smithi139 bash[43301]: cluster 2022-04-28T21:36:40.063546+0000 mgr.y (mgr.24961) 3506 : cluster [DBG] pgmap v4021: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:42.036Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:42 smithi059 bash[59745]: cluster 2022-04-28T21:36:41.527292+0000 mon.a (mon.0) 3333 : cluster [DBG] osdmap e1148: 8 total, 8 up, 8 in 2022-04-28T21:36:42.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:42 smithi059 bash[59745]: audit 2022-04-28T21:36:41.536045+0000 mon.a (mon.0) 3334 : audit [INF] from='client.? 172.21.15.59:0/214427465' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84128-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:42 smithi059 bash[61152]: cluster 2022-04-28T21:36:41.527292+0000 mon.a (mon.0) 3333 : cluster [DBG] osdmap e1148: 8 total, 8 up, 8 in 2022-04-28T21:36:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:42 smithi059 bash[61152]: audit 2022-04-28T21:36:41.536045+0000 mon.a (mon.0) 3334 : audit [INF] from='client.? 172.21.15.59:0/214427465' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84128-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:42 smithi139 bash[43301]: cluster 2022-04-28T21:36:41.527292+0000 mon.a (mon.0) 3333 : cluster [DBG] osdmap e1148: 8 total, 8 up, 8 in 2022-04-28T21:36:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:42 smithi139 bash[43301]: audit 2022-04-28T21:36:41.536045+0000 mon.a (mon.0) 3334 : audit [INF] from='client.? 172.21.15.59:0/214427465' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84128-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:36:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[59745]: cluster 2022-04-28T21:36:42.064144+0000 mgr.y (mgr.24961) 3507 : cluster [DBG] pgmap v4023: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[59745]: audit 2022-04-28T21:36:42.531983+0000 mon.a (mon.0) 3335 : audit [INF] from='client.? 172.21.15.59:0/214427465' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84128-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[59745]: cluster 2022-04-28T21:36:42.532222+0000 mon.a (mon.0) 3336 : cluster [DBG] osdmap e1149: 8 total, 8 up, 8 in 2022-04-28T21:36:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[61152]: cluster 2022-04-28T21:36:42.064144+0000 mgr.y (mgr.24961) 3507 : cluster [DBG] pgmap v4023: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[61152]: audit 2022-04-28T21:36:42.531983+0000 mon.a (mon.0) 3335 : audit [INF] from='client.? 172.21.15.59:0/214427465' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84128-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:43.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[61152]: cluster 2022-04-28T21:36:42.532222+0000 mon.a (mon.0) 3336 : cluster [DBG] osdmap e1149: 8 total, 8 up, 8 in 2022-04-28T21:36:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:43 smithi139 bash[43301]: cluster 2022-04-28T21:36:42.064144+0000 mgr.y (mgr.24961) 3507 : cluster [DBG] pgmap v4023: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:43 smithi139 bash[43301]: audit 2022-04-28T21:36:42.531983+0000 mon.a (mon.0) 3335 : audit [INF] from='client.? 172.21.15.59:0/214427465' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84128-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:36:43.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:43 smithi139 bash[43301]: cluster 2022-04-28T21:36:42.532222+0000 mon.a (mon.0) 3336 : cluster [DBG] osdmap e1149: 8 total, 8 up, 8 in 2022-04-28T21:36:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:43.930Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: cluster 2022-04-28T21:36:43.552015+0000 mon.a (mon.0) 3337 : cluster [DBG] osdmap e1150: 8 total, 8 up, 8 in 2022-04-28T21:36:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: audit 2022-04-28T21:36:43.567264+0000 mon.b (mon.2) 19 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: audit 2022-04-28T21:36:43.568337+0000 mon.b (mon.2) 20 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: audit 2022-04-28T21:36:43.569275+0000 mon.b (mon.2) 21 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: audit 2022-04-28T21:36:43.570838+0000 mon.a (mon.0) 3338 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: audit 2022-04-28T21:36:43.571870+0000 mon.a (mon.0) 3339 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[61152]: audit 2022-04-28T21:36:43.572854+0000 mon.a (mon.0) 3340 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:44.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: cluster 2022-04-28T21:36:43.552015+0000 mon.a (mon.0) 3337 : cluster [DBG] osdmap e1150: 8 total, 8 up, 8 in 2022-04-28T21:36:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: audit 2022-04-28T21:36:43.567264+0000 mon.b (mon.2) 19 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: audit 2022-04-28T21:36:43.568337+0000 mon.b (mon.2) 20 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: audit 2022-04-28T21:36:43.569275+0000 mon.b (mon.2) 21 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:44.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: audit 2022-04-28T21:36:43.570838+0000 mon.a (mon.0) 3338 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: audit 2022-04-28T21:36:43.571870+0000 mon.a (mon.0) 3339 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:44 smithi059 bash[59745]: audit 2022-04-28T21:36:43.572854+0000 mon.a (mon.0) 3340 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: cluster 2022-04-28T21:36:43.552015+0000 mon.a (mon.0) 3337 : cluster [DBG] osdmap e1150: 8 total, 8 up, 8 in 2022-04-28T21:36:44.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: audit 2022-04-28T21:36:43.567264+0000 mon.b (mon.2) 19 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: audit 2022-04-28T21:36:43.568337+0000 mon.b (mon.2) 20 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: audit 2022-04-28T21:36:43.569275+0000 mon.b (mon.2) 21 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:44.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: audit 2022-04-28T21:36:43.570838+0000 mon.a (mon.0) 3338 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: audit 2022-04-28T21:36:43.571870+0000 mon.a (mon.0) 3339 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:44.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:44 smithi139 bash[43301]: audit 2022-04-28T21:36:43.572854+0000 mon.a (mon.0) 3340 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:36:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[61152]: cluster 2022-04-28T21:36:44.064978+0000 mgr.y (mgr.24961) 3508 : cluster [DBG] pgmap v4026: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[61152]: audit 2022-04-28T21:36:44.573073+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:46.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[61152]: audit 2022-04-28T21:36:44.575294+0000 mon.a (mon.0) 3341 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:36:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[61152]: cluster 2022-04-28T21:36:44.575471+0000 mon.a (mon.0) 3342 : cluster [DBG] osdmap e1151: 8 total, 8 up, 8 in 2022-04-28T21:36:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[61152]: audit 2022-04-28T21:36:44.576709+0000 mon.a (mon.0) 3343 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[59745]: cluster 2022-04-28T21:36:44.064978+0000 mgr.y (mgr.24961) 3508 : cluster [DBG] pgmap v4026: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:46.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[59745]: audit 2022-04-28T21:36:44.573073+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[59745]: audit 2022-04-28T21:36:44.575294+0000 mon.a (mon.0) 3341 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:36:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[59745]: cluster 2022-04-28T21:36:44.575471+0000 mon.a (mon.0) 3342 : cluster [DBG] osdmap e1151: 8 total, 8 up, 8 in 2022-04-28T21:36:46.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:46 smithi059 bash[59745]: audit 2022-04-28T21:36:44.576709+0000 mon.a (mon.0) 3343 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:46.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:46 smithi139 bash[43301]: cluster 2022-04-28T21:36:44.064978+0000 mgr.y (mgr.24961) 3508 : cluster [DBG] pgmap v4026: 137 pgs: 137 active+clean; 24 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:46 smithi139 bash[43301]: audit 2022-04-28T21:36:44.573073+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:46 smithi139 bash[43301]: audit 2022-04-28T21:36:44.575294+0000 mon.a (mon.0) 3341 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84128-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:36:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:46 smithi139 bash[43301]: cluster 2022-04-28T21:36:44.575471+0000 mon.a (mon.0) 3342 : cluster [DBG] osdmap e1151: 8 total, 8 up, 8 in 2022-04-28T21:36:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:46 smithi139 bash[43301]: audit 2022-04-28T21:36:44.576709+0000 mon.a (mon.0) 3343 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:47.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[59745]: cluster 2022-04-28T21:36:46.044224+0000 mon.a (mon.0) 3344 : cluster [DBG] osdmap e1152: 8 total, 8 up, 8 in 2022-04-28T21:36:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[59745]: cluster 2022-04-28T21:36:46.065897+0000 mgr.y (mgr.24961) 3509 : cluster [DBG] pgmap v4029: 137 pgs: 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[59745]: audit 2022-04-28T21:36:46.589852+0000 mon.a (mon.0) 3345 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[59745]: cluster 2022-04-28T21:36:46.590063+0000 mon.a (mon.0) 3346 : cluster [DBG] osdmap e1153: 8 total, 8 up, 8 in 2022-04-28T21:36:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[61152]: cluster 2022-04-28T21:36:46.044224+0000 mon.a (mon.0) 3344 : cluster [DBG] osdmap e1152: 8 total, 8 up, 8 in 2022-04-28T21:36:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[61152]: cluster 2022-04-28T21:36:46.065897+0000 mgr.y (mgr.24961) 3509 : cluster [DBG] pgmap v4029: 137 pgs: 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[61152]: audit 2022-04-28T21:36:46.589852+0000 mon.a (mon.0) 3345 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:47 smithi059 bash[61152]: cluster 2022-04-28T21:36:46.590063+0000 mon.a (mon.0) 3346 : cluster [DBG] osdmap e1153: 8 total, 8 up, 8 in 2022-04-28T21:36:47.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:47 smithi139 bash[43301]: cluster 2022-04-28T21:36:46.044224+0000 mon.a (mon.0) 3344 : cluster [DBG] osdmap e1152: 8 total, 8 up, 8 in 2022-04-28T21:36:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:47 smithi139 bash[43301]: cluster 2022-04-28T21:36:46.065897+0000 mgr.y (mgr.24961) 3509 : cluster [DBG] pgmap v4029: 137 pgs: 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:47 smithi139 bash[43301]: audit 2022-04-28T21:36:46.589852+0000 mon.a (mon.0) 3345 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84128-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:47 smithi139 bash[43301]: cluster 2022-04-28T21:36:46.590063+0000 mon.a (mon.0) 3346 : cluster [DBG] osdmap e1153: 8 total, 8 up, 8 in 2022-04-28T21:36:48.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[59745]: cluster 2022-04-28T21:36:47.591171+0000 mon.a (mon.0) 3347 : cluster [DBG] osdmap e1154: 8 total, 8 up, 8 in 2022-04-28T21:36:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[59745]: audit 2022-04-28T21:36:48.485656+0000 mon.a (mon.0) 3348 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:36:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[59745]: audit 2022-04-28T21:36:48.488625+0000 mon.a (mon.0) 3349 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:36:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[59745]: audit 2022-04-28T21:36:48.490553+0000 mon.a (mon.0) 3350 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:36:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[61152]: cluster 2022-04-28T21:36:47.591171+0000 mon.a (mon.0) 3347 : cluster [DBG] osdmap e1154: 8 total, 8 up, 8 in 2022-04-28T21:36:48.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[61152]: audit 2022-04-28T21:36:48.485656+0000 mon.a (mon.0) 3348 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:36:48.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[61152]: audit 2022-04-28T21:36:48.488625+0000 mon.a (mon.0) 3349 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:36:48.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:48 smithi059 bash[61152]: audit 2022-04-28T21:36:48.490553+0000 mon.a (mon.0) 3350 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:36:48.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:48 smithi139 bash[43301]: cluster 2022-04-28T21:36:47.591171+0000 mon.a (mon.0) 3347 : cluster [DBG] osdmap e1154: 8 total, 8 up, 8 in 2022-04-28T21:36:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:48 smithi139 bash[43301]: audit 2022-04-28T21:36:48.485656+0000 mon.a (mon.0) 3348 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:36:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:48 smithi139 bash[43301]: audit 2022-04-28T21:36:48.488625+0000 mon.a (mon.0) 3349 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:36:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:48 smithi139 bash[43301]: audit 2022-04-28T21:36:48.490553+0000 mon.a (mon.0) 3350 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:36:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:49 smithi139 bash[43301]: cluster 2022-04-28T21:36:48.066588+0000 mgr.y (mgr.24961) 3510 : cluster [DBG] pgmap v4032: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:49 smithi139 bash[43301]: cephadm 2022-04-28T21:36:48.492200+0000 mgr.y (mgr.24961) 3511 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:36:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:49 smithi139 bash[43301]: cluster 2022-04-28T21:36:48.595181+0000 mon.a (mon.0) 3351 : cluster [DBG] osdmap e1155: 8 total, 8 up, 8 in 2022-04-28T21:36:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:49 smithi139 bash[43301]: audit 2022-04-28T21:36:48.597005+0000 mon.b (mon.2) 23 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:49 smithi139 bash[43301]: audit 2022-04-28T21:36:48.600739+0000 mon.a (mon.0) 3352 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:49 smithi139 bash[43301]: audit 2022-04-28T21:36:48.812225+0000 mon.a (mon.0) 3353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:36:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[61152]: cluster 2022-04-28T21:36:48.066588+0000 mgr.y (mgr.24961) 3510 : cluster [DBG] pgmap v4032: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[61152]: cephadm 2022-04-28T21:36:48.492200+0000 mgr.y (mgr.24961) 3511 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:36:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[61152]: cluster 2022-04-28T21:36:48.595181+0000 mon.a (mon.0) 3351 : cluster [DBG] osdmap e1155: 8 total, 8 up, 8 in 2022-04-28T21:36:50.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[61152]: audit 2022-04-28T21:36:48.597005+0000 mon.b (mon.2) 23 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:50.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[61152]: audit 2022-04-28T21:36:48.600739+0000 mon.a (mon.0) 3352 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:50.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[61152]: audit 2022-04-28T21:36:48.812225+0000 mon.a (mon.0) 3353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:36:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[59745]: cluster 2022-04-28T21:36:48.066588+0000 mgr.y (mgr.24961) 3510 : cluster [DBG] pgmap v4032: 145 pgs: 8 unknown, 137 active+clean; 24 KiB data, 821 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[59745]: cephadm 2022-04-28T21:36:48.492200+0000 mgr.y (mgr.24961) 3511 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:36:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[59745]: cluster 2022-04-28T21:36:48.595181+0000 mon.a (mon.0) 3351 : cluster [DBG] osdmap e1155: 8 total, 8 up, 8 in 2022-04-28T21:36:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[59745]: audit 2022-04-28T21:36:48.597005+0000 mon.b (mon.2) 23 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[59745]: audit 2022-04-28T21:36:48.600739+0000 mon.a (mon.0) 3352 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:49 smithi059 bash[59745]: audit 2022-04-28T21:36:48.812225+0000 mon.a (mon.0) 3353 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:36:50.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:50 smithi139 bash[43301]: audit 2022-04-28T21:36:49.603566+0000 mon.a (mon.0) 3354 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:50 smithi139 bash[43301]: cluster 2022-04-28T21:36:49.603651+0000 mon.a (mon.0) 3355 : cluster [DBG] osdmap e1156: 8 total, 8 up, 8 in 2022-04-28T21:36:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:50 smithi139 bash[43301]: audit 2022-04-28T21:36:49.605608+0000 mon.b (mon.2) 24 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:50 smithi139 bash[43301]: audit 2022-04-28T21:36:49.609268+0000 mon.a (mon.0) 3356 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:50 smithi139 bash[43301]: cluster 2022-04-28T21:36:50.067414+0000 mgr.y (mgr.24961) 3512 : cluster [DBG] pgmap v4035: 137 pgs: 137 active+clean; 24 KiB data, 826 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[61152]: audit 2022-04-28T21:36:49.603566+0000 mon.a (mon.0) 3354 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[61152]: cluster 2022-04-28T21:36:49.603651+0000 mon.a (mon.0) 3355 : cluster [DBG] osdmap e1156: 8 total, 8 up, 8 in 2022-04-28T21:36:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[61152]: audit 2022-04-28T21:36:49.605608+0000 mon.b (mon.2) 24 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[61152]: audit 2022-04-28T21:36:49.609268+0000 mon.a (mon.0) 3356 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[61152]: cluster 2022-04-28T21:36:50.067414+0000 mgr.y (mgr.24961) 3512 : cluster [DBG] pgmap v4035: 137 pgs: 137 active+clean; 24 KiB data, 826 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[59745]: audit 2022-04-28T21:36:49.603566+0000 mon.a (mon.0) 3354 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[59745]: cluster 2022-04-28T21:36:49.603651+0000 mon.a (mon.0) 3355 : cluster [DBG] osdmap e1156: 8 total, 8 up, 8 in 2022-04-28T21:36:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[59745]: audit 2022-04-28T21:36:49.605608+0000 mon.b (mon.2) 24 : audit [INF] from='client.? 172.21.15.59:0/1772812333' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[59745]: audit 2022-04-28T21:36:49.609268+0000 mon.a (mon.0) 3356 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]: dispatch 2022-04-28T21:36:51.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:50 smithi059 bash[59745]: cluster 2022-04-28T21:36:50.067414+0000 mgr.y (mgr.24961) 3512 : cluster [DBG] pgmap v4035: 137 pgs: 137 active+clean; 24 KiB data, 826 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:52.038 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:51 smithi059 bash[61152]: audit 2022-04-28T21:36:50.610762+0000 mon.a (mon.0) 3357 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:52.039 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:51 smithi059 bash[61152]: cluster 2022-04-28T21:36:50.610895+0000 mon.a (mon.0) 3358 : cluster [DBG] osdmap e1157: 8 total, 8 up, 8 in 2022-04-28T21:36:52.039 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:51 smithi059 bash[59745]: audit 2022-04-28T21:36:50.610762+0000 mon.a (mon.0) 3357 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:52.040 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:51 smithi059 bash[59745]: cluster 2022-04-28T21:36:50.610895+0000 mon.a (mon.0) 3358 : cluster [DBG] osdmap e1157: 8 total, 8 up, 8 in 2022-04-28T21:36:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:51 smithi139 bash[43301]: audit 2022-04-28T21:36:50.610762+0000 mon.a (mon.0) 3357 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84128-22"}]': finished 2022-04-28T21:36:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:51 smithi139 bash[43301]: cluster 2022-04-28T21:36:50.610895+0000 mon.a (mon.0) 3358 : cluster [DBG] osdmap e1157: 8 total, 8 up, 8 in 2022-04-28T21:36:52.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:52.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:53.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:52 smithi059 bash[61152]: cluster 2022-04-28T21:36:52.068458+0000 mgr.y (mgr.24961) 3513 : cluster [DBG] pgmap v4037: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:52 smithi059 bash[59745]: cluster 2022-04-28T21:36:52.068458+0000 mgr.y (mgr.24961) 3513 : cluster [DBG] pgmap v4037: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:53.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:52 smithi139 bash[43301]: cluster 2022-04-28T21:36:52.068458+0000 mgr.y (mgr.24961) 3513 : cluster [DBG] pgmap v4037: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:54.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:36:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:36:53.933Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:36:55.590 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:55 smithi059 bash[61152]: cluster 2022-04-28T21:36:54.069343+0000 mgr.y (mgr.24961) 3514 : cluster [DBG] pgmap v4038: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:55 smithi059 bash[59745]: cluster 2022-04-28T21:36:54.069343+0000 mgr.y (mgr.24961) 3514 : cluster [DBG] pgmap v4038: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:55 smithi139 bash[43301]: cluster 2022-04-28T21:36:54.069343+0000 mgr.y (mgr.24961) 3514 : cluster [DBG] pgmap v4038: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:57.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:57 smithi059 bash[61152]: cluster 2022-04-28T21:36:56.070817+0000 mgr.y (mgr.24961) 3515 : cluster [DBG] pgmap v4039: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:57.850 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:57 smithi059 bash[59745]: cluster 2022-04-28T21:36:56.070817+0000 mgr.y (mgr.24961) 3515 : cluster [DBG] pgmap v4039: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:57 smithi139 bash[43301]: cluster 2022-04-28T21:36:56.070817+0000 mgr.y (mgr.24961) 3515 : cluster [DBG] pgmap v4039: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:36:59 smithi059 bash[61152]: cluster 2022-04-28T21:36:58.071544+0000 mgr.y (mgr.24961) 3516 : cluster [DBG] pgmap v4040: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:36:59 smithi059 bash[59745]: cluster 2022-04-28T21:36:58.071544+0000 mgr.y (mgr.24961) 3516 : cluster [DBG] pgmap v4040: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:36:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:36:59 smithi139 bash[43301]: cluster 2022-04-28T21:36:58.071544+0000 mgr.y (mgr.24961) 3516 : cluster [DBG] pgmap v4040: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:01.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:01 smithi059 bash[61152]: cluster 2022-04-28T21:37:00.072743+0000 mgr.y (mgr.24961) 3517 : cluster [DBG] pgmap v4041: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:01 smithi059 bash[59745]: cluster 2022-04-28T21:37:00.072743+0000 mgr.y (mgr.24961) 3517 : cluster [DBG] pgmap v4041: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:01 smithi139 bash[43301]: cluster 2022-04-28T21:37:00.072743+0000 mgr.y (mgr.24961) 3517 : cluster [DBG] pgmap v4041: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:02.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:02.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:03.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:03 smithi059 bash[59745]: cluster 2022-04-28T21:37:02.073978+0000 mgr.y (mgr.24961) 3518 : cluster [DBG] pgmap v4042: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:03 smithi059 bash[61152]: cluster 2022-04-28T21:37:02.073978+0000 mgr.y (mgr.24961) 3518 : cluster [DBG] pgmap v4042: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:03 smithi139 bash[43301]: cluster 2022-04-28T21:37:02.073978+0000 mgr.y (mgr.24961) 3518 : cluster [DBG] pgmap v4042: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:03.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:05 smithi059 bash[61152]: cluster 2022-04-28T21:37:04.074945+0000 mgr.y (mgr.24961) 3519 : cluster [DBG] pgmap v4043: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:05 smithi059 bash[59745]: cluster 2022-04-28T21:37:04.074945+0000 mgr.y (mgr.24961) 3519 : cluster [DBG] pgmap v4043: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:05 smithi139 bash[43301]: cluster 2022-04-28T21:37:04.074945+0000 mgr.y (mgr.24961) 3519 : cluster [DBG] pgmap v4043: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:07.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:07 smithi059 bash[61152]: cluster 2022-04-28T21:37:06.076521+0000 mgr.y (mgr.24961) 3520 : cluster [DBG] pgmap v4044: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:07 smithi059 bash[59745]: cluster 2022-04-28T21:37:06.076521+0000 mgr.y (mgr.24961) 3520 : cluster [DBG] pgmap v4044: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:07 smithi139 bash[43301]: cluster 2022-04-28T21:37:06.076521+0000 mgr.y (mgr.24961) 3520 : cluster [DBG] pgmap v4044: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:09.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:09 smithi059 bash[61152]: cluster 2022-04-28T21:37:08.077251+0000 mgr.y (mgr.24961) 3521 : cluster [DBG] pgmap v4045: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:09 smithi059 bash[59745]: cluster 2022-04-28T21:37:08.077251+0000 mgr.y (mgr.24961) 3521 : cluster [DBG] pgmap v4045: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:09 smithi139 bash[43301]: cluster 2022-04-28T21:37:08.077251+0000 mgr.y (mgr.24961) 3521 : cluster [DBG] pgmap v4045: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:11.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:11 smithi059 bash[61152]: cluster 2022-04-28T21:37:10.078438+0000 mgr.y (mgr.24961) 3522 : cluster [DBG] pgmap v4046: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:11.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:11 smithi059 bash[59745]: cluster 2022-04-28T21:37:10.078438+0000 mgr.y (mgr.24961) 3522 : cluster [DBG] pgmap v4046: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:11.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:11 smithi139 bash[43301]: cluster 2022-04-28T21:37:10.078438+0000 mgr.y (mgr.24961) 3522 : cluster [DBG] pgmap v4046: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:12.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:13.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:13 smithi059 bash[61152]: cluster 2022-04-28T21:37:12.079687+0000 mgr.y (mgr.24961) 3523 : cluster [DBG] pgmap v4047: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:13 smithi059 bash[59745]: cluster 2022-04-28T21:37:12.079687+0000 mgr.y (mgr.24961) 3523 : cluster [DBG] pgmap v4047: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:13 smithi139 bash[43301]: cluster 2022-04-28T21:37:12.079687+0000 mgr.y (mgr.24961) 3523 : cluster [DBG] pgmap v4047: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:14.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:13.932Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:15 smithi059 bash[61152]: cluster 2022-04-28T21:37:14.080591+0000 mgr.y (mgr.24961) 3524 : cluster [DBG] pgmap v4048: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:15 smithi059 bash[59745]: cluster 2022-04-28T21:37:14.080591+0000 mgr.y (mgr.24961) 3524 : cluster [DBG] pgmap v4048: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:15 smithi139 bash[43301]: cluster 2022-04-28T21:37:14.080591+0000 mgr.y (mgr.24961) 3524 : cluster [DBG] pgmap v4048: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:17 smithi059 bash[61152]: cluster 2022-04-28T21:37:16.082075+0000 mgr.y (mgr.24961) 3525 : cluster [DBG] pgmap v4049: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:17.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:17 smithi059 bash[59745]: cluster 2022-04-28T21:37:16.082075+0000 mgr.y (mgr.24961) 3525 : cluster [DBG] pgmap v4049: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:17 smithi139 bash[43301]: cluster 2022-04-28T21:37:16.082075+0000 mgr.y (mgr.24961) 3525 : cluster [DBG] pgmap v4049: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: cluster 2022-04-28T21:37:18.082797+0000 mgr.y (mgr.24961) 3526 : cluster [DBG] pgmap v4050: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: audit 2022-04-28T21:37:19.313811+0000 mon.a (mon.0) 3359 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:37:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: audit 2022-04-28T21:37:19.317442+0000 mon.a (mon.0) 3360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:37:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: audit 2022-04-28T21:37:19.327170+0000 mon.a (mon.0) 3361 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:37:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: audit 2022-04-28T21:37:19.327769+0000 mon.a (mon.0) 3362 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:37:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: audit 2022-04-28T21:37:19.328332+0000 mon.a (mon.0) 3363 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:37:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[61152]: audit 2022-04-28T21:37:19.328838+0000 mon.a (mon.0) 3364 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:37:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: cluster 2022-04-28T21:37:18.082797+0000 mgr.y (mgr.24961) 3526 : cluster [DBG] pgmap v4050: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: audit 2022-04-28T21:37:19.313811+0000 mon.a (mon.0) 3359 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:37:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: audit 2022-04-28T21:37:19.317442+0000 mon.a (mon.0) 3360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:37:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: audit 2022-04-28T21:37:19.327170+0000 mon.a (mon.0) 3361 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:37:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: audit 2022-04-28T21:37:19.327769+0000 mon.a (mon.0) 3362 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:37:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: audit 2022-04-28T21:37:19.328332+0000 mon.a (mon.0) 3363 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:37:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:19 smithi059 bash[59745]: audit 2022-04-28T21:37:19.328838+0000 mon.a (mon.0) 3364 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:37:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: cluster 2022-04-28T21:37:18.082797+0000 mgr.y (mgr.24961) 3526 : cluster [DBG] pgmap v4050: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: audit 2022-04-28T21:37:19.313811+0000 mon.a (mon.0) 3359 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:37:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: audit 2022-04-28T21:37:19.317442+0000 mon.a (mon.0) 3360 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:37:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: audit 2022-04-28T21:37:19.327170+0000 mon.a (mon.0) 3361 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:37:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: audit 2022-04-28T21:37:19.327769+0000 mon.a (mon.0) 3362 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:37:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: audit 2022-04-28T21:37:19.328332+0000 mon.a (mon.0) 3363 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:37:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:19 smithi139 bash[43301]: audit 2022-04-28T21:37:19.328838+0000 mon.a (mon.0) 3364 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:37:21.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:21 smithi059 bash[59745]: cluster 2022-04-28T21:37:20.084128+0000 mgr.y (mgr.24961) 3527 : cluster [DBG] pgmap v4051: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:21.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:21 smithi059 bash[61152]: cluster 2022-04-28T21:37:20.084128+0000 mgr.y (mgr.24961) 3527 : cluster [DBG] pgmap v4051: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:21 smithi139 bash[43301]: cluster 2022-04-28T21:37:20.084128+0000 mgr.y (mgr.24961) 3527 : cluster [DBG] pgmap v4051: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:22.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:23 smithi059 bash[61152]: cluster 2022-04-28T21:37:22.085348+0000 mgr.y (mgr.24961) 3528 : cluster [DBG] pgmap v4052: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:23 smithi059 bash[59745]: cluster 2022-04-28T21:37:22.085348+0000 mgr.y (mgr.24961) 3528 : cluster [DBG] pgmap v4052: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:23 smithi139 bash[43301]: cluster 2022-04-28T21:37:22.085348+0000 mgr.y (mgr.24961) 3528 : cluster [DBG] pgmap v4052: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:23.934Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:25.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:25 smithi059 bash[61152]: cluster 2022-04-28T21:37:24.086262+0000 mgr.y (mgr.24961) 3529 : cluster [DBG] pgmap v4053: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:25 smithi059 bash[59745]: cluster 2022-04-28T21:37:24.086262+0000 mgr.y (mgr.24961) 3529 : cluster [DBG] pgmap v4053: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:25.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:25 smithi139 bash[43301]: cluster 2022-04-28T21:37:24.086262+0000 mgr.y (mgr.24961) 3529 : cluster [DBG] pgmap v4053: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:27.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:27 smithi059 bash[61152]: cluster 2022-04-28T21:37:26.087783+0000 mgr.y (mgr.24961) 3530 : cluster [DBG] pgmap v4054: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:27 smithi059 bash[59745]: cluster 2022-04-28T21:37:26.087783+0000 mgr.y (mgr.24961) 3530 : cluster [DBG] pgmap v4054: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:27 smithi139 bash[43301]: cluster 2022-04-28T21:37:26.087783+0000 mgr.y (mgr.24961) 3530 : cluster [DBG] pgmap v4054: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:29.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:29 smithi059 bash[61152]: cluster 2022-04-28T21:37:28.088487+0000 mgr.y (mgr.24961) 3531 : cluster [DBG] pgmap v4055: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:29 smithi059 bash[59745]: cluster 2022-04-28T21:37:28.088487+0000 mgr.y (mgr.24961) 3531 : cluster [DBG] pgmap v4055: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:29.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:29 smithi139 bash[43301]: cluster 2022-04-28T21:37:28.088487+0000 mgr.y (mgr.24961) 3531 : cluster [DBG] pgmap v4055: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:31.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:31 smithi059 bash[59745]: cluster 2022-04-28T21:37:30.089748+0000 mgr.y (mgr.24961) 3532 : cluster [DBG] pgmap v4056: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:31 smithi059 bash[61152]: cluster 2022-04-28T21:37:30.089748+0000 mgr.y (mgr.24961) 3532 : cluster [DBG] pgmap v4056: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:31.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:31 smithi139 bash[43301]: cluster 2022-04-28T21:37:30.089748+0000 mgr.y (mgr.24961) 3532 : cluster [DBG] pgmap v4056: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:32.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:33.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:33 smithi059 bash[61152]: cluster 2022-04-28T21:37:32.090828+0000 mgr.y (mgr.24961) 3533 : cluster [DBG] pgmap v4057: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:33 smithi059 bash[59745]: cluster 2022-04-28T21:37:32.090828+0000 mgr.y (mgr.24961) 3533 : cluster [DBG] pgmap v4057: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:33 smithi139 bash[43301]: cluster 2022-04-28T21:37:32.090828+0000 mgr.y (mgr.24961) 3533 : cluster [DBG] pgmap v4057: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:34.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:33.935Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:35 smithi059 bash[61152]: cluster 2022-04-28T21:37:34.091859+0000 mgr.y (mgr.24961) 3534 : cluster [DBG] pgmap v4058: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:35 smithi059 bash[59745]: cluster 2022-04-28T21:37:34.091859+0000 mgr.y (mgr.24961) 3534 : cluster [DBG] pgmap v4058: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:35 smithi139 bash[43301]: cluster 2022-04-28T21:37:34.091859+0000 mgr.y (mgr.24961) 3534 : cluster [DBG] pgmap v4058: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:37 smithi139 bash[43301]: cluster 2022-04-28T21:37:36.093249+0000 mgr.y (mgr.24961) 3535 : cluster [DBG] pgmap v4059: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:38.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:37 smithi059 bash[61152]: cluster 2022-04-28T21:37:36.093249+0000 mgr.y (mgr.24961) 3535 : cluster [DBG] pgmap v4059: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:37 smithi059 bash[59745]: cluster 2022-04-28T21:37:36.093249+0000 mgr.y (mgr.24961) 3535 : cluster [DBG] pgmap v4059: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:39 smithi139 bash[43301]: cluster 2022-04-28T21:37:38.094001+0000 mgr.y (mgr.24961) 3536 : cluster [DBG] pgmap v4060: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:39 smithi059 bash[59745]: cluster 2022-04-28T21:37:38.094001+0000 mgr.y (mgr.24961) 3536 : cluster [DBG] pgmap v4060: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:40.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:39 smithi059 bash[61152]: cluster 2022-04-28T21:37:38.094001+0000 mgr.y (mgr.24961) 3536 : cluster [DBG] pgmap v4060: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:41 smithi139 bash[43301]: cluster 2022-04-28T21:37:40.095245+0000 mgr.y (mgr.24961) 3537 : cluster [DBG] pgmap v4061: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:42.043 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:41 smithi059 bash[61152]: cluster 2022-04-28T21:37:40.095245+0000 mgr.y (mgr.24961) 3537 : cluster [DBG] pgmap v4061: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:42.043 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:41 smithi059 bash[59745]: cluster 2022-04-28T21:37:40.095245+0000 mgr.y (mgr.24961) 3537 : cluster [DBG] pgmap v4061: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:43 smithi139 bash[43301]: cluster 2022-04-28T21:37:42.096519+0000 mgr.y (mgr.24961) 3538 : cluster [DBG] pgmap v4062: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:43.936 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:43 smithi059 bash[61152]: cluster 2022-04-28T21:37:42.096519+0000 mgr.y (mgr.24961) 3538 : cluster [DBG] pgmap v4062: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:43.937 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:42.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:43.937 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:43 smithi059 bash[59745]: cluster 2022-04-28T21:37:42.096519+0000 mgr.y (mgr.24961) 3538 : cluster [DBG] pgmap v4062: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:43.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:45.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:44 smithi059 bash[61152]: cluster 2022-04-28T21:37:44.097381+0000 mgr.y (mgr.24961) 3539 : cluster [DBG] pgmap v4063: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:45.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:44 smithi059 bash[59745]: cluster 2022-04-28T21:37:44.097381+0000 mgr.y (mgr.24961) 3539 : cluster [DBG] pgmap v4063: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:45.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:44 smithi139 bash[43301]: cluster 2022-04-28T21:37:44.097381+0000 mgr.y (mgr.24961) 3539 : cluster [DBG] pgmap v4063: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:47 smithi059 bash[61152]: cluster 2022-04-28T21:37:46.098880+0000 mgr.y (mgr.24961) 3540 : cluster [DBG] pgmap v4064: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:47 smithi059 bash[59745]: cluster 2022-04-28T21:37:46.098880+0000 mgr.y (mgr.24961) 3540 : cluster [DBG] pgmap v4064: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:47.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:47 smithi139 bash[43301]: cluster 2022-04-28T21:37:46.098880+0000 mgr.y (mgr.24961) 3540 : cluster [DBG] pgmap v4064: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[59745]: cluster 2022-04-28T21:37:48.099629+0000 mgr.y (mgr.24961) 3541 : cluster [DBG] pgmap v4065: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[59745]: audit 2022-04-28T21:37:48.821481+0000 mon.a (mon.0) 3365 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:37:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[59745]: audit 2022-04-28T21:37:48.824455+0000 mon.a (mon.0) 3366 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:37:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[59745]: audit 2022-04-28T21:37:48.826406+0000 mon.a (mon.0) 3367 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:37:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[59745]: audit 2022-04-28T21:37:48.829717+0000 mon.a (mon.0) 3368 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:37:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[59745]: audit 2022-04-28T21:37:48.830335+0000 mon.a (mon.0) 3369 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:37:49.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[61152]: cluster 2022-04-28T21:37:48.099629+0000 mgr.y (mgr.24961) 3541 : cluster [DBG] pgmap v4065: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[61152]: audit 2022-04-28T21:37:48.821481+0000 mon.a (mon.0) 3365 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:37:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[61152]: audit 2022-04-28T21:37:48.824455+0000 mon.a (mon.0) 3366 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:37:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[61152]: audit 2022-04-28T21:37:48.826406+0000 mon.a (mon.0) 3367 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:37:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[61152]: audit 2022-04-28T21:37:48.829717+0000 mon.a (mon.0) 3368 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:37:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:49 smithi059 bash[61152]: audit 2022-04-28T21:37:48.830335+0000 mon.a (mon.0) 3369 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:37:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:49 smithi139 bash[43301]: cluster 2022-04-28T21:37:48.099629+0000 mgr.y (mgr.24961) 3541 : cluster [DBG] pgmap v4065: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:49.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:49 smithi139 bash[43301]: audit 2022-04-28T21:37:48.821481+0000 mon.a (mon.0) 3365 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:37:49.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:49 smithi139 bash[43301]: audit 2022-04-28T21:37:48.824455+0000 mon.a (mon.0) 3366 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:37:49.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:49 smithi139 bash[43301]: audit 2022-04-28T21:37:48.826406+0000 mon.a (mon.0) 3367 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:37:49.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:49 smithi139 bash[43301]: audit 2022-04-28T21:37:48.829717+0000 mon.a (mon.0) 3368 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:37:49.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:49 smithi139 bash[43301]: audit 2022-04-28T21:37:48.830335+0000 mon.a (mon.0) 3369 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:37:50.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:50 smithi059 bash[61152]: cephadm 2022-04-28T21:37:48.828221+0000 mgr.y (mgr.24961) 3542 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:37:50.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:50 smithi059 bash[59745]: cephadm 2022-04-28T21:37:48.828221+0000 mgr.y (mgr.24961) 3542 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:37:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:50 smithi139 bash[43301]: cephadm 2022-04-28T21:37:48.828221+0000 mgr.y (mgr.24961) 3542 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:37:51.840 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:51 smithi059 bash[59745]: cluster 2022-04-28T21:37:50.100827+0000 mgr.y (mgr.24961) 3543 : cluster [DBG] pgmap v4066: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:51 smithi059 bash[61152]: cluster 2022-04-28T21:37:50.100827+0000 mgr.y (mgr.24961) 3543 : cluster [DBG] pgmap v4066: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:51 smithi139 bash[43301]: cluster 2022-04-28T21:37:50.100827+0000 mgr.y (mgr.24961) 3543 : cluster [DBG] pgmap v4066: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:52.041Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[61152]: cluster 2022-04-28T21:37:52.101714+0000 mgr.y (mgr.24961) 3544 : cluster [DBG] pgmap v4067: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[61152]: audit 2022-04-28T21:37:52.937862+0000 mon.a (mon.0) 3370 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[61152]: audit 2022-04-28T21:37:53.133634+0000 mon.a (mon.0) 3371 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[61152]: audit 2022-04-28T21:37:53.427455+0000 mon.a (mon.0) 3372 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[59745]: cluster 2022-04-28T21:37:52.101714+0000 mgr.y (mgr.24961) 3544 : cluster [DBG] pgmap v4067: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[59745]: audit 2022-04-28T21:37:52.937862+0000 mon.a (mon.0) 3370 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[59745]: audit 2022-04-28T21:37:53.133634+0000 mon.a (mon.0) 3371 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[59745]: audit 2022-04-28T21:37:53.427455+0000 mon.a (mon.0) 3372 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:53 smithi139 bash[43301]: cluster 2022-04-28T21:37:52.101714+0000 mgr.y (mgr.24961) 3544 : cluster [DBG] pgmap v4067: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:53 smithi139 bash[43301]: audit 2022-04-28T21:37:52.937862+0000 mon.a (mon.0) 3370 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:53 smithi139 bash[43301]: audit 2022-04-28T21:37:53.133634+0000 mon.a (mon.0) 3371 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:53 smithi139 bash[43301]: audit 2022-04-28T21:37:53.427455+0000 mon.a (mon.0) 3372 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:37:54.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:37:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:37:53.937Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:37:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:55 smithi059 bash[61152]: cluster 2022-04-28T21:37:54.102570+0000 mgr.y (mgr.24961) 3545 : cluster [DBG] pgmap v4068: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:55 smithi059 bash[59745]: cluster 2022-04-28T21:37:54.102570+0000 mgr.y (mgr.24961) 3545 : cluster [DBG] pgmap v4068: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:55 smithi139 bash[43301]: cluster 2022-04-28T21:37:54.102570+0000 mgr.y (mgr.24961) 3545 : cluster [DBG] pgmap v4068: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:57.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:57 smithi059 bash[61152]: cluster 2022-04-28T21:37:56.104084+0000 mgr.y (mgr.24961) 3546 : cluster [DBG] pgmap v4069: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:57 smithi059 bash[59745]: cluster 2022-04-28T21:37:56.104084+0000 mgr.y (mgr.24961) 3546 : cluster [DBG] pgmap v4069: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:57.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:57 smithi139 bash[43301]: cluster 2022-04-28T21:37:56.104084+0000 mgr.y (mgr.24961) 3546 : cluster [DBG] pgmap v4069: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:59.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:37:59 smithi059 bash[61152]: cluster 2022-04-28T21:37:58.104807+0000 mgr.y (mgr.24961) 3547 : cluster [DBG] pgmap v4070: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:37:59 smithi059 bash[59745]: cluster 2022-04-28T21:37:58.104807+0000 mgr.y (mgr.24961) 3547 : cluster [DBG] pgmap v4070: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:37:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:37:59 smithi139 bash[43301]: cluster 2022-04-28T21:37:58.104807+0000 mgr.y (mgr.24961) 3547 : cluster [DBG] pgmap v4070: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:01 smithi059 bash[61152]: cluster 2022-04-28T21:38:00.105985+0000 mgr.y (mgr.24961) 3548 : cluster [DBG] pgmap v4071: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:01 smithi059 bash[59745]: cluster 2022-04-28T21:38:00.105985+0000 mgr.y (mgr.24961) 3548 : cluster [DBG] pgmap v4071: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:01 smithi139 bash[43301]: cluster 2022-04-28T21:38:00.105985+0000 mgr.y (mgr.24961) 3548 : cluster [DBG] pgmap v4071: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:02.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:02.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:03.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:03 smithi059 bash[61152]: cluster 2022-04-28T21:38:02.107097+0000 mgr.y (mgr.24961) 3549 : cluster [DBG] pgmap v4072: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:03 smithi059 bash[59745]: cluster 2022-04-28T21:38:02.107097+0000 mgr.y (mgr.24961) 3549 : cluster [DBG] pgmap v4072: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:03 smithi139 bash[43301]: cluster 2022-04-28T21:38:02.107097+0000 mgr.y (mgr.24961) 3549 : cluster [DBG] pgmap v4072: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:03.936Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:05.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:05 smithi059 bash[61152]: cluster 2022-04-28T21:38:04.108048+0000 mgr.y (mgr.24961) 3550 : cluster [DBG] pgmap v4073: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:05 smithi059 bash[59745]: cluster 2022-04-28T21:38:04.108048+0000 mgr.y (mgr.24961) 3550 : cluster [DBG] pgmap v4073: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:05 smithi139 bash[43301]: cluster 2022-04-28T21:38:04.108048+0000 mgr.y (mgr.24961) 3550 : cluster [DBG] pgmap v4073: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:07 smithi059 bash[61152]: cluster 2022-04-28T21:38:06.109580+0000 mgr.y (mgr.24961) 3551 : cluster [DBG] pgmap v4074: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:07 smithi059 bash[59745]: cluster 2022-04-28T21:38:06.109580+0000 mgr.y (mgr.24961) 3551 : cluster [DBG] pgmap v4074: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:07 smithi139 bash[43301]: cluster 2022-04-28T21:38:06.109580+0000 mgr.y (mgr.24961) 3551 : cluster [DBG] pgmap v4074: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:09.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:09 smithi059 bash[61152]: cluster 2022-04-28T21:38:08.110292+0000 mgr.y (mgr.24961) 3552 : cluster [DBG] pgmap v4075: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:09 smithi059 bash[59745]: cluster 2022-04-28T21:38:08.110292+0000 mgr.y (mgr.24961) 3552 : cluster [DBG] pgmap v4075: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:09.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:09 smithi139 bash[43301]: cluster 2022-04-28T21:38:08.110292+0000 mgr.y (mgr.24961) 3552 : cluster [DBG] pgmap v4075: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:11.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:11 smithi059 bash[61152]: cluster 2022-04-28T21:38:10.111621+0000 mgr.y (mgr.24961) 3553 : cluster [DBG] pgmap v4076: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:11.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:11 smithi059 bash[59745]: cluster 2022-04-28T21:38:10.111621+0000 mgr.y (mgr.24961) 3553 : cluster [DBG] pgmap v4076: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:11 smithi139 bash[43301]: cluster 2022-04-28T21:38:10.111621+0000 mgr.y (mgr.24961) 3553 : cluster [DBG] pgmap v4076: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:12.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:13 smithi059 bash[61152]: cluster 2022-04-28T21:38:12.112757+0000 mgr.y (mgr.24961) 3554 : cluster [DBG] pgmap v4077: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:13 smithi059 bash[59745]: cluster 2022-04-28T21:38:12.112757+0000 mgr.y (mgr.24961) 3554 : cluster [DBG] pgmap v4077: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:13.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:13 smithi139 bash[43301]: cluster 2022-04-28T21:38:12.112757+0000 mgr.y (mgr.24961) 3554 : cluster [DBG] pgmap v4077: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:13.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:15.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:15 smithi059 bash[61152]: cluster 2022-04-28T21:38:14.113766+0000 mgr.y (mgr.24961) 3555 : cluster [DBG] pgmap v4078: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:15.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:15 smithi059 bash[59745]: cluster 2022-04-28T21:38:14.113766+0000 mgr.y (mgr.24961) 3555 : cluster [DBG] pgmap v4078: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:15.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:15 smithi139 bash[43301]: cluster 2022-04-28T21:38:14.113766+0000 mgr.y (mgr.24961) 3555 : cluster [DBG] pgmap v4078: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:17 smithi139 bash[43301]: cluster 2022-04-28T21:38:16.115215+0000 mgr.y (mgr.24961) 3556 : cluster [DBG] pgmap v4079: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:18.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:17 smithi059 bash[61152]: cluster 2022-04-28T21:38:16.115215+0000 mgr.y (mgr.24961) 3556 : cluster [DBG] pgmap v4079: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:17 smithi059 bash[59745]: cluster 2022-04-28T21:38:16.115215+0000 mgr.y (mgr.24961) 3556 : cluster [DBG] pgmap v4079: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: cluster 2022-04-28T21:38:18.115977+0000 mgr.y (mgr.24961) 3557 : cluster [DBG] pgmap v4080: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: audit 2022-04-28T21:38:19.320987+0000 mon.a (mon.0) 3373 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:38:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: audit 2022-04-28T21:38:19.326028+0000 mon.a (mon.0) 3374 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:38:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: audit 2022-04-28T21:38:19.336804+0000 mon.a (mon.0) 3375 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:38:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: audit 2022-04-28T21:38:19.337430+0000 mon.a (mon.0) 3376 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:38:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: audit 2022-04-28T21:38:19.337987+0000 mon.a (mon.0) 3377 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:38:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:19 smithi139 bash[43301]: audit 2022-04-28T21:38:19.338547+0000 mon.a (mon.0) 3378 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:38:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: cluster 2022-04-28T21:38:18.115977+0000 mgr.y (mgr.24961) 3557 : cluster [DBG] pgmap v4080: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: audit 2022-04-28T21:38:19.320987+0000 mon.a (mon.0) 3373 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:38:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: audit 2022-04-28T21:38:19.326028+0000 mon.a (mon.0) 3374 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:38:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: audit 2022-04-28T21:38:19.336804+0000 mon.a (mon.0) 3375 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:38:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: audit 2022-04-28T21:38:19.337430+0000 mon.a (mon.0) 3376 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:38:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: audit 2022-04-28T21:38:19.337987+0000 mon.a (mon.0) 3377 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:38:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[61152]: audit 2022-04-28T21:38:19.338547+0000 mon.a (mon.0) 3378 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:38:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: cluster 2022-04-28T21:38:18.115977+0000 mgr.y (mgr.24961) 3557 : cluster [DBG] pgmap v4080: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: audit 2022-04-28T21:38:19.320987+0000 mon.a (mon.0) 3373 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:38:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: audit 2022-04-28T21:38:19.326028+0000 mon.a (mon.0) 3374 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:38:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: audit 2022-04-28T21:38:19.336804+0000 mon.a (mon.0) 3375 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:38:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: audit 2022-04-28T21:38:19.337430+0000 mon.a (mon.0) 3376 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:38:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: audit 2022-04-28T21:38:19.337987+0000 mon.a (mon.0) 3377 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:38:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:19 smithi059 bash[59745]: audit 2022-04-28T21:38:19.338547+0000 mon.a (mon.0) 3378 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T21:38:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:21 smithi139 bash[43301]: cluster 2022-04-28T21:38:20.117200+0000 mgr.y (mgr.24961) 3558 : cluster [DBG] pgmap v4081: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:22.044 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:21 smithi059 bash[61152]: cluster 2022-04-28T21:38:20.117200+0000 mgr.y (mgr.24961) 3558 : cluster [DBG] pgmap v4081: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:22.045 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:21 smithi059 bash[59745]: cluster 2022-04-28T21:38:20.117200+0000 mgr.y (mgr.24961) 3558 : cluster [DBG] pgmap v4081: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:22.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:23.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:23 smithi139 bash[43301]: cluster 2022-04-28T21:38:22.118316+0000 mgr.y (mgr.24961) 3559 : cluster [DBG] pgmap v4082: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:23.939 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:23 smithi059 bash[61152]: cluster 2022-04-28T21:38:22.118316+0000 mgr.y (mgr.24961) 3559 : cluster [DBG] pgmap v4082: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:23.939 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:23 smithi059 bash[59745]: cluster 2022-04-28T21:38:22.118316+0000 mgr.y (mgr.24961) 3559 : cluster [DBG] pgmap v4082: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:23.938Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:25.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:25 smithi139 bash[43301]: cluster 2022-04-28T21:38:24.119348+0000 mgr.y (mgr.24961) 3560 : cluster [DBG] pgmap v4083: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:26.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:25 smithi059 bash[61152]: cluster 2022-04-28T21:38:24.119348+0000 mgr.y (mgr.24961) 3560 : cluster [DBG] pgmap v4083: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:25 smithi059 bash[59745]: cluster 2022-04-28T21:38:24.119348+0000 mgr.y (mgr.24961) 3560 : cluster [DBG] pgmap v4083: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:27 smithi059 bash[61152]: cluster 2022-04-28T21:38:26.120680+0000 mgr.y (mgr.24961) 3561 : cluster [DBG] pgmap v4084: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:27 smithi059 bash[59745]: cluster 2022-04-28T21:38:26.120680+0000 mgr.y (mgr.24961) 3561 : cluster [DBG] pgmap v4084: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:27 smithi139 bash[43301]: cluster 2022-04-28T21:38:26.120680+0000 mgr.y (mgr.24961) 3561 : cluster [DBG] pgmap v4084: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:29 smithi059 bash[61152]: cluster 2022-04-28T21:38:28.121410+0000 mgr.y (mgr.24961) 3562 : cluster [DBG] pgmap v4085: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:29 smithi059 bash[59745]: cluster 2022-04-28T21:38:28.121410+0000 mgr.y (mgr.24961) 3562 : cluster [DBG] pgmap v4085: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:29 smithi139 bash[43301]: cluster 2022-04-28T21:38:28.121410+0000 mgr.y (mgr.24961) 3562 : cluster [DBG] pgmap v4085: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:31.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:31 smithi059 bash[61152]: cluster 2022-04-28T21:38:30.122657+0000 mgr.y (mgr.24961) 3563 : cluster [DBG] pgmap v4086: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:31 smithi059 bash[59745]: cluster 2022-04-28T21:38:30.122657+0000 mgr.y (mgr.24961) 3563 : cluster [DBG] pgmap v4086: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:31 smithi139 bash[43301]: cluster 2022-04-28T21:38:30.122657+0000 mgr.y (mgr.24961) 3563 : cluster [DBG] pgmap v4086: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:32.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:33 smithi059 bash[61152]: cluster 2022-04-28T21:38:32.123856+0000 mgr.y (mgr.24961) 3564 : cluster [DBG] pgmap v4087: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:33 smithi059 bash[59745]: cluster 2022-04-28T21:38:32.123856+0000 mgr.y (mgr.24961) 3564 : cluster [DBG] pgmap v4087: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:33.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:33 smithi139 bash[43301]: cluster 2022-04-28T21:38:32.123856+0000 mgr.y (mgr.24961) 3564 : cluster [DBG] pgmap v4087: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:33.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:35.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:35 smithi059 bash[61152]: cluster 2022-04-28T21:38:34.124864+0000 mgr.y (mgr.24961) 3565 : cluster [DBG] pgmap v4088: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:35.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:35 smithi059 bash[59745]: cluster 2022-04-28T21:38:34.124864+0000 mgr.y (mgr.24961) 3565 : cluster [DBG] pgmap v4088: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:35 smithi139 bash[43301]: cluster 2022-04-28T21:38:34.124864+0000 mgr.y (mgr.24961) 3565 : cluster [DBG] pgmap v4088: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:37.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:37 smithi059 bash[61152]: cluster 2022-04-28T21:38:36.126196+0000 mgr.y (mgr.24961) 3566 : cluster [DBG] pgmap v4089: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:37.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:37 smithi059 bash[59745]: cluster 2022-04-28T21:38:36.126196+0000 mgr.y (mgr.24961) 3566 : cluster [DBG] pgmap v4089: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:37.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:37 smithi139 bash[43301]: cluster 2022-04-28T21:38:36.126196+0000 mgr.y (mgr.24961) 3566 : cluster [DBG] pgmap v4089: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:39.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:39 smithi059 bash[61152]: cluster 2022-04-28T21:38:38.126969+0000 mgr.y (mgr.24961) 3567 : cluster [DBG] pgmap v4090: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:39 smithi059 bash[59745]: cluster 2022-04-28T21:38:38.126969+0000 mgr.y (mgr.24961) 3567 : cluster [DBG] pgmap v4090: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:39 smithi139 bash[43301]: cluster 2022-04-28T21:38:38.126969+0000 mgr.y (mgr.24961) 3567 : cluster [DBG] pgmap v4090: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:41.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:41 smithi059 bash[61152]: cluster 2022-04-28T21:38:40.128289+0000 mgr.y (mgr.24961) 3568 : cluster [DBG] pgmap v4091: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:41.858 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:41 smithi059 bash[59745]: cluster 2022-04-28T21:38:40.128289+0000 mgr.y (mgr.24961) 3568 : cluster [DBG] pgmap v4091: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:41 smithi139 bash[43301]: cluster 2022-04-28T21:38:40.128289+0000 mgr.y (mgr.24961) 3568 : cluster [DBG] pgmap v4091: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:42.044Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:43 smithi059 bash[59745]: cluster 2022-04-28T21:38:42.129376+0000 mgr.y (mgr.24961) 3569 : cluster [DBG] pgmap v4092: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:43 smithi059 bash[61152]: cluster 2022-04-28T21:38:42.129376+0000 mgr.y (mgr.24961) 3569 : cluster [DBG] pgmap v4092: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:43.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:43 smithi139 bash[43301]: cluster 2022-04-28T21:38:42.129376+0000 mgr.y (mgr.24961) 3569 : cluster [DBG] pgmap v4092: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:44.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:43.939Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:45 smithi059 bash[61152]: cluster 2022-04-28T21:38:44.130517+0000 mgr.y (mgr.24961) 3570 : cluster [DBG] pgmap v4093: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:45 smithi059 bash[59745]: cluster 2022-04-28T21:38:44.130517+0000 mgr.y (mgr.24961) 3570 : cluster [DBG] pgmap v4093: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:45 smithi139 bash[43301]: cluster 2022-04-28T21:38:44.130517+0000 mgr.y (mgr.24961) 3570 : cluster [DBG] pgmap v4093: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:47.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:47 smithi059 bash[61152]: cluster 2022-04-28T21:38:46.131890+0000 mgr.y (mgr.24961) 3571 : cluster [DBG] pgmap v4094: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:47 smithi059 bash[59745]: cluster 2022-04-28T21:38:46.131890+0000 mgr.y (mgr.24961) 3571 : cluster [DBG] pgmap v4094: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:47 smithi139 bash[43301]: cluster 2022-04-28T21:38:46.131890+0000 mgr.y (mgr.24961) 3571 : cluster [DBG] pgmap v4094: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:49 smithi059 bash[61152]: cluster 2022-04-28T21:38:48.132597+0000 mgr.y (mgr.24961) 3572 : cluster [DBG] pgmap v4095: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:49 smithi059 bash[59745]: cluster 2022-04-28T21:38:48.132597+0000 mgr.y (mgr.24961) 3572 : cluster [DBG] pgmap v4095: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:49.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:49 smithi139 bash[43301]: cluster 2022-04-28T21:38:48.132597+0000 mgr.y (mgr.24961) 3572 : cluster [DBG] pgmap v4095: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:51 smithi059 bash[61152]: cluster 2022-04-28T21:38:50.133838+0000 mgr.y (mgr.24961) 3573 : cluster [DBG] pgmap v4096: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:51 smithi059 bash[59745]: cluster 2022-04-28T21:38:50.133838+0000 mgr.y (mgr.24961) 3573 : cluster [DBG] pgmap v4096: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:51.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:51 smithi139 bash[43301]: cluster 2022-04-28T21:38:50.133838+0000 mgr.y (mgr.24961) 3573 : cluster [DBG] pgmap v4096: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:52.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:52.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[59745]: cluster 2022-04-28T21:38:52.134952+0000 mgr.y (mgr.24961) 3574 : cluster [DBG] pgmap v4097: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[59745]: audit 2022-04-28T21:38:53.438197+0000 mon.a (mon.0) 3379 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:38:53.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[59745]: audit 2022-04-28T21:38:53.441156+0000 mon.a (mon.0) 3380 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:38:53.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[59745]: audit 2022-04-28T21:38:53.443018+0000 mon.a (mon.0) 3381 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:38:53.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[61152]: cluster 2022-04-28T21:38:52.134952+0000 mgr.y (mgr.24961) 3574 : cluster [DBG] pgmap v4097: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:53.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[61152]: audit 2022-04-28T21:38:53.438197+0000 mon.a (mon.0) 3379 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:38:53.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[61152]: audit 2022-04-28T21:38:53.441156+0000 mon.a (mon.0) 3380 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:38:53.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[61152]: audit 2022-04-28T21:38:53.443018+0000 mon.a (mon.0) 3381 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:38:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:53 smithi139 bash[43301]: cluster 2022-04-28T21:38:52.134952+0000 mgr.y (mgr.24961) 3574 : cluster [DBG] pgmap v4097: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:53.887 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:53 smithi139 bash[43301]: audit 2022-04-28T21:38:53.438197+0000 mon.a (mon.0) 3379 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:38:53.887 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:53 smithi139 bash[43301]: audit 2022-04-28T21:38:53.441156+0000 mon.a (mon.0) 3380 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:38:53.887 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:53 smithi139 bash[43301]: audit 2022-04-28T21:38:53.443018+0000 mon.a (mon.0) 3381 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:38:54.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:38:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:38:53.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:38:54.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:54 smithi059 bash[61152]: cephadm 2022-04-28T21:38:53.444761+0000 mgr.y (mgr.24961) 3575 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:38:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:54 smithi059 bash[61152]: audit 2022-04-28T21:38:53.773288+0000 mon.a (mon.0) 3382 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:38:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:54 smithi059 bash[59745]: cephadm 2022-04-28T21:38:53.444761+0000 mgr.y (mgr.24961) 3575 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:38:54.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:54 smithi059 bash[59745]: audit 2022-04-28T21:38:53.773288+0000 mon.a (mon.0) 3382 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:38:54.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:54 smithi139 bash[43301]: cephadm 2022-04-28T21:38:53.444761+0000 mgr.y (mgr.24961) 3575 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:38:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:54 smithi139 bash[43301]: audit 2022-04-28T21:38:53.773288+0000 mon.a (mon.0) 3382 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:38:55.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:55 smithi059 bash[61152]: cluster 2022-04-28T21:38:54.136081+0000 mgr.y (mgr.24961) 3576 : cluster [DBG] pgmap v4098: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:55.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:55 smithi059 bash[59745]: cluster 2022-04-28T21:38:54.136081+0000 mgr.y (mgr.24961) 3576 : cluster [DBG] pgmap v4098: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:55.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:55 smithi139 bash[43301]: cluster 2022-04-28T21:38:54.136081+0000 mgr.y (mgr.24961) 3576 : cluster [DBG] pgmap v4098: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:57.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:57 smithi139 bash[43301]: cluster 2022-04-28T21:38:56.137442+0000 mgr.y (mgr.24961) 3577 : cluster [DBG] pgmap v4099: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:57 smithi059 bash[61152]: cluster 2022-04-28T21:38:56.137442+0000 mgr.y (mgr.24961) 3577 : cluster [DBG] pgmap v4099: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:57 smithi059 bash[59745]: cluster 2022-04-28T21:38:56.137442+0000 mgr.y (mgr.24961) 3577 : cluster [DBG] pgmap v4099: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:38:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:38:59 smithi139 bash[43301]: cluster 2022-04-28T21:38:58.138166+0000 mgr.y (mgr.24961) 3578 : cluster [DBG] pgmap v4100: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:00.090 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:38:59 smithi059 bash[61152]: cluster 2022-04-28T21:38:58.138166+0000 mgr.y (mgr.24961) 3578 : cluster [DBG] pgmap v4100: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:38:59 smithi059 bash[59745]: cluster 2022-04-28T21:38:58.138166+0000 mgr.y (mgr.24961) 3578 : cluster [DBG] pgmap v4100: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:01.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:00 smithi059 bash[59745]: cluster 2022-04-28T21:39:00.139444+0000 mgr.y (mgr.24961) 3579 : cluster [DBG] pgmap v4101: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:00 smithi059 bash[61152]: cluster 2022-04-28T21:39:00.139444+0000 mgr.y (mgr.24961) 3579 : cluster [DBG] pgmap v4101: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:01.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:00 smithi139 bash[43301]: cluster 2022-04-28T21:39:00.139444+0000 mgr.y (mgr.24961) 3579 : cluster [DBG] pgmap v4101: 137 pgs: 137 active+clean; 24 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[59745]: cluster 2022-04-28T21:39:00.971132+0000 mon.a (mon.0) 3383 : cluster [DBG] osdmap e1158: 8 total, 8 up, 8 in 2022-04-28T21:39:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[59745]: audit 2022-04-28T21:39:00.974138+0000 mon.c (mon.1) 131 : audit [INF] from='client.? 172.21.15.59:0/3177316918' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:39:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[59745]: audit 2022-04-28T21:39:00.978722+0000 mon.a (mon.0) 3384 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:39:02.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[59745]: audit 2022-04-28T21:39:01.972273+0000 mon.a (mon.0) 3385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:39:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[59745]: cluster 2022-04-28T21:39:01.972425+0000 mon.a (mon.0) 3386 : cluster [DBG] osdmap e1159: 8 total, 8 up, 8 in 2022-04-28T21:39:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[61152]: cluster 2022-04-28T21:39:00.971132+0000 mon.a (mon.0) 3383 : cluster [DBG] osdmap e1158: 8 total, 8 up, 8 in 2022-04-28T21:39:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[61152]: audit 2022-04-28T21:39:00.974138+0000 mon.c (mon.1) 131 : audit [INF] from='client.? 172.21.15.59:0/3177316918' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:39:02.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[61152]: audit 2022-04-28T21:39:00.978722+0000 mon.a (mon.0) 3384 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:39:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[61152]: audit 2022-04-28T21:39:01.972273+0000 mon.a (mon.0) 3385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:39:02.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:01 smithi059 bash[61152]: cluster 2022-04-28T21:39:01.972425+0000 mon.a (mon.0) 3386 : cluster [DBG] osdmap e1159: 8 total, 8 up, 8 in 2022-04-28T21:39:02.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:02.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:02.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:01 smithi139 bash[43301]: cluster 2022-04-28T21:39:00.971132+0000 mon.a (mon.0) 3383 : cluster [DBG] osdmap e1158: 8 total, 8 up, 8 in 2022-04-28T21:39:02.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:01 smithi139 bash[43301]: audit 2022-04-28T21:39:00.974138+0000 mon.c (mon.1) 131 : audit [INF] from='client.? 172.21.15.59:0/3177316918' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:39:02.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:01 smithi139 bash[43301]: audit 2022-04-28T21:39:00.978722+0000 mon.a (mon.0) 3384 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:39:02.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:01 smithi139 bash[43301]: audit 2022-04-28T21:39:01.972273+0000 mon.a (mon.0) 3385 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84549-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:39:02.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:01 smithi139 bash[43301]: cluster 2022-04-28T21:39:01.972425+0000 mon.a (mon.0) 3386 : cluster [DBG] osdmap e1159: 8 total, 8 up, 8 in 2022-04-28T21:39:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:03 smithi059 bash[61152]: cluster 2022-04-28T21:39:02.140213+0000 mgr.y (mgr.24961) 3580 : cluster [DBG] pgmap v4104: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:03 smithi059 bash[59745]: cluster 2022-04-28T21:39:02.140213+0000 mgr.y (mgr.24961) 3580 : cluster [DBG] pgmap v4104: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:03.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:02 smithi139 bash[43301]: cluster 2022-04-28T21:39:02.140213+0000 mgr.y (mgr.24961) 3580 : cluster [DBG] pgmap v4104: 169 pgs: 6 creating+peering, 26 unknown, 137 active+clean; 24 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:04.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:03.940Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:04.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[61152]: cluster 2022-04-28T21:39:02.990397+0000 mon.a (mon.0) 3387 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2022-04-28T21:39:04.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[61152]: audit 2022-04-28T21:39:03.010005+0000 mon.a (mon.0) 3388 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:04.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[61152]: audit 2022-04-28T21:39:03.011085+0000 mon.a (mon.0) 3389 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:04.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[61152]: audit 2022-04-28T21:39:03.011471+0000 mon.a (mon.0) 3390 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84549-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:39:04.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[59745]: cluster 2022-04-28T21:39:02.990397+0000 mon.a (mon.0) 3387 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2022-04-28T21:39:04.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[59745]: audit 2022-04-28T21:39:03.010005+0000 mon.a (mon.0) 3388 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:04.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[59745]: audit 2022-04-28T21:39:03.011085+0000 mon.a (mon.0) 3389 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:04.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:04 smithi059 bash[59745]: audit 2022-04-28T21:39:03.011471+0000 mon.a (mon.0) 3390 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84549-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:39:04.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:04 smithi139 bash[43301]: cluster 2022-04-28T21:39:02.990397+0000 mon.a (mon.0) 3387 : cluster [DBG] osdmap e1160: 8 total, 8 up, 8 in 2022-04-28T21:39:04.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:04 smithi139 bash[43301]: audit 2022-04-28T21:39:03.010005+0000 mon.a (mon.0) 3388 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:04.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:04 smithi139 bash[43301]: audit 2022-04-28T21:39:03.011085+0000 mon.a (mon.0) 3389 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:04.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:04 smithi139 bash[43301]: audit 2022-04-28T21:39:03.011471+0000 mon.a (mon.0) 3390 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84549-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:39:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[61152]: audit 2022-04-28T21:39:04.017428+0000 mon.a (mon.0) 3391 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84549-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:39:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[61152]: cluster 2022-04-28T21:39:04.017523+0000 mon.a (mon.0) 3392 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2022-04-28T21:39:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[61152]: audit 2022-04-28T21:39:04.018199+0000 mon.a (mon.0) 3393 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84549-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[61152]: cluster 2022-04-28T21:39:04.141070+0000 mgr.y (mgr.24961) 3581 : cluster [DBG] pgmap v4107: 137 pgs: 137 active+clean; 24 KiB data, 832 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[59745]: audit 2022-04-28T21:39:04.017428+0000 mon.a (mon.0) 3391 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84549-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:39:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[59745]: cluster 2022-04-28T21:39:04.017523+0000 mon.a (mon.0) 3392 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2022-04-28T21:39:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[59745]: audit 2022-04-28T21:39:04.018199+0000 mon.a (mon.0) 3393 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84549-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:05 smithi059 bash[59745]: cluster 2022-04-28T21:39:04.141070+0000 mgr.y (mgr.24961) 3581 : cluster [DBG] pgmap v4107: 137 pgs: 137 active+clean; 24 KiB data, 832 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:05 smithi139 bash[43301]: audit 2022-04-28T21:39:04.017428+0000 mon.a (mon.0) 3391 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84549-22", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:39:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:05 smithi139 bash[43301]: cluster 2022-04-28T21:39:04.017523+0000 mon.a (mon.0) 3392 : cluster [DBG] osdmap e1161: 8 total, 8 up, 8 in 2022-04-28T21:39:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:05 smithi139 bash[43301]: audit 2022-04-28T21:39:04.018199+0000 mon.a (mon.0) 3393 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84549-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:05 smithi139 bash[43301]: cluster 2022-04-28T21:39:04.141070+0000 mgr.y (mgr.24961) 3581 : cluster [DBG] pgmap v4107: 137 pgs: 137 active+clean; 24 KiB data, 832 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:06 smithi059 bash[61152]: cluster 2022-04-28T21:39:05.496548+0000 mon.a (mon.0) 3394 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2022-04-28T21:39:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:06 smithi059 bash[61152]: audit 2022-04-28T21:39:06.023296+0000 mon.a (mon.0) 3395 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84549-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:06.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:06 smithi059 bash[61152]: cluster 2022-04-28T21:39:06.023500+0000 mon.a (mon.0) 3396 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2022-04-28T21:39:06.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:06 smithi059 bash[59745]: cluster 2022-04-28T21:39:05.496548+0000 mon.a (mon.0) 3394 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2022-04-28T21:39:06.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:06 smithi059 bash[59745]: audit 2022-04-28T21:39:06.023296+0000 mon.a (mon.0) 3395 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84549-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:06.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:06 smithi059 bash[59745]: cluster 2022-04-28T21:39:06.023500+0000 mon.a (mon.0) 3396 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2022-04-28T21:39:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:06 smithi139 bash[43301]: cluster 2022-04-28T21:39:05.496548+0000 mon.a (mon.0) 3394 : cluster [DBG] osdmap e1162: 8 total, 8 up, 8 in 2022-04-28T21:39:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:06 smithi139 bash[43301]: audit 2022-04-28T21:39:06.023296+0000 mon.a (mon.0) 3395 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84549-22", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:06 smithi139 bash[43301]: cluster 2022-04-28T21:39:06.023500+0000 mon.a (mon.0) 3396 : cluster [DBG] osdmap e1163: 8 total, 8 up, 8 in 2022-04-28T21:39:07.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:07 smithi059 bash[61152]: cluster 2022-04-28T21:39:06.142058+0000 mgr.y (mgr.24961) 3582 : cluster [DBG] pgmap v4110: 145 pgs: 4 creating+peering, 4 unknown, 137 active+clean; 24 KiB data, 833 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:07.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:07 smithi059 bash[61152]: cluster 2022-04-28T21:39:07.025540+0000 mon.a (mon.0) 3397 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2022-04-28T21:39:07.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:07 smithi059 bash[59745]: cluster 2022-04-28T21:39:06.142058+0000 mgr.y (mgr.24961) 3582 : cluster [DBG] pgmap v4110: 145 pgs: 4 creating+peering, 4 unknown, 137 active+clean; 24 KiB data, 833 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:07.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:07 smithi059 bash[59745]: cluster 2022-04-28T21:39:07.025540+0000 mon.a (mon.0) 3397 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2022-04-28T21:39:07.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:07 smithi139 bash[43301]: cluster 2022-04-28T21:39:06.142058+0000 mgr.y (mgr.24961) 3582 : cluster [DBG] pgmap v4110: 145 pgs: 4 creating+peering, 4 unknown, 137 active+clean; 24 KiB data, 833 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:07.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:07 smithi139 bash[43301]: cluster 2022-04-28T21:39:07.025540+0000 mon.a (mon.0) 3397 : cluster [DBG] osdmap e1164: 8 total, 8 up, 8 in 2022-04-28T21:39:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:09 smithi059 bash[59745]: cluster 2022-04-28T21:39:08.037798+0000 mon.a (mon.0) 3398 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2022-04-28T21:39:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:09 smithi059 bash[59745]: audit 2022-04-28T21:39:08.039186+0000 mon.a (mon.0) 3399 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:09.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:09 smithi059 bash[59745]: cluster 2022-04-28T21:39:08.142739+0000 mgr.y (mgr.24961) 3583 : cluster [DBG] pgmap v4113: 137 pgs: 137 active+clean; 24 KiB data, 833 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:09 smithi059 bash[61152]: cluster 2022-04-28T21:39:08.037798+0000 mon.a (mon.0) 3398 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2022-04-28T21:39:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:09 smithi059 bash[61152]: audit 2022-04-28T21:39:08.039186+0000 mon.a (mon.0) 3399 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:09.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:09 smithi059 bash[61152]: cluster 2022-04-28T21:39:08.142739+0000 mgr.y (mgr.24961) 3583 : cluster [DBG] pgmap v4113: 137 pgs: 137 active+clean; 24 KiB data, 833 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:09 smithi139 bash[43301]: cluster 2022-04-28T21:39:08.037798+0000 mon.a (mon.0) 3398 : cluster [DBG] osdmap e1165: 8 total, 8 up, 8 in 2022-04-28T21:39:09.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:09 smithi139 bash[43301]: audit 2022-04-28T21:39:08.039186+0000 mon.a (mon.0) 3399 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:09.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:09 smithi139 bash[43301]: cluster 2022-04-28T21:39:08.142739+0000 mgr.y (mgr.24961) 3583 : cluster [DBG] pgmap v4113: 137 pgs: 137 active+clean; 24 KiB data, 833 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:10 smithi059 bash[59745]: audit 2022-04-28T21:39:09.041796+0000 mon.a (mon.0) 3400 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:10 smithi059 bash[59745]: cluster 2022-04-28T21:39:09.042111+0000 mon.a (mon.0) 3401 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2022-04-28T21:39:10.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:10 smithi059 bash[59745]: audit 2022-04-28T21:39:09.043284+0000 mon.a (mon.0) 3402 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:10.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:10 smithi059 bash[61152]: audit 2022-04-28T21:39:09.041796+0000 mon.a (mon.0) 3400 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:10 smithi059 bash[61152]: cluster 2022-04-28T21:39:09.042111+0000 mon.a (mon.0) 3401 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2022-04-28T21:39:10.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:10 smithi059 bash[61152]: audit 2022-04-28T21:39:09.043284+0000 mon.a (mon.0) 3402 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:10 smithi139 bash[43301]: audit 2022-04-28T21:39:09.041796+0000 mon.a (mon.0) 3400 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:10 smithi139 bash[43301]: cluster 2022-04-28T21:39:09.042111+0000 mon.a (mon.0) 3401 : cluster [DBG] osdmap e1166: 8 total, 8 up, 8 in 2022-04-28T21:39:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:10 smithi139 bash[43301]: audit 2022-04-28T21:39:09.043284+0000 mon.a (mon.0) 3402 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]: dispatch 2022-04-28T21:39:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:11 smithi059 bash[61152]: audit 2022-04-28T21:39:10.045030+0000 mon.a (mon.0) 3403 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:11 smithi059 bash[61152]: cluster 2022-04-28T21:39:10.045210+0000 mon.a (mon.0) 3404 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2022-04-28T21:39:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:11 smithi059 bash[61152]: cluster 2022-04-28T21:39:10.143465+0000 mgr.y (mgr.24961) 3584 : cluster [DBG] pgmap v4116: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:11 smithi059 bash[59745]: audit 2022-04-28T21:39:10.045030+0000 mon.a (mon.0) 3403 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:11 smithi059 bash[59745]: cluster 2022-04-28T21:39:10.045210+0000 mon.a (mon.0) 3404 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2022-04-28T21:39:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:11 smithi059 bash[59745]: cluster 2022-04-28T21:39:10.143465+0000 mgr.y (mgr.24961) 3584 : cluster [DBG] pgmap v4116: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:11.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:11 smithi139 bash[43301]: audit 2022-04-28T21:39:10.045030+0000 mon.a (mon.0) 3403 : audit [INF] from='client.? 172.21.15.59:0/1424221290' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84549-22"}]': finished 2022-04-28T21:39:11.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:11 smithi139 bash[43301]: cluster 2022-04-28T21:39:10.045210+0000 mon.a (mon.0) 3404 : cluster [DBG] osdmap e1167: 8 total, 8 up, 8 in 2022-04-28T21:39:11.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:11 smithi139 bash[43301]: cluster 2022-04-28T21:39:10.143465+0000 mgr.y (mgr.24961) 3584 : cluster [DBG] pgmap v4116: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:12.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:12.046Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:13 smithi059 bash[61152]: cluster 2022-04-28T21:39:12.144585+0000 mgr.y (mgr.24961) 3585 : cluster [DBG] pgmap v4117: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:13.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:13 smithi059 bash[59745]: cluster 2022-04-28T21:39:12.144585+0000 mgr.y (mgr.24961) 3585 : cluster [DBG] pgmap v4117: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:13.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:13 smithi139 bash[43301]: cluster 2022-04-28T21:39:12.144585+0000 mgr.y (mgr.24961) 3585 : cluster [DBG] pgmap v4117: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:13.941Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:15 smithi059 bash[61152]: cluster 2022-04-28T21:39:14.145709+0000 mgr.y (mgr.24961) 3586 : cluster [DBG] pgmap v4118: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:15.919 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:15 smithi059 bash[59745]: cluster 2022-04-28T21:39:14.145709+0000 mgr.y (mgr.24961) 3586 : cluster [DBG] pgmap v4118: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:15.920 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:15 smithi139 bash[43301]: cluster 2022-04-28T21:39:14.145709+0000 mgr.y (mgr.24961) 3586 : cluster [DBG] pgmap v4118: 137 pgs: 137 active+clean; 24 KiB data, 834 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:17 smithi059 bash[61152]: cluster 2022-04-28T21:39:16.146887+0000 mgr.y (mgr.24961) 3587 : cluster [DBG] pgmap v4119: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:17 smithi059 bash[59745]: cluster 2022-04-28T21:39:16.146887+0000 mgr.y (mgr.24961) 3587 : cluster [DBG] pgmap v4119: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:17 smithi139 bash[43301]: cluster 2022-04-28T21:39:16.146887+0000 mgr.y (mgr.24961) 3587 : cluster [DBG] pgmap v4119: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: cluster 2022-04-28T21:39:18.147651+0000 mgr.y (mgr.24961) 3588 : cluster [DBG] pgmap v4120: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: audit 2022-04-28T21:39:19.326102+0000 mon.a (mon.0) 3405 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:39:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: audit 2022-04-28T21:39:19.331622+0000 mon.a (mon.0) 3406 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:39:19.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: audit 2022-04-28T21:39:19.345642+0000 mon.a (mon.0) 3407 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:39:19.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: audit 2022-04-28T21:39:19.346473+0000 mon.a (mon.0) 3408 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:39:19.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: audit 2022-04-28T21:39:19.346942+0000 mon.a (mon.0) 3409 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:39:19.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[61152]: audit 2022-04-28T21:39:19.347602+0000 mon.a (mon.0) 3410 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:39:19.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: cluster 2022-04-28T21:39:18.147651+0000 mgr.y (mgr.24961) 3588 : cluster [DBG] pgmap v4120: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:19.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: audit 2022-04-28T21:39:19.326102+0000 mon.a (mon.0) 3405 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:39:19.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: audit 2022-04-28T21:39:19.331622+0000 mon.a (mon.0) 3406 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:39:19.850 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: audit 2022-04-28T21:39:19.345642+0000 mon.a (mon.0) 3407 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:39:19.850 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: audit 2022-04-28T21:39:19.346473+0000 mon.a (mon.0) 3408 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:39:19.852 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: audit 2022-04-28T21:39:19.346942+0000 mon.a (mon.0) 3409 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:39:19.853 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:19 smithi059 bash[59745]: audit 2022-04-28T21:39:19.347602+0000 mon.a (mon.0) 3410 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:39:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: cluster 2022-04-28T21:39:18.147651+0000 mgr.y (mgr.24961) 3588 : cluster [DBG] pgmap v4120: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: audit 2022-04-28T21:39:19.326102+0000 mon.a (mon.0) 3405 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:39:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: audit 2022-04-28T21:39:19.331622+0000 mon.a (mon.0) 3406 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:39:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: audit 2022-04-28T21:39:19.345642+0000 mon.a (mon.0) 3407 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:39:19.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: audit 2022-04-28T21:39:19.346473+0000 mon.a (mon.0) 3408 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:39:19.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: audit 2022-04-28T21:39:19.346942+0000 mon.a (mon.0) 3409 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:39:19.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:19 smithi139 bash[43301]: audit 2022-04-28T21:39:19.347602+0000 mon.a (mon.0) 3410 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:39:21.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:21 smithi059 bash[61152]: cluster 2022-04-28T21:39:20.148903+0000 mgr.y (mgr.24961) 3589 : cluster [DBG] pgmap v4121: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:21.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:21 smithi059 bash[59745]: cluster 2022-04-28T21:39:20.148903+0000 mgr.y (mgr.24961) 3589 : cluster [DBG] pgmap v4121: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:21.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:21 smithi139 bash[43301]: cluster 2022-04-28T21:39:20.148903+0000 mgr.y (mgr.24961) 3589 : cluster [DBG] pgmap v4121: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:22.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:22.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:23 smithi059 bash[61152]: cluster 2022-04-28T21:39:22.150054+0000 mgr.y (mgr.24961) 3590 : cluster [DBG] pgmap v4122: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:23.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:23 smithi059 bash[59745]: cluster 2022-04-28T21:39:22.150054+0000 mgr.y (mgr.24961) 3590 : cluster [DBG] pgmap v4122: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:23.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:23 smithi139 bash[43301]: cluster 2022-04-28T21:39:22.150054+0000 mgr.y (mgr.24961) 3590 : cluster [DBG] pgmap v4122: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:24.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:23.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:25.840 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:25 smithi059 bash[61152]: cluster 2022-04-28T21:39:24.151172+0000 mgr.y (mgr.24961) 3591 : cluster [DBG] pgmap v4123: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:25.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:25 smithi059 bash[59745]: cluster 2022-04-28T21:39:24.151172+0000 mgr.y (mgr.24961) 3591 : cluster [DBG] pgmap v4123: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:25.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:25 smithi139 bash[43301]: cluster 2022-04-28T21:39:24.151172+0000 mgr.y (mgr.24961) 3591 : cluster [DBG] pgmap v4123: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:27 smithi059 bash[61152]: cluster 2022-04-28T21:39:26.152423+0000 mgr.y (mgr.24961) 3592 : cluster [DBG] pgmap v4124: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:27.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:27 smithi059 bash[59745]: cluster 2022-04-28T21:39:26.152423+0000 mgr.y (mgr.24961) 3592 : cluster [DBG] pgmap v4124: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:27 smithi139 bash[43301]: cluster 2022-04-28T21:39:26.152423+0000 mgr.y (mgr.24961) 3592 : cluster [DBG] pgmap v4124: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:29 smithi059 bash[61152]: cluster 2022-04-28T21:39:28.153148+0000 mgr.y (mgr.24961) 3593 : cluster [DBG] pgmap v4125: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:29 smithi059 bash[59745]: cluster 2022-04-28T21:39:28.153148+0000 mgr.y (mgr.24961) 3593 : cluster [DBG] pgmap v4125: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:29.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:29 smithi139 bash[43301]: cluster 2022-04-28T21:39:28.153148+0000 mgr.y (mgr.24961) 3593 : cluster [DBG] pgmap v4125: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:31 smithi059 bash[59745]: cluster 2022-04-28T21:39:30.154445+0000 mgr.y (mgr.24961) 3594 : cluster [DBG] pgmap v4126: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:31 smithi059 bash[61152]: cluster 2022-04-28T21:39:30.154445+0000 mgr.y (mgr.24961) 3594 : cluster [DBG] pgmap v4126: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:31 smithi139 bash[43301]: cluster 2022-04-28T21:39:30.154445+0000 mgr.y (mgr.24961) 3594 : cluster [DBG] pgmap v4126: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:32.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:33 smithi059 bash[61152]: cluster 2022-04-28T21:39:32.155609+0000 mgr.y (mgr.24961) 3595 : cluster [DBG] pgmap v4127: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:33 smithi059 bash[59745]: cluster 2022-04-28T21:39:32.155609+0000 mgr.y (mgr.24961) 3595 : cluster [DBG] pgmap v4127: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:33 smithi139 bash[43301]: cluster 2022-04-28T21:39:32.155609+0000 mgr.y (mgr.24961) 3595 : cluster [DBG] pgmap v4127: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:33.942Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:35.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:35 smithi139 bash[43301]: cluster 2022-04-28T21:39:34.156709+0000 mgr.y (mgr.24961) 3596 : cluster [DBG] pgmap v4128: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:36.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:35 smithi059 bash[61152]: cluster 2022-04-28T21:39:34.156709+0000 mgr.y (mgr.24961) 3596 : cluster [DBG] pgmap v4128: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:36.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:35 smithi059 bash[59745]: cluster 2022-04-28T21:39:34.156709+0000 mgr.y (mgr.24961) 3596 : cluster [DBG] pgmap v4128: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:37 smithi139 bash[43301]: cluster 2022-04-28T21:39:36.157897+0000 mgr.y (mgr.24961) 3597 : cluster [DBG] pgmap v4129: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:37 smithi059 bash[61152]: cluster 2022-04-28T21:39:36.157897+0000 mgr.y (mgr.24961) 3597 : cluster [DBG] pgmap v4129: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:38.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:37 smithi059 bash[59745]: cluster 2022-04-28T21:39:36.157897+0000 mgr.y (mgr.24961) 3597 : cluster [DBG] pgmap v4129: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:39 smithi139 bash[43301]: cluster 2022-04-28T21:39:38.158690+0000 mgr.y (mgr.24961) 3598 : cluster [DBG] pgmap v4130: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:39 smithi059 bash[61152]: cluster 2022-04-28T21:39:38.158690+0000 mgr.y (mgr.24961) 3598 : cluster [DBG] pgmap v4130: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:40.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:39 smithi059 bash[59745]: cluster 2022-04-28T21:39:38.158690+0000 mgr.y (mgr.24961) 3598 : cluster [DBG] pgmap v4130: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:41 smithi139 bash[43301]: cluster 2022-04-28T21:39:40.159994+0000 mgr.y (mgr.24961) 3599 : cluster [DBG] pgmap v4131: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:42.049 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:41 smithi059 bash[61152]: cluster 2022-04-28T21:39:40.159994+0000 mgr.y (mgr.24961) 3599 : cluster [DBG] pgmap v4131: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:42.049 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:41 smithi059 bash[59745]: cluster 2022-04-28T21:39:40.159994+0000 mgr.y (mgr.24961) 3599 : cluster [DBG] pgmap v4131: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:42.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:42.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:43 smithi139 bash[43301]: cluster 2022-04-28T21:39:42.161150+0000 mgr.y (mgr.24961) 3600 : cluster [DBG] pgmap v4132: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:43.944 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:43 smithi059 bash[61152]: cluster 2022-04-28T21:39:42.161150+0000 mgr.y (mgr.24961) 3600 : cluster [DBG] pgmap v4132: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:43.993 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:43.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:43.993 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:43 smithi059 bash[59745]: cluster 2022-04-28T21:39:42.161150+0000 mgr.y (mgr.24961) 3600 : cluster [DBG] pgmap v4132: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:46.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:45 smithi059 bash[61152]: cluster 2022-04-28T21:39:44.162256+0000 mgr.y (mgr.24961) 3601 : cluster [DBG] pgmap v4133: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:46.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:45 smithi059 bash[59745]: cluster 2022-04-28T21:39:44.162256+0000 mgr.y (mgr.24961) 3601 : cluster [DBG] pgmap v4133: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:46.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:45 smithi139 bash[43301]: cluster 2022-04-28T21:39:44.162256+0000 mgr.y (mgr.24961) 3601 : cluster [DBG] pgmap v4133: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:47 smithi059 bash[61152]: cluster 2022-04-28T21:39:46.163627+0000 mgr.y (mgr.24961) 3602 : cluster [DBG] pgmap v4134: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:48.104 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:47 smithi059 bash[59745]: cluster 2022-04-28T21:39:46.163627+0000 mgr.y (mgr.24961) 3602 : cluster [DBG] pgmap v4134: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:48.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:47 smithi139 bash[43301]: cluster 2022-04-28T21:39:46.163627+0000 mgr.y (mgr.24961) 3602 : cluster [DBG] pgmap v4134: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:49 smithi059 bash[61152]: cluster 2022-04-28T21:39:48.164387+0000 mgr.y (mgr.24961) 3603 : cluster [DBG] pgmap v4135: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:49 smithi059 bash[59745]: cluster 2022-04-28T21:39:48.164387+0000 mgr.y (mgr.24961) 3603 : cluster [DBG] pgmap v4135: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:50.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:49 smithi139 bash[43301]: cluster 2022-04-28T21:39:48.164387+0000 mgr.y (mgr.24961) 3603 : cluster [DBG] pgmap v4135: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:52.050 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:51 smithi059 bash[61152]: cluster 2022-04-28T21:39:50.165777+0000 mgr.y (mgr.24961) 3604 : cluster [DBG] pgmap v4136: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:52.050 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:51 smithi059 bash[59745]: cluster 2022-04-28T21:39:50.165777+0000 mgr.y (mgr.24961) 3604 : cluster [DBG] pgmap v4136: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:52.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:51 smithi139 bash[43301]: cluster 2022-04-28T21:39:50.165777+0000 mgr.y (mgr.24961) 3604 : cluster [DBG] pgmap v4136: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:52.049Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:52 smithi059 bash[61152]: cluster 2022-04-28T21:39:52.166766+0000 mgr.y (mgr.24961) 3605 : cluster [DBG] pgmap v4137: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:52 smithi059 bash[59745]: cluster 2022-04-28T21:39:52.166766+0000 mgr.y (mgr.24961) 3605 : cluster [DBG] pgmap v4137: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:53.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:52 smithi139 bash[43301]: cluster 2022-04-28T21:39:52.166766+0000 mgr.y (mgr.24961) 3605 : cluster [DBG] pgmap v4137: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[59745]: audit 2022-04-28T21:39:53.781747+0000 mon.a (mon.0) 3411 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:39:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[59745]: audit 2022-04-28T21:39:53.784611+0000 mon.a (mon.0) 3412 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:39:54.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[59745]: audit 2022-04-28T21:39:53.786573+0000 mon.a (mon.0) 3413 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:39:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:39:53.943Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:39:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[61152]: audit 2022-04-28T21:39:53.781747+0000 mon.a (mon.0) 3411 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:39:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[61152]: audit 2022-04-28T21:39:53.784611+0000 mon.a (mon.0) 3412 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:39:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:53 smithi059 bash[61152]: audit 2022-04-28T21:39:53.786573+0000 mon.a (mon.0) 3413 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:39:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:53 smithi139 bash[43301]: audit 2022-04-28T21:39:53.781747+0000 mon.a (mon.0) 3411 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:39:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:53 smithi139 bash[43301]: audit 2022-04-28T21:39:53.784611+0000 mon.a (mon.0) 3412 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:39:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:53 smithi139 bash[43301]: audit 2022-04-28T21:39:53.786573+0000 mon.a (mon.0) 3413 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:39:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[59745]: cephadm 2022-04-28T21:39:53.788287+0000 mgr.y (mgr.24961) 3606 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:39:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[59745]: audit 2022-04-28T21:39:54.087505+0000 mon.a (mon.0) 3414 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[59745]: audit 2022-04-28T21:39:54.093956+0000 mon.a (mon.0) 3415 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[59745]: cluster 2022-04-28T21:39:54.167882+0000 mgr.y (mgr.24961) 3607 : cluster [DBG] pgmap v4138: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[59745]: audit 2022-04-28T21:39:54.401567+0000 mon.a (mon.0) 3416 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[61152]: cephadm 2022-04-28T21:39:53.788287+0000 mgr.y (mgr.24961) 3606 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:39:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[61152]: audit 2022-04-28T21:39:54.087505+0000 mon.a (mon.0) 3414 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[61152]: audit 2022-04-28T21:39:54.093956+0000 mon.a (mon.0) 3415 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[61152]: cluster 2022-04-28T21:39:54.167882+0000 mgr.y (mgr.24961) 3607 : cluster [DBG] pgmap v4138: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:55.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:55 smithi059 bash[61152]: audit 2022-04-28T21:39:54.401567+0000 mon.a (mon.0) 3416 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:55 smithi139 bash[43301]: cephadm 2022-04-28T21:39:53.788287+0000 mgr.y (mgr.24961) 3606 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:39:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:55 smithi139 bash[43301]: audit 2022-04-28T21:39:54.087505+0000 mon.a (mon.0) 3414 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:55 smithi139 bash[43301]: audit 2022-04-28T21:39:54.093956+0000 mon.a (mon.0) 3415 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:55.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:55 smithi139 bash[43301]: cluster 2022-04-28T21:39:54.167882+0000 mgr.y (mgr.24961) 3607 : cluster [DBG] pgmap v4138: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:55.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:55 smithi139 bash[43301]: audit 2022-04-28T21:39:54.401567+0000 mon.a (mon.0) 3416 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:39:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:57 smithi059 bash[61152]: cluster 2022-04-28T21:39:56.169106+0000 mgr.y (mgr.24961) 3608 : cluster [DBG] pgmap v4139: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:57 smithi059 bash[59745]: cluster 2022-04-28T21:39:56.169106+0000 mgr.y (mgr.24961) 3608 : cluster [DBG] pgmap v4139: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:57 smithi139 bash[43301]: cluster 2022-04-28T21:39:56.169106+0000 mgr.y (mgr.24961) 3608 : cluster [DBG] pgmap v4139: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:39:59 smithi059 bash[61152]: cluster 2022-04-28T21:39:58.169834+0000 mgr.y (mgr.24961) 3609 : cluster [DBG] pgmap v4140: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:39:59 smithi059 bash[59745]: cluster 2022-04-28T21:39:58.169834+0000 mgr.y (mgr.24961) 3609 : cluster [DBG] pgmap v4140: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:39:59.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:39:59 smithi139 bash[43301]: cluster 2022-04-28T21:39:58.169834+0000 mgr.y (mgr.24961) 3609 : cluster [DBG] pgmap v4140: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:00 smithi059 bash[61152]: cluster 2022-04-28T21:40:00.000150+0000 mon.a (mon.0) 3417 : cluster [INF] overall HEALTH_OK 2022-04-28T21:40:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:00 smithi059 bash[59745]: cluster 2022-04-28T21:40:00.000150+0000 mon.a (mon.0) 3417 : cluster [INF] overall HEALTH_OK 2022-04-28T21:40:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:00 smithi139 bash[43301]: cluster 2022-04-28T21:40:00.000150+0000 mon.a (mon.0) 3417 : cluster [INF] overall HEALTH_OK 2022-04-28T21:40:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:01 smithi059 bash[61152]: cluster 2022-04-28T21:40:00.171316+0000 mgr.y (mgr.24961) 3610 : cluster [DBG] pgmap v4141: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:01.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:01 smithi059 bash[59745]: cluster 2022-04-28T21:40:00.171316+0000 mgr.y (mgr.24961) 3610 : cluster [DBG] pgmap v4141: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:01.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:01 smithi139 bash[43301]: cluster 2022-04-28T21:40:00.171316+0000 mgr.y (mgr.24961) 3610 : cluster [DBG] pgmap v4141: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:02.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:03 smithi059 bash[61152]: cluster 2022-04-28T21:40:02.172322+0000 mgr.y (mgr.24961) 3611 : cluster [DBG] pgmap v4142: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:03.855 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:03 smithi059 bash[59745]: cluster 2022-04-28T21:40:02.172322+0000 mgr.y (mgr.24961) 3611 : cluster [DBG] pgmap v4142: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:03 smithi139 bash[43301]: cluster 2022-04-28T21:40:02.172322+0000 mgr.y (mgr.24961) 3611 : cluster [DBG] pgmap v4142: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:04.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:03.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:05.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:05 smithi059 bash[59745]: cluster 2022-04-28T21:40:04.173433+0000 mgr.y (mgr.24961) 3612 : cluster [DBG] pgmap v4143: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:05 smithi059 bash[61152]: cluster 2022-04-28T21:40:04.173433+0000 mgr.y (mgr.24961) 3612 : cluster [DBG] pgmap v4143: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:05.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:05 smithi139 bash[43301]: cluster 2022-04-28T21:40:04.173433+0000 mgr.y (mgr.24961) 3612 : cluster [DBG] pgmap v4143: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:07 smithi059 bash[61152]: cluster 2022-04-28T21:40:06.174685+0000 mgr.y (mgr.24961) 3613 : cluster [DBG] pgmap v4144: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:07.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:07 smithi059 bash[59745]: cluster 2022-04-28T21:40:06.174685+0000 mgr.y (mgr.24961) 3613 : cluster [DBG] pgmap v4144: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:07.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:07 smithi139 bash[43301]: cluster 2022-04-28T21:40:06.174685+0000 mgr.y (mgr.24961) 3613 : cluster [DBG] pgmap v4144: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:09 smithi059 bash[61152]: cluster 2022-04-28T21:40:08.175415+0000 mgr.y (mgr.24961) 3614 : cluster [DBG] pgmap v4145: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:09 smithi059 bash[59745]: cluster 2022-04-28T21:40:08.175415+0000 mgr.y (mgr.24961) 3614 : cluster [DBG] pgmap v4145: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:09.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:09 smithi139 bash[43301]: cluster 2022-04-28T21:40:08.175415+0000 mgr.y (mgr.24961) 3614 : cluster [DBG] pgmap v4145: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:11 smithi059 bash[61152]: cluster 2022-04-28T21:40:10.176832+0000 mgr.y (mgr.24961) 3615 : cluster [DBG] pgmap v4146: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:11.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:11 smithi059 bash[59745]: cluster 2022-04-28T21:40:10.176832+0000 mgr.y (mgr.24961) 3615 : cluster [DBG] pgmap v4146: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:11 smithi139 bash[43301]: cluster 2022-04-28T21:40:10.176832+0000 mgr.y (mgr.24961) 3615 : cluster [DBG] pgmap v4146: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:12.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:13 smithi139 bash[43301]: cluster 2022-04-28T21:40:12.177882+0000 mgr.y (mgr.24961) 3616 : cluster [DBG] pgmap v4147: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:13.946 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:13 smithi059 bash[61152]: cluster 2022-04-28T21:40:12.177882+0000 mgr.y (mgr.24961) 3616 : cluster [DBG] pgmap v4147: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:13.947 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:13 smithi059 bash[59745]: cluster 2022-04-28T21:40:12.177882+0000 mgr.y (mgr.24961) 3616 : cluster [DBG] pgmap v4147: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:14.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:13.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:15 smithi139 bash[43301]: cluster 2022-04-28T21:40:14.178999+0000 mgr.y (mgr.24961) 3617 : cluster [DBG] pgmap v4148: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:15 smithi059 bash[61152]: cluster 2022-04-28T21:40:14.178999+0000 mgr.y (mgr.24961) 3617 : cluster [DBG] pgmap v4148: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:16.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:15 smithi059 bash[59745]: cluster 2022-04-28T21:40:14.178999+0000 mgr.y (mgr.24961) 3617 : cluster [DBG] pgmap v4148: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:17.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:17 smithi139 bash[43301]: cluster 2022-04-28T21:40:16.180267+0000 mgr.y (mgr.24961) 3618 : cluster [DBG] pgmap v4149: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:17 smithi059 bash[61152]: cluster 2022-04-28T21:40:16.180267+0000 mgr.y (mgr.24961) 3618 : cluster [DBG] pgmap v4149: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:17 smithi059 bash[59745]: cluster 2022-04-28T21:40:16.180267+0000 mgr.y (mgr.24961) 3618 : cluster [DBG] pgmap v4149: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: cluster 2022-04-28T21:40:18.181041+0000 mgr.y (mgr.24961) 3619 : cluster [DBG] pgmap v4150: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:19.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: audit 2022-04-28T21:40:19.331322+0000 mon.a (mon.0) 3418 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:40:19.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: audit 2022-04-28T21:40:19.337769+0000 mon.a (mon.0) 3419 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:40:19.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: audit 2022-04-28T21:40:19.354779+0000 mon.a (mon.0) 3420 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:40:19.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: audit 2022-04-28T21:40:19.355546+0000 mon.a (mon.0) 3421 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:40:19.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: audit 2022-04-28T21:40:19.356215+0000 mon.a (mon.0) 3422 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:40:19.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:19 smithi139 bash[43301]: audit 2022-04-28T21:40:19.356841+0000 mon.a (mon.0) 3423 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:40:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: cluster 2022-04-28T21:40:18.181041+0000 mgr.y (mgr.24961) 3619 : cluster [DBG] pgmap v4150: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: audit 2022-04-28T21:40:19.331322+0000 mon.a (mon.0) 3418 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:40:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: audit 2022-04-28T21:40:19.337769+0000 mon.a (mon.0) 3419 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:40:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: audit 2022-04-28T21:40:19.354779+0000 mon.a (mon.0) 3420 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:40:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: audit 2022-04-28T21:40:19.355546+0000 mon.a (mon.0) 3421 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:40:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: audit 2022-04-28T21:40:19.356215+0000 mon.a (mon.0) 3422 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:40:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[61152]: audit 2022-04-28T21:40:19.356841+0000 mon.a (mon.0) 3423 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:40:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: cluster 2022-04-28T21:40:18.181041+0000 mgr.y (mgr.24961) 3619 : cluster [DBG] pgmap v4150: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: audit 2022-04-28T21:40:19.331322+0000 mon.a (mon.0) 3418 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:40:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: audit 2022-04-28T21:40:19.337769+0000 mon.a (mon.0) 3419 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:40:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: audit 2022-04-28T21:40:19.354779+0000 mon.a (mon.0) 3420 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:40:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: audit 2022-04-28T21:40:19.355546+0000 mon.a (mon.0) 3421 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:40:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: audit 2022-04-28T21:40:19.356215+0000 mon.a (mon.0) 3422 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:40:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:19 smithi059 bash[59745]: audit 2022-04-28T21:40:19.356841+0000 mon.a (mon.0) 3423 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:40:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:21 smithi139 bash[43301]: cluster 2022-04-28T21:40:20.182451+0000 mgr.y (mgr.24961) 3620 : cluster [DBG] pgmap v4151: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:22.053 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:21 smithi059 bash[61152]: cluster 2022-04-28T21:40:20.182451+0000 mgr.y (mgr.24961) 3620 : cluster [DBG] pgmap v4151: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:22.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:21 smithi059 bash[59745]: cluster 2022-04-28T21:40:20.182451+0000 mgr.y (mgr.24961) 3620 : cluster [DBG] pgmap v4151: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:22.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:23 smithi139 bash[43301]: cluster 2022-04-28T21:40:22.183465+0000 mgr.y (mgr.24961) 3621 : cluster [DBG] pgmap v4152: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:23.945 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:23 smithi059 bash[61152]: cluster 2022-04-28T21:40:22.183465+0000 mgr.y (mgr.24961) 3621 : cluster [DBG] pgmap v4152: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:23.947 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:23.945Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:23.947 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:23 smithi059 bash[59745]: cluster 2022-04-28T21:40:22.183465+0000 mgr.y (mgr.24961) 3621 : cluster [DBG] pgmap v4152: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:25 smithi059 bash[59745]: cluster 2022-04-28T21:40:24.184698+0000 mgr.y (mgr.24961) 3622 : cluster [DBG] pgmap v4153: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:25 smithi059 bash[61152]: cluster 2022-04-28T21:40:24.184698+0000 mgr.y (mgr.24961) 3622 : cluster [DBG] pgmap v4153: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:26.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:25 smithi139 bash[43301]: cluster 2022-04-28T21:40:24.184698+0000 mgr.y (mgr.24961) 3622 : cluster [DBG] pgmap v4153: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:27 smithi059 bash[61152]: cluster 2022-04-28T21:40:26.185960+0000 mgr.y (mgr.24961) 3623 : cluster [DBG] pgmap v4154: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:28.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:27 smithi059 bash[59745]: cluster 2022-04-28T21:40:26.185960+0000 mgr.y (mgr.24961) 3623 : cluster [DBG] pgmap v4154: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:28.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:27 smithi139 bash[43301]: cluster 2022-04-28T21:40:26.185960+0000 mgr.y (mgr.24961) 3623 : cluster [DBG] pgmap v4154: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:29 smithi059 bash[61152]: cluster 2022-04-28T21:40:28.186696+0000 mgr.y (mgr.24961) 3624 : cluster [DBG] pgmap v4155: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:30.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:29 smithi059 bash[59745]: cluster 2022-04-28T21:40:28.186696+0000 mgr.y (mgr.24961) 3624 : cluster [DBG] pgmap v4155: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:30.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:29 smithi139 bash[43301]: cluster 2022-04-28T21:40:28.186696+0000 mgr.y (mgr.24961) 3624 : cluster [DBG] pgmap v4155: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:32.053 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:31 smithi059 bash[61152]: cluster 2022-04-28T21:40:30.188334+0000 mgr.y (mgr.24961) 3625 : cluster [DBG] pgmap v4156: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:32.054 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:31 smithi059 bash[59745]: cluster 2022-04-28T21:40:30.188334+0000 mgr.y (mgr.24961) 3625 : cluster [DBG] pgmap v4156: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:32.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:31 smithi139 bash[43301]: cluster 2022-04-28T21:40:30.188334+0000 mgr.y (mgr.24961) 3625 : cluster [DBG] pgmap v4156: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:32.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:32.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:32 smithi059 bash[59745]: cluster 2022-04-28T21:40:32.189444+0000 mgr.y (mgr.24961) 3626 : cluster [DBG] pgmap v4157: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:32 smithi059 bash[61152]: cluster 2022-04-28T21:40:32.189444+0000 mgr.y (mgr.24961) 3626 : cluster [DBG] pgmap v4157: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:33.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:32 smithi139 bash[43301]: cluster 2022-04-28T21:40:32.189444+0000 mgr.y (mgr.24961) 3626 : cluster [DBG] pgmap v4157: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:33.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:35.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:35 smithi059 bash[61152]: cluster 2022-04-28T21:40:34.190562+0000 mgr.y (mgr.24961) 3627 : cluster [DBG] pgmap v4158: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:35.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:35 smithi059 bash[59745]: cluster 2022-04-28T21:40:34.190562+0000 mgr.y (mgr.24961) 3627 : cluster [DBG] pgmap v4158: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:35.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:35 smithi139 bash[43301]: cluster 2022-04-28T21:40:34.190562+0000 mgr.y (mgr.24961) 3627 : cluster [DBG] pgmap v4158: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[61152]: cluster 2022-04-28T21:40:36.191824+0000 mgr.y (mgr.24961) 3628 : cluster [DBG] pgmap v4159: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[61152]: cluster 2022-04-28T21:40:36.270987+0000 mon.a (mon.0) 3424 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2022-04-28T21:40:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[61152]: audit 2022-04-28T21:40:36.277882+0000 mon.c (mon.1) 132 : audit [INF] from='client.? 172.21.15.59:0/471337723' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:37.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[61152]: audit 2022-04-28T21:40:36.279994+0000 mon.a (mon.0) 3425 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:37.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[59745]: cluster 2022-04-28T21:40:36.191824+0000 mgr.y (mgr.24961) 3628 : cluster [DBG] pgmap v4159: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:37.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[59745]: cluster 2022-04-28T21:40:36.270987+0000 mon.a (mon.0) 3424 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2022-04-28T21:40:37.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[59745]: audit 2022-04-28T21:40:36.277882+0000 mon.c (mon.1) 132 : audit [INF] from='client.? 172.21.15.59:0/471337723' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:37.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:37 smithi059 bash[59745]: audit 2022-04-28T21:40:36.279994+0000 mon.a (mon.0) 3425 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:37.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:37 smithi139 bash[43301]: cluster 2022-04-28T21:40:36.191824+0000 mgr.y (mgr.24961) 3628 : cluster [DBG] pgmap v4159: 137 pgs: 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:37.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:37 smithi139 bash[43301]: cluster 2022-04-28T21:40:36.270987+0000 mon.a (mon.0) 3424 : cluster [DBG] osdmap e1168: 8 total, 8 up, 8 in 2022-04-28T21:40:37.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:37 smithi139 bash[43301]: audit 2022-04-28T21:40:36.277882+0000 mon.c (mon.1) 132 : audit [INF] from='client.? 172.21.15.59:0/471337723' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:37.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:37 smithi139 bash[43301]: audit 2022-04-28T21:40:36.279994+0000 mon.a (mon.0) 3425 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:38 smithi059 bash[61152]: audit 2022-04-28T21:40:37.272697+0000 mon.a (mon.0) 3426 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:38 smithi059 bash[61152]: cluster 2022-04-28T21:40:37.272846+0000 mon.a (mon.0) 3427 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2022-04-28T21:40:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:38 smithi059 bash[59745]: audit 2022-04-28T21:40:37.272697+0000 mon.a (mon.0) 3426 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:38 smithi059 bash[59745]: cluster 2022-04-28T21:40:37.272846+0000 mon.a (mon.0) 3427 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2022-04-28T21:40:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:38 smithi139 bash[43301]: audit 2022-04-28T21:40:37.272697+0000 mon.a (mon.0) 3426 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:38 smithi139 bash[43301]: cluster 2022-04-28T21:40:37.272846+0000 mon.a (mon.0) 3427 : cluster [DBG] osdmap e1169: 8 total, 8 up, 8 in 2022-04-28T21:40:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:39 smithi059 bash[61152]: cluster 2022-04-28T21:40:38.192672+0000 mgr.y (mgr.24961) 3629 : cluster [DBG] pgmap v4162: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:39.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:39 smithi059 bash[61152]: cluster 2022-04-28T21:40:38.294416+0000 mon.a (mon.0) 3428 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2022-04-28T21:40:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:39 smithi059 bash[59745]: cluster 2022-04-28T21:40:38.192672+0000 mgr.y (mgr.24961) 3629 : cluster [DBG] pgmap v4162: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:39 smithi059 bash[59745]: cluster 2022-04-28T21:40:38.294416+0000 mon.a (mon.0) 3428 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2022-04-28T21:40:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:39 smithi139 bash[43301]: cluster 2022-04-28T21:40:38.192672+0000 mgr.y (mgr.24961) 3629 : cluster [DBG] pgmap v4162: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 835 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:39 smithi139 bash[43301]: cluster 2022-04-28T21:40:38.294416+0000 mon.a (mon.0) 3428 : cluster [DBG] osdmap e1170: 8 total, 8 up, 8 in 2022-04-28T21:40:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:40 smithi059 bash[61152]: cluster 2022-04-28T21:40:39.307194+0000 mon.a (mon.0) 3429 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2022-04-28T21:40:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:40 smithi059 bash[61152]: audit 2022-04-28T21:40:39.310779+0000 mon.c (mon.1) 133 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:40 smithi059 bash[61152]: audit 2022-04-28T21:40:39.312341+0000 mon.a (mon.0) 3430 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:40 smithi059 bash[59745]: cluster 2022-04-28T21:40:39.307194+0000 mon.a (mon.0) 3429 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2022-04-28T21:40:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:40 smithi059 bash[59745]: audit 2022-04-28T21:40:39.310779+0000 mon.c (mon.1) 133 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:40 smithi059 bash[59745]: audit 2022-04-28T21:40:39.312341+0000 mon.a (mon.0) 3430 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:40.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:40 smithi139 bash[43301]: cluster 2022-04-28T21:40:39.307194+0000 mon.a (mon.0) 3429 : cluster [DBG] osdmap e1171: 8 total, 8 up, 8 in 2022-04-28T21:40:40.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:40 smithi139 bash[43301]: audit 2022-04-28T21:40:39.310779+0000 mon.c (mon.1) 133 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:40.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:40 smithi139 bash[43301]: audit 2022-04-28T21:40:39.312341+0000 mon.a (mon.0) 3430 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:41 smithi139 bash[43301]: cluster 2022-04-28T21:40:40.193831+0000 mgr.y (mgr.24961) 3630 : cluster [DBG] pgmap v4165: 169 pgs: 12 creating+peering, 20 unknown, 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:41 smithi139 bash[43301]: audit 2022-04-28T21:40:40.311067+0000 mon.a (mon.0) 3431 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:41 smithi139 bash[43301]: cluster 2022-04-28T21:40:40.311197+0000 mon.a (mon.0) 3432 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2022-04-28T21:40:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:41 smithi059 bash[61152]: cluster 2022-04-28T21:40:40.193831+0000 mgr.y (mgr.24961) 3630 : cluster [DBG] pgmap v4165: 169 pgs: 12 creating+peering, 20 unknown, 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:41 smithi059 bash[61152]: audit 2022-04-28T21:40:40.311067+0000 mon.a (mon.0) 3431 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:41.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:41 smithi059 bash[61152]: cluster 2022-04-28T21:40:40.311197+0000 mon.a (mon.0) 3432 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2022-04-28T21:40:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:41 smithi059 bash[59745]: cluster 2022-04-28T21:40:40.193831+0000 mgr.y (mgr.24961) 3630 : cluster [DBG] pgmap v4165: 169 pgs: 12 creating+peering, 20 unknown, 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:41 smithi059 bash[59745]: audit 2022-04-28T21:40:40.311067+0000 mon.a (mon.0) 3431 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-2","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:41.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:41 smithi059 bash[59745]: cluster 2022-04-28T21:40:40.311197+0000 mon.a (mon.0) 3432 : cluster [DBG] osdmap e1172: 8 total, 8 up, 8 in 2022-04-28T21:40:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:42.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: cluster 2022-04-28T21:40:41.335334+0000 mon.a (mon.0) 3433 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2022-04-28T21:40:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: audit 2022-04-28T21:40:41.336512+0000 mon.c (mon.1) 134 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: audit 2022-04-28T21:40:41.338614+0000 mon.a (mon.0) 3434 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: audit 2022-04-28T21:40:42.333401+0000 mon.a (mon.0) 3435 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: cluster 2022-04-28T21:40:42.333910+0000 mon.a (mon.0) 3436 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2022-04-28T21:40:42.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: audit 2022-04-28T21:40:42.340205+0000 mon.c (mon.1) 135 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]: dispatch 2022-04-28T21:40:42.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:42 smithi139 bash[43301]: audit 2022-04-28T21:40:42.341067+0000 mon.a (mon.0) 3437 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]: dispatch 2022-04-28T21:40:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: cluster 2022-04-28T21:40:41.335334+0000 mon.a (mon.0) 3433 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2022-04-28T21:40:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: audit 2022-04-28T21:40:41.336512+0000 mon.c (mon.1) 134 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: audit 2022-04-28T21:40:41.338614+0000 mon.a (mon.0) 3434 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: audit 2022-04-28T21:40:42.333401+0000 mon.a (mon.0) 3435 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: cluster 2022-04-28T21:40:42.333910+0000 mon.a (mon.0) 3436 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2022-04-28T21:40:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: audit 2022-04-28T21:40:42.340205+0000 mon.c (mon.1) 135 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]: dispatch 2022-04-28T21:40:42.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[61152]: audit 2022-04-28T21:40:42.341067+0000 mon.a (mon.0) 3437 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]: dispatch 2022-04-28T21:40:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: cluster 2022-04-28T21:40:41.335334+0000 mon.a (mon.0) 3433 : cluster [DBG] osdmap e1173: 8 total, 8 up, 8 in 2022-04-28T21:40:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: audit 2022-04-28T21:40:41.336512+0000 mon.c (mon.1) 134 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: audit 2022-04-28T21:40:41.338614+0000 mon.a (mon.0) 3434 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:40:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: audit 2022-04-28T21:40:42.333401+0000 mon.a (mon.0) 3435 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-3","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:40:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: cluster 2022-04-28T21:40:42.333910+0000 mon.a (mon.0) 3436 : cluster [DBG] osdmap e1174: 8 total, 8 up, 8 in 2022-04-28T21:40:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: audit 2022-04-28T21:40:42.340205+0000 mon.c (mon.1) 135 : audit [INF] from='client.? 172.21.15.59:0/1677400632' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]: dispatch 2022-04-28T21:40:42.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:42 smithi059 bash[59745]: audit 2022-04-28T21:40:42.341067+0000 mon.a (mon.0) 3437 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]: dispatch 2022-04-28T21:40:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[59745]: cluster 2022-04-28T21:40:42.194636+0000 mgr.y (mgr.24961) 3631 : cluster [DBG] pgmap v4168: 201 pgs: 12 creating+peering, 41 unknown, 148 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[59745]: audit 2022-04-28T21:40:43.336602+0000 mon.a (mon.0) 3438 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]': finished 2022-04-28T21:40:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[59745]: cluster 2022-04-28T21:40:43.336831+0000 mon.a (mon.0) 3439 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2022-04-28T21:40:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[61152]: cluster 2022-04-28T21:40:42.194636+0000 mgr.y (mgr.24961) 3631 : cluster [DBG] pgmap v4168: 201 pgs: 12 creating+peering, 41 unknown, 148 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[61152]: audit 2022-04-28T21:40:43.336602+0000 mon.a (mon.0) 3438 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]': finished 2022-04-28T21:40:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[61152]: cluster 2022-04-28T21:40:43.336831+0000 mon.a (mon.0) 3439 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2022-04-28T21:40:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:43 smithi139 bash[43301]: cluster 2022-04-28T21:40:42.194636+0000 mgr.y (mgr.24961) 3631 : cluster [DBG] pgmap v4168: 201 pgs: 12 creating+peering, 41 unknown, 148 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:43 smithi139 bash[43301]: audit 2022-04-28T21:40:43.336602+0000 mon.a (mon.0) 3438 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-rados-api-smithi059-84622-3", "field": "max_bytes", "val": "4096"}]': finished 2022-04-28T21:40:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:43 smithi139 bash[43301]: cluster 2022-04-28T21:40:43.336831+0000 mon.a (mon.0) 3439 : cluster [DBG] osdmap e1175: 8 total, 8 up, 8 in 2022-04-28T21:40:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:43.946Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:45.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:45 smithi059 bash[61152]: cluster 2022-04-28T21:40:44.195937+0000 mgr.y (mgr.24961) 3632 : cluster [DBG] pgmap v4171: 201 pgs: 12 creating+peering, 22 unknown, 167 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:45.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:45 smithi059 bash[59745]: cluster 2022-04-28T21:40:44.195937+0000 mgr.y (mgr.24961) 3632 : cluster [DBG] pgmap v4171: 201 pgs: 12 creating+peering, 22 unknown, 167 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:45.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:45 smithi139 bash[43301]: cluster 2022-04-28T21:40:44.195937+0000 mgr.y (mgr.24961) 3632 : cluster [DBG] pgmap v4171: 201 pgs: 12 creating+peering, 22 unknown, 167 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:40:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:47 smithi059 bash[61152]: cluster 2022-04-28T21:40:46.197495+0000 mgr.y (mgr.24961) 3633 : cluster [DBG] pgmap v4172: 201 pgs: 201 active+clean; 36 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s wr, 0 op/s 2022-04-28T21:40:47.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:47 smithi059 bash[59745]: cluster 2022-04-28T21:40:46.197495+0000 mgr.y (mgr.24961) 3633 : cluster [DBG] pgmap v4172: 201 pgs: 201 active+clean; 36 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s wr, 0 op/s 2022-04-28T21:40:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:47 smithi139 bash[43301]: cluster 2022-04-28T21:40:46.197495+0000 mgr.y (mgr.24961) 3633 : cluster [DBG] pgmap v4172: 201 pgs: 201 active+clean; 36 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s wr, 0 op/s 2022-04-28T21:40:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:49 smithi059 bash[61152]: cluster 2022-04-28T21:40:48.198268+0000 mgr.y (mgr.24961) 3634 : cluster [DBG] pgmap v4173: 201 pgs: 201 active+clean; 36 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s wr, 0 op/s 2022-04-28T21:40:49.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:49 smithi059 bash[59745]: cluster 2022-04-28T21:40:48.198268+0000 mgr.y (mgr.24961) 3634 : cluster [DBG] pgmap v4173: 201 pgs: 201 active+clean; 36 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s wr, 0 op/s 2022-04-28T21:40:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:49 smithi139 bash[43301]: cluster 2022-04-28T21:40:48.198268+0000 mgr.y (mgr.24961) 3634 : cluster [DBG] pgmap v4173: 201 pgs: 201 active+clean; 36 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s wr, 0 op/s 2022-04-28T21:40:51.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[59745]: cluster 2022-04-28T21:40:50.200009+0000 mgr.y (mgr.24961) 3635 : cluster [DBG] pgmap v4174: 201 pgs: 201 active+clean; 48 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s wr, 0 op/s 2022-04-28T21:40:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[59745]: cluster 2022-04-28T21:40:51.554494+0000 mon.a (mon.0) 3440 : cluster [WRN] pool 'test-rados-api-smithi059-84622-3' is full (reached quota's max_bytes: 4 KiB) 2022-04-28T21:40:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[59745]: cluster 2022-04-28T21:40:51.554973+0000 mon.a (mon.0) 3441 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2022-04-28T21:40:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[59745]: cluster 2022-04-28T21:40:51.566153+0000 mon.a (mon.0) 3442 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2022-04-28T21:40:51.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[61152]: cluster 2022-04-28T21:40:50.200009+0000 mgr.y (mgr.24961) 3635 : cluster [DBG] pgmap v4174: 201 pgs: 201 active+clean; 48 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s wr, 0 op/s 2022-04-28T21:40:51.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[61152]: cluster 2022-04-28T21:40:51.554494+0000 mon.a (mon.0) 3440 : cluster [WRN] pool 'test-rados-api-smithi059-84622-3' is full (reached quota's max_bytes: 4 KiB) 2022-04-28T21:40:51.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[61152]: cluster 2022-04-28T21:40:51.554973+0000 mon.a (mon.0) 3441 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2022-04-28T21:40:51.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:51 smithi059 bash[61152]: cluster 2022-04-28T21:40:51.566153+0000 mon.a (mon.0) 3442 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2022-04-28T21:40:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:51 smithi139 bash[43301]: cluster 2022-04-28T21:40:50.200009+0000 mgr.y (mgr.24961) 3635 : cluster [DBG] pgmap v4174: 201 pgs: 201 active+clean; 48 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s wr, 0 op/s 2022-04-28T21:40:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:51 smithi139 bash[43301]: cluster 2022-04-28T21:40:51.554494+0000 mon.a (mon.0) 3440 : cluster [WRN] pool 'test-rados-api-smithi059-84622-3' is full (reached quota's max_bytes: 4 KiB) 2022-04-28T21:40:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:51 smithi139 bash[43301]: cluster 2022-04-28T21:40:51.554973+0000 mon.a (mon.0) 3441 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2022-04-28T21:40:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:51 smithi139 bash[43301]: cluster 2022-04-28T21:40:51.566153+0000 mon.a (mon.0) 3442 : cluster [DBG] osdmap e1176: 8 total, 8 up, 8 in 2022-04-28T21:40:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:52.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:53 smithi059 bash[61152]: cluster 2022-04-28T21:40:52.200807+0000 mgr.y (mgr.24961) 3636 : cluster [DBG] pgmap v4176: 201 pgs: 201 active+clean; 52 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s 2022-04-28T21:40:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:53 smithi059 bash[59745]: cluster 2022-04-28T21:40:52.200807+0000 mgr.y (mgr.24961) 3636 : cluster [DBG] pgmap v4176: 201 pgs: 201 active+clean; 52 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s 2022-04-28T21:40:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:53 smithi139 bash[43301]: cluster 2022-04-28T21:40:52.200807+0000 mgr.y (mgr.24961) 3636 : cluster [DBG] pgmap v4176: 201 pgs: 201 active+clean; 52 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s 2022-04-28T21:40:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:40:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:40:53.947Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:40:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:54 smithi139 bash[43301]: audit 2022-04-28T21:40:54.410369+0000 mon.a (mon.0) 3443 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:40:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:54 smithi139 bash[43301]: audit 2022-04-28T21:40:54.413264+0000 mon.a (mon.0) 3444 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:40:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:54 smithi139 bash[43301]: audit 2022-04-28T21:40:54.415284+0000 mon.a (mon.0) 3445 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:40:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:54 smithi139 bash[43301]: audit 2022-04-28T21:40:54.418477+0000 mon.a (mon.0) 3446 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:40:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[59745]: audit 2022-04-28T21:40:54.410369+0000 mon.a (mon.0) 3443 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:40:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[59745]: audit 2022-04-28T21:40:54.413264+0000 mon.a (mon.0) 3444 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:40:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[59745]: audit 2022-04-28T21:40:54.415284+0000 mon.a (mon.0) 3445 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:40:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[59745]: audit 2022-04-28T21:40:54.418477+0000 mon.a (mon.0) 3446 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:40:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[61152]: audit 2022-04-28T21:40:54.410369+0000 mon.a (mon.0) 3443 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:40:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[61152]: audit 2022-04-28T21:40:54.413264+0000 mon.a (mon.0) 3444 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:40:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[61152]: audit 2022-04-28T21:40:54.415284+0000 mon.a (mon.0) 3445 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:40:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:54 smithi059 bash[61152]: audit 2022-04-28T21:40:54.418477+0000 mon.a (mon.0) 3446 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:40:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:55 smithi139 bash[43301]: cluster 2022-04-28T21:40:54.202116+0000 mgr.y (mgr.24961) 3637 : cluster [DBG] pgmap v4177: 201 pgs: 201 active+clean; 56 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s 2022-04-28T21:40:55.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:55 smithi139 bash[43301]: cephadm 2022-04-28T21:40:54.416944+0000 mgr.y (mgr.24961) 3638 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:40:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:55 smithi059 bash[61152]: cluster 2022-04-28T21:40:54.202116+0000 mgr.y (mgr.24961) 3637 : cluster [DBG] pgmap v4177: 201 pgs: 201 active+clean; 56 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s 2022-04-28T21:40:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:55 smithi059 bash[61152]: cephadm 2022-04-28T21:40:54.416944+0000 mgr.y (mgr.24961) 3638 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:40:56.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:55 smithi059 bash[59745]: cluster 2022-04-28T21:40:54.202116+0000 mgr.y (mgr.24961) 3637 : cluster [DBG] pgmap v4177: 201 pgs: 201 active+clean; 56 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s wr, 0 op/s 2022-04-28T21:40:56.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:55 smithi059 bash[59745]: cephadm 2022-04-28T21:40:54.416944+0000 mgr.y (mgr.24961) 3638 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:40:58.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:57 smithi059 bash[59745]: cluster 2022-04-28T21:40:56.203720+0000 mgr.y (mgr.24961) 3639 : cluster [DBG] pgmap v4178: 201 pgs: 201 active+clean; 60 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s wr, 0 op/s 2022-04-28T21:40:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:57 smithi059 bash[61152]: cluster 2022-04-28T21:40:56.203720+0000 mgr.y (mgr.24961) 3639 : cluster [DBG] pgmap v4178: 201 pgs: 201 active+clean; 60 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s wr, 0 op/s 2022-04-28T21:40:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:57 smithi139 bash[43301]: cluster 2022-04-28T21:40:56.203720+0000 mgr.y (mgr.24961) 3639 : cluster [DBG] pgmap v4178: 201 pgs: 201 active+clean; 60 KiB data, 846 MiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s wr, 0 op/s 2022-04-28T21:40:58.661 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:58 smithi059 bash[61152]: cluster 2022-04-28T21:40:57.602151+0000 mon.a (mon.0) 3447 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2022-04-28T21:40:58.661 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:58 smithi059 bash[61152]: cluster 2022-04-28T21:40:57.602223+0000 mon.a (mon.0) 3448 : cluster [INF] Cluster is now healthy 2022-04-28T21:40:58.661 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:58 smithi059 bash[61152]: cluster 2022-04-28T21:40:57.612081+0000 mon.a (mon.0) 3449 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2022-04-28T21:40:58.662 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:58 smithi059 bash[59745]: cluster 2022-04-28T21:40:57.602151+0000 mon.a (mon.0) 3447 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2022-04-28T21:40:58.662 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:58 smithi059 bash[59745]: cluster 2022-04-28T21:40:57.602223+0000 mon.a (mon.0) 3448 : cluster [INF] Cluster is now healthy 2022-04-28T21:40:58.663 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:58 smithi059 bash[59745]: cluster 2022-04-28T21:40:57.612081+0000 mon.a (mon.0) 3449 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2022-04-28T21:40:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:58 smithi139 bash[43301]: cluster 2022-04-28T21:40:57.602151+0000 mon.a (mon.0) 3447 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2022-04-28T21:40:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:58 smithi139 bash[43301]: cluster 2022-04-28T21:40:57.602223+0000 mon.a (mon.0) 3448 : cluster [INF] Cluster is now healthy 2022-04-28T21:40:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:58 smithi139 bash[43301]: cluster 2022-04-28T21:40:57.612081+0000 mon.a (mon.0) 3449 : cluster [DBG] osdmap e1177: 8 total, 8 up, 8 in 2022-04-28T21:41:00.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:59 smithi059 bash[61152]: cluster 2022-04-28T21:40:58.204522+0000 mgr.y (mgr.24961) 3640 : cluster [DBG] pgmap v4180: 169 pgs: 169 active+clean; 24 KiB data, 846 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:00.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:40:59 smithi059 bash[61152]: cluster 2022-04-28T21:40:58.634792+0000 mon.a (mon.0) 3450 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2022-04-28T21:41:00.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:59 smithi059 bash[59745]: cluster 2022-04-28T21:40:58.204522+0000 mgr.y (mgr.24961) 3640 : cluster [DBG] pgmap v4180: 169 pgs: 169 active+clean; 24 KiB data, 846 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:00.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:40:59 smithi059 bash[59745]: cluster 2022-04-28T21:40:58.634792+0000 mon.a (mon.0) 3450 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2022-04-28T21:41:00.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:59 smithi139 bash[43301]: cluster 2022-04-28T21:40:58.204522+0000 mgr.y (mgr.24961) 3640 : cluster [DBG] pgmap v4180: 169 pgs: 169 active+clean; 24 KiB data, 846 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:00.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:40:59 smithi139 bash[43301]: cluster 2022-04-28T21:40:58.634792+0000 mon.a (mon.0) 3450 : cluster [DBG] osdmap e1178: 8 total, 8 up, 8 in 2022-04-28T21:41:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[61152]: cluster 2022-04-28T21:40:59.641632+0000 mon.a (mon.0) 3451 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2022-04-28T21:41:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[61152]: audit 2022-04-28T21:40:59.644032+0000 mon.c (mon.1) 136 : audit [INF] from='client.? 172.21.15.59:0/3142876468' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[61152]: audit 2022-04-28T21:40:59.645332+0000 mon.a (mon.0) 3452 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[61152]: audit 2022-04-28T21:41:00.642151+0000 mon.a (mon.0) 3453 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[61152]: cluster 2022-04-28T21:41:00.642643+0000 mon.a (mon.0) 3454 : cluster [DBG] osdmap e1180: 8 total, 8 up, 8 in 2022-04-28T21:41:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[59745]: cluster 2022-04-28T21:40:59.641632+0000 mon.a (mon.0) 3451 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2022-04-28T21:41:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[59745]: audit 2022-04-28T21:40:59.644032+0000 mon.c (mon.1) 136 : audit [INF] from='client.? 172.21.15.59:0/3142876468' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[59745]: audit 2022-04-28T21:40:59.645332+0000 mon.a (mon.0) 3452 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[59745]: audit 2022-04-28T21:41:00.642151+0000 mon.a (mon.0) 3453 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:00.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:00 smithi059 bash[59745]: cluster 2022-04-28T21:41:00.642643+0000 mon.a (mon.0) 3454 : cluster [DBG] osdmap e1180: 8 total, 8 up, 8 in 2022-04-28T21:41:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:00 smithi139 bash[43301]: cluster 2022-04-28T21:40:59.641632+0000 mon.a (mon.0) 3451 : cluster [DBG] osdmap e1179: 8 total, 8 up, 8 in 2022-04-28T21:41:01.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:00 smithi139 bash[43301]: audit 2022-04-28T21:40:59.644032+0000 mon.c (mon.1) 136 : audit [INF] from='client.? 172.21.15.59:0/3142876468' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:01.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:00 smithi139 bash[43301]: audit 2022-04-28T21:40:59.645332+0000 mon.a (mon.0) 3452 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:01.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:00 smithi139 bash[43301]: audit 2022-04-28T21:41:00.642151+0000 mon.a (mon.0) 3453 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-4","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:01.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:00 smithi139 bash[43301]: cluster 2022-04-28T21:41:00.642643+0000 mon.a (mon.0) 3454 : cluster [DBG] osdmap e1180: 8 total, 8 up, 8 in 2022-04-28T21:41:02.055 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:01 smithi059 bash[61152]: cluster 2022-04-28T21:41:00.205561+0000 mgr.y (mgr.24961) 3641 : cluster [DBG] pgmap v4183: 169 pgs: 7 creating+peering, 25 unknown, 137 active+clean; 24 KiB data, 847 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:02.055 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:01 smithi059 bash[61152]: cluster 2022-04-28T21:41:01.643079+0000 mon.a (mon.0) 3455 : cluster [DBG] osdmap e1181: 8 total, 8 up, 8 in 2022-04-28T21:41:02.055 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:01 smithi059 bash[59745]: cluster 2022-04-28T21:41:00.205561+0000 mgr.y (mgr.24961) 3641 : cluster [DBG] pgmap v4183: 169 pgs: 7 creating+peering, 25 unknown, 137 active+clean; 24 KiB data, 847 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:02.056 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:01 smithi059 bash[59745]: cluster 2022-04-28T21:41:01.643079+0000 mon.a (mon.0) 3455 : cluster [DBG] osdmap e1181: 8 total, 8 up, 8 in 2022-04-28T21:41:02.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:01 smithi139 bash[43301]: cluster 2022-04-28T21:41:00.205561+0000 mgr.y (mgr.24961) 3641 : cluster [DBG] pgmap v4183: 169 pgs: 7 creating+peering, 25 unknown, 137 active+clean; 24 KiB data, 847 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:01 smithi139 bash[43301]: cluster 2022-04-28T21:41:01.643079+0000 mon.a (mon.0) 3455 : cluster [DBG] osdmap e1181: 8 total, 8 up, 8 in 2022-04-28T21:41:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:02.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:03.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[59745]: audit 2022-04-28T21:41:02.137737+0000 mon.a (mon.0) 3456 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:41:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[59745]: cluster 2022-04-28T21:41:02.206047+0000 mgr.y (mgr.24961) 3642 : cluster [DBG] pgmap v4186: 137 pgs: 137 active+clean; 24 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[59745]: cephadm 2022-04-28T21:41:02.348063+0000 mgr.y (mgr.24961) 3643 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T21:41:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[59745]: audit 2022-04-28T21:41:02.357908+0000 mon.a (mon.0) 3457 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:41:03.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[61152]: audit 2022-04-28T21:41:02.137737+0000 mon.a (mon.0) 3456 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:41:03.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[61152]: cluster 2022-04-28T21:41:02.206047+0000 mgr.y (mgr.24961) 3642 : cluster [DBG] pgmap v4186: 137 pgs: 137 active+clean; 24 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:03.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[61152]: cephadm 2022-04-28T21:41:02.348063+0000 mgr.y (mgr.24961) 3643 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T21:41:03.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:02 smithi059 bash[61152]: audit 2022-04-28T21:41:02.357908+0000 mon.a (mon.0) 3457 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:41:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:02 smithi139 bash[43301]: audit 2022-04-28T21:41:02.137737+0000 mon.a (mon.0) 3456 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:41:03.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:02 smithi139 bash[43301]: cluster 2022-04-28T21:41:02.206047+0000 mgr.y (mgr.24961) 3642 : cluster [DBG] pgmap v4186: 137 pgs: 137 active+clean; 24 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:03.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:02 smithi139 bash[43301]: cephadm 2022-04-28T21:41:02.348063+0000 mgr.y (mgr.24961) 3643 : cephadm [INF] Detected new or changed devices on smithi059 2022-04-28T21:41:03.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:02 smithi139 bash[43301]: audit 2022-04-28T21:41:02.357908+0000 mon.a (mon.0) 3457 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:41:03.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[59745]: cluster 2022-04-28T21:41:02.676239+0000 mon.a (mon.0) 3458 : cluster [DBG] osdmap e1182: 8 total, 8 up, 8 in 2022-04-28T21:41:03.949 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[59745]: audit 2022-04-28T21:41:02.679303+0000 mon.a (mon.0) 3459 : audit [INF] from='client.? 172.21.15.59:0/3359369100' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:03.950 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[59745]: audit 2022-04-28T21:41:02.684926+0000 mon.a (mon.0) 3460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:41:03.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[61152]: cluster 2022-04-28T21:41:02.676239+0000 mon.a (mon.0) 3458 : cluster [DBG] osdmap e1182: 8 total, 8 up, 8 in 2022-04-28T21:41:03.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[61152]: audit 2022-04-28T21:41:02.679303+0000 mon.a (mon.0) 3459 : audit [INF] from='client.? 172.21.15.59:0/3359369100' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:03.950 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[61152]: audit 2022-04-28T21:41:02.684926+0000 mon.a (mon.0) 3460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:41:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:03 smithi139 bash[43301]: cluster 2022-04-28T21:41:02.676239+0000 mon.a (mon.0) 3458 : cluster [DBG] osdmap e1182: 8 total, 8 up, 8 in 2022-04-28T21:41:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:03 smithi139 bash[43301]: audit 2022-04-28T21:41:02.679303+0000 mon.a (mon.0) 3459 : audit [INF] from='client.? 172.21.15.59:0/3359369100' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:03 smithi139 bash[43301]: audit 2022-04-28T21:41:02.684926+0000 mon.a (mon.0) 3460 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:41:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:03.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:04 smithi059 bash[61152]: audit 2022-04-28T21:41:03.686476+0000 mon.a (mon.0) 3461 : audit [INF] from='client.? 172.21.15.59:0/3359369100' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:04 smithi059 bash[61152]: cluster 2022-04-28T21:41:03.686753+0000 mon.a (mon.0) 3462 : cluster [DBG] osdmap e1183: 8 total, 8 up, 8 in 2022-04-28T21:41:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:04 smithi059 bash[61152]: cluster 2022-04-28T21:41:04.207188+0000 mgr.y (mgr.24961) 3644 : cluster [DBG] pgmap v4189: 169 pgs: 21 unknown, 148 active+clean; 25 KiB data, 836 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:41:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:04 smithi059 bash[59745]: audit 2022-04-28T21:41:03.686476+0000 mon.a (mon.0) 3461 : audit [INF] from='client.? 172.21.15.59:0/3359369100' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:04 smithi059 bash[59745]: cluster 2022-04-28T21:41:03.686753+0000 mon.a (mon.0) 3462 : cluster [DBG] osdmap e1183: 8 total, 8 up, 8 in 2022-04-28T21:41:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:04 smithi059 bash[59745]: cluster 2022-04-28T21:41:04.207188+0000 mgr.y (mgr.24961) 3644 : cluster [DBG] pgmap v4189: 169 pgs: 21 unknown, 148 active+clean; 25 KiB data, 836 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:41:05.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:04 smithi139 bash[43301]: audit 2022-04-28T21:41:03.686476+0000 mon.a (mon.0) 3461 : audit [INF] from='client.? 172.21.15.59:0/3359369100' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-5","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:05.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:04 smithi139 bash[43301]: cluster 2022-04-28T21:41:03.686753+0000 mon.a (mon.0) 3462 : cluster [DBG] osdmap e1183: 8 total, 8 up, 8 in 2022-04-28T21:41:05.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:04 smithi139 bash[43301]: cluster 2022-04-28T21:41:04.207188+0000 mgr.y (mgr.24961) 3644 : cluster [DBG] pgmap v4189: 169 pgs: 21 unknown, 148 active+clean; 25 KiB data, 836 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:41:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:05 smithi059 bash[61152]: cluster 2022-04-28T21:41:04.689908+0000 mon.a (mon.0) 3463 : cluster [DBG] osdmap e1184: 8 total, 8 up, 8 in 2022-04-28T21:41:06.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:05 smithi059 bash[59745]: cluster 2022-04-28T21:41:04.689908+0000 mon.a (mon.0) 3463 : cluster [DBG] osdmap e1184: 8 total, 8 up, 8 in 2022-04-28T21:41:06.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:05 smithi139 bash[43301]: cluster 2022-04-28T21:41:04.689908+0000 mon.a (mon.0) 3463 : cluster [DBG] osdmap e1184: 8 total, 8 up, 8 in 2022-04-28T21:41:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[61152]: cluster 2022-04-28T21:41:05.713656+0000 mon.a (mon.0) 3464 : cluster [DBG] osdmap e1185: 8 total, 8 up, 8 in 2022-04-28T21:41:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[61152]: audit 2022-04-28T21:41:05.717157+0000 mon.c (mon.1) 137 : audit [INF] from='client.? 172.21.15.59:0/3268764996' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[61152]: audit 2022-04-28T21:41:05.718423+0000 mon.a (mon.0) 3465 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:07.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[61152]: cluster 2022-04-28T21:41:06.208243+0000 mgr.y (mgr.24961) 3645 : cluster [DBG] pgmap v4192: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[59745]: cluster 2022-04-28T21:41:05.713656+0000 mon.a (mon.0) 3464 : cluster [DBG] osdmap e1185: 8 total, 8 up, 8 in 2022-04-28T21:41:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[59745]: audit 2022-04-28T21:41:05.717157+0000 mon.c (mon.1) 137 : audit [INF] from='client.? 172.21.15.59:0/3268764996' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[59745]: audit 2022-04-28T21:41:05.718423+0000 mon.a (mon.0) 3465 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:06 smithi059 bash[59745]: cluster 2022-04-28T21:41:06.208243+0000 mgr.y (mgr.24961) 3645 : cluster [DBG] pgmap v4192: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:06 smithi139 bash[43301]: cluster 2022-04-28T21:41:05.713656+0000 mon.a (mon.0) 3464 : cluster [DBG] osdmap e1185: 8 total, 8 up, 8 in 2022-04-28T21:41:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:06 smithi139 bash[43301]: audit 2022-04-28T21:41:05.717157+0000 mon.c (mon.1) 137 : audit [INF] from='client.? 172.21.15.59:0/3268764996' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:06 smithi139 bash[43301]: audit 2022-04-28T21:41:05.718423+0000 mon.a (mon.0) 3465 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:06 smithi139 bash[43301]: cluster 2022-04-28T21:41:06.208243+0000 mgr.y (mgr.24961) 3645 : cluster [DBG] pgmap v4192: 169 pgs: 4 creating+peering, 28 unknown, 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:07 smithi059 bash[61152]: audit 2022-04-28T21:41:06.716502+0000 mon.a (mon.0) 3466 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:07 smithi059 bash[61152]: cluster 2022-04-28T21:41:06.716666+0000 mon.a (mon.0) 3467 : cluster [DBG] osdmap e1186: 8 total, 8 up, 8 in 2022-04-28T21:41:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:07 smithi059 bash[59745]: audit 2022-04-28T21:41:06.716502+0000 mon.a (mon.0) 3466 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:07 smithi059 bash[59745]: cluster 2022-04-28T21:41:06.716666+0000 mon.a (mon.0) 3467 : cluster [DBG] osdmap e1186: 8 total, 8 up, 8 in 2022-04-28T21:41:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:07 smithi139 bash[43301]: audit 2022-04-28T21:41:06.716502+0000 mon.a (mon.0) 3466 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-6","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:07 smithi139 bash[43301]: cluster 2022-04-28T21:41:06.716666+0000 mon.a (mon.0) 3467 : cluster [DBG] osdmap e1186: 8 total, 8 up, 8 in 2022-04-28T21:41:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:08 smithi059 bash[59745]: cluster 2022-04-28T21:41:07.741813+0000 mon.a (mon.0) 3468 : cluster [DBG] osdmap e1187: 8 total, 8 up, 8 in 2022-04-28T21:41:09.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:08 smithi059 bash[59745]: cluster 2022-04-28T21:41:08.208685+0000 mgr.y (mgr.24961) 3646 : cluster [DBG] pgmap v4195: 137 pgs: 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:08 smithi059 bash[61152]: cluster 2022-04-28T21:41:07.741813+0000 mon.a (mon.0) 3468 : cluster [DBG] osdmap e1187: 8 total, 8 up, 8 in 2022-04-28T21:41:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:08 smithi059 bash[61152]: cluster 2022-04-28T21:41:08.208685+0000 mgr.y (mgr.24961) 3646 : cluster [DBG] pgmap v4195: 137 pgs: 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:08 smithi139 bash[43301]: cluster 2022-04-28T21:41:07.741813+0000 mon.a (mon.0) 3468 : cluster [DBG] osdmap e1187: 8 total, 8 up, 8 in 2022-04-28T21:41:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:08 smithi139 bash[43301]: cluster 2022-04-28T21:41:08.208685+0000 mgr.y (mgr.24961) 3646 : cluster [DBG] pgmap v4195: 137 pgs: 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:09 smithi059 bash[61152]: cluster 2022-04-28T21:41:08.755272+0000 mon.a (mon.0) 3469 : cluster [DBG] osdmap e1188: 8 total, 8 up, 8 in 2022-04-28T21:41:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:09 smithi059 bash[61152]: audit 2022-04-28T21:41:08.765490+0000 mon.a (mon.0) 3470 : audit [INF] from='client.? 172.21.15.59:0/354390529' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:10.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:09 smithi059 bash[59745]: cluster 2022-04-28T21:41:08.755272+0000 mon.a (mon.0) 3469 : cluster [DBG] osdmap e1188: 8 total, 8 up, 8 in 2022-04-28T21:41:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:09 smithi059 bash[59745]: audit 2022-04-28T21:41:08.765490+0000 mon.a (mon.0) 3470 : audit [INF] from='client.? 172.21.15.59:0/354390529' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:10.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:09 smithi139 bash[43301]: cluster 2022-04-28T21:41:08.755272+0000 mon.a (mon.0) 3469 : cluster [DBG] osdmap e1188: 8 total, 8 up, 8 in 2022-04-28T21:41:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:09 smithi139 bash[43301]: audit 2022-04-28T21:41:08.765490+0000 mon.a (mon.0) 3470 : audit [INF] from='client.? 172.21.15.59:0/354390529' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:11.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:10 smithi059 bash[61152]: audit 2022-04-28T21:41:09.756799+0000 mon.a (mon.0) 3471 : audit [INF] from='client.? 172.21.15.59:0/354390529' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-7","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:11.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:10 smithi059 bash[61152]: cluster 2022-04-28T21:41:09.756926+0000 mon.a (mon.0) 3472 : cluster [DBG] osdmap e1189: 8 total, 8 up, 8 in 2022-04-28T21:41:11.131 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:10 smithi059 bash[61152]: cluster 2022-04-28T21:41:10.209712+0000 mgr.y (mgr.24961) 3647 : cluster [DBG] pgmap v4198: 169 pgs: 13 creating+peering, 10 unknown, 146 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:11.131 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:10 smithi059 bash[59745]: audit 2022-04-28T21:41:09.756799+0000 mon.a (mon.0) 3471 : audit [INF] from='client.? 172.21.15.59:0/354390529' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-7","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:11.132 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:10 smithi059 bash[59745]: cluster 2022-04-28T21:41:09.756926+0000 mon.a (mon.0) 3472 : cluster [DBG] osdmap e1189: 8 total, 8 up, 8 in 2022-04-28T21:41:11.132 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:10 smithi059 bash[59745]: cluster 2022-04-28T21:41:10.209712+0000 mgr.y (mgr.24961) 3647 : cluster [DBG] pgmap v4198: 169 pgs: 13 creating+peering, 10 unknown, 146 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:11.132 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:10 smithi139 bash[43301]: audit 2022-04-28T21:41:09.756799+0000 mon.a (mon.0) 3471 : audit [INF] from='client.? 172.21.15.59:0/354390529' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-7","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:11.133 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:10 smithi139 bash[43301]: cluster 2022-04-28T21:41:09.756926+0000 mon.a (mon.0) 3472 : cluster [DBG] osdmap e1189: 8 total, 8 up, 8 in 2022-04-28T21:41:11.148 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:10 smithi139 bash[43301]: cluster 2022-04-28T21:41:10.209712+0000 mgr.y (mgr.24961) 3647 : cluster [DBG] pgmap v4198: 169 pgs: 13 creating+peering, 10 unknown, 146 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:12.059 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:11 smithi059 bash[61152]: cluster 2022-04-28T21:41:10.771802+0000 mon.a (mon.0) 3473 : cluster [DBG] osdmap e1190: 8 total, 8 up, 8 in 2022-04-28T21:41:12.059 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:11 smithi059 bash[59745]: cluster 2022-04-28T21:41:10.771802+0000 mon.a (mon.0) 3473 : cluster [DBG] osdmap e1190: 8 total, 8 up, 8 in 2022-04-28T21:41:12.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:11 smithi139 bash[43301]: cluster 2022-04-28T21:41:10.771802+0000 mon.a (mon.0) 3473 : cluster [DBG] osdmap e1190: 8 total, 8 up, 8 in 2022-04-28T21:41:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:12.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[61152]: cluster 2022-04-28T21:41:11.796412+0000 mon.a (mon.0) 3474 : cluster [DBG] osdmap e1191: 8 total, 8 up, 8 in 2022-04-28T21:41:13.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[61152]: audit 2022-04-28T21:41:11.801958+0000 mon.c (mon.1) 138 : audit [INF] from='client.? 172.21.15.59:0/3880058207' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:13.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[61152]: audit 2022-04-28T21:41:11.805624+0000 mon.a (mon.0) 3475 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:13.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[61152]: cluster 2022-04-28T21:41:12.210524+0000 mgr.y (mgr.24961) 3648 : cluster [DBG] pgmap v4201: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 843 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:13.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[59745]: cluster 2022-04-28T21:41:11.796412+0000 mon.a (mon.0) 3474 : cluster [DBG] osdmap e1191: 8 total, 8 up, 8 in 2022-04-28T21:41:13.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[59745]: audit 2022-04-28T21:41:11.801958+0000 mon.c (mon.1) 138 : audit [INF] from='client.? 172.21.15.59:0/3880058207' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:13.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[59745]: audit 2022-04-28T21:41:11.805624+0000 mon.a (mon.0) 3475 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:13.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:12 smithi059 bash[59745]: cluster 2022-04-28T21:41:12.210524+0000 mgr.y (mgr.24961) 3648 : cluster [DBG] pgmap v4201: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 843 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:12 smithi139 bash[43301]: cluster 2022-04-28T21:41:11.796412+0000 mon.a (mon.0) 3474 : cluster [DBG] osdmap e1191: 8 total, 8 up, 8 in 2022-04-28T21:41:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:12 smithi139 bash[43301]: audit 2022-04-28T21:41:11.801958+0000 mon.c (mon.1) 138 : audit [INF] from='client.? 172.21.15.59:0/3880058207' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:12 smithi139 bash[43301]: audit 2022-04-28T21:41:11.805624+0000 mon.a (mon.0) 3475 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:12 smithi139 bash[43301]: cluster 2022-04-28T21:41:12.210524+0000 mgr.y (mgr.24961) 3648 : cluster [DBG] pgmap v4201: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 843 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:13 smithi059 bash[61152]: audit 2022-04-28T21:41:12.807073+0000 mon.a (mon.0) 3476 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:13 smithi059 bash[61152]: cluster 2022-04-28T21:41:12.807361+0000 mon.a (mon.0) 3477 : cluster [DBG] osdmap e1192: 8 total, 8 up, 8 in 2022-04-28T21:41:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:13.948Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:13 smithi059 bash[59745]: audit 2022-04-28T21:41:12.807073+0000 mon.a (mon.0) 3476 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:13 smithi059 bash[59745]: cluster 2022-04-28T21:41:12.807361+0000 mon.a (mon.0) 3477 : cluster [DBG] osdmap e1192: 8 total, 8 up, 8 in 2022-04-28T21:41:14.119 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:13 smithi139 bash[43301]: audit 2022-04-28T21:41:12.807073+0000 mon.a (mon.0) 3476 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-8","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:13 smithi139 bash[43301]: cluster 2022-04-28T21:41:12.807361+0000 mon.a (mon.0) 3477 : cluster [DBG] osdmap e1192: 8 total, 8 up, 8 in 2022-04-28T21:41:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:14 smithi059 bash[61152]: cluster 2022-04-28T21:41:13.815375+0000 mon.a (mon.0) 3478 : cluster [DBG] osdmap e1193: 8 total, 8 up, 8 in 2022-04-28T21:41:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:14 smithi059 bash[61152]: cluster 2022-04-28T21:41:14.211436+0000 mgr.y (mgr.24961) 3649 : cluster [DBG] pgmap v4204: 137 pgs: 137 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:14 smithi059 bash[59745]: cluster 2022-04-28T21:41:13.815375+0000 mon.a (mon.0) 3478 : cluster [DBG] osdmap e1193: 8 total, 8 up, 8 in 2022-04-28T21:41:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:14 smithi059 bash[59745]: cluster 2022-04-28T21:41:14.211436+0000 mgr.y (mgr.24961) 3649 : cluster [DBG] pgmap v4204: 137 pgs: 137 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:15.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:14 smithi139 bash[43301]: cluster 2022-04-28T21:41:13.815375+0000 mon.a (mon.0) 3478 : cluster [DBG] osdmap e1193: 8 total, 8 up, 8 in 2022-04-28T21:41:15.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:14 smithi139 bash[43301]: cluster 2022-04-28T21:41:14.211436+0000 mgr.y (mgr.24961) 3649 : cluster [DBG] pgmap v4204: 137 pgs: 137 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:15 smithi139 bash[43301]: cluster 2022-04-28T21:41:14.831493+0000 mon.a (mon.0) 3479 : cluster [DBG] osdmap e1194: 8 total, 8 up, 8 in 2022-04-28T21:41:16.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:15 smithi139 bash[43301]: audit 2022-04-28T21:41:14.832980+0000 mon.a (mon.0) 3480 : audit [INF] from='client.? 172.21.15.59:0/3428659581' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:15 smithi059 bash[61152]: cluster 2022-04-28T21:41:14.831493+0000 mon.a (mon.0) 3479 : cluster [DBG] osdmap e1194: 8 total, 8 up, 8 in 2022-04-28T21:41:16.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:15 smithi059 bash[61152]: audit 2022-04-28T21:41:14.832980+0000 mon.a (mon.0) 3480 : audit [INF] from='client.? 172.21.15.59:0/3428659581' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:15 smithi059 bash[59745]: cluster 2022-04-28T21:41:14.831493+0000 mon.a (mon.0) 3479 : cluster [DBG] osdmap e1194: 8 total, 8 up, 8 in 2022-04-28T21:41:16.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:15 smithi059 bash[59745]: audit 2022-04-28T21:41:14.832980+0000 mon.a (mon.0) 3480 : audit [INF] from='client.? 172.21.15.59:0/3428659581' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:16 smithi139 bash[43301]: audit 2022-04-28T21:41:15.838027+0000 mon.a (mon.0) 3481 : audit [INF] from='client.? 172.21.15.59:0/3428659581' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-9","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:16 smithi139 bash[43301]: cluster 2022-04-28T21:41:15.839514+0000 mon.a (mon.0) 3482 : cluster [DBG] osdmap e1195: 8 total, 8 up, 8 in 2022-04-28T21:41:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:16 smithi139 bash[43301]: cluster 2022-04-28T21:41:16.212447+0000 mgr.y (mgr.24961) 3650 : cluster [DBG] pgmap v4207: 169 pgs: 13 creating+peering, 12 unknown, 144 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:41:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:16 smithi059 bash[61152]: audit 2022-04-28T21:41:15.838027+0000 mon.a (mon.0) 3481 : audit [INF] from='client.? 172.21.15.59:0/3428659581' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-9","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:16 smithi059 bash[61152]: cluster 2022-04-28T21:41:15.839514+0000 mon.a (mon.0) 3482 : cluster [DBG] osdmap e1195: 8 total, 8 up, 8 in 2022-04-28T21:41:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:16 smithi059 bash[61152]: cluster 2022-04-28T21:41:16.212447+0000 mgr.y (mgr.24961) 3650 : cluster [DBG] pgmap v4207: 169 pgs: 13 creating+peering, 12 unknown, 144 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:41:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:16 smithi059 bash[59745]: audit 2022-04-28T21:41:15.838027+0000 mon.a (mon.0) 3481 : audit [INF] from='client.? 172.21.15.59:0/3428659581' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-9","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:16 smithi059 bash[59745]: cluster 2022-04-28T21:41:15.839514+0000 mon.a (mon.0) 3482 : cluster [DBG] osdmap e1195: 8 total, 8 up, 8 in 2022-04-28T21:41:17.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:16 smithi059 bash[59745]: cluster 2022-04-28T21:41:16.212447+0000 mgr.y (mgr.24961) 3650 : cluster [DBG] pgmap v4207: 169 pgs: 13 creating+peering, 12 unknown, 144 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:41:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:17 smithi139 bash[43301]: cluster 2022-04-28T21:41:16.857624+0000 mon.a (mon.0) 3483 : cluster [DBG] osdmap e1196: 8 total, 8 up, 8 in 2022-04-28T21:41:18.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:17 smithi059 bash[61152]: cluster 2022-04-28T21:41:16.857624+0000 mon.a (mon.0) 3483 : cluster [DBG] osdmap e1196: 8 total, 8 up, 8 in 2022-04-28T21:41:18.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:17 smithi059 bash[59745]: cluster 2022-04-28T21:41:16.857624+0000 mon.a (mon.0) 3483 : cluster [DBG] osdmap e1196: 8 total, 8 up, 8 in 2022-04-28T21:41:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:18 smithi059 bash[61152]: cluster 2022-04-28T21:41:17.873067+0000 mon.a (mon.0) 3484 : cluster [DBG] osdmap e1197: 8 total, 8 up, 8 in 2022-04-28T21:41:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:18 smithi059 bash[61152]: audit 2022-04-28T21:41:17.875902+0000 mon.a (mon.0) 3485 : audit [INF] from='client.? 172.21.15.59:0/2525299981' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:18 smithi059 bash[61152]: cluster 2022-04-28T21:41:18.213218+0000 mgr.y (mgr.24961) 3651 : cluster [DBG] pgmap v4210: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:18 smithi059 bash[59745]: cluster 2022-04-28T21:41:17.873067+0000 mon.a (mon.0) 3484 : cluster [DBG] osdmap e1197: 8 total, 8 up, 8 in 2022-04-28T21:41:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:18 smithi059 bash[59745]: audit 2022-04-28T21:41:17.875902+0000 mon.a (mon.0) 3485 : audit [INF] from='client.? 172.21.15.59:0/2525299981' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:18 smithi059 bash[59745]: cluster 2022-04-28T21:41:18.213218+0000 mgr.y (mgr.24961) 3651 : cluster [DBG] pgmap v4210: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:18 smithi139 bash[43301]: cluster 2022-04-28T21:41:17.873067+0000 mon.a (mon.0) 3484 : cluster [DBG] osdmap e1197: 8 total, 8 up, 8 in 2022-04-28T21:41:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:18 smithi139 bash[43301]: audit 2022-04-28T21:41:17.875902+0000 mon.a (mon.0) 3485 : audit [INF] from='client.? 172.21.15.59:0/2525299981' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:18 smithi139 bash[43301]: cluster 2022-04-28T21:41:18.213218+0000 mgr.y (mgr.24961) 3651 : cluster [DBG] pgmap v4210: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 837 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[61152]: audit 2022-04-28T21:41:18.886939+0000 mon.a (mon.0) 3486 : audit [INF] from='client.? 172.21.15.59:0/2525299981' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-10","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[61152]: cluster 2022-04-28T21:41:18.887435+0000 mon.a (mon.0) 3487 : cluster [DBG] osdmap e1198: 8 total, 8 up, 8 in 2022-04-28T21:41:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[61152]: audit 2022-04-28T21:41:19.336218+0000 mon.a (mon.0) 3488 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:41:20.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[61152]: audit 2022-04-28T21:41:19.343966+0000 mon.a (mon.0) 3489 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:41:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[59745]: audit 2022-04-28T21:41:18.886939+0000 mon.a (mon.0) 3486 : audit [INF] from='client.? 172.21.15.59:0/2525299981' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-10","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[59745]: cluster 2022-04-28T21:41:18.887435+0000 mon.a (mon.0) 3487 : cluster [DBG] osdmap e1198: 8 total, 8 up, 8 in 2022-04-28T21:41:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[59745]: audit 2022-04-28T21:41:19.336218+0000 mon.a (mon.0) 3488 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:41:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:19 smithi059 bash[59745]: audit 2022-04-28T21:41:19.343966+0000 mon.a (mon.0) 3489 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:41:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:19 smithi139 bash[43301]: audit 2022-04-28T21:41:18.886939+0000 mon.a (mon.0) 3486 : audit [INF] from='client.? 172.21.15.59:0/2525299981' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-10","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:19 smithi139 bash[43301]: cluster 2022-04-28T21:41:18.887435+0000 mon.a (mon.0) 3487 : cluster [DBG] osdmap e1198: 8 total, 8 up, 8 in 2022-04-28T21:41:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:19 smithi139 bash[43301]: audit 2022-04-28T21:41:19.336218+0000 mon.a (mon.0) 3488 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:41:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:19 smithi139 bash[43301]: audit 2022-04-28T21:41:19.343966+0000 mon.a (mon.0) 3489 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:41:21.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:20 smithi059 bash[59745]: cluster 2022-04-28T21:41:19.897810+0000 mon.a (mon.0) 3490 : cluster [DBG] osdmap e1199: 8 total, 8 up, 8 in 2022-04-28T21:41:21.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:20 smithi059 bash[59745]: cluster 2022-04-28T21:41:20.214241+0000 mgr.y (mgr.24961) 3652 : cluster [DBG] pgmap v4213: 137 pgs: 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:21.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:20 smithi059 bash[61152]: cluster 2022-04-28T21:41:19.897810+0000 mon.a (mon.0) 3490 : cluster [DBG] osdmap e1199: 8 total, 8 up, 8 in 2022-04-28T21:41:21.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:20 smithi059 bash[61152]: cluster 2022-04-28T21:41:20.214241+0000 mgr.y (mgr.24961) 3652 : cluster [DBG] pgmap v4213: 137 pgs: 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:20 smithi139 bash[43301]: cluster 2022-04-28T21:41:19.897810+0000 mon.a (mon.0) 3490 : cluster [DBG] osdmap e1199: 8 total, 8 up, 8 in 2022-04-28T21:41:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:20 smithi139 bash[43301]: cluster 2022-04-28T21:41:20.214241+0000 mgr.y (mgr.24961) 3652 : cluster [DBG] pgmap v4213: 137 pgs: 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:22.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:22.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:21 smithi059 bash[61152]: cluster 2022-04-28T21:41:20.907601+0000 mon.a (mon.0) 3491 : cluster [DBG] osdmap e1200: 8 total, 8 up, 8 in 2022-04-28T21:41:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:21 smithi059 bash[61152]: audit 2022-04-28T21:41:20.910330+0000 mon.c (mon.1) 139 : audit [INF] from='client.? 172.21.15.59:0/3988055837' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:21 smithi059 bash[61152]: audit 2022-04-28T21:41:20.919453+0000 mon.a (mon.0) 3492 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:21 smithi059 bash[59745]: cluster 2022-04-28T21:41:20.907601+0000 mon.a (mon.0) 3491 : cluster [DBG] osdmap e1200: 8 total, 8 up, 8 in 2022-04-28T21:41:22.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:21 smithi059 bash[59745]: audit 2022-04-28T21:41:20.910330+0000 mon.c (mon.1) 139 : audit [INF] from='client.? 172.21.15.59:0/3988055837' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:21 smithi059 bash[59745]: audit 2022-04-28T21:41:20.919453+0000 mon.a (mon.0) 3492 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:21 smithi139 bash[43301]: cluster 2022-04-28T21:41:20.907601+0000 mon.a (mon.0) 3491 : cluster [DBG] osdmap e1200: 8 total, 8 up, 8 in 2022-04-28T21:41:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:21 smithi139 bash[43301]: audit 2022-04-28T21:41:20.910330+0000 mon.c (mon.1) 139 : audit [INF] from='client.? 172.21.15.59:0/3988055837' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:22.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:21 smithi139 bash[43301]: audit 2022-04-28T21:41:20.919453+0000 mon.a (mon.0) 3492 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[61152]: audit 2022-04-28T21:41:21.913308+0000 mon.a (mon.0) 3493 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[61152]: cluster 2022-04-28T21:41:21.913757+0000 mon.a (mon.0) 3494 : cluster [DBG] osdmap e1201: 8 total, 8 up, 8 in 2022-04-28T21:41:23.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[61152]: cluster 2022-04-28T21:41:22.215180+0000 mgr.y (mgr.24961) 3653 : cluster [DBG] pgmap v4216: 169 pgs: 12 creating+peering, 20 unknown, 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[59745]: audit 2022-04-28T21:41:21.913308+0000 mon.a (mon.0) 3493 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[59745]: cluster 2022-04-28T21:41:21.913757+0000 mon.a (mon.0) 3494 : cluster [DBG] osdmap e1201: 8 total, 8 up, 8 in 2022-04-28T21:41:23.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:22 smithi059 bash[59745]: cluster 2022-04-28T21:41:22.215180+0000 mgr.y (mgr.24961) 3653 : cluster [DBG] pgmap v4216: 169 pgs: 12 creating+peering, 20 unknown, 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:22 smithi139 bash[43301]: audit 2022-04-28T21:41:21.913308+0000 mon.a (mon.0) 3493 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-11","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:22 smithi139 bash[43301]: cluster 2022-04-28T21:41:21.913757+0000 mon.a (mon.0) 3494 : cluster [DBG] osdmap e1201: 8 total, 8 up, 8 in 2022-04-28T21:41:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:22 smithi139 bash[43301]: cluster 2022-04-28T21:41:22.215180+0000 mgr.y (mgr.24961) 3653 : cluster [DBG] pgmap v4216: 169 pgs: 12 creating+peering, 20 unknown, 137 active+clean; 24 KiB data, 839 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:23 smithi059 bash[61152]: cluster 2022-04-28T21:41:22.936289+0000 mon.a (mon.0) 3495 : cluster [DBG] osdmap e1202: 8 total, 8 up, 8 in 2022-04-28T21:41:24.379 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:23.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:24.380 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:23 smithi059 bash[59745]: cluster 2022-04-28T21:41:22.936289+0000 mon.a (mon.0) 3495 : cluster [DBG] osdmap e1202: 8 total, 8 up, 8 in 2022-04-28T21:41:24.381 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:23 smithi139 bash[43301]: cluster 2022-04-28T21:41:22.936289+0000 mon.a (mon.0) 3495 : cluster [DBG] osdmap e1202: 8 total, 8 up, 8 in 2022-04-28T21:41:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:24 smithi059 bash[61152]: cluster 2022-04-28T21:41:23.947683+0000 mon.a (mon.0) 3496 : cluster [DBG] osdmap e1203: 8 total, 8 up, 8 in 2022-04-28T21:41:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:24 smithi059 bash[61152]: audit 2022-04-28T21:41:23.949561+0000 mon.a (mon.0) 3497 : audit [INF] from='client.? 172.21.15.59:0/3089923545' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:24 smithi059 bash[61152]: cluster 2022-04-28T21:41:24.216401+0000 mgr.y (mgr.24961) 3654 : cluster [DBG] pgmap v4219: 169 pgs: 11 creating+peering, 21 unknown, 137 active+clean; 24 KiB data, 840 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:24 smithi059 bash[59745]: cluster 2022-04-28T21:41:23.947683+0000 mon.a (mon.0) 3496 : cluster [DBG] osdmap e1203: 8 total, 8 up, 8 in 2022-04-28T21:41:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:24 smithi059 bash[59745]: audit 2022-04-28T21:41:23.949561+0000 mon.a (mon.0) 3497 : audit [INF] from='client.? 172.21.15.59:0/3089923545' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:24 smithi059 bash[59745]: cluster 2022-04-28T21:41:24.216401+0000 mgr.y (mgr.24961) 3654 : cluster [DBG] pgmap v4219: 169 pgs: 11 creating+peering, 21 unknown, 137 active+clean; 24 KiB data, 840 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:24 smithi139 bash[43301]: cluster 2022-04-28T21:41:23.947683+0000 mon.a (mon.0) 3496 : cluster [DBG] osdmap e1203: 8 total, 8 up, 8 in 2022-04-28T21:41:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:24 smithi139 bash[43301]: audit 2022-04-28T21:41:23.949561+0000 mon.a (mon.0) 3497 : audit [INF] from='client.? 172.21.15.59:0/3089923545' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:24 smithi139 bash[43301]: cluster 2022-04-28T21:41:24.216401+0000 mgr.y (mgr.24961) 3654 : cluster [DBG] pgmap v4219: 169 pgs: 11 creating+peering, 21 unknown, 137 active+clean; 24 KiB data, 840 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:25 smithi059 bash[61152]: audit 2022-04-28T21:41:24.958962+0000 mon.a (mon.0) 3498 : audit [INF] from='client.? 172.21.15.59:0/3089923545' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-12","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:26.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:25 smithi059 bash[61152]: cluster 2022-04-28T21:41:24.959207+0000 mon.a (mon.0) 3499 : cluster [DBG] osdmap e1204: 8 total, 8 up, 8 in 2022-04-28T21:41:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:25 smithi059 bash[59745]: audit 2022-04-28T21:41:24.958962+0000 mon.a (mon.0) 3498 : audit [INF] from='client.? 172.21.15.59:0/3089923545' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-12","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:26.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:25 smithi059 bash[59745]: cluster 2022-04-28T21:41:24.959207+0000 mon.a (mon.0) 3499 : cluster [DBG] osdmap e1204: 8 total, 8 up, 8 in 2022-04-28T21:41:26.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:25 smithi139 bash[43301]: audit 2022-04-28T21:41:24.958962+0000 mon.a (mon.0) 3498 : audit [INF] from='client.? 172.21.15.59:0/3089923545' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-12","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:26.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:25 smithi139 bash[43301]: cluster 2022-04-28T21:41:24.959207+0000 mon.a (mon.0) 3499 : cluster [DBG] osdmap e1204: 8 total, 8 up, 8 in 2022-04-28T21:41:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:26 smithi059 bash[61152]: cluster 2022-04-28T21:41:25.971180+0000 mon.a (mon.0) 3500 : cluster [DBG] osdmap e1205: 8 total, 8 up, 8 in 2022-04-28T21:41:27.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:26 smithi059 bash[61152]: cluster 2022-04-28T21:41:26.217004+0000 mgr.y (mgr.24961) 3655 : cluster [DBG] pgmap v4222: 137 pgs: 137 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:26 smithi059 bash[59745]: cluster 2022-04-28T21:41:25.971180+0000 mon.a (mon.0) 3500 : cluster [DBG] osdmap e1205: 8 total, 8 up, 8 in 2022-04-28T21:41:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:26 smithi059 bash[59745]: cluster 2022-04-28T21:41:26.217004+0000 mgr.y (mgr.24961) 3655 : cluster [DBG] pgmap v4222: 137 pgs: 137 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:26 smithi139 bash[43301]: cluster 2022-04-28T21:41:25.971180+0000 mon.a (mon.0) 3500 : cluster [DBG] osdmap e1205: 8 total, 8 up, 8 in 2022-04-28T21:41:27.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:26 smithi139 bash[43301]: cluster 2022-04-28T21:41:26.217004+0000 mgr.y (mgr.24961) 3655 : cluster [DBG] pgmap v4222: 137 pgs: 137 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:27 smithi059 bash[59745]: cluster 2022-04-28T21:41:26.985545+0000 mon.a (mon.0) 3501 : cluster [DBG] osdmap e1206: 8 total, 8 up, 8 in 2022-04-28T21:41:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:27 smithi059 bash[59745]: audit 2022-04-28T21:41:26.992859+0000 mon.a (mon.0) 3502 : audit [INF] from='client.? 172.21.15.59:0/4176470587' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:27 smithi059 bash[61152]: cluster 2022-04-28T21:41:26.985545+0000 mon.a (mon.0) 3501 : cluster [DBG] osdmap e1206: 8 total, 8 up, 8 in 2022-04-28T21:41:28.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:27 smithi059 bash[61152]: audit 2022-04-28T21:41:26.992859+0000 mon.a (mon.0) 3502 : audit [INF] from='client.? 172.21.15.59:0/4176470587' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:27 smithi139 bash[43301]: cluster 2022-04-28T21:41:26.985545+0000 mon.a (mon.0) 3501 : cluster [DBG] osdmap e1206: 8 total, 8 up, 8 in 2022-04-28T21:41:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:27 smithi139 bash[43301]: audit 2022-04-28T21:41:26.992859+0000 mon.a (mon.0) 3502 : audit [INF] from='client.? 172.21.15.59:0/4176470587' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:29 smithi059 bash[59745]: audit 2022-04-28T21:41:27.988566+0000 mon.a (mon.0) 3503 : audit [INF] from='client.? 172.21.15.59:0/4176470587' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-13","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:29 smithi059 bash[59745]: cluster 2022-04-28T21:41:27.989277+0000 mon.a (mon.0) 3504 : cluster [DBG] osdmap e1207: 8 total, 8 up, 8 in 2022-04-28T21:41:29.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:29 smithi059 bash[59745]: cluster 2022-04-28T21:41:28.217703+0000 mgr.y (mgr.24961) 3656 : cluster [DBG] pgmap v4225: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:29 smithi059 bash[61152]: audit 2022-04-28T21:41:27.988566+0000 mon.a (mon.0) 3503 : audit [INF] from='client.? 172.21.15.59:0/4176470587' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-13","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:29 smithi059 bash[61152]: cluster 2022-04-28T21:41:27.989277+0000 mon.a (mon.0) 3504 : cluster [DBG] osdmap e1207: 8 total, 8 up, 8 in 2022-04-28T21:41:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:29 smithi059 bash[61152]: cluster 2022-04-28T21:41:28.217703+0000 mgr.y (mgr.24961) 3656 : cluster [DBG] pgmap v4225: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:29.369 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:29 smithi139 bash[43301]: audit 2022-04-28T21:41:27.988566+0000 mon.a (mon.0) 3503 : audit [INF] from='client.? 172.21.15.59:0/4176470587' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-13","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:29 smithi139 bash[43301]: cluster 2022-04-28T21:41:27.989277+0000 mon.a (mon.0) 3504 : cluster [DBG] osdmap e1207: 8 total, 8 up, 8 in 2022-04-28T21:41:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:29 smithi139 bash[43301]: cluster 2022-04-28T21:41:28.217703+0000 mgr.y (mgr.24961) 3656 : cluster [DBG] pgmap v4225: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:30.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:30 smithi059 bash[61152]: cluster 2022-04-28T21:41:29.002800+0000 mon.a (mon.0) 3505 : cluster [DBG] osdmap e1208: 8 total, 8 up, 8 in 2022-04-28T21:41:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:30 smithi059 bash[59745]: cluster 2022-04-28T21:41:29.002800+0000 mon.a (mon.0) 3505 : cluster [DBG] osdmap e1208: 8 total, 8 up, 8 in 2022-04-28T21:41:30.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:30 smithi139 bash[43301]: cluster 2022-04-28T21:41:29.002800+0000 mon.a (mon.0) 3505 : cluster [DBG] osdmap e1208: 8 total, 8 up, 8 in 2022-04-28T21:41:31.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:31 smithi059 bash[61152]: cluster 2022-04-28T21:41:30.020156+0000 mon.a (mon.0) 3506 : cluster [DBG] osdmap e1209: 8 total, 8 up, 8 in 2022-04-28T21:41:31.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:31 smithi059 bash[61152]: audit 2022-04-28T21:41:30.021327+0000 mon.a (mon.0) 3507 : audit [INF] from='client.? 172.21.15.59:0/4123184861' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:31 smithi059 bash[61152]: cluster 2022-04-28T21:41:30.218415+0000 mgr.y (mgr.24961) 3657 : cluster [DBG] pgmap v4228: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 847 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:31.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:31 smithi059 bash[59745]: cluster 2022-04-28T21:41:30.020156+0000 mon.a (mon.0) 3506 : cluster [DBG] osdmap e1209: 8 total, 8 up, 8 in 2022-04-28T21:41:31.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:31 smithi059 bash[59745]: audit 2022-04-28T21:41:30.021327+0000 mon.a (mon.0) 3507 : audit [INF] from='client.? 172.21.15.59:0/4123184861' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:31.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:31 smithi059 bash[59745]: cluster 2022-04-28T21:41:30.218415+0000 mgr.y (mgr.24961) 3657 : cluster [DBG] pgmap v4228: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 847 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:31 smithi139 bash[43301]: cluster 2022-04-28T21:41:30.020156+0000 mon.a (mon.0) 3506 : cluster [DBG] osdmap e1209: 8 total, 8 up, 8 in 2022-04-28T21:41:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:31 smithi139 bash[43301]: audit 2022-04-28T21:41:30.021327+0000 mon.a (mon.0) 3507 : audit [INF] from='client.? 172.21.15.59:0/4123184861' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:31.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:31 smithi139 bash[43301]: cluster 2022-04-28T21:41:30.218415+0000 mgr.y (mgr.24961) 3657 : cluster [DBG] pgmap v4228: 169 pgs: 9 creating+peering, 23 unknown, 137 active+clean; 24 KiB data, 847 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:32.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:32 smithi059 bash[61152]: audit 2022-04-28T21:41:31.033271+0000 mon.a (mon.0) 3508 : audit [INF] from='client.? 172.21.15.59:0/4123184861' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-14","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:32 smithi059 bash[61152]: cluster 2022-04-28T21:41:31.033559+0000 mon.a (mon.0) 3509 : cluster [DBG] osdmap e1210: 8 total, 8 up, 8 in 2022-04-28T21:41:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:32 smithi059 bash[59745]: audit 2022-04-28T21:41:31.033271+0000 mon.a (mon.0) 3508 : audit [INF] from='client.? 172.21.15.59:0/4123184861' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-14","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:32 smithi059 bash[59745]: cluster 2022-04-28T21:41:31.033559+0000 mon.a (mon.0) 3509 : cluster [DBG] osdmap e1210: 8 total, 8 up, 8 in 2022-04-28T21:41:32.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:32.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:32 smithi139 bash[43301]: audit 2022-04-28T21:41:31.033271+0000 mon.a (mon.0) 3508 : audit [INF] from='client.? 172.21.15.59:0/4123184861' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-14","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:32 smithi139 bash[43301]: cluster 2022-04-28T21:41:31.033559+0000 mon.a (mon.0) 3509 : cluster [DBG] osdmap e1210: 8 total, 8 up, 8 in 2022-04-28T21:41:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:33 smithi059 bash[59745]: cluster 2022-04-28T21:41:32.045971+0000 mon.a (mon.0) 3510 : cluster [DBG] osdmap e1211: 8 total, 8 up, 8 in 2022-04-28T21:41:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:33 smithi059 bash[59745]: cluster 2022-04-28T21:41:32.219065+0000 mgr.y (mgr.24961) 3658 : cluster [DBG] pgmap v4231: 137 pgs: 137 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:33 smithi059 bash[61152]: cluster 2022-04-28T21:41:32.045971+0000 mon.a (mon.0) 3510 : cluster [DBG] osdmap e1211: 8 total, 8 up, 8 in 2022-04-28T21:41:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:33 smithi059 bash[61152]: cluster 2022-04-28T21:41:32.219065+0000 mgr.y (mgr.24961) 3658 : cluster [DBG] pgmap v4231: 137 pgs: 137 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:33 smithi139 bash[43301]: cluster 2022-04-28T21:41:32.045971+0000 mon.a (mon.0) 3510 : cluster [DBG] osdmap e1211: 8 total, 8 up, 8 in 2022-04-28T21:41:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:33 smithi139 bash[43301]: cluster 2022-04-28T21:41:32.219065+0000 mgr.y (mgr.24961) 3658 : cluster [DBG] pgmap v4231: 137 pgs: 137 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:33.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:34 smithi059 bash[61152]: cluster 2022-04-28T21:41:33.058994+0000 mon.a (mon.0) 3511 : cluster [DBG] osdmap e1212: 8 total, 8 up, 8 in 2022-04-28T21:41:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:34 smithi059 bash[61152]: audit 2022-04-28T21:41:33.062006+0000 mon.b (mon.2) 25 : audit [INF] from='client.? 172.21.15.59:0/3235078920' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:34 smithi059 bash[61152]: audit 2022-04-28T21:41:33.065755+0000 mon.a (mon.0) 3512 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:34.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:34 smithi059 bash[59745]: cluster 2022-04-28T21:41:33.058994+0000 mon.a (mon.0) 3511 : cluster [DBG] osdmap e1212: 8 total, 8 up, 8 in 2022-04-28T21:41:34.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:34 smithi059 bash[59745]: audit 2022-04-28T21:41:33.062006+0000 mon.b (mon.2) 25 : audit [INF] from='client.? 172.21.15.59:0/3235078920' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:34.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:34 smithi059 bash[59745]: audit 2022-04-28T21:41:33.065755+0000 mon.a (mon.0) 3512 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:34 smithi139 bash[43301]: cluster 2022-04-28T21:41:33.058994+0000 mon.a (mon.0) 3511 : cluster [DBG] osdmap e1212: 8 total, 8 up, 8 in 2022-04-28T21:41:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:34 smithi139 bash[43301]: audit 2022-04-28T21:41:33.062006+0000 mon.b (mon.2) 25 : audit [INF] from='client.? 172.21.15.59:0/3235078920' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:34 smithi139 bash[43301]: audit 2022-04-28T21:41:33.065755+0000 mon.a (mon.0) 3512 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[59745]: audit 2022-04-28T21:41:34.063107+0000 mon.a (mon.0) 3513 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:35.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[59745]: cluster 2022-04-28T21:41:34.063774+0000 mon.a (mon.0) 3514 : cluster [DBG] osdmap e1213: 8 total, 8 up, 8 in 2022-04-28T21:41:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[59745]: cluster 2022-04-28T21:41:34.220170+0000 mgr.y (mgr.24961) 3659 : cluster [DBG] pgmap v4234: 169 pgs: 4 creating+activating, 3 creating+peering, 23 unknown, 139 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[59745]: cluster 2022-04-28T21:41:35.066623+0000 mon.a (mon.0) 3515 : cluster [DBG] osdmap e1214: 8 total, 8 up, 8 in 2022-04-28T21:41:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[61152]: audit 2022-04-28T21:41:34.063107+0000 mon.a (mon.0) 3513 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[61152]: cluster 2022-04-28T21:41:34.063774+0000 mon.a (mon.0) 3514 : cluster [DBG] osdmap e1213: 8 total, 8 up, 8 in 2022-04-28T21:41:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[61152]: cluster 2022-04-28T21:41:34.220170+0000 mgr.y (mgr.24961) 3659 : cluster [DBG] pgmap v4234: 169 pgs: 4 creating+activating, 3 creating+peering, 23 unknown, 139 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:35 smithi059 bash[61152]: cluster 2022-04-28T21:41:35.066623+0000 mon.a (mon.0) 3515 : cluster [DBG] osdmap e1214: 8 total, 8 up, 8 in 2022-04-28T21:41:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:35 smithi139 bash[43301]: audit 2022-04-28T21:41:34.063107+0000 mon.a (mon.0) 3513 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-15","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:35 smithi139 bash[43301]: cluster 2022-04-28T21:41:34.063774+0000 mon.a (mon.0) 3514 : cluster [DBG] osdmap e1213: 8 total, 8 up, 8 in 2022-04-28T21:41:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:35 smithi139 bash[43301]: cluster 2022-04-28T21:41:34.220170+0000 mgr.y (mgr.24961) 3659 : cluster [DBG] pgmap v4234: 169 pgs: 4 creating+activating, 3 creating+peering, 23 unknown, 139 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:35 smithi139 bash[43301]: cluster 2022-04-28T21:41:35.066623+0000 mon.a (mon.0) 3515 : cluster [DBG] osdmap e1214: 8 total, 8 up, 8 in 2022-04-28T21:41:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:37 smithi059 bash[61152]: cluster 2022-04-28T21:41:36.098705+0000 mon.a (mon.0) 3516 : cluster [DBG] osdmap e1215: 8 total, 8 up, 8 in 2022-04-28T21:41:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:37 smithi059 bash[61152]: audit 2022-04-28T21:41:36.101877+0000 mon.a (mon.0) 3517 : audit [INF] from='client.? 172.21.15.59:0/2835567557' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:37.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:37 smithi059 bash[61152]: cluster 2022-04-28T21:41:36.220924+0000 mgr.y (mgr.24961) 3660 : cluster [DBG] pgmap v4237: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:37 smithi059 bash[59745]: cluster 2022-04-28T21:41:36.098705+0000 mon.a (mon.0) 3516 : cluster [DBG] osdmap e1215: 8 total, 8 up, 8 in 2022-04-28T21:41:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:37 smithi059 bash[59745]: audit 2022-04-28T21:41:36.101877+0000 mon.a (mon.0) 3517 : audit [INF] from='client.? 172.21.15.59:0/2835567557' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:37.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:37 smithi059 bash[59745]: cluster 2022-04-28T21:41:36.220924+0000 mgr.y (mgr.24961) 3660 : cluster [DBG] pgmap v4237: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:37.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:37 smithi139 bash[43301]: cluster 2022-04-28T21:41:36.098705+0000 mon.a (mon.0) 3516 : cluster [DBG] osdmap e1215: 8 total, 8 up, 8 in 2022-04-28T21:41:37.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:37 smithi139 bash[43301]: audit 2022-04-28T21:41:36.101877+0000 mon.a (mon.0) 3517 : audit [INF] from='client.? 172.21.15.59:0/2835567557' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:37.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:37 smithi139 bash[43301]: cluster 2022-04-28T21:41:36.220924+0000 mgr.y (mgr.24961) 3660 : cluster [DBG] pgmap v4237: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:38 smithi059 bash[61152]: audit 2022-04-28T21:41:37.104288+0000 mon.a (mon.0) 3518 : audit [INF] from='client.? 172.21.15.59:0/2835567557' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-16","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:38 smithi059 bash[61152]: cluster 2022-04-28T21:41:37.104691+0000 mon.a (mon.0) 3519 : cluster [DBG] osdmap e1216: 8 total, 8 up, 8 in 2022-04-28T21:41:38.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:38 smithi059 bash[59745]: audit 2022-04-28T21:41:37.104288+0000 mon.a (mon.0) 3518 : audit [INF] from='client.? 172.21.15.59:0/2835567557' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-16","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:38 smithi059 bash[59745]: cluster 2022-04-28T21:41:37.104691+0000 mon.a (mon.0) 3519 : cluster [DBG] osdmap e1216: 8 total, 8 up, 8 in 2022-04-28T21:41:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:38 smithi139 bash[43301]: audit 2022-04-28T21:41:37.104288+0000 mon.a (mon.0) 3518 : audit [INF] from='client.? 172.21.15.59:0/2835567557' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-16","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:38 smithi139 bash[43301]: cluster 2022-04-28T21:41:37.104691+0000 mon.a (mon.0) 3519 : cluster [DBG] osdmap e1216: 8 total, 8 up, 8 in 2022-04-28T21:41:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:39 smithi059 bash[59745]: cluster 2022-04-28T21:41:38.127308+0000 mon.a (mon.0) 3520 : cluster [DBG] osdmap e1217: 8 total, 8 up, 8 in 2022-04-28T21:41:39.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:39 smithi059 bash[59745]: cluster 2022-04-28T21:41:38.221416+0000 mgr.y (mgr.24961) 3661 : cluster [DBG] pgmap v4240: 137 pgs: 137 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:39 smithi059 bash[61152]: cluster 2022-04-28T21:41:38.127308+0000 mon.a (mon.0) 3520 : cluster [DBG] osdmap e1217: 8 total, 8 up, 8 in 2022-04-28T21:41:39.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:39 smithi059 bash[61152]: cluster 2022-04-28T21:41:38.221416+0000 mgr.y (mgr.24961) 3661 : cluster [DBG] pgmap v4240: 137 pgs: 137 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:39 smithi139 bash[43301]: cluster 2022-04-28T21:41:38.127308+0000 mon.a (mon.0) 3520 : cluster [DBG] osdmap e1217: 8 total, 8 up, 8 in 2022-04-28T21:41:39.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:39 smithi139 bash[43301]: cluster 2022-04-28T21:41:38.221416+0000 mgr.y (mgr.24961) 3661 : cluster [DBG] pgmap v4240: 137 pgs: 137 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:40.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:40 smithi059 bash[59745]: audit 2022-04-28T21:41:39.156122+0000 mon.b (mon.2) 26 : audit [INF] from='client.? 172.21.15.59:0/1440118778' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:40 smithi059 bash[59745]: cluster 2022-04-28T21:41:39.157364+0000 mon.a (mon.0) 3521 : cluster [DBG] osdmap e1218: 8 total, 8 up, 8 in 2022-04-28T21:41:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:40 smithi059 bash[59745]: audit 2022-04-28T21:41:39.160387+0000 mon.a (mon.0) 3522 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:40 smithi059 bash[61152]: audit 2022-04-28T21:41:39.156122+0000 mon.b (mon.2) 26 : audit [INF] from='client.? 172.21.15.59:0/1440118778' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:40 smithi059 bash[61152]: cluster 2022-04-28T21:41:39.157364+0000 mon.a (mon.0) 3521 : cluster [DBG] osdmap e1218: 8 total, 8 up, 8 in 2022-04-28T21:41:40.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:40 smithi059 bash[61152]: audit 2022-04-28T21:41:39.160387+0000 mon.a (mon.0) 3522 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:40 smithi139 bash[43301]: audit 2022-04-28T21:41:39.156122+0000 mon.b (mon.2) 26 : audit [INF] from='client.? 172.21.15.59:0/1440118778' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:40 smithi139 bash[43301]: cluster 2022-04-28T21:41:39.157364+0000 mon.a (mon.0) 3521 : cluster [DBG] osdmap e1218: 8 total, 8 up, 8 in 2022-04-28T21:41:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:40 smithi139 bash[43301]: audit 2022-04-28T21:41:39.160387+0000 mon.a (mon.0) 3522 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:41 smithi059 bash[61152]: audit 2022-04-28T21:41:40.157156+0000 mon.a (mon.0) 3523 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:41 smithi059 bash[61152]: cluster 2022-04-28T21:41:40.157327+0000 mon.a (mon.0) 3524 : cluster [DBG] osdmap e1219: 8 total, 8 up, 8 in 2022-04-28T21:41:41.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:41 smithi059 bash[61152]: cluster 2022-04-28T21:41:40.222353+0000 mgr.y (mgr.24961) 3662 : cluster [DBG] pgmap v4243: 169 pgs: 3 creating+activating, 5 creating+peering, 21 unknown, 140 active+clean; 24 KiB data, 855 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:41 smithi059 bash[59745]: audit 2022-04-28T21:41:40.157156+0000 mon.a (mon.0) 3523 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:41 smithi059 bash[59745]: cluster 2022-04-28T21:41:40.157327+0000 mon.a (mon.0) 3524 : cluster [DBG] osdmap e1219: 8 total, 8 up, 8 in 2022-04-28T21:41:41.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:41 smithi059 bash[59745]: cluster 2022-04-28T21:41:40.222353+0000 mgr.y (mgr.24961) 3662 : cluster [DBG] pgmap v4243: 169 pgs: 3 creating+activating, 5 creating+peering, 21 unknown, 140 active+clean; 24 KiB data, 855 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:41 smithi139 bash[43301]: audit 2022-04-28T21:41:40.157156+0000 mon.a (mon.0) 3523 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-17","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:41 smithi139 bash[43301]: cluster 2022-04-28T21:41:40.157327+0000 mon.a (mon.0) 3524 : cluster [DBG] osdmap e1219: 8 total, 8 up, 8 in 2022-04-28T21:41:41.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:41 smithi139 bash[43301]: cluster 2022-04-28T21:41:40.222353+0000 mgr.y (mgr.24961) 3662 : cluster [DBG] pgmap v4243: 169 pgs: 3 creating+activating, 5 creating+peering, 21 unknown, 140 active+clean; 24 KiB data, 855 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:42 smithi059 bash[61152]: cluster 2022-04-28T21:41:41.180761+0000 mon.a (mon.0) 3525 : cluster [DBG] osdmap e1220: 8 total, 8 up, 8 in 2022-04-28T21:41:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:42.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:42 smithi059 bash[59745]: cluster 2022-04-28T21:41:41.180761+0000 mon.a (mon.0) 3525 : cluster [DBG] osdmap e1220: 8 total, 8 up, 8 in 2022-04-28T21:41:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:42 smithi139 bash[43301]: cluster 2022-04-28T21:41:41.180761+0000 mon.a (mon.0) 3525 : cluster [DBG] osdmap e1220: 8 total, 8 up, 8 in 2022-04-28T21:41:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[61152]: cluster 2022-04-28T21:41:42.181338+0000 mon.a (mon.0) 3526 : cluster [DBG] osdmap e1221: 8 total, 8 up, 8 in 2022-04-28T21:41:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[61152]: audit 2022-04-28T21:41:42.187194+0000 mon.c (mon.1) 140 : audit [INF] from='client.? 172.21.15.59:0/1677732714' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:43.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[61152]: audit 2022-04-28T21:41:42.195861+0000 mon.a (mon.0) 3527 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[61152]: cluster 2022-04-28T21:41:42.223002+0000 mgr.y (mgr.24961) 3663 : cluster [DBG] pgmap v4246: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[61152]: audit 2022-04-28T21:41:43.184986+0000 mon.a (mon.0) 3528 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:43.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[61152]: cluster 2022-04-28T21:41:43.185269+0000 mon.a (mon.0) 3529 : cluster [DBG] osdmap e1222: 8 total, 8 up, 8 in 2022-04-28T21:41:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[59745]: cluster 2022-04-28T21:41:42.181338+0000 mon.a (mon.0) 3526 : cluster [DBG] osdmap e1221: 8 total, 8 up, 8 in 2022-04-28T21:41:43.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[59745]: audit 2022-04-28T21:41:42.187194+0000 mon.c (mon.1) 140 : audit [INF] from='client.? 172.21.15.59:0/1677732714' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[59745]: audit 2022-04-28T21:41:42.195861+0000 mon.a (mon.0) 3527 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[59745]: cluster 2022-04-28T21:41:42.223002+0000 mgr.y (mgr.24961) 3663 : cluster [DBG] pgmap v4246: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[59745]: audit 2022-04-28T21:41:43.184986+0000 mon.a (mon.0) 3528 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:43.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[59745]: cluster 2022-04-28T21:41:43.185269+0000 mon.a (mon.0) 3529 : cluster [DBG] osdmap e1222: 8 total, 8 up, 8 in 2022-04-28T21:41:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:43 smithi139 bash[43301]: cluster 2022-04-28T21:41:42.181338+0000 mon.a (mon.0) 3526 : cluster [DBG] osdmap e1221: 8 total, 8 up, 8 in 2022-04-28T21:41:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:43 smithi139 bash[43301]: audit 2022-04-28T21:41:42.187194+0000 mon.c (mon.1) 140 : audit [INF] from='client.? 172.21.15.59:0/1677732714' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:43 smithi139 bash[43301]: audit 2022-04-28T21:41:42.195861+0000 mon.a (mon.0) 3527 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:43.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:43 smithi139 bash[43301]: cluster 2022-04-28T21:41:42.223002+0000 mgr.y (mgr.24961) 3663 : cluster [DBG] pgmap v4246: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:43 smithi139 bash[43301]: audit 2022-04-28T21:41:43.184986+0000 mon.a (mon.0) 3528 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-18","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:43.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:43 smithi139 bash[43301]: cluster 2022-04-28T21:41:43.185269+0000 mon.a (mon.0) 3529 : cluster [DBG] osdmap e1222: 8 total, 8 up, 8 in 2022-04-28T21:41:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:43.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:45 smithi059 bash[61152]: cluster 2022-04-28T21:41:44.187687+0000 mon.a (mon.0) 3530 : cluster [DBG] osdmap e1223: 8 total, 8 up, 8 in 2022-04-28T21:41:45.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:45 smithi059 bash[61152]: cluster 2022-04-28T21:41:44.223746+0000 mgr.y (mgr.24961) 3664 : cluster [DBG] pgmap v4249: 137 pgs: 137 active+clean; 24 KiB data, 861 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:45 smithi059 bash[59745]: cluster 2022-04-28T21:41:44.187687+0000 mon.a (mon.0) 3530 : cluster [DBG] osdmap e1223: 8 total, 8 up, 8 in 2022-04-28T21:41:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:45 smithi059 bash[59745]: cluster 2022-04-28T21:41:44.223746+0000 mgr.y (mgr.24961) 3664 : cluster [DBG] pgmap v4249: 137 pgs: 137 active+clean; 24 KiB data, 861 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:45 smithi139 bash[43301]: cluster 2022-04-28T21:41:44.187687+0000 mon.a (mon.0) 3530 : cluster [DBG] osdmap e1223: 8 total, 8 up, 8 in 2022-04-28T21:41:45.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:45 smithi139 bash[43301]: cluster 2022-04-28T21:41:44.223746+0000 mgr.y (mgr.24961) 3664 : cluster [DBG] pgmap v4249: 137 pgs: 137 active+clean; 24 KiB data, 861 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:46 smithi059 bash[61152]: cluster 2022-04-28T21:41:45.202049+0000 mon.a (mon.0) 3531 : cluster [DBG] osdmap e1224: 8 total, 8 up, 8 in 2022-04-28T21:41:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:46 smithi059 bash[61152]: audit 2022-04-28T21:41:45.208134+0000 mon.c (mon.1) 141 : audit [INF] from='client.? 172.21.15.59:0/1538447316' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:46 smithi059 bash[61152]: audit 2022-04-28T21:41:45.210536+0000 mon.a (mon.0) 3532 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:46 smithi059 bash[59745]: cluster 2022-04-28T21:41:45.202049+0000 mon.a (mon.0) 3531 : cluster [DBG] osdmap e1224: 8 total, 8 up, 8 in 2022-04-28T21:41:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:46 smithi059 bash[59745]: audit 2022-04-28T21:41:45.208134+0000 mon.c (mon.1) 141 : audit [INF] from='client.? 172.21.15.59:0/1538447316' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:46 smithi059 bash[59745]: audit 2022-04-28T21:41:45.210536+0000 mon.a (mon.0) 3532 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:46 smithi139 bash[43301]: cluster 2022-04-28T21:41:45.202049+0000 mon.a (mon.0) 3531 : cluster [DBG] osdmap e1224: 8 total, 8 up, 8 in 2022-04-28T21:41:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:46 smithi139 bash[43301]: audit 2022-04-28T21:41:45.208134+0000 mon.c (mon.1) 141 : audit [INF] from='client.? 172.21.15.59:0/1538447316' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:46 smithi139 bash[43301]: audit 2022-04-28T21:41:45.210536+0000 mon.a (mon.0) 3532 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:47 smithi059 bash[61152]: audit 2022-04-28T21:41:46.205688+0000 mon.a (mon.0) 3533 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:47.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:47 smithi059 bash[61152]: cluster 2022-04-28T21:41:46.205963+0000 mon.a (mon.0) 3534 : cluster [DBG] osdmap e1225: 8 total, 8 up, 8 in 2022-04-28T21:41:47.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:47 smithi059 bash[61152]: cluster 2022-04-28T21:41:46.224330+0000 mgr.y (mgr.24961) 3665 : cluster [DBG] pgmap v4252: 169 pgs: 8 creating+peering, 24 unknown, 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:47 smithi059 bash[59745]: audit 2022-04-28T21:41:46.205688+0000 mon.a (mon.0) 3533 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:47 smithi059 bash[59745]: cluster 2022-04-28T21:41:46.205963+0000 mon.a (mon.0) 3534 : cluster [DBG] osdmap e1225: 8 total, 8 up, 8 in 2022-04-28T21:41:47.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:47 smithi059 bash[59745]: cluster 2022-04-28T21:41:46.224330+0000 mgr.y (mgr.24961) 3665 : cluster [DBG] pgmap v4252: 169 pgs: 8 creating+peering, 24 unknown, 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:47.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:47 smithi139 bash[43301]: audit 2022-04-28T21:41:46.205688+0000 mon.a (mon.0) 3533 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-19","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:47.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:47 smithi139 bash[43301]: cluster 2022-04-28T21:41:46.205963+0000 mon.a (mon.0) 3534 : cluster [DBG] osdmap e1225: 8 total, 8 up, 8 in 2022-04-28T21:41:47.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:47 smithi139 bash[43301]: cluster 2022-04-28T21:41:46.224330+0000 mgr.y (mgr.24961) 3665 : cluster [DBG] pgmap v4252: 169 pgs: 8 creating+peering, 24 unknown, 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:48 smithi059 bash[61152]: cluster 2022-04-28T21:41:47.229657+0000 mon.a (mon.0) 3535 : cluster [DBG] osdmap e1226: 8 total, 8 up, 8 in 2022-04-28T21:41:48.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:48 smithi059 bash[59745]: cluster 2022-04-28T21:41:47.229657+0000 mon.a (mon.0) 3535 : cluster [DBG] osdmap e1226: 8 total, 8 up, 8 in 2022-04-28T21:41:48.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:48 smithi139 bash[43301]: cluster 2022-04-28T21:41:47.229657+0000 mon.a (mon.0) 3535 : cluster [DBG] osdmap e1226: 8 total, 8 up, 8 in 2022-04-28T21:41:49.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[61152]: cluster 2022-04-28T21:41:48.224962+0000 mgr.y (mgr.24961) 3666 : cluster [DBG] pgmap v4254: 137 pgs: 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[61152]: cluster 2022-04-28T21:41:48.242965+0000 mon.a (mon.0) 3536 : cluster [DBG] osdmap e1227: 8 total, 8 up, 8 in 2022-04-28T21:41:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[61152]: audit 2022-04-28T21:41:48.247053+0000 mon.c (mon.1) 142 : audit [INF] from='client.? 172.21.15.59:0/1649104021' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:49.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[61152]: audit 2022-04-28T21:41:48.247688+0000 mon.a (mon.0) 3537 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[59745]: cluster 2022-04-28T21:41:48.224962+0000 mgr.y (mgr.24961) 3666 : cluster [DBG] pgmap v4254: 137 pgs: 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[59745]: cluster 2022-04-28T21:41:48.242965+0000 mon.a (mon.0) 3536 : cluster [DBG] osdmap e1227: 8 total, 8 up, 8 in 2022-04-28T21:41:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[59745]: audit 2022-04-28T21:41:48.247053+0000 mon.c (mon.1) 142 : audit [INF] from='client.? 172.21.15.59:0/1649104021' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:49.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:49 smithi059 bash[59745]: audit 2022-04-28T21:41:48.247688+0000 mon.a (mon.0) 3537 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:49 smithi139 bash[43301]: cluster 2022-04-28T21:41:48.224962+0000 mgr.y (mgr.24961) 3666 : cluster [DBG] pgmap v4254: 137 pgs: 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:49 smithi139 bash[43301]: cluster 2022-04-28T21:41:48.242965+0000 mon.a (mon.0) 3536 : cluster [DBG] osdmap e1227: 8 total, 8 up, 8 in 2022-04-28T21:41:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:49 smithi139 bash[43301]: audit 2022-04-28T21:41:48.247053+0000 mon.c (mon.1) 142 : audit [INF] from='client.? 172.21.15.59:0/1649104021' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:49.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:49 smithi139 bash[43301]: audit 2022-04-28T21:41:48.247688+0000 mon.a (mon.0) 3537 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:50 smithi059 bash[61152]: audit 2022-04-28T21:41:49.244789+0000 mon.a (mon.0) 3538 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:50 smithi059 bash[61152]: cluster 2022-04-28T21:41:49.244908+0000 mon.a (mon.0) 3539 : cluster [DBG] osdmap e1228: 8 total, 8 up, 8 in 2022-04-28T21:41:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:50 smithi059 bash[59745]: audit 2022-04-28T21:41:49.244789+0000 mon.a (mon.0) 3538 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:50 smithi059 bash[59745]: cluster 2022-04-28T21:41:49.244908+0000 mon.a (mon.0) 3539 : cluster [DBG] osdmap e1228: 8 total, 8 up, 8 in 2022-04-28T21:41:50.619 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:50 smithi139 bash[43301]: audit 2022-04-28T21:41:49.244789+0000 mon.a (mon.0) 3538 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-20","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:50.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:50 smithi139 bash[43301]: cluster 2022-04-28T21:41:49.244908+0000 mon.a (mon.0) 3539 : cluster [DBG] osdmap e1228: 8 total, 8 up, 8 in 2022-04-28T21:41:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:51 smithi059 bash[61152]: cluster 2022-04-28T21:41:50.226141+0000 mgr.y (mgr.24961) 3667 : cluster [DBG] pgmap v4257: 169 pgs: 8 creating+peering, 11 unknown, 150 active+clean; 25 KiB data, 855 MiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 254 B/s wr, 0 op/s 2022-04-28T21:41:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:51 smithi059 bash[61152]: cluster 2022-04-28T21:41:50.260411+0000 mon.a (mon.0) 3540 : cluster [DBG] osdmap e1229: 8 total, 8 up, 8 in 2022-04-28T21:41:51.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:51 smithi059 bash[59745]: cluster 2022-04-28T21:41:50.226141+0000 mgr.y (mgr.24961) 3667 : cluster [DBG] pgmap v4257: 169 pgs: 8 creating+peering, 11 unknown, 150 active+clean; 25 KiB data, 855 MiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 254 B/s wr, 0 op/s 2022-04-28T21:41:51.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:51 smithi059 bash[59745]: cluster 2022-04-28T21:41:50.260411+0000 mon.a (mon.0) 3540 : cluster [DBG] osdmap e1229: 8 total, 8 up, 8 in 2022-04-28T21:41:51.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:51 smithi139 bash[43301]: cluster 2022-04-28T21:41:50.226141+0000 mgr.y (mgr.24961) 3667 : cluster [DBG] pgmap v4257: 169 pgs: 8 creating+peering, 11 unknown, 150 active+clean; 25 KiB data, 855 MiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 254 B/s wr, 0 op/s 2022-04-28T21:41:51.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:51 smithi139 bash[43301]: cluster 2022-04-28T21:41:50.260411+0000 mon.a (mon.0) 3540 : cluster [DBG] osdmap e1229: 8 total, 8 up, 8 in 2022-04-28T21:41:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:52.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:52.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[59745]: audit 2022-04-28T21:41:51.283321+0000 mon.b (mon.2) 27 : audit [INF] from='client.? 172.21.15.59:0/2576960986' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[59745]: cluster 2022-04-28T21:41:51.284022+0000 mon.a (mon.0) 3541 : cluster [DBG] osdmap e1230: 8 total, 8 up, 8 in 2022-04-28T21:41:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[59745]: audit 2022-04-28T21:41:51.287273+0000 mon.a (mon.0) 3542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[61152]: audit 2022-04-28T21:41:51.283321+0000 mon.b (mon.2) 27 : audit [INF] from='client.? 172.21.15.59:0/2576960986' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[61152]: cluster 2022-04-28T21:41:51.284022+0000 mon.a (mon.0) 3541 : cluster [DBG] osdmap e1230: 8 total, 8 up, 8 in 2022-04-28T21:41:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:52 smithi059 bash[61152]: audit 2022-04-28T21:41:51.287273+0000 mon.a (mon.0) 3542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:52 smithi139 bash[43301]: audit 2022-04-28T21:41:51.283321+0000 mon.b (mon.2) 27 : audit [INF] from='client.? 172.21.15.59:0/2576960986' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:52 smithi139 bash[43301]: cluster 2022-04-28T21:41:51.284022+0000 mon.a (mon.0) 3541 : cluster [DBG] osdmap e1230: 8 total, 8 up, 8 in 2022-04-28T21:41:52.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:52 smithi139 bash[43301]: audit 2022-04-28T21:41:51.287273+0000 mon.a (mon.0) 3542 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[59745]: cluster 2022-04-28T21:41:52.226952+0000 mgr.y (mgr.24961) 3668 : cluster [DBG] pgmap v4260: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 856 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[59745]: audit 2022-04-28T21:41:52.287454+0000 mon.a (mon.0) 3543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:53.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[59745]: cluster 2022-04-28T21:41:52.287867+0000 mon.a (mon.0) 3544 : cluster [DBG] osdmap e1231: 8 total, 8 up, 8 in 2022-04-28T21:41:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[61152]: cluster 2022-04-28T21:41:52.226952+0000 mgr.y (mgr.24961) 3668 : cluster [DBG] pgmap v4260: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 856 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[61152]: audit 2022-04-28T21:41:52.287454+0000 mon.a (mon.0) 3543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[61152]: cluster 2022-04-28T21:41:52.287867+0000 mon.a (mon.0) 3544 : cluster [DBG] osdmap e1231: 8 total, 8 up, 8 in 2022-04-28T21:41:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:53 smithi139 bash[43301]: cluster 2022-04-28T21:41:52.226952+0000 mgr.y (mgr.24961) 3668 : cluster [DBG] pgmap v4260: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 856 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:53 smithi139 bash[43301]: audit 2022-04-28T21:41:52.287454+0000 mon.a (mon.0) 3543 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-21","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:53 smithi139 bash[43301]: cluster 2022-04-28T21:41:52.287867+0000 mon.a (mon.0) 3544 : cluster [DBG] osdmap e1231: 8 total, 8 up, 8 in 2022-04-28T21:41:54.313 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:41:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:41:53.951Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:41:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:54 smithi059 bash[61152]: cluster 2022-04-28T21:41:53.303028+0000 mon.a (mon.0) 3545 : cluster [DBG] osdmap e1232: 8 total, 8 up, 8 in 2022-04-28T21:41:54.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:54 smithi059 bash[59745]: cluster 2022-04-28T21:41:53.303028+0000 mon.a (mon.0) 3545 : cluster [DBG] osdmap e1232: 8 total, 8 up, 8 in 2022-04-28T21:41:54.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:54 smithi139 bash[43301]: cluster 2022-04-28T21:41:53.303028+0000 mon.a (mon.0) 3545 : cluster [DBG] osdmap e1232: 8 total, 8 up, 8 in 2022-04-28T21:41:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:55 smithi139 bash[43301]: cluster 2022-04-28T21:41:54.228145+0000 mgr.y (mgr.24961) 3669 : cluster [DBG] pgmap v4263: 137 pgs: 137 active+clean; 24 KiB data, 861 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:55 smithi139 bash[43301]: cluster 2022-04-28T21:41:54.325889+0000 mon.a (mon.0) 3546 : cluster [DBG] osdmap e1233: 8 total, 8 up, 8 in 2022-04-28T21:41:55.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:55 smithi139 bash[43301]: audit 2022-04-28T21:41:54.327948+0000 mon.b (mon.2) 28 : audit [INF] from='client.? 172.21.15.59:0/4137754020' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:55.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:55 smithi139 bash[43301]: audit 2022-04-28T21:41:54.340087+0000 mon.a (mon.0) 3547 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:55.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[61152]: cluster 2022-04-28T21:41:54.228145+0000 mgr.y (mgr.24961) 3669 : cluster [DBG] pgmap v4263: 137 pgs: 137 active+clean; 24 KiB data, 861 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[61152]: cluster 2022-04-28T21:41:54.325889+0000 mon.a (mon.0) 3546 : cluster [DBG] osdmap e1233: 8 total, 8 up, 8 in 2022-04-28T21:41:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[61152]: audit 2022-04-28T21:41:54.327948+0000 mon.b (mon.2) 28 : audit [INF] from='client.? 172.21.15.59:0/4137754020' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:55.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[61152]: audit 2022-04-28T21:41:54.340087+0000 mon.a (mon.0) 3547 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:55.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[59745]: cluster 2022-04-28T21:41:54.228145+0000 mgr.y (mgr.24961) 3669 : cluster [DBG] pgmap v4263: 137 pgs: 137 active+clean; 24 KiB data, 861 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:55.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[59745]: cluster 2022-04-28T21:41:54.325889+0000 mon.a (mon.0) 3546 : cluster [DBG] osdmap e1233: 8 total, 8 up, 8 in 2022-04-28T21:41:55.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[59745]: audit 2022-04-28T21:41:54.327948+0000 mon.b (mon.2) 28 : audit [INF] from='client.? 172.21.15.59:0/4137754020' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:55.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:55 smithi059 bash[59745]: audit 2022-04-28T21:41:54.340087+0000 mon.a (mon.0) 3547 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:56 smithi139 bash[43301]: audit 2022-04-28T21:41:55.331163+0000 mon.a (mon.0) 3548 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:56 smithi139 bash[43301]: cluster 2022-04-28T21:41:55.331734+0000 mon.a (mon.0) 3549 : cluster [DBG] osdmap e1234: 8 total, 8 up, 8 in 2022-04-28T21:41:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:56 smithi059 bash[61152]: audit 2022-04-28T21:41:55.331163+0000 mon.a (mon.0) 3548 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:56.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:56 smithi059 bash[61152]: cluster 2022-04-28T21:41:55.331734+0000 mon.a (mon.0) 3549 : cluster [DBG] osdmap e1234: 8 total, 8 up, 8 in 2022-04-28T21:41:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:56 smithi059 bash[59745]: audit 2022-04-28T21:41:55.331163+0000 mon.a (mon.0) 3548 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-22","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:56 smithi059 bash[59745]: cluster 2022-04-28T21:41:55.331734+0000 mon.a (mon.0) 3549 : cluster [DBG] osdmap e1234: 8 total, 8 up, 8 in 2022-04-28T21:41:57.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:57 smithi139 bash[43301]: cluster 2022-04-28T21:41:56.229188+0000 mgr.y (mgr.24961) 3670 : cluster [DBG] pgmap v4266: 169 pgs: 14 creating+peering, 10 unknown, 145 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:57.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:57 smithi139 bash[43301]: cluster 2022-04-28T21:41:56.349384+0000 mon.a (mon.0) 3550 : cluster [DBG] osdmap e1235: 8 total, 8 up, 8 in 2022-04-28T21:41:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:57 smithi059 bash[61152]: cluster 2022-04-28T21:41:56.229188+0000 mgr.y (mgr.24961) 3670 : cluster [DBG] pgmap v4266: 169 pgs: 14 creating+peering, 10 unknown, 145 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:57 smithi059 bash[61152]: cluster 2022-04-28T21:41:56.349384+0000 mon.a (mon.0) 3550 : cluster [DBG] osdmap e1235: 8 total, 8 up, 8 in 2022-04-28T21:41:57.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:57 smithi059 bash[59745]: cluster 2022-04-28T21:41:56.229188+0000 mgr.y (mgr.24961) 3670 : cluster [DBG] pgmap v4266: 169 pgs: 14 creating+peering, 10 unknown, 145 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:57 smithi059 bash[59745]: cluster 2022-04-28T21:41:56.349384+0000 mon.a (mon.0) 3550 : cluster [DBG] osdmap e1235: 8 total, 8 up, 8 in 2022-04-28T21:41:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:58 smithi059 bash[61152]: cluster 2022-04-28T21:41:57.373576+0000 mon.a (mon.0) 3551 : cluster [DBG] osdmap e1236: 8 total, 8 up, 8 in 2022-04-28T21:41:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:58 smithi059 bash[61152]: audit 2022-04-28T21:41:57.375556+0000 mon.a (mon.0) 3552 : audit [INF] from='client.? 172.21.15.59:0/3234100782' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:58 smithi059 bash[59745]: cluster 2022-04-28T21:41:57.373576+0000 mon.a (mon.0) 3551 : cluster [DBG] osdmap e1236: 8 total, 8 up, 8 in 2022-04-28T21:41:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:58 smithi059 bash[59745]: audit 2022-04-28T21:41:57.375556+0000 mon.a (mon.0) 3552 : audit [INF] from='client.? 172.21.15.59:0/3234100782' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:58 smithi139 bash[43301]: cluster 2022-04-28T21:41:57.373576+0000 mon.a (mon.0) 3551 : cluster [DBG] osdmap e1236: 8 total, 8 up, 8 in 2022-04-28T21:41:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:58 smithi139 bash[43301]: audit 2022-04-28T21:41:57.375556+0000 mon.a (mon.0) 3552 : audit [INF] from='client.? 172.21.15.59:0/3234100782' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:41:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:59 smithi059 bash[59745]: cluster 2022-04-28T21:41:58.229960+0000 mgr.y (mgr.24961) 3671 : cluster [DBG] pgmap v4269: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:59 smithi059 bash[59745]: audit 2022-04-28T21:41:58.376906+0000 mon.a (mon.0) 3553 : audit [INF] from='client.? 172.21.15.59:0/3234100782' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-23","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:41:59 smithi059 bash[59745]: cluster 2022-04-28T21:41:58.377992+0000 mon.a (mon.0) 3554 : cluster [DBG] osdmap e1237: 8 total, 8 up, 8 in 2022-04-28T21:41:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:59 smithi059 bash[61152]: cluster 2022-04-28T21:41:58.229960+0000 mgr.y (mgr.24961) 3671 : cluster [DBG] pgmap v4269: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:59 smithi059 bash[61152]: audit 2022-04-28T21:41:58.376906+0000 mon.a (mon.0) 3553 : audit [INF] from='client.? 172.21.15.59:0/3234100782' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-23","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:59.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:41:59 smithi059 bash[61152]: cluster 2022-04-28T21:41:58.377992+0000 mon.a (mon.0) 3554 : cluster [DBG] osdmap e1237: 8 total, 8 up, 8 in 2022-04-28T21:41:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:59 smithi139 bash[43301]: cluster 2022-04-28T21:41:58.229960+0000 mgr.y (mgr.24961) 3671 : cluster [DBG] pgmap v4269: 169 pgs: 32 unknown, 137 active+clean; 24 KiB data, 866 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:41:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:59 smithi139 bash[43301]: audit 2022-04-28T21:41:58.376906+0000 mon.a (mon.0) 3553 : audit [INF] from='client.? 172.21.15.59:0/3234100782' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-23","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:41:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:41:59 smithi139 bash[43301]: cluster 2022-04-28T21:41:58.377992+0000 mon.a (mon.0) 3554 : cluster [DBG] osdmap e1237: 8 total, 8 up, 8 in 2022-04-28T21:42:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:00 smithi059 bash[61152]: cluster 2022-04-28T21:41:59.390591+0000 mon.a (mon.0) 3555 : cluster [DBG] osdmap e1238: 8 total, 8 up, 8 in 2022-04-28T21:42:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:00 smithi059 bash[59745]: cluster 2022-04-28T21:41:59.390591+0000 mon.a (mon.0) 3555 : cluster [DBG] osdmap e1238: 8 total, 8 up, 8 in 2022-04-28T21:42:00.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:00 smithi139 bash[43301]: cluster 2022-04-28T21:41:59.390591+0000 mon.a (mon.0) 3555 : cluster [DBG] osdmap e1238: 8 total, 8 up, 8 in 2022-04-28T21:42:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:01 smithi059 bash[61152]: cluster 2022-04-28T21:42:00.230953+0000 mgr.y (mgr.24961) 3672 : cluster [DBG] pgmap v4272: 137 pgs: 137 active+clean; 24 KiB data, 836 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:01.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:01 smithi059 bash[61152]: cluster 2022-04-28T21:42:00.417397+0000 mon.a (mon.0) 3556 : cluster [DBG] osdmap e1239: 8 total, 8 up, 8 in 2022-04-28T21:42:01.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:01 smithi059 bash[61152]: audit 2022-04-28T21:42:00.421175+0000 mon.a (mon.0) 3557 : audit [INF] from='client.? 172.21.15.59:0/2303999013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:01.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:01 smithi059 bash[59745]: cluster 2022-04-28T21:42:00.230953+0000 mgr.y (mgr.24961) 3672 : cluster [DBG] pgmap v4272: 137 pgs: 137 active+clean; 24 KiB data, 836 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:01.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:01 smithi059 bash[59745]: cluster 2022-04-28T21:42:00.417397+0000 mon.a (mon.0) 3556 : cluster [DBG] osdmap e1239: 8 total, 8 up, 8 in 2022-04-28T21:42:01.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:01 smithi059 bash[59745]: audit 2022-04-28T21:42:00.421175+0000 mon.a (mon.0) 3557 : audit [INF] from='client.? 172.21.15.59:0/2303999013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:01.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:01 smithi139 bash[43301]: cluster 2022-04-28T21:42:00.230953+0000 mgr.y (mgr.24961) 3672 : cluster [DBG] pgmap v4272: 137 pgs: 137 active+clean; 24 KiB data, 836 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:01.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:01 smithi139 bash[43301]: cluster 2022-04-28T21:42:00.417397+0000 mon.a (mon.0) 3556 : cluster [DBG] osdmap e1239: 8 total, 8 up, 8 in 2022-04-28T21:42:01.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:01 smithi139 bash[43301]: audit 2022-04-28T21:42:00.421175+0000 mon.a (mon.0) 3557 : audit [INF] from='client.? 172.21.15.59:0/2303999013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:02.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:02 smithi059 bash[61152]: audit 2022-04-28T21:42:01.420914+0000 mon.a (mon.0) 3558 : audit [INF] from='client.? 172.21.15.59:0/2303999013' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:02 smithi059 bash[61152]: cluster 2022-04-28T21:42:01.421298+0000 mon.a (mon.0) 3559 : cluster [DBG] osdmap e1240: 8 total, 8 up, 8 in 2022-04-28T21:42:02.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:02 smithi059 bash[59745]: audit 2022-04-28T21:42:01.420914+0000 mon.a (mon.0) 3558 : audit [INF] from='client.? 172.21.15.59:0/2303999013' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:02 smithi059 bash[59745]: cluster 2022-04-28T21:42:01.421298+0000 mon.a (mon.0) 3559 : cluster [DBG] osdmap e1240: 8 total, 8 up, 8 in 2022-04-28T21:42:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:02 smithi139 bash[43301]: audit 2022-04-28T21:42:01.420914+0000 mon.a (mon.0) 3558 : audit [INF] from='client.? 172.21.15.59:0/2303999013' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-24","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:02 smithi139 bash[43301]: cluster 2022-04-28T21:42:01.421298+0000 mon.a (mon.0) 3559 : cluster [DBG] osdmap e1240: 8 total, 8 up, 8 in 2022-04-28T21:42:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: cluster 2022-04-28T21:42:02.231718+0000 mgr.y (mgr.24961) 3673 : cluster [DBG] pgmap v4275: 169 pgs: 11 creating+peering, 21 unknown, 137 active+clean; 24 KiB data, 836 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: cluster 2022-04-28T21:42:02.438930+0000 mon.a (mon.0) 3560 : cluster [DBG] osdmap e1241: 8 total, 8 up, 8 in 2022-04-28T21:42:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:02.442484+0000 mon.c (mon.1) 143 : audit [INF] from='client.? 172.21.15.59:0/1138865777' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:02.450947+0000 mon.a (mon.0) 3561 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:02.690164+0000 mon.a (mon.0) 3562 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:42:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:02.693163+0000 mon.a (mon.0) 3563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:42:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:02.695187+0000 mon.a (mon.0) 3564 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:42:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:02.699294+0000 mon.a (mon.0) 3565 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:42:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.010725+0000 mon.a (mon.0) 3566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.011616+0000 mon.a (mon.0) 3567 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.012359+0000 mon.a (mon.0) 3568 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.013048+0000 mon.a (mon.0) 3569 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.013776+0000 mon.a (mon.0) 3570 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.014454+0000 mon.a (mon.0) 3571 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.015182+0000 mon.a (mon.0) 3572 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.015850+0000 mon.a (mon.0) 3573 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.016573+0000 mon.a (mon.0) 3574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:42:03.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[59745]: audit 2022-04-28T21:42:03.017125+0000 mon.a (mon.0) 3575 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: cluster 2022-04-28T21:42:02.231718+0000 mgr.y (mgr.24961) 3673 : cluster [DBG] pgmap v4275: 169 pgs: 11 creating+peering, 21 unknown, 137 active+clean; 24 KiB data, 836 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:03.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: cluster 2022-04-28T21:42:02.438930+0000 mon.a (mon.0) 3560 : cluster [DBG] osdmap e1241: 8 total, 8 up, 8 in 2022-04-28T21:42:03.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:02.442484+0000 mon.c (mon.1) 143 : audit [INF] from='client.? 172.21.15.59:0/1138865777' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:03.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:02.450947+0000 mon.a (mon.0) 3561 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:03.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:02.690164+0000 mon.a (mon.0) 3562 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:42:03.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:02.693163+0000 mon.a (mon.0) 3563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:42:03.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:02.695187+0000 mon.a (mon.0) 3564 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:42:03.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:02.699294+0000 mon.a (mon.0) 3565 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:42:03.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.010725+0000 mon.a (mon.0) 3566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.011616+0000 mon.a (mon.0) 3567 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.012359+0000 mon.a (mon.0) 3568 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.013048+0000 mon.a (mon.0) 3569 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.013776+0000 mon.a (mon.0) 3570 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.014454+0000 mon.a (mon.0) 3571 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.015182+0000 mon.a (mon.0) 3572 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.015850+0000 mon.a (mon.0) 3573 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.016573+0000 mon.a (mon.0) 3574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:42:03.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[61152]: audit 2022-04-28T21:42:03.017125+0000 mon.a (mon.0) 3575 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: cluster 2022-04-28T21:42:02.231718+0000 mgr.y (mgr.24961) 3673 : cluster [DBG] pgmap v4275: 169 pgs: 11 creating+peering, 21 unknown, 137 active+clean; 24 KiB data, 836 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:03.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: cluster 2022-04-28T21:42:02.438930+0000 mon.a (mon.0) 3560 : cluster [DBG] osdmap e1241: 8 total, 8 up, 8 in 2022-04-28T21:42:03.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:02.442484+0000 mon.c (mon.1) 143 : audit [INF] from='client.? 172.21.15.59:0/1138865777' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:03.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:02.450947+0000 mon.a (mon.0) 3561 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:03.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:02.690164+0000 mon.a (mon.0) 3562 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:42:03.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:02.693163+0000 mon.a (mon.0) 3563 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:42:03.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:02.695187+0000 mon.a (mon.0) 3564 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:42:03.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:02.699294+0000 mon.a (mon.0) 3565 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd","key": "container_image"}]: dispatch 2022-04-28T21:42:03.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.010725+0000 mon.a (mon.0) 3566 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.011616+0000 mon.a (mon.0) 3567 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.012359+0000 mon.a (mon.0) 3568 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.013048+0000 mon.a (mon.0) 3569 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.013776+0000 mon.a (mon.0) 3570 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.014454+0000 mon.a (mon.0) 3571 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.015182+0000 mon.a (mon.0) 3572 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:03.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.015850+0000 mon.a (mon.0) 3573 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:03.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.016573+0000 mon.a (mon.0) 3574 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:42:03.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:03 smithi139 bash[43301]: audit 2022-04-28T21:42:03.017125+0000 mon.a (mon.0) 3575 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:42:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:03.953Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:04.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:04 smithi059 bash[61152]: cephadm 2022-04-28T21:42:02.696896+0000 mgr.y (mgr.24961) 3674 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:42:04.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:04 smithi059 bash[61152]: audit 2022-04-28T21:42:03.438512+0000 mon.a (mon.0) 3576 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:04.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:04 smithi059 bash[61152]: cluster 2022-04-28T21:42:03.438685+0000 mon.a (mon.0) 3577 : cluster [DBG] osdmap e1242: 8 total, 8 up, 8 in 2022-04-28T21:42:04.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:04 smithi059 bash[59745]: cephadm 2022-04-28T21:42:02.696896+0000 mgr.y (mgr.24961) 3674 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:42:04.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:04 smithi059 bash[59745]: audit 2022-04-28T21:42:03.438512+0000 mon.a (mon.0) 3576 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:04.860 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:04 smithi059 bash[59745]: cluster 2022-04-28T21:42:03.438685+0000 mon.a (mon.0) 3577 : cluster [DBG] osdmap e1242: 8 total, 8 up, 8 in 2022-04-28T21:42:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:04 smithi139 bash[43301]: cephadm 2022-04-28T21:42:02.696896+0000 mgr.y (mgr.24961) 3674 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:42:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:04 smithi139 bash[43301]: audit 2022-04-28T21:42:03.438512+0000 mon.a (mon.0) 3576 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-25","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:04 smithi139 bash[43301]: cluster 2022-04-28T21:42:03.438685+0000 mon.a (mon.0) 3577 : cluster [DBG] osdmap e1242: 8 total, 8 up, 8 in 2022-04-28T21:42:05.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:05 smithi059 bash[61152]: cluster 2022-04-28T21:42:04.233002+0000 mgr.y (mgr.24961) 3675 : cluster [DBG] pgmap v4278: 201 pgs: 11 creating+peering, 30 unknown, 160 active+clean; 25 KiB data, 837 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 0 op/s 2022-04-28T21:42:05.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:05 smithi059 bash[61152]: cluster 2022-04-28T21:42:04.463117+0000 mon.a (mon.0) 3578 : cluster [DBG] osdmap e1243: 8 total, 8 up, 8 in 2022-04-28T21:42:05.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:05 smithi059 bash[59745]: cluster 2022-04-28T21:42:04.233002+0000 mgr.y (mgr.24961) 3675 : cluster [DBG] pgmap v4278: 201 pgs: 11 creating+peering, 30 unknown, 160 active+clean; 25 KiB data, 837 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 0 op/s 2022-04-28T21:42:05.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:05 smithi059 bash[59745]: cluster 2022-04-28T21:42:04.463117+0000 mon.a (mon.0) 3578 : cluster [DBG] osdmap e1243: 8 total, 8 up, 8 in 2022-04-28T21:42:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:05 smithi139 bash[43301]: cluster 2022-04-28T21:42:04.233002+0000 mgr.y (mgr.24961) 3675 : cluster [DBG] pgmap v4278: 201 pgs: 11 creating+peering, 30 unknown, 160 active+clean; 25 KiB data, 837 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 0 op/s 2022-04-28T21:42:05.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:05 smithi139 bash[43301]: cluster 2022-04-28T21:42:04.463117+0000 mon.a (mon.0) 3578 : cluster [DBG] osdmap e1243: 8 total, 8 up, 8 in 2022-04-28T21:42:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:06 smithi059 bash[59745]: audit 2022-04-28T21:42:05.486114+0000 mon.b (mon.2) 29 : audit [INF] from='client.? 172.21.15.59:0/3068500198' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:06 smithi059 bash[59745]: cluster 2022-04-28T21:42:05.486740+0000 mon.a (mon.0) 3579 : cluster [DBG] osdmap e1244: 8 total, 8 up, 8 in 2022-04-28T21:42:06.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:06 smithi059 bash[59745]: audit 2022-04-28T21:42:05.489837+0000 mon.a (mon.0) 3580 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:06 smithi059 bash[61152]: audit 2022-04-28T21:42:05.486114+0000 mon.b (mon.2) 29 : audit [INF] from='client.? 172.21.15.59:0/3068500198' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:06 smithi059 bash[61152]: cluster 2022-04-28T21:42:05.486740+0000 mon.a (mon.0) 3579 : cluster [DBG] osdmap e1244: 8 total, 8 up, 8 in 2022-04-28T21:42:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:06 smithi059 bash[61152]: audit 2022-04-28T21:42:05.489837+0000 mon.a (mon.0) 3580 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:06.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:06 smithi139 bash[43301]: audit 2022-04-28T21:42:05.486114+0000 mon.b (mon.2) 29 : audit [INF] from='client.? 172.21.15.59:0/3068500198' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:06 smithi139 bash[43301]: cluster 2022-04-28T21:42:05.486740+0000 mon.a (mon.0) 3579 : cluster [DBG] osdmap e1244: 8 total, 8 up, 8 in 2022-04-28T21:42:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:06 smithi139 bash[43301]: audit 2022-04-28T21:42:05.489837+0000 mon.a (mon.0) 3580 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:07 smithi059 bash[61152]: cluster 2022-04-28T21:42:06.234163+0000 mgr.y (mgr.24961) 3676 : cluster [DBG] pgmap v4281: 201 pgs: 9 creating+peering, 23 unknown, 169 active+clean; 24 KiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 255 B/s wr, 1 op/s 2022-04-28T21:42:07.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:07 smithi059 bash[61152]: audit 2022-04-28T21:42:06.490039+0000 mon.a (mon.0) 3581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:07.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:07 smithi059 bash[61152]: cluster 2022-04-28T21:42:06.490229+0000 mon.a (mon.0) 3582 : cluster [DBG] osdmap e1245: 8 total, 8 up, 8 in 2022-04-28T21:42:07.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:07 smithi059 bash[59745]: cluster 2022-04-28T21:42:06.234163+0000 mgr.y (mgr.24961) 3676 : cluster [DBG] pgmap v4281: 201 pgs: 9 creating+peering, 23 unknown, 169 active+clean; 24 KiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 255 B/s wr, 1 op/s 2022-04-28T21:42:07.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:07 smithi059 bash[59745]: audit 2022-04-28T21:42:06.490039+0000 mon.a (mon.0) 3581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:07.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:07 smithi059 bash[59745]: cluster 2022-04-28T21:42:06.490229+0000 mon.a (mon.0) 3582 : cluster [DBG] osdmap e1245: 8 total, 8 up, 8 in 2022-04-28T21:42:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:07 smithi139 bash[43301]: cluster 2022-04-28T21:42:06.234163+0000 mgr.y (mgr.24961) 3676 : cluster [DBG] pgmap v4281: 201 pgs: 9 creating+peering, 23 unknown, 169 active+clean; 24 KiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 255 B/s wr, 1 op/s 2022-04-28T21:42:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:07 smithi139 bash[43301]: audit 2022-04-28T21:42:06.490039+0000 mon.a (mon.0) 3581 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-26","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:07 smithi139 bash[43301]: cluster 2022-04-28T21:42:06.490229+0000 mon.a (mon.0) 3582 : cluster [DBG] osdmap e1245: 8 total, 8 up, 8 in 2022-04-28T21:42:08.820 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:08 smithi139 bash[43301]: cluster 2022-04-28T21:42:07.513534+0000 mon.a (mon.0) 3583 : cluster [DBG] osdmap e1246: 8 total, 8 up, 8 in 2022-04-28T21:42:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:08 smithi059 bash[61152]: cluster 2022-04-28T21:42:07.513534+0000 mon.a (mon.0) 3583 : cluster [DBG] osdmap e1246: 8 total, 8 up, 8 in 2022-04-28T21:42:08.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:08 smithi059 bash[59745]: cluster 2022-04-28T21:42:07.513534+0000 mon.a (mon.0) 3583 : cluster [DBG] osdmap e1246: 8 total, 8 up, 8 in 2022-04-28T21:42:09.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:09 smithi059 bash[59745]: cluster 2022-04-28T21:42:08.234896+0000 mgr.y (mgr.24961) 3677 : cluster [DBG] pgmap v4284: 169 pgs: 169 active+clean; 24 KiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2022-04-28T21:42:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:09 smithi059 bash[59745]: cluster 2022-04-28T21:42:08.527477+0000 mon.a (mon.0) 3584 : cluster [DBG] osdmap e1247: 8 total, 8 up, 8 in 2022-04-28T21:42:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:09 smithi059 bash[59745]: audit 2022-04-28T21:42:08.529622+0000 mon.a (mon.0) 3585 : audit [INF] from='client.? 172.21.15.59:0/802294790' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:09.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:09 smithi059 bash[61152]: cluster 2022-04-28T21:42:08.234896+0000 mgr.y (mgr.24961) 3677 : cluster [DBG] pgmap v4284: 169 pgs: 169 active+clean; 24 KiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2022-04-28T21:42:09.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:09 smithi059 bash[61152]: cluster 2022-04-28T21:42:08.527477+0000 mon.a (mon.0) 3584 : cluster [DBG] osdmap e1247: 8 total, 8 up, 8 in 2022-04-28T21:42:09.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:09 smithi059 bash[61152]: audit 2022-04-28T21:42:08.529622+0000 mon.a (mon.0) 3585 : audit [INF] from='client.? 172.21.15.59:0/802294790' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:09 smithi139 bash[43301]: cluster 2022-04-28T21:42:08.234896+0000 mgr.y (mgr.24961) 3677 : cluster [DBG] pgmap v4284: 169 pgs: 169 active+clean; 24 KiB data, 838 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2022-04-28T21:42:09.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:09 smithi139 bash[43301]: cluster 2022-04-28T21:42:08.527477+0000 mon.a (mon.0) 3584 : cluster [DBG] osdmap e1247: 8 total, 8 up, 8 in 2022-04-28T21:42:09.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:09 smithi139 bash[43301]: audit 2022-04-28T21:42:08.529622+0000 mon.a (mon.0) 3585 : audit [INF] from='client.? 172.21.15.59:0/802294790' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:10.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:10 smithi059 bash[61152]: audit 2022-04-28T21:42:09.542134+0000 mon.a (mon.0) 3586 : audit [INF] from='client.? 172.21.15.59:0/802294790' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-27","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:10.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:10 smithi059 bash[61152]: cluster 2022-04-28T21:42:09.542507+0000 mon.a (mon.0) 3587 : cluster [DBG] osdmap e1248: 8 total, 8 up, 8 in 2022-04-28T21:42:10.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:10 smithi059 bash[61152]: audit 2022-04-28T21:42:10.437697+0000 mon.a (mon.0) 3588 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:42:10.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:10 smithi059 bash[59745]: audit 2022-04-28T21:42:09.542134+0000 mon.a (mon.0) 3586 : audit [INF] from='client.? 172.21.15.59:0/802294790' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-27","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:10.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:10 smithi059 bash[59745]: cluster 2022-04-28T21:42:09.542507+0000 mon.a (mon.0) 3587 : cluster [DBG] osdmap e1248: 8 total, 8 up, 8 in 2022-04-28T21:42:10.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:10 smithi059 bash[59745]: audit 2022-04-28T21:42:10.437697+0000 mon.a (mon.0) 3588 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:42:10.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:10 smithi139 bash[43301]: audit 2022-04-28T21:42:09.542134+0000 mon.a (mon.0) 3586 : audit [INF] from='client.? 172.21.15.59:0/802294790' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-27","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:10.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:10 smithi139 bash[43301]: cluster 2022-04-28T21:42:09.542507+0000 mon.a (mon.0) 3587 : cluster [DBG] osdmap e1248: 8 total, 8 up, 8 in 2022-04-28T21:42:10.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:10 smithi139 bash[43301]: audit 2022-04-28T21:42:10.437697+0000 mon.a (mon.0) 3588 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:42:11.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: cluster 2022-04-28T21:42:10.236117+0000 mgr.y (mgr.24961) 3678 : cluster [DBG] pgmap v4287: 201 pgs: 11 creating+peering, 10 unknown, 180 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:11.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: cluster 2022-04-28T21:42:10.540954+0000 mon.a (mon.0) 3589 : cluster [DBG] osdmap e1249: 8 total, 8 up, 8 in 2022-04-28T21:42:11.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: cephadm 2022-04-28T21:42:10.659030+0000 mgr.y (mgr.24961) 3679 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T21:42:11.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.048350+0000 mon.a (mon.0) 3590 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:42:11.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.354663+0000 mon.a (mon.0) 3591 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.355824+0000 mon.a (mon.0) 3592 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.356887+0000 mon.a (mon.0) 3593 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.357900+0000 mon.a (mon.0) 3594 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.358867+0000 mon.a (mon.0) 3595 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.359866+0000 mon.a (mon.0) 3596 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.360836+0000 mon.a (mon.0) 3597 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.361795+0000 mon.a (mon.0) 3598 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.362796+0000 mon.a (mon.0) 3599 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:42:11.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.363841+0000 mon.a (mon.0) 3600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[59745]: audit 2022-04-28T21:42:11.371948+0000 mon.a (mon.0) 3601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:42:11.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: cluster 2022-04-28T21:42:10.236117+0000 mgr.y (mgr.24961) 3678 : cluster [DBG] pgmap v4287: 201 pgs: 11 creating+peering, 10 unknown, 180 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:11.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: cluster 2022-04-28T21:42:10.540954+0000 mon.a (mon.0) 3589 : cluster [DBG] osdmap e1249: 8 total, 8 up, 8 in 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: cephadm 2022-04-28T21:42:10.659030+0000 mgr.y (mgr.24961) 3679 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.048350+0000 mon.a (mon.0) 3590 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.354663+0000 mon.a (mon.0) 3591 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.355824+0000 mon.a (mon.0) 3592 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.356887+0000 mon.a (mon.0) 3593 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.357900+0000 mon.a (mon.0) 3594 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.358867+0000 mon.a (mon.0) 3595 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.359866+0000 mon.a (mon.0) 3596 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.360836+0000 mon.a (mon.0) 3597 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.361795+0000 mon.a (mon.0) 3598 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.362796+0000 mon.a (mon.0) 3599 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:42:11.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.363841+0000 mon.a (mon.0) 3600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.848 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:11 smithi059 bash[61152]: audit 2022-04-28T21:42:11.371948+0000 mon.a (mon.0) 3601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:42:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: cluster 2022-04-28T21:42:10.236117+0000 mgr.y (mgr.24961) 3678 : cluster [DBG] pgmap v4287: 201 pgs: 11 creating+peering, 10 unknown, 180 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: cluster 2022-04-28T21:42:10.540954+0000 mon.a (mon.0) 3589 : cluster [DBG] osdmap e1249: 8 total, 8 up, 8 in 2022-04-28T21:42:11.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: cephadm 2022-04-28T21:42:10.659030+0000 mgr.y (mgr.24961) 3679 : cephadm [INF] Detected new or changed devices on smithi139 2022-04-28T21:42:11.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.048350+0000 mon.a (mon.0) 3590 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:42:11.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.354663+0000 mon.a (mon.0) 3591 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.355824+0000 mon.a (mon.0) 3592 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.356887+0000 mon.a (mon.0) 3593 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.357900+0000 mon.a (mon.0) 3594 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.358867+0000 mon.a (mon.0) 3595 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.359866+0000 mon.a (mon.0) 3596 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.360836+0000 mon.a (mon.0) 3597 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:42:11.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.361795+0000 mon.a (mon.0) 3598 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.362796+0000 mon.a (mon.0) 3599 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:42:11.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.363841+0000 mon.a (mon.0) 3600 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:42:11.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:11 smithi139 bash[43301]: audit 2022-04-28T21:42:11.371948+0000 mon.a (mon.0) 3601 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:42:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:12.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:12 smithi059 bash[61152]: cluster 2022-04-28T21:42:11.565470+0000 mon.a (mon.0) 3602 : cluster [DBG] osdmap e1250: 8 total, 8 up, 8 in 2022-04-28T21:42:12.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:12 smithi059 bash[61152]: audit 2022-04-28T21:42:11.568707+0000 mon.a (mon.0) 3603 : audit [INF] from='client.? 172.21.15.59:0/1625191060' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:12 smithi059 bash[59745]: cluster 2022-04-28T21:42:11.565470+0000 mon.a (mon.0) 3602 : cluster [DBG] osdmap e1250: 8 total, 8 up, 8 in 2022-04-28T21:42:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:12 smithi059 bash[59745]: audit 2022-04-28T21:42:11.568707+0000 mon.a (mon.0) 3603 : audit [INF] from='client.? 172.21.15.59:0/1625191060' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:12 smithi139 bash[43301]: cluster 2022-04-28T21:42:11.565470+0000 mon.a (mon.0) 3602 : cluster [DBG] osdmap e1250: 8 total, 8 up, 8 in 2022-04-28T21:42:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:12 smithi139 bash[43301]: audit 2022-04-28T21:42:11.568707+0000 mon.a (mon.0) 3603 : audit [INF] from='client.? 172.21.15.59:0/1625191060' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[61152]: cluster 2022-04-28T21:42:12.236885+0000 mgr.y (mgr.24961) 3680 : cluster [DBG] pgmap v4290: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[61152]: audit 2022-04-28T21:42:12.571949+0000 mon.a (mon.0) 3604 : audit [INF] from='client.? 172.21.15.59:0/1625191060' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-28","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[61152]: cluster 2022-04-28T21:42:12.572480+0000 mon.a (mon.0) 3605 : cluster [DBG] osdmap e1251: 8 total, 8 up, 8 in 2022-04-28T21:42:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[59745]: cluster 2022-04-28T21:42:12.236885+0000 mgr.y (mgr.24961) 3680 : cluster [DBG] pgmap v4290: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[59745]: audit 2022-04-28T21:42:12.571949+0000 mon.a (mon.0) 3604 : audit [INF] from='client.? 172.21.15.59:0/1625191060' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-28","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[59745]: cluster 2022-04-28T21:42:12.572480+0000 mon.a (mon.0) 3605 : cluster [DBG] osdmap e1251: 8 total, 8 up, 8 in 2022-04-28T21:42:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:13 smithi139 bash[43301]: cluster 2022-04-28T21:42:12.236885+0000 mgr.y (mgr.24961) 3680 : cluster [DBG] pgmap v4290: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 844 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:13 smithi139 bash[43301]: audit 2022-04-28T21:42:12.571949+0000 mon.a (mon.0) 3604 : audit [INF] from='client.? 172.21.15.59:0/1625191060' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-28","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:13 smithi139 bash[43301]: cluster 2022-04-28T21:42:12.572480+0000 mon.a (mon.0) 3605 : cluster [DBG] osdmap e1251: 8 total, 8 up, 8 in 2022-04-28T21:42:14.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:13.952Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:14 smithi139 bash[43301]: cluster 2022-04-28T21:42:13.596401+0000 mon.a (mon.0) 3606 : cluster [DBG] osdmap e1252: 8 total, 8 up, 8 in 2022-04-28T21:42:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:14 smithi059 bash[61152]: cluster 2022-04-28T21:42:13.596401+0000 mon.a (mon.0) 3606 : cluster [DBG] osdmap e1252: 8 total, 8 up, 8 in 2022-04-28T21:42:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:14 smithi059 bash[59745]: cluster 2022-04-28T21:42:13.596401+0000 mon.a (mon.0) 3606 : cluster [DBG] osdmap e1252: 8 total, 8 up, 8 in 2022-04-28T21:42:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:15 smithi139 bash[43301]: cluster 2022-04-28T21:42:14.238058+0000 mgr.y (mgr.24961) 3681 : cluster [DBG] pgmap v4293: 169 pgs: 169 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:15 smithi139 bash[43301]: cluster 2022-04-28T21:42:14.607404+0000 mon.a (mon.0) 3607 : cluster [DBG] osdmap e1253: 8 total, 8 up, 8 in 2022-04-28T21:42:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:15 smithi139 bash[43301]: audit 2022-04-28T21:42:14.613799+0000 mon.c (mon.1) 144 : audit [INF] from='client.? 172.21.15.59:0/1272539452' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:15 smithi139 bash[43301]: audit 2022-04-28T21:42:14.619500+0000 mon.a (mon.0) 3608 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:16.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[61152]: cluster 2022-04-28T21:42:14.238058+0000 mgr.y (mgr.24961) 3681 : cluster [DBG] pgmap v4293: 169 pgs: 169 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:16.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[61152]: cluster 2022-04-28T21:42:14.607404+0000 mon.a (mon.0) 3607 : cluster [DBG] osdmap e1253: 8 total, 8 up, 8 in 2022-04-28T21:42:16.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[61152]: audit 2022-04-28T21:42:14.613799+0000 mon.c (mon.1) 144 : audit [INF] from='client.? 172.21.15.59:0/1272539452' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:16.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[61152]: audit 2022-04-28T21:42:14.619500+0000 mon.a (mon.0) 3608 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:16.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[59745]: cluster 2022-04-28T21:42:14.238058+0000 mgr.y (mgr.24961) 3681 : cluster [DBG] pgmap v4293: 169 pgs: 169 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:16.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[59745]: cluster 2022-04-28T21:42:14.607404+0000 mon.a (mon.0) 3607 : cluster [DBG] osdmap e1253: 8 total, 8 up, 8 in 2022-04-28T21:42:16.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[59745]: audit 2022-04-28T21:42:14.613799+0000 mon.c (mon.1) 144 : audit [INF] from='client.? 172.21.15.59:0/1272539452' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:16.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:15 smithi059 bash[59745]: audit 2022-04-28T21:42:14.619500+0000 mon.a (mon.0) 3608 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:17.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:16 smithi059 bash[61152]: audit 2022-04-28T21:42:15.612308+0000 mon.a (mon.0) 3609 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:17.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:16 smithi059 bash[61152]: cluster 2022-04-28T21:42:15.612745+0000 mon.a (mon.0) 3610 : cluster [DBG] osdmap e1254: 8 total, 8 up, 8 in 2022-04-28T21:42:17.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:16 smithi059 bash[59745]: audit 2022-04-28T21:42:15.612308+0000 mon.a (mon.0) 3609 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:17.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:16 smithi059 bash[59745]: cluster 2022-04-28T21:42:15.612745+0000 mon.a (mon.0) 3610 : cluster [DBG] osdmap e1254: 8 total, 8 up, 8 in 2022-04-28T21:42:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:16 smithi139 bash[43301]: audit 2022-04-28T21:42:15.612308+0000 mon.a (mon.0) 3609 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-29","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:16 smithi139 bash[43301]: cluster 2022-04-28T21:42:15.612745+0000 mon.a (mon.0) 3610 : cluster [DBG] osdmap e1254: 8 total, 8 up, 8 in 2022-04-28T21:42:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:17 smithi059 bash[59745]: cluster 2022-04-28T21:42:16.239146+0000 mgr.y (mgr.24961) 3682 : cluster [DBG] pgmap v4296: 201 pgs: 12 creating+peering, 14 unknown, 175 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:18.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:17 smithi059 bash[59745]: cluster 2022-04-28T21:42:16.623366+0000 mon.a (mon.0) 3611 : cluster [DBG] osdmap e1255: 8 total, 8 up, 8 in 2022-04-28T21:42:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:17 smithi059 bash[61152]: cluster 2022-04-28T21:42:16.239146+0000 mgr.y (mgr.24961) 3682 : cluster [DBG] pgmap v4296: 201 pgs: 12 creating+peering, 14 unknown, 175 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:17 smithi059 bash[61152]: cluster 2022-04-28T21:42:16.623366+0000 mon.a (mon.0) 3611 : cluster [DBG] osdmap e1255: 8 total, 8 up, 8 in 2022-04-28T21:42:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:17 smithi139 bash[43301]: cluster 2022-04-28T21:42:16.239146+0000 mgr.y (mgr.24961) 3682 : cluster [DBG] pgmap v4296: 201 pgs: 12 creating+peering, 14 unknown, 175 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:17 smithi139 bash[43301]: cluster 2022-04-28T21:42:16.623366+0000 mon.a (mon.0) 3611 : cluster [DBG] osdmap e1255: 8 total, 8 up, 8 in 2022-04-28T21:42:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:18 smithi059 bash[61152]: cluster 2022-04-28T21:42:17.647720+0000 mon.a (mon.0) 3612 : cluster [DBG] osdmap e1256: 8 total, 8 up, 8 in 2022-04-28T21:42:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:18 smithi059 bash[61152]: audit 2022-04-28T21:42:17.654143+0000 mon.c (mon.1) 145 : audit [INF] from='client.? 172.21.15.59:0/1431424272' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:18 smithi059 bash[61152]: audit 2022-04-28T21:42:17.654980+0000 mon.a (mon.0) 3613 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:18 smithi059 bash[59745]: cluster 2022-04-28T21:42:17.647720+0000 mon.a (mon.0) 3612 : cluster [DBG] osdmap e1256: 8 total, 8 up, 8 in 2022-04-28T21:42:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:18 smithi059 bash[59745]: audit 2022-04-28T21:42:17.654143+0000 mon.c (mon.1) 145 : audit [INF] from='client.? 172.21.15.59:0/1431424272' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:18 smithi059 bash[59745]: audit 2022-04-28T21:42:17.654980+0000 mon.a (mon.0) 3613 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:19.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:18 smithi139 bash[43301]: cluster 2022-04-28T21:42:17.647720+0000 mon.a (mon.0) 3612 : cluster [DBG] osdmap e1256: 8 total, 8 up, 8 in 2022-04-28T21:42:19.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:18 smithi139 bash[43301]: audit 2022-04-28T21:42:17.654143+0000 mon.c (mon.1) 145 : audit [INF] from='client.? 172.21.15.59:0/1431424272' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:19.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:18 smithi139 bash[43301]: audit 2022-04-28T21:42:17.654980+0000 mon.a (mon.0) 3613 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[61152]: cluster 2022-04-28T21:42:18.239956+0000 mgr.y (mgr.24961) 3683 : cluster [DBG] pgmap v4299: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[61152]: audit 2022-04-28T21:42:18.657885+0000 mon.a (mon.0) 3614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[61152]: cluster 2022-04-28T21:42:18.658106+0000 mon.a (mon.0) 3615 : cluster [DBG] osdmap e1257: 8 total, 8 up, 8 in 2022-04-28T21:42:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[61152]: audit 2022-04-28T21:42:19.342341+0000 mon.a (mon.0) 3616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:42:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[61152]: audit 2022-04-28T21:42:19.350279+0000 mon.a (mon.0) 3617 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:42:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[59745]: cluster 2022-04-28T21:42:18.239956+0000 mgr.y (mgr.24961) 3683 : cluster [DBG] pgmap v4299: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[59745]: audit 2022-04-28T21:42:18.657885+0000 mon.a (mon.0) 3614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[59745]: cluster 2022-04-28T21:42:18.658106+0000 mon.a (mon.0) 3615 : cluster [DBG] osdmap e1257: 8 total, 8 up, 8 in 2022-04-28T21:42:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[59745]: audit 2022-04-28T21:42:19.342341+0000 mon.a (mon.0) 3616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:42:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:19 smithi059 bash[59745]: audit 2022-04-28T21:42:19.350279+0000 mon.a (mon.0) 3617 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:42:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:19 smithi139 bash[43301]: cluster 2022-04-28T21:42:18.239956+0000 mgr.y (mgr.24961) 3683 : cluster [DBG] pgmap v4299: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 850 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:19 smithi139 bash[43301]: audit 2022-04-28T21:42:18.657885+0000 mon.a (mon.0) 3614 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-30","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:19 smithi139 bash[43301]: cluster 2022-04-28T21:42:18.658106+0000 mon.a (mon.0) 3615 : cluster [DBG] osdmap e1257: 8 total, 8 up, 8 in 2022-04-28T21:42:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:19 smithi139 bash[43301]: audit 2022-04-28T21:42:19.342341+0000 mon.a (mon.0) 3616 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:42:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:19 smithi139 bash[43301]: audit 2022-04-28T21:42:19.350279+0000 mon.a (mon.0) 3617 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:42:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:20 smithi059 bash[59745]: cluster 2022-04-28T21:42:19.663469+0000 mon.a (mon.0) 3618 : cluster [DBG] osdmap e1258: 8 total, 8 up, 8 in 2022-04-28T21:42:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:20 smithi059 bash[59745]: cluster 2022-04-28T21:42:20.241093+0000 mgr.y (mgr.24961) 3684 : cluster [DBG] pgmap v4302: 169 pgs: 169 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:20 smithi059 bash[61152]: cluster 2022-04-28T21:42:19.663469+0000 mon.a (mon.0) 3618 : cluster [DBG] osdmap e1258: 8 total, 8 up, 8 in 2022-04-28T21:42:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:20 smithi059 bash[61152]: cluster 2022-04-28T21:42:20.241093+0000 mgr.y (mgr.24961) 3684 : cluster [DBG] pgmap v4302: 169 pgs: 169 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:20 smithi139 bash[43301]: cluster 2022-04-28T21:42:19.663469+0000 mon.a (mon.0) 3618 : cluster [DBG] osdmap e1258: 8 total, 8 up, 8 in 2022-04-28T21:42:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:20 smithi139 bash[43301]: cluster 2022-04-28T21:42:20.241093+0000 mgr.y (mgr.24961) 3684 : cluster [DBG] pgmap v4302: 169 pgs: 169 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:22.063 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:21 smithi059 bash[61152]: cluster 2022-04-28T21:42:20.685188+0000 mon.a (mon.0) 3619 : cluster [DBG] osdmap e1259: 8 total, 8 up, 8 in 2022-04-28T21:42:22.064 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:21 smithi059 bash[61152]: audit 2022-04-28T21:42:20.698643+0000 mon.c (mon.1) 146 : audit [INF] from='client.? 172.21.15.59:0/3377606658' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:22.064 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:21 smithi059 bash[61152]: audit 2022-04-28T21:42:20.699232+0000 mon.a (mon.0) 3620 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:22.064 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:21 smithi059 bash[59745]: cluster 2022-04-28T21:42:20.685188+0000 mon.a (mon.0) 3619 : cluster [DBG] osdmap e1259: 8 total, 8 up, 8 in 2022-04-28T21:42:22.064 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:21 smithi059 bash[59745]: audit 2022-04-28T21:42:20.698643+0000 mon.c (mon.1) 146 : audit [INF] from='client.? 172.21.15.59:0/3377606658' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:22.065 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:21 smithi059 bash[59745]: audit 2022-04-28T21:42:20.699232+0000 mon.a (mon.0) 3620 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:21 smithi139 bash[43301]: cluster 2022-04-28T21:42:20.685188+0000 mon.a (mon.0) 3619 : cluster [DBG] osdmap e1259: 8 total, 8 up, 8 in 2022-04-28T21:42:22.160 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:21 smithi139 bash[43301]: audit 2022-04-28T21:42:20.698643+0000 mon.c (mon.1) 146 : audit [INF] from='client.? 172.21.15.59:0/3377606658' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:22.161 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:21 smithi139 bash[43301]: audit 2022-04-28T21:42:20.699232+0000 mon.a (mon.0) 3620 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:22.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[59745]: audit 2022-04-28T21:42:21.690507+0000 mon.a (mon.0) 3621 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[59745]: cluster 2022-04-28T21:42:21.690715+0000 mon.a (mon.0) 3622 : cluster [DBG] osdmap e1260: 8 total, 8 up, 8 in 2022-04-28T21:42:23.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[59745]: cluster 2022-04-28T21:42:22.241751+0000 mgr.y (mgr.24961) 3685 : cluster [DBG] pgmap v4305: 201 pgs: 12 creating+peering, 20 unknown, 169 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[61152]: audit 2022-04-28T21:42:21.690507+0000 mon.a (mon.0) 3621 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[61152]: cluster 2022-04-28T21:42:21.690715+0000 mon.a (mon.0) 3622 : cluster [DBG] osdmap e1260: 8 total, 8 up, 8 in 2022-04-28T21:42:23.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:22 smithi059 bash[61152]: cluster 2022-04-28T21:42:22.241751+0000 mgr.y (mgr.24961) 3685 : cluster [DBG] pgmap v4305: 201 pgs: 12 creating+peering, 20 unknown, 169 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:22 smithi139 bash[43301]: audit 2022-04-28T21:42:21.690507+0000 mon.a (mon.0) 3621 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-31","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:22 smithi139 bash[43301]: cluster 2022-04-28T21:42:21.690715+0000 mon.a (mon.0) 3622 : cluster [DBG] osdmap e1260: 8 total, 8 up, 8 in 2022-04-28T21:42:23.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:22 smithi139 bash[43301]: cluster 2022-04-28T21:42:22.241751+0000 mgr.y (mgr.24961) 3685 : cluster [DBG] pgmap v4305: 201 pgs: 12 creating+peering, 20 unknown, 169 active+clean; 24 KiB data, 860 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:23 smithi059 bash[61152]: cluster 2022-04-28T21:42:22.704068+0000 mon.a (mon.0) 3623 : cluster [DBG] osdmap e1261: 8 total, 8 up, 8 in 2022-04-28T21:42:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:23.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:23 smithi059 bash[59745]: cluster 2022-04-28T21:42:22.704068+0000 mon.a (mon.0) 3623 : cluster [DBG] osdmap e1261: 8 total, 8 up, 8 in 2022-04-28T21:42:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:23 smithi139 bash[43301]: cluster 2022-04-28T21:42:22.704068+0000 mon.a (mon.0) 3623 : cluster [DBG] osdmap e1261: 8 total, 8 up, 8 in 2022-04-28T21:42:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:24 smithi059 bash[61152]: cluster 2022-04-28T21:42:23.725208+0000 mon.a (mon.0) 3624 : cluster [DBG] osdmap e1262: 8 total, 8 up, 8 in 2022-04-28T21:42:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:24 smithi059 bash[61152]: audit 2022-04-28T21:42:23.726928+0000 mon.a (mon.0) 3625 : audit [INF] from='client.? 172.21.15.59:0/3305245456' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:25.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:24 smithi059 bash[61152]: cluster 2022-04-28T21:42:24.242990+0000 mgr.y (mgr.24961) 3686 : cluster [DBG] pgmap v4308: 201 pgs: 16 creating+peering, 16 unknown, 169 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:24 smithi059 bash[59745]: cluster 2022-04-28T21:42:23.725208+0000 mon.a (mon.0) 3624 : cluster [DBG] osdmap e1262: 8 total, 8 up, 8 in 2022-04-28T21:42:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:24 smithi059 bash[59745]: audit 2022-04-28T21:42:23.726928+0000 mon.a (mon.0) 3625 : audit [INF] from='client.? 172.21.15.59:0/3305245456' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:24 smithi059 bash[59745]: cluster 2022-04-28T21:42:24.242990+0000 mgr.y (mgr.24961) 3686 : cluster [DBG] pgmap v4308: 201 pgs: 16 creating+peering, 16 unknown, 169 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:24 smithi139 bash[43301]: cluster 2022-04-28T21:42:23.725208+0000 mon.a (mon.0) 3624 : cluster [DBG] osdmap e1262: 8 total, 8 up, 8 in 2022-04-28T21:42:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:24 smithi139 bash[43301]: audit 2022-04-28T21:42:23.726928+0000 mon.a (mon.0) 3625 : audit [INF] from='client.? 172.21.15.59:0/3305245456' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:24 smithi139 bash[43301]: cluster 2022-04-28T21:42:24.242990+0000 mgr.y (mgr.24961) 3686 : cluster [DBG] pgmap v4308: 201 pgs: 16 creating+peering, 16 unknown, 169 active+clean; 24 KiB data, 845 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:26.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:25 smithi059 bash[59745]: audit 2022-04-28T21:42:24.731648+0000 mon.a (mon.0) 3626 : audit [INF] from='client.? 172.21.15.59:0/3305245456' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-32","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:26.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:25 smithi059 bash[59745]: cluster 2022-04-28T21:42:24.731965+0000 mon.a (mon.0) 3627 : cluster [DBG] osdmap e1263: 8 total, 8 up, 8 in 2022-04-28T21:42:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:25 smithi059 bash[61152]: audit 2022-04-28T21:42:24.731648+0000 mon.a (mon.0) 3626 : audit [INF] from='client.? 172.21.15.59:0/3305245456' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-32","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:26.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:25 smithi059 bash[61152]: cluster 2022-04-28T21:42:24.731965+0000 mon.a (mon.0) 3627 : cluster [DBG] osdmap e1263: 8 total, 8 up, 8 in 2022-04-28T21:42:26.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:25 smithi139 bash[43301]: audit 2022-04-28T21:42:24.731648+0000 mon.a (mon.0) 3626 : audit [INF] from='client.? 172.21.15.59:0/3305245456' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-32","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:26.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:25 smithi139 bash[43301]: cluster 2022-04-28T21:42:24.731965+0000 mon.a (mon.0) 3627 : cluster [DBG] osdmap e1263: 8 total, 8 up, 8 in 2022-04-28T21:42:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[61152]: cluster 2022-04-28T21:42:25.742032+0000 mon.a (mon.0) 3628 : cluster [DBG] osdmap e1264: 8 total, 8 up, 8 in 2022-04-28T21:42:27.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[61152]: audit 2022-04-28T21:42:25.742812+0000 mon.c (mon.1) 147 : audit [INF] from='client.? 172.21.15.59:0/351396473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:27.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[61152]: audit 2022-04-28T21:42:25.747743+0000 mon.a (mon.0) 3629 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:27.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[61152]: cluster 2022-04-28T21:42:26.244280+0000 mgr.y (mgr.24961) 3687 : cluster [DBG] pgmap v4311: 233 pgs: 22 creating+peering, 26 unknown, 185 active+clean; 25 KiB data, 847 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:27.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[61152]: audit 2022-04-28T21:42:26.739714+0000 mon.a (mon.0) 3630 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:27.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[61152]: cluster 2022-04-28T21:42:26.739953+0000 mon.a (mon.0) 3631 : cluster [DBG] osdmap e1265: 8 total, 8 up, 8 in 2022-04-28T21:42:27.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[59745]: cluster 2022-04-28T21:42:25.742032+0000 mon.a (mon.0) 3628 : cluster [DBG] osdmap e1264: 8 total, 8 up, 8 in 2022-04-28T21:42:27.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[59745]: audit 2022-04-28T21:42:25.742812+0000 mon.c (mon.1) 147 : audit [INF] from='client.? 172.21.15.59:0/351396473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:27.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[59745]: audit 2022-04-28T21:42:25.747743+0000 mon.a (mon.0) 3629 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:27.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[59745]: cluster 2022-04-28T21:42:26.244280+0000 mgr.y (mgr.24961) 3687 : cluster [DBG] pgmap v4311: 233 pgs: 22 creating+peering, 26 unknown, 185 active+clean; 25 KiB data, 847 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:27.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[59745]: audit 2022-04-28T21:42:26.739714+0000 mon.a (mon.0) 3630 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:27.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:26 smithi059 bash[59745]: cluster 2022-04-28T21:42:26.739953+0000 mon.a (mon.0) 3631 : cluster [DBG] osdmap e1265: 8 total, 8 up, 8 in 2022-04-28T21:42:27.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:26 smithi139 bash[43301]: cluster 2022-04-28T21:42:25.742032+0000 mon.a (mon.0) 3628 : cluster [DBG] osdmap e1264: 8 total, 8 up, 8 in 2022-04-28T21:42:27.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:26 smithi139 bash[43301]: audit 2022-04-28T21:42:25.742812+0000 mon.c (mon.1) 147 : audit [INF] from='client.? 172.21.15.59:0/351396473' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:27.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:26 smithi139 bash[43301]: audit 2022-04-28T21:42:25.747743+0000 mon.a (mon.0) 3629 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:27.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:26 smithi139 bash[43301]: cluster 2022-04-28T21:42:26.244280+0000 mgr.y (mgr.24961) 3687 : cluster [DBG] pgmap v4311: 233 pgs: 22 creating+peering, 26 unknown, 185 active+clean; 25 KiB data, 847 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:27.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:26 smithi139 bash[43301]: audit 2022-04-28T21:42:26.739714+0000 mon.a (mon.0) 3630 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-33","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:27.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:26 smithi139 bash[43301]: cluster 2022-04-28T21:42:26.739953+0000 mon.a (mon.0) 3631 : cluster [DBG] osdmap e1265: 8 total, 8 up, 8 in 2022-04-28T21:42:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:28 smithi059 bash[61152]: cluster 2022-04-28T21:42:27.745020+0000 mon.a (mon.0) 3632 : cluster [DBG] osdmap e1266: 8 total, 8 up, 8 in 2022-04-28T21:42:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:28 smithi059 bash[61152]: audit 2022-04-28T21:42:27.748171+0000 mon.a (mon.0) 3633 : audit [INF] from='client.? 172.21.15.59:0/1591740995' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:28 smithi059 bash[61152]: cluster 2022-04-28T21:42:28.245146+0000 mgr.y (mgr.24961) 3688 : cluster [DBG] pgmap v4314: 265 pgs: 22 creating+peering, 58 unknown, 185 active+clean; 25 KiB data, 847 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:28 smithi059 bash[59745]: cluster 2022-04-28T21:42:27.745020+0000 mon.a (mon.0) 3632 : cluster [DBG] osdmap e1266: 8 total, 8 up, 8 in 2022-04-28T21:42:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:28 smithi059 bash[59745]: audit 2022-04-28T21:42:27.748171+0000 mon.a (mon.0) 3633 : audit [INF] from='client.? 172.21.15.59:0/1591740995' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:28 smithi059 bash[59745]: cluster 2022-04-28T21:42:28.245146+0000 mgr.y (mgr.24961) 3688 : cluster [DBG] pgmap v4314: 265 pgs: 22 creating+peering, 58 unknown, 185 active+clean; 25 KiB data, 847 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:29.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:28 smithi139 bash[43301]: cluster 2022-04-28T21:42:27.745020+0000 mon.a (mon.0) 3632 : cluster [DBG] osdmap e1266: 8 total, 8 up, 8 in 2022-04-28T21:42:29.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:28 smithi139 bash[43301]: audit 2022-04-28T21:42:27.748171+0000 mon.a (mon.0) 3633 : audit [INF] from='client.? 172.21.15.59:0/1591740995' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:29.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:28 smithi139 bash[43301]: cluster 2022-04-28T21:42:28.245146+0000 mgr.y (mgr.24961) 3688 : cluster [DBG] pgmap v4314: 265 pgs: 22 creating+peering, 58 unknown, 185 active+clean; 25 KiB data, 847 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:29 smithi059 bash[61152]: audit 2022-04-28T21:42:28.757443+0000 mon.a (mon.0) 3634 : audit [INF] from='client.? 172.21.15.59:0/1591740995' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-34","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:29 smithi059 bash[61152]: cluster 2022-04-28T21:42:28.757757+0000 mon.a (mon.0) 3635 : cluster [DBG] osdmap e1267: 8 total, 8 up, 8 in 2022-04-28T21:42:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:29 smithi059 bash[59745]: audit 2022-04-28T21:42:28.757443+0000 mon.a (mon.0) 3634 : audit [INF] from='client.? 172.21.15.59:0/1591740995' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-34","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:29 smithi059 bash[59745]: cluster 2022-04-28T21:42:28.757757+0000 mon.a (mon.0) 3635 : cluster [DBG] osdmap e1267: 8 total, 8 up, 8 in 2022-04-28T21:42:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:29 smithi139 bash[43301]: audit 2022-04-28T21:42:28.757443+0000 mon.a (mon.0) 3634 : audit [INF] from='client.? 172.21.15.59:0/1591740995' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-34","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:29 smithi139 bash[43301]: cluster 2022-04-28T21:42:28.757757+0000 mon.a (mon.0) 3635 : cluster [DBG] osdmap e1267: 8 total, 8 up, 8 in 2022-04-28T21:42:31.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:30 smithi059 bash[61152]: cluster 2022-04-28T21:42:29.766889+0000 mon.a (mon.0) 3636 : cluster [DBG] osdmap e1268: 8 total, 8 up, 8 in 2022-04-28T21:42:31.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:30 smithi059 bash[61152]: cluster 2022-04-28T21:42:30.246528+0000 mgr.y (mgr.24961) 3689 : cluster [DBG] pgmap v4317: 233 pgs: 6 creating+peering, 227 active+clean; 25 KiB data, 848 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2022-04-28T21:42:31.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:30 smithi059 bash[59745]: cluster 2022-04-28T21:42:29.766889+0000 mon.a (mon.0) 3636 : cluster [DBG] osdmap e1268: 8 total, 8 up, 8 in 2022-04-28T21:42:31.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:30 smithi059 bash[59745]: cluster 2022-04-28T21:42:30.246528+0000 mgr.y (mgr.24961) 3689 : cluster [DBG] pgmap v4317: 233 pgs: 6 creating+peering, 227 active+clean; 25 KiB data, 848 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2022-04-28T21:42:31.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:30 smithi139 bash[43301]: cluster 2022-04-28T21:42:29.766889+0000 mon.a (mon.0) 3636 : cluster [DBG] osdmap e1268: 8 total, 8 up, 8 in 2022-04-28T21:42:31.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:30 smithi139 bash[43301]: cluster 2022-04-28T21:42:30.246528+0000 mgr.y (mgr.24961) 3689 : cluster [DBG] pgmap v4317: 233 pgs: 6 creating+peering, 227 active+clean; 25 KiB data, 848 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2022-04-28T21:42:32.062 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:31 smithi059 bash[61152]: cluster 2022-04-28T21:42:30.774374+0000 mon.a (mon.0) 3637 : cluster [DBG] osdmap e1269: 8 total, 8 up, 8 in 2022-04-28T21:42:32.062 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:31 smithi059 bash[59745]: cluster 2022-04-28T21:42:30.774374+0000 mon.a (mon.0) 3637 : cluster [DBG] osdmap e1269: 8 total, 8 up, 8 in 2022-04-28T21:42:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:31 smithi139 bash[43301]: cluster 2022-04-28T21:42:30.774374+0000 mon.a (mon.0) 3637 : cluster [DBG] osdmap e1269: 8 total, 8 up, 8 in 2022-04-28T21:42:32.354 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:32.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:32 smithi059 bash[59745]: cluster 2022-04-28T21:42:31.793630+0000 mon.a (mon.0) 3638 : cluster [DBG] osdmap e1270: 8 total, 8 up, 8 in 2022-04-28T21:42:33.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:32 smithi059 bash[59745]: cluster 2022-04-28T21:42:32.247235+0000 mgr.y (mgr.24961) 3690 : cluster [DBG] pgmap v4320: 169 pgs: 169 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:32 smithi059 bash[61152]: cluster 2022-04-28T21:42:31.793630+0000 mon.a (mon.0) 3638 : cluster [DBG] osdmap e1270: 8 total, 8 up, 8 in 2022-04-28T21:42:33.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:32 smithi059 bash[61152]: cluster 2022-04-28T21:42:32.247235+0000 mgr.y (mgr.24961) 3690 : cluster [DBG] pgmap v4320: 169 pgs: 169 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:32 smithi139 bash[43301]: cluster 2022-04-28T21:42:31.793630+0000 mon.a (mon.0) 3638 : cluster [DBG] osdmap e1270: 8 total, 8 up, 8 in 2022-04-28T21:42:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:32 smithi139 bash[43301]: cluster 2022-04-28T21:42:32.247235+0000 mgr.y (mgr.24961) 3690 : cluster [DBG] pgmap v4320: 169 pgs: 169 active+clean; 24 KiB data, 848 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[61152]: cluster 2022-04-28T21:42:32.814764+0000 mon.a (mon.0) 3639 : cluster [DBG] osdmap e1271: 8 total, 8 up, 8 in 2022-04-28T21:42:34.139 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[61152]: audit 2022-04-28T21:42:32.817273+0000 mon.c (mon.1) 148 : audit [INF] from='client.? 172.21.15.59:0/1914064506' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:34.140 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[61152]: audit 2022-04-28T21:42:32.819114+0000 mon.a (mon.0) 3640 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:34.141 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:33.954Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:34.141 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[59745]: cluster 2022-04-28T21:42:32.814764+0000 mon.a (mon.0) 3639 : cluster [DBG] osdmap e1271: 8 total, 8 up, 8 in 2022-04-28T21:42:34.146 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[59745]: audit 2022-04-28T21:42:32.817273+0000 mon.c (mon.1) 148 : audit [INF] from='client.? 172.21.15.59:0/1914064506' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:34.147 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:33 smithi059 bash[59745]: audit 2022-04-28T21:42:32.819114+0000 mon.a (mon.0) 3640 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:34.148 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:33 smithi139 bash[43301]: cluster 2022-04-28T21:42:32.814764+0000 mon.a (mon.0) 3639 : cluster [DBG] osdmap e1271: 8 total, 8 up, 8 in 2022-04-28T21:42:34.148 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:33 smithi139 bash[43301]: audit 2022-04-28T21:42:32.817273+0000 mon.c (mon.1) 148 : audit [INF] from='client.? 172.21.15.59:0/1914064506' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:34.149 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:33 smithi139 bash[43301]: audit 2022-04-28T21:42:32.819114+0000 mon.a (mon.0) 3640 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:35.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:34 smithi139 bash[43301]: audit 2022-04-28T21:42:33.820034+0000 mon.a (mon.0) 3641 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:35.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:34 smithi139 bash[43301]: cluster 2022-04-28T21:42:33.820558+0000 mon.a (mon.0) 3642 : cluster [DBG] osdmap e1272: 8 total, 8 up, 8 in 2022-04-28T21:42:35.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:34 smithi139 bash[43301]: cluster 2022-04-28T21:42:34.248464+0000 mgr.y (mgr.24961) 3691 : cluster [DBG] pgmap v4323: 201 pgs: 22 unknown, 179 active+clean; 24 KiB data, 853 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:34 smithi059 bash[61152]: audit 2022-04-28T21:42:33.820034+0000 mon.a (mon.0) 3641 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:34 smithi059 bash[61152]: cluster 2022-04-28T21:42:33.820558+0000 mon.a (mon.0) 3642 : cluster [DBG] osdmap e1272: 8 total, 8 up, 8 in 2022-04-28T21:42:35.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:34 smithi059 bash[61152]: cluster 2022-04-28T21:42:34.248464+0000 mgr.y (mgr.24961) 3691 : cluster [DBG] pgmap v4323: 201 pgs: 22 unknown, 179 active+clean; 24 KiB data, 853 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:34 smithi059 bash[59745]: audit 2022-04-28T21:42:33.820034+0000 mon.a (mon.0) 3641 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-35","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:34 smithi059 bash[59745]: cluster 2022-04-28T21:42:33.820558+0000 mon.a (mon.0) 3642 : cluster [DBG] osdmap e1272: 8 total, 8 up, 8 in 2022-04-28T21:42:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:34 smithi059 bash[59745]: cluster 2022-04-28T21:42:34.248464+0000 mgr.y (mgr.24961) 3691 : cluster [DBG] pgmap v4323: 201 pgs: 22 unknown, 179 active+clean; 24 KiB data, 853 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:36.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:35 smithi139 bash[43301]: cluster 2022-04-28T21:42:34.845327+0000 mon.a (mon.0) 3643 : cluster [DBG] osdmap e1273: 8 total, 8 up, 8 in 2022-04-28T21:42:36.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:35 smithi139 bash[43301]: audit 2022-04-28T21:42:34.849742+0000 mon.c (mon.1) 149 : audit [INF] from='client.? 172.21.15.59:0/3383702682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:36.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:35 smithi139 bash[43301]: audit 2022-04-28T21:42:34.857688+0000 mon.a (mon.0) 3644 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:36.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:35 smithi139 bash[43301]: audit 2022-04-28T21:42:35.840917+0000 mon.a (mon.0) 3645 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:36.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:35 smithi139 bash[43301]: cluster 2022-04-28T21:42:35.841061+0000 mon.a (mon.0) 3646 : cluster [DBG] osdmap e1274: 8 total, 8 up, 8 in 2022-04-28T21:42:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[59745]: cluster 2022-04-28T21:42:34.845327+0000 mon.a (mon.0) 3643 : cluster [DBG] osdmap e1273: 8 total, 8 up, 8 in 2022-04-28T21:42:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[59745]: audit 2022-04-28T21:42:34.849742+0000 mon.c (mon.1) 149 : audit [INF] from='client.? 172.21.15.59:0/3383702682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[59745]: audit 2022-04-28T21:42:34.857688+0000 mon.a (mon.0) 3644 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[59745]: audit 2022-04-28T21:42:35.840917+0000 mon.a (mon.0) 3645 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:36.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[59745]: cluster 2022-04-28T21:42:35.841061+0000 mon.a (mon.0) 3646 : cluster [DBG] osdmap e1274: 8 total, 8 up, 8 in 2022-04-28T21:42:36.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[61152]: cluster 2022-04-28T21:42:34.845327+0000 mon.a (mon.0) 3643 : cluster [DBG] osdmap e1273: 8 total, 8 up, 8 in 2022-04-28T21:42:36.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[61152]: audit 2022-04-28T21:42:34.849742+0000 mon.c (mon.1) 149 : audit [INF] from='client.? 172.21.15.59:0/3383702682' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:36.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[61152]: audit 2022-04-28T21:42:34.857688+0000 mon.a (mon.0) 3644 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:36.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[61152]: audit 2022-04-28T21:42:35.840917+0000 mon.a (mon.0) 3645 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-36","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:36.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:35 smithi059 bash[61152]: cluster 2022-04-28T21:42:35.841061+0000 mon.a (mon.0) 3646 : cluster [DBG] osdmap e1274: 8 total, 8 up, 8 in 2022-04-28T21:42:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:36 smithi139 bash[43301]: cluster 2022-04-28T21:42:36.249562+0000 mgr.y (mgr.24961) 3692 : cluster [DBG] pgmap v4326: 233 pgs: 10 creating+peering, 16 unknown, 207 active+clean; 25 KiB data, 859 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:36 smithi139 bash[43301]: cluster 2022-04-28T21:42:36.845863+0000 mon.a (mon.0) 3647 : cluster [DBG] osdmap e1275: 8 total, 8 up, 8 in 2022-04-28T21:42:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:36 smithi139 bash[43301]: audit 2022-04-28T21:42:36.847874+0000 mon.c (mon.1) 150 : audit [INF] from='client.? 172.21.15.59:0/3918668128' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:36 smithi139 bash[43301]: audit 2022-04-28T21:42:36.849686+0000 mon.a (mon.0) 3648 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[59745]: cluster 2022-04-28T21:42:36.249562+0000 mgr.y (mgr.24961) 3692 : cluster [DBG] pgmap v4326: 233 pgs: 10 creating+peering, 16 unknown, 207 active+clean; 25 KiB data, 859 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[59745]: cluster 2022-04-28T21:42:36.845863+0000 mon.a (mon.0) 3647 : cluster [DBG] osdmap e1275: 8 total, 8 up, 8 in 2022-04-28T21:42:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[59745]: audit 2022-04-28T21:42:36.847874+0000 mon.c (mon.1) 150 : audit [INF] from='client.? 172.21.15.59:0/3918668128' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[59745]: audit 2022-04-28T21:42:36.849686+0000 mon.a (mon.0) 3648 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[61152]: cluster 2022-04-28T21:42:36.249562+0000 mgr.y (mgr.24961) 3692 : cluster [DBG] pgmap v4326: 233 pgs: 10 creating+peering, 16 unknown, 207 active+clean; 25 KiB data, 859 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[61152]: cluster 2022-04-28T21:42:36.845863+0000 mon.a (mon.0) 3647 : cluster [DBG] osdmap e1275: 8 total, 8 up, 8 in 2022-04-28T21:42:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[61152]: audit 2022-04-28T21:42:36.847874+0000 mon.c (mon.1) 150 : audit [INF] from='client.? 172.21.15.59:0/3918668128' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:36 smithi059 bash[61152]: audit 2022-04-28T21:42:36.849686+0000 mon.a (mon.0) 3648 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:38 smithi139 bash[43301]: audit 2022-04-28T21:42:37.849133+0000 mon.a (mon.0) 3649 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:38 smithi139 bash[43301]: cluster 2022-04-28T21:42:37.849352+0000 mon.a (mon.0) 3650 : cluster [DBG] osdmap e1276: 8 total, 8 up, 8 in 2022-04-28T21:42:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:38 smithi139 bash[43301]: cluster 2022-04-28T21:42:38.250315+0000 mgr.y (mgr.24961) 3693 : cluster [DBG] pgmap v4329: 265 pgs: 10 creating+peering, 48 unknown, 207 active+clean; 25 KiB data, 859 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:38 smithi059 bash[59745]: audit 2022-04-28T21:42:37.849133+0000 mon.a (mon.0) 3649 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:38 smithi059 bash[59745]: cluster 2022-04-28T21:42:37.849352+0000 mon.a (mon.0) 3650 : cluster [DBG] osdmap e1276: 8 total, 8 up, 8 in 2022-04-28T21:42:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:38 smithi059 bash[59745]: cluster 2022-04-28T21:42:38.250315+0000 mgr.y (mgr.24961) 3693 : cluster [DBG] pgmap v4329: 265 pgs: 10 creating+peering, 48 unknown, 207 active+clean; 25 KiB data, 859 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:38 smithi059 bash[61152]: audit 2022-04-28T21:42:37.849133+0000 mon.a (mon.0) 3649 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-37","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:38 smithi059 bash[61152]: cluster 2022-04-28T21:42:37.849352+0000 mon.a (mon.0) 3650 : cluster [DBG] osdmap e1276: 8 total, 8 up, 8 in 2022-04-28T21:42:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:38 smithi059 bash[61152]: cluster 2022-04-28T21:42:38.250315+0000 mgr.y (mgr.24961) 3693 : cluster [DBG] pgmap v4329: 265 pgs: 10 creating+peering, 48 unknown, 207 active+clean; 25 KiB data, 859 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:40.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:39 smithi059 bash[59745]: cluster 2022-04-28T21:42:38.859143+0000 mon.a (mon.0) 3651 : cluster [DBG] osdmap e1277: 8 total, 8 up, 8 in 2022-04-28T21:42:40.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:39 smithi059 bash[59745]: audit 2022-04-28T21:42:38.864670+0000 mon.c (mon.1) 151 : audit [INF] from='client.? 172.21.15.59:0/1685146982' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:40.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:39 smithi059 bash[59745]: audit 2022-04-28T21:42:38.865640+0000 mon.a (mon.0) 3652 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:39 smithi059 bash[61152]: cluster 2022-04-28T21:42:38.859143+0000 mon.a (mon.0) 3651 : cluster [DBG] osdmap e1277: 8 total, 8 up, 8 in 2022-04-28T21:42:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:39 smithi059 bash[61152]: audit 2022-04-28T21:42:38.864670+0000 mon.c (mon.1) 151 : audit [INF] from='client.? 172.21.15.59:0/1685146982' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:40.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:39 smithi059 bash[61152]: audit 2022-04-28T21:42:38.865640+0000 mon.a (mon.0) 3652 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:40.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:39 smithi139 bash[43301]: cluster 2022-04-28T21:42:38.859143+0000 mon.a (mon.0) 3651 : cluster [DBG] osdmap e1277: 8 total, 8 up, 8 in 2022-04-28T21:42:40.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:39 smithi139 bash[43301]: audit 2022-04-28T21:42:38.864670+0000 mon.c (mon.1) 151 : audit [INF] from='client.? 172.21.15.59:0/1685146982' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:40.373 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:39 smithi139 bash[43301]: audit 2022-04-28T21:42:38.865640+0000 mon.a (mon.0) 3652 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:40 smithi059 bash[59745]: audit 2022-04-28T21:42:39.864053+0000 mon.a (mon.0) 3653 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:40 smithi059 bash[59745]: cluster 2022-04-28T21:42:39.864349+0000 mon.a (mon.0) 3654 : cluster [DBG] osdmap e1278: 8 total, 8 up, 8 in 2022-04-28T21:42:41.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:40 smithi059 bash[59745]: cluster 2022-04-28T21:42:40.251748+0000 mgr.y (mgr.24961) 3694 : cluster [DBG] pgmap v4332: 297 pgs: 16 creating+peering, 13 unknown, 268 active+clean; 25 KiB data, 860 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:40 smithi059 bash[61152]: audit 2022-04-28T21:42:39.864053+0000 mon.a (mon.0) 3653 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:40 smithi059 bash[61152]: cluster 2022-04-28T21:42:39.864349+0000 mon.a (mon.0) 3654 : cluster [DBG] osdmap e1278: 8 total, 8 up, 8 in 2022-04-28T21:42:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:40 smithi059 bash[61152]: cluster 2022-04-28T21:42:40.251748+0000 mgr.y (mgr.24961) 3694 : cluster [DBG] pgmap v4332: 297 pgs: 16 creating+peering, 13 unknown, 268 active+clean; 25 KiB data, 860 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:40 smithi139 bash[43301]: audit 2022-04-28T21:42:39.864053+0000 mon.a (mon.0) 3653 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-38","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:41.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:40 smithi139 bash[43301]: cluster 2022-04-28T21:42:39.864349+0000 mon.a (mon.0) 3654 : cluster [DBG] osdmap e1278: 8 total, 8 up, 8 in 2022-04-28T21:42:41.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:40 smithi139 bash[43301]: cluster 2022-04-28T21:42:40.251748+0000 mgr.y (mgr.24961) 3694 : cluster [DBG] pgmap v4332: 297 pgs: 16 creating+peering, 13 unknown, 268 active+clean; 25 KiB data, 860 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:42.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:41 smithi059 bash[59745]: cluster 2022-04-28T21:42:40.891639+0000 mon.a (mon.0) 3655 : cluster [DBG] osdmap e1279: 8 total, 8 up, 8 in 2022-04-28T21:42:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:41 smithi059 bash[59745]: audit 2022-04-28T21:42:40.896422+0000 mon.a (mon.0) 3656 : audit [INF] from='client.? 172.21.15.59:0/3823246797' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:41 smithi059 bash[61152]: cluster 2022-04-28T21:42:40.891639+0000 mon.a (mon.0) 3655 : cluster [DBG] osdmap e1279: 8 total, 8 up, 8 in 2022-04-28T21:42:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:41 smithi059 bash[61152]: audit 2022-04-28T21:42:40.896422+0000 mon.a (mon.0) 3656 : audit [INF] from='client.? 172.21.15.59:0/3823246797' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:42.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:42.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:41 smithi139 bash[43301]: cluster 2022-04-28T21:42:40.891639+0000 mon.a (mon.0) 3655 : cluster [DBG] osdmap e1279: 8 total, 8 up, 8 in 2022-04-28T21:42:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:41 smithi139 bash[43301]: audit 2022-04-28T21:42:40.896422+0000 mon.a (mon.0) 3656 : audit [INF] from='client.? 172.21.15.59:0/3823246797' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[61152]: audit 2022-04-28T21:42:41.899928+0000 mon.a (mon.0) 3657 : audit [INF] from='client.? 172.21.15.59:0/3823246797' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-39","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[61152]: cluster 2022-04-28T21:42:41.900576+0000 mon.a (mon.0) 3658 : cluster [DBG] osdmap e1280: 8 total, 8 up, 8 in 2022-04-28T21:42:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[61152]: cluster 2022-04-28T21:42:42.252651+0000 mgr.y (mgr.24961) 3695 : cluster [DBG] pgmap v4335: 329 pgs: 21 creating+peering, 27 unknown, 281 active+clean; 25 KiB data, 861 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[59745]: audit 2022-04-28T21:42:41.899928+0000 mon.a (mon.0) 3657 : audit [INF] from='client.? 172.21.15.59:0/3823246797' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-39","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[59745]: cluster 2022-04-28T21:42:41.900576+0000 mon.a (mon.0) 3658 : cluster [DBG] osdmap e1280: 8 total, 8 up, 8 in 2022-04-28T21:42:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:42 smithi059 bash[59745]: cluster 2022-04-28T21:42:42.252651+0000 mgr.y (mgr.24961) 3695 : cluster [DBG] pgmap v4335: 329 pgs: 21 creating+peering, 27 unknown, 281 active+clean; 25 KiB data, 861 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:42 smithi139 bash[43301]: audit 2022-04-28T21:42:41.899928+0000 mon.a (mon.0) 3657 : audit [INF] from='client.? 172.21.15.59:0/3823246797' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-39","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:42 smithi139 bash[43301]: cluster 2022-04-28T21:42:41.900576+0000 mon.a (mon.0) 3658 : cluster [DBG] osdmap e1280: 8 total, 8 up, 8 in 2022-04-28T21:42:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:42 smithi139 bash[43301]: cluster 2022-04-28T21:42:42.252651+0000 mgr.y (mgr.24961) 3695 : cluster [DBG] pgmap v4335: 329 pgs: 21 creating+peering, 27 unknown, 281 active+clean; 25 KiB data, 861 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2022-04-28T21:42:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:43 smithi059 bash[61152]: cluster 2022-04-28T21:42:42.914695+0000 mon.a (mon.0) 3659 : cluster [DBG] osdmap e1281: 8 total, 8 up, 8 in 2022-04-28T21:42:44.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:43.950Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:43 smithi059 bash[59745]: cluster 2022-04-28T21:42:42.914695+0000 mon.a (mon.0) 3659 : cluster [DBG] osdmap e1281: 8 total, 8 up, 8 in 2022-04-28T21:42:44.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:43 smithi139 bash[43301]: cluster 2022-04-28T21:42:42.914695+0000 mon.a (mon.0) 3659 : cluster [DBG] osdmap e1281: 8 total, 8 up, 8 in 2022-04-28T21:42:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:44 smithi059 bash[61152]: cluster 2022-04-28T21:42:43.934954+0000 mon.a (mon.0) 3660 : cluster [DBG] osdmap e1282: 8 total, 8 up, 8 in 2022-04-28T21:42:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:44 smithi059 bash[61152]: cluster 2022-04-28T21:42:44.253629+0000 mgr.y (mgr.24961) 3696 : cluster [DBG] pgmap v4338: 265 pgs: 265 active+clean; 25 KiB data, 862 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:44 smithi059 bash[59745]: cluster 2022-04-28T21:42:43.934954+0000 mon.a (mon.0) 3660 : cluster [DBG] osdmap e1282: 8 total, 8 up, 8 in 2022-04-28T21:42:45.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:44 smithi059 bash[59745]: cluster 2022-04-28T21:42:44.253629+0000 mgr.y (mgr.24961) 3696 : cluster [DBG] pgmap v4338: 265 pgs: 265 active+clean; 25 KiB data, 862 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:44 smithi139 bash[43301]: cluster 2022-04-28T21:42:43.934954+0000 mon.a (mon.0) 3660 : cluster [DBG] osdmap e1282: 8 total, 8 up, 8 in 2022-04-28T21:42:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:44 smithi139 bash[43301]: cluster 2022-04-28T21:42:44.253629+0000 mgr.y (mgr.24961) 3696 : cluster [DBG] pgmap v4338: 265 pgs: 265 active+clean; 25 KiB data, 862 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:46.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:45 smithi059 bash[61152]: cluster 2022-04-28T21:42:44.949215+0000 mon.a (mon.0) 3661 : cluster [DBG] osdmap e1283: 8 total, 8 up, 8 in 2022-04-28T21:42:46.389 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:45 smithi059 bash[59745]: cluster 2022-04-28T21:42:44.949215+0000 mon.a (mon.0) 3661 : cluster [DBG] osdmap e1283: 8 total, 8 up, 8 in 2022-04-28T21:42:46.391 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:45 smithi139 bash[43301]: cluster 2022-04-28T21:42:44.949215+0000 mon.a (mon.0) 3661 : cluster [DBG] osdmap e1283: 8 total, 8 up, 8 in 2022-04-28T21:42:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:46 smithi059 bash[61152]: cluster 2022-04-28T21:42:45.952649+0000 mon.a (mon.0) 3662 : cluster [DBG] osdmap e1284: 8 total, 8 up, 8 in 2022-04-28T21:42:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:46 smithi059 bash[61152]: cluster 2022-04-28T21:42:46.254765+0000 mgr.y (mgr.24961) 3697 : cluster [DBG] pgmap v4341: 201 pgs: 201 active+clean; 25 KiB data, 863 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:46 smithi059 bash[59745]: cluster 2022-04-28T21:42:45.952649+0000 mon.a (mon.0) 3662 : cluster [DBG] osdmap e1284: 8 total, 8 up, 8 in 2022-04-28T21:42:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:46 smithi059 bash[59745]: cluster 2022-04-28T21:42:46.254765+0000 mgr.y (mgr.24961) 3697 : cluster [DBG] pgmap v4341: 201 pgs: 201 active+clean; 25 KiB data, 863 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:46 smithi139 bash[43301]: cluster 2022-04-28T21:42:45.952649+0000 mon.a (mon.0) 3662 : cluster [DBG] osdmap e1284: 8 total, 8 up, 8 in 2022-04-28T21:42:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:46 smithi139 bash[43301]: cluster 2022-04-28T21:42:46.254765+0000 mgr.y (mgr.24961) 3697 : cluster [DBG] pgmap v4341: 201 pgs: 201 active+clean; 25 KiB data, 863 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:48.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:47 smithi059 bash[61152]: cluster 2022-04-28T21:42:46.966310+0000 mon.a (mon.0) 3663 : cluster [DBG] osdmap e1285: 8 total, 8 up, 8 in 2022-04-28T21:42:48.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:47 smithi059 bash[59745]: cluster 2022-04-28T21:42:46.966310+0000 mon.a (mon.0) 3663 : cluster [DBG] osdmap e1285: 8 total, 8 up, 8 in 2022-04-28T21:42:48.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:47 smithi139 bash[43301]: cluster 2022-04-28T21:42:46.966310+0000 mon.a (mon.0) 3663 : cluster [DBG] osdmap e1285: 8 total, 8 up, 8 in 2022-04-28T21:42:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:48 smithi059 bash[61152]: cluster 2022-04-28T21:42:47.987874+0000 mon.a (mon.0) 3664 : cluster [DBG] osdmap e1286: 8 total, 8 up, 8 in 2022-04-28T21:42:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:48 smithi059 bash[61152]: audit 2022-04-28T21:42:47.988734+0000 mon.a (mon.0) 3665 : audit [INF] from='client.? 172.21.15.59:0/1233654483' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:48 smithi059 bash[61152]: cluster 2022-04-28T21:42:48.255565+0000 mgr.y (mgr.24961) 3698 : cluster [DBG] pgmap v4344: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 863 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:49.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:48 smithi059 bash[59745]: cluster 2022-04-28T21:42:47.987874+0000 mon.a (mon.0) 3664 : cluster [DBG] osdmap e1286: 8 total, 8 up, 8 in 2022-04-28T21:42:49.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:48 smithi059 bash[59745]: audit 2022-04-28T21:42:47.988734+0000 mon.a (mon.0) 3665 : audit [INF] from='client.? 172.21.15.59:0/1233654483' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:49.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:48 smithi059 bash[59745]: cluster 2022-04-28T21:42:48.255565+0000 mgr.y (mgr.24961) 3698 : cluster [DBG] pgmap v4344: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 863 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:49.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:48 smithi139 bash[43301]: cluster 2022-04-28T21:42:47.987874+0000 mon.a (mon.0) 3664 : cluster [DBG] osdmap e1286: 8 total, 8 up, 8 in 2022-04-28T21:42:49.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:48 smithi139 bash[43301]: audit 2022-04-28T21:42:47.988734+0000 mon.a (mon.0) 3665 : audit [INF] from='client.? 172.21.15.59:0/1233654483' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:49.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:48 smithi139 bash[43301]: cluster 2022-04-28T21:42:48.255565+0000 mgr.y (mgr.24961) 3698 : cluster [DBG] pgmap v4344: 201 pgs: 32 unknown, 169 active+clean; 24 KiB data, 863 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:50 smithi059 bash[61152]: audit 2022-04-28T21:42:49.003540+0000 mon.a (mon.0) 3666 : audit [INF] from='client.? 172.21.15.59:0/1233654483' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-40","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:50.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:50 smithi059 bash[61152]: cluster 2022-04-28T21:42:49.003838+0000 mon.a (mon.0) 3667 : cluster [DBG] osdmap e1287: 8 total, 8 up, 8 in 2022-04-28T21:42:50.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:50 smithi059 bash[59745]: audit 2022-04-28T21:42:49.003540+0000 mon.a (mon.0) 3666 : audit [INF] from='client.? 172.21.15.59:0/1233654483' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-40","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:50.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:50 smithi059 bash[59745]: cluster 2022-04-28T21:42:49.003838+0000 mon.a (mon.0) 3667 : cluster [DBG] osdmap e1287: 8 total, 8 up, 8 in 2022-04-28T21:42:50.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:50 smithi139 bash[43301]: audit 2022-04-28T21:42:49.003540+0000 mon.a (mon.0) 3666 : audit [INF] from='client.? 172.21.15.59:0/1233654483' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-40","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:50.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:50 smithi139 bash[43301]: cluster 2022-04-28T21:42:49.003838+0000 mon.a (mon.0) 3667 : cluster [DBG] osdmap e1287: 8 total, 8 up, 8 in 2022-04-28T21:42:51.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:51 smithi059 bash[61152]: cluster 2022-04-28T21:42:50.004329+0000 mon.a (mon.0) 3668 : cluster [DBG] osdmap e1288: 8 total, 8 up, 8 in 2022-04-28T21:42:51.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:51 smithi059 bash[61152]: cluster 2022-04-28T21:42:50.256295+0000 mgr.y (mgr.24961) 3699 : cluster [DBG] pgmap v4347: 169 pgs: 169 active+clean; 24 KiB data, 877 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:51.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:51 smithi059 bash[59745]: cluster 2022-04-28T21:42:50.004329+0000 mon.a (mon.0) 3668 : cluster [DBG] osdmap e1288: 8 total, 8 up, 8 in 2022-04-28T21:42:51.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:51 smithi059 bash[59745]: cluster 2022-04-28T21:42:50.256295+0000 mgr.y (mgr.24961) 3699 : cluster [DBG] pgmap v4347: 169 pgs: 169 active+clean; 24 KiB data, 877 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:51 smithi139 bash[43301]: cluster 2022-04-28T21:42:50.004329+0000 mon.a (mon.0) 3668 : cluster [DBG] osdmap e1288: 8 total, 8 up, 8 in 2022-04-28T21:42:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:51 smithi139 bash[43301]: cluster 2022-04-28T21:42:50.256295+0000 mgr.y (mgr.24961) 3699 : cluster [DBG] pgmap v4347: 169 pgs: 169 active+clean; 24 KiB data, 877 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:52 smithi059 bash[61152]: cluster 2022-04-28T21:42:51.032882+0000 mon.a (mon.0) 3669 : cluster [DBG] osdmap e1289: 8 total, 8 up, 8 in 2022-04-28T21:42:52.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:52 smithi059 bash[61152]: audit 2022-04-28T21:42:51.035443+0000 mon.a (mon.0) 3670 : audit [INF] from='client.? 172.21.15.59:0/1060697690' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:52.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:52.063Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:52 smithi059 bash[59745]: cluster 2022-04-28T21:42:51.032882+0000 mon.a (mon.0) 3669 : cluster [DBG] osdmap e1289: 8 total, 8 up, 8 in 2022-04-28T21:42:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:52 smithi059 bash[59745]: audit 2022-04-28T21:42:51.035443+0000 mon.a (mon.0) 3670 : audit [INF] from='client.? 172.21.15.59:0/1060697690' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:52 smithi139 bash[43301]: cluster 2022-04-28T21:42:51.032882+0000 mon.a (mon.0) 3669 : cluster [DBG] osdmap e1289: 8 total, 8 up, 8 in 2022-04-28T21:42:52.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:52 smithi139 bash[43301]: audit 2022-04-28T21:42:51.035443+0000 mon.a (mon.0) 3670 : audit [INF] from='client.? 172.21.15.59:0/1060697690' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:42:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[61152]: audit 2022-04-28T21:42:52.038146+0000 mon.a (mon.0) 3671 : audit [INF] from='client.? 172.21.15.59:0/1060697690' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-41","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[61152]: cluster 2022-04-28T21:42:52.038454+0000 mon.a (mon.0) 3672 : cluster [DBG] osdmap e1290: 8 total, 8 up, 8 in 2022-04-28T21:42:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[61152]: cluster 2022-04-28T21:42:52.257035+0000 mgr.y (mgr.24961) 3700 : cluster [DBG] pgmap v4350: 201 pgs: 8 creating+peering, 24 unknown, 169 active+clean; 24 KiB data, 881 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[59745]: audit 2022-04-28T21:42:52.038146+0000 mon.a (mon.0) 3671 : audit [INF] from='client.? 172.21.15.59:0/1060697690' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-41","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[59745]: cluster 2022-04-28T21:42:52.038454+0000 mon.a (mon.0) 3672 : cluster [DBG] osdmap e1290: 8 total, 8 up, 8 in 2022-04-28T21:42:53.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[59745]: cluster 2022-04-28T21:42:52.257035+0000 mgr.y (mgr.24961) 3700 : cluster [DBG] pgmap v4350: 201 pgs: 8 creating+peering, 24 unknown, 169 active+clean; 24 KiB data, 881 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:53 smithi139 bash[43301]: audit 2022-04-28T21:42:52.038146+0000 mon.a (mon.0) 3671 : audit [INF] from='client.? 172.21.15.59:0/1060697690' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-41","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:42:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:53 smithi139 bash[43301]: cluster 2022-04-28T21:42:52.038454+0000 mon.a (mon.0) 3672 : cluster [DBG] osdmap e1290: 8 total, 8 up, 8 in 2022-04-28T21:42:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:53 smithi139 bash[43301]: cluster 2022-04-28T21:42:52.257035+0000 mgr.y (mgr.24961) 3700 : cluster [DBG] pgmap v4350: 201 pgs: 8 creating+peering, 24 unknown, 169 active+clean; 24 KiB data, 881 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:42:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:42:53.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:42:54.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: cluster 2022-04-28T21:42:53.058104+0000 mon.a (mon.0) 3673 : cluster [DBG] osdmap e1291: 8 total, 8 up, 8 in 2022-04-28T21:42:54.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: audit 2022-04-28T21:42:53.081414+0000 mon.c (mon.1) 152 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: audit 2022-04-28T21:42:53.082242+0000 mon.a (mon.0) 3674 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: audit 2022-04-28T21:42:53.082779+0000 mon.c (mon.1) 153 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: audit 2022-04-28T21:42:53.083086+0000 mon.a (mon.0) 3675 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: audit 2022-04-28T21:42:53.083631+0000 mon.c (mon.1) 154 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:42:54.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[61152]: audit 2022-04-28T21:42:53.083967+0000 mon.a (mon.0) 3676 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:42:54.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: cluster 2022-04-28T21:42:53.058104+0000 mon.a (mon.0) 3673 : cluster [DBG] osdmap e1291: 8 total, 8 up, 8 in 2022-04-28T21:42:54.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: audit 2022-04-28T21:42:53.081414+0000 mon.c (mon.1) 152 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: audit 2022-04-28T21:42:53.082242+0000 mon.a (mon.0) 3674 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: audit 2022-04-28T21:42:53.082779+0000 mon.c (mon.1) 153 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: audit 2022-04-28T21:42:53.083086+0000 mon.a (mon.0) 3675 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: audit 2022-04-28T21:42:53.083631+0000 mon.c (mon.1) 154 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:42:54.346 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:54 smithi059 bash[59745]: audit 2022-04-28T21:42:53.083967+0000 mon.a (mon.0) 3676 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:42:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: cluster 2022-04-28T21:42:53.058104+0000 mon.a (mon.0) 3673 : cluster [DBG] osdmap e1291: 8 total, 8 up, 8 in 2022-04-28T21:42:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: audit 2022-04-28T21:42:53.081414+0000 mon.c (mon.1) 152 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: audit 2022-04-28T21:42:53.082242+0000 mon.a (mon.0) 3674 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: audit 2022-04-28T21:42:53.082779+0000 mon.c (mon.1) 153 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: audit 2022-04-28T21:42:53.083086+0000 mon.a (mon.0) 3675 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:54.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: audit 2022-04-28T21:42:53.083631+0000 mon.c (mon.1) 154 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:42:54.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:54 smithi139 bash[43301]: audit 2022-04-28T21:42:53.083967+0000 mon.a (mon.0) 3676 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:42:55.383 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:55 smithi139 bash[43301]: audit 2022-04-28T21:42:54.094046+0000 mon.a (mon.0) 3677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:42:55.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:55 smithi139 bash[43301]: cluster 2022-04-28T21:42:54.094278+0000 mon.a (mon.0) 3678 : cluster [DBG] osdmap e1292: 8 total, 8 up, 8 in 2022-04-28T21:42:55.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:55 smithi139 bash[43301]: audit 2022-04-28T21:42:54.095242+0000 mon.c (mon.1) 155 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:55.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:55 smithi139 bash[43301]: audit 2022-04-28T21:42:54.095812+0000 mon.a (mon.0) 3679 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:55.384 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:55 smithi139 bash[43301]: cluster 2022-04-28T21:42:54.257823+0000 mgr.y (mgr.24961) 3701 : cluster [DBG] pgmap v4353: 169 pgs: 169 active+clean; 24 KiB data, 882 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[59745]: audit 2022-04-28T21:42:54.094046+0000 mon.a (mon.0) 3677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:42:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[59745]: cluster 2022-04-28T21:42:54.094278+0000 mon.a (mon.0) 3678 : cluster [DBG] osdmap e1292: 8 total, 8 up, 8 in 2022-04-28T21:42:55.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[59745]: audit 2022-04-28T21:42:54.095242+0000 mon.c (mon.1) 155 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[59745]: audit 2022-04-28T21:42:54.095812+0000 mon.a (mon.0) 3679 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[59745]: cluster 2022-04-28T21:42:54.257823+0000 mgr.y (mgr.24961) 3701 : cluster [DBG] pgmap v4353: 169 pgs: 169 active+clean; 24 KiB data, 882 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[61152]: audit 2022-04-28T21:42:54.094046+0000 mon.a (mon.0) 3677 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-42", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:42:55.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[61152]: cluster 2022-04-28T21:42:54.094278+0000 mon.a (mon.0) 3678 : cluster [DBG] osdmap e1292: 8 total, 8 up, 8 in 2022-04-28T21:42:55.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[61152]: audit 2022-04-28T21:42:54.095242+0000 mon.c (mon.1) 155 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:55.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[61152]: audit 2022-04-28T21:42:54.095812+0000 mon.a (mon.0) 3679 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:55.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:55 smithi059 bash[61152]: cluster 2022-04-28T21:42:54.257823+0000 mgr.y (mgr.24961) 3701 : cluster [DBG] pgmap v4353: 169 pgs: 169 active+clean; 24 KiB data, 882 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:56.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:56 smithi139 bash[43301]: cluster 2022-04-28T21:42:55.564612+0000 mon.a (mon.0) 3680 : cluster [DBG] osdmap e1293: 8 total, 8 up, 8 in 2022-04-28T21:42:56.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:56 smithi059 bash[61152]: cluster 2022-04-28T21:42:55.564612+0000 mon.a (mon.0) 3680 : cluster [DBG] osdmap e1293: 8 total, 8 up, 8 in 2022-04-28T21:42:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:56 smithi059 bash[59745]: cluster 2022-04-28T21:42:55.564612+0000 mon.a (mon.0) 3680 : cluster [DBG] osdmap e1293: 8 total, 8 up, 8 in 2022-04-28T21:42:57.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:57 smithi139 bash[43301]: audit 2022-04-28T21:42:56.106948+0000 mon.a (mon.0) 3681 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:42:57.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:57 smithi139 bash[43301]: cluster 2022-04-28T21:42:56.107172+0000 mon.a (mon.0) 3682 : cluster [DBG] osdmap e1294: 8 total, 8 up, 8 in 2022-04-28T21:42:57.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:57 smithi139 bash[43301]: cluster 2022-04-28T21:42:56.258543+0000 mgr.y (mgr.24961) 3702 : cluster [DBG] pgmap v4356: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 883 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:57.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:57 smithi139 bash[43301]: cluster 2022-04-28T21:42:57.108558+0000 mon.a (mon.0) 3683 : cluster [DBG] osdmap e1295: 8 total, 8 up, 8 in 2022-04-28T21:42:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[61152]: audit 2022-04-28T21:42:56.106948+0000 mon.a (mon.0) 3681 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:42:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[61152]: cluster 2022-04-28T21:42:56.107172+0000 mon.a (mon.0) 3682 : cluster [DBG] osdmap e1294: 8 total, 8 up, 8 in 2022-04-28T21:42:57.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[61152]: cluster 2022-04-28T21:42:56.258543+0000 mgr.y (mgr.24961) 3702 : cluster [DBG] pgmap v4356: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 883 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[61152]: cluster 2022-04-28T21:42:57.108558+0000 mon.a (mon.0) 3683 : cluster [DBG] osdmap e1295: 8 total, 8 up, 8 in 2022-04-28T21:42:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[59745]: audit 2022-04-28T21:42:56.106948+0000 mon.a (mon.0) 3681 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-42", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:42:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[59745]: cluster 2022-04-28T21:42:56.107172+0000 mon.a (mon.0) 3682 : cluster [DBG] osdmap e1294: 8 total, 8 up, 8 in 2022-04-28T21:42:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[59745]: cluster 2022-04-28T21:42:56.258543+0000 mgr.y (mgr.24961) 3702 : cluster [DBG] pgmap v4356: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 883 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:57.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:57 smithi059 bash[59745]: cluster 2022-04-28T21:42:57.108558+0000 mon.a (mon.0) 3683 : cluster [DBG] osdmap e1295: 8 total, 8 up, 8 in 2022-04-28T21:42:59.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[61152]: cluster 2022-04-28T21:42:58.127846+0000 mon.a (mon.0) 3684 : cluster [DBG] osdmap e1296: 8 total, 8 up, 8 in 2022-04-28T21:42:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[61152]: audit 2022-04-28T21:42:58.132119+0000 mon.c (mon.1) 156 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[61152]: audit 2022-04-28T21:42:58.132921+0000 mon.a (mon.0) 3685 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[61152]: cluster 2022-04-28T21:42:58.259308+0000 mgr.y (mgr.24961) 3703 : cluster [DBG] pgmap v4359: 169 pgs: 169 active+clean; 24 KiB data, 883 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[59745]: cluster 2022-04-28T21:42:58.127846+0000 mon.a (mon.0) 3684 : cluster [DBG] osdmap e1296: 8 total, 8 up, 8 in 2022-04-28T21:42:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[59745]: audit 2022-04-28T21:42:58.132119+0000 mon.c (mon.1) 156 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:59.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[59745]: audit 2022-04-28T21:42:58.132921+0000 mon.a (mon.0) 3685 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:59.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:42:59 smithi059 bash[59745]: cluster 2022-04-28T21:42:58.259308+0000 mgr.y (mgr.24961) 3703 : cluster [DBG] pgmap v4359: 169 pgs: 169 active+clean; 24 KiB data, 883 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:42:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:59 smithi139 bash[43301]: cluster 2022-04-28T21:42:58.127846+0000 mon.a (mon.0) 3684 : cluster [DBG] osdmap e1296: 8 total, 8 up, 8 in 2022-04-28T21:42:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:59 smithi139 bash[43301]: audit 2022-04-28T21:42:58.132119+0000 mon.c (mon.1) 156 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:59 smithi139 bash[43301]: audit 2022-04-28T21:42:58.132921+0000 mon.a (mon.0) 3685 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:42:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:42:59 smithi139 bash[43301]: cluster 2022-04-28T21:42:58.259308+0000 mgr.y (mgr.24961) 3703 : cluster [DBG] pgmap v4359: 169 pgs: 169 active+clean; 24 KiB data, 883 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[59745]: audit 2022-04-28T21:42:59.137366+0000 mon.a (mon.0) 3686 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:43:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[59745]: cluster 2022-04-28T21:42:59.137486+0000 mon.a (mon.0) 3687 : cluster [DBG] osdmap e1297: 8 total, 8 up, 8 in 2022-04-28T21:43:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[59745]: audit 2022-04-28T21:42:59.138883+0000 mon.c (mon.1) 157 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:43:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[59745]: audit 2022-04-28T21:42:59.139303+0000 mon.a (mon.0) 3688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:43:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[61152]: audit 2022-04-28T21:42:59.137366+0000 mon.a (mon.0) 3686 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:43:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[61152]: cluster 2022-04-28T21:42:59.137486+0000 mon.a (mon.0) 3687 : cluster [DBG] osdmap e1297: 8 total, 8 up, 8 in 2022-04-28T21:43:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[61152]: audit 2022-04-28T21:42:59.138883+0000 mon.c (mon.1) 157 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:43:00.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:00 smithi059 bash[61152]: audit 2022-04-28T21:42:59.139303+0000 mon.a (mon.0) 3688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:43:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:00 smithi139 bash[43301]: audit 2022-04-28T21:42:59.137366+0000 mon.a (mon.0) 3686 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:43:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:00 smithi139 bash[43301]: cluster 2022-04-28T21:42:59.137486+0000 mon.a (mon.0) 3687 : cluster [DBG] osdmap e1297: 8 total, 8 up, 8 in 2022-04-28T21:43:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:00 smithi139 bash[43301]: audit 2022-04-28T21:42:59.138883+0000 mon.c (mon.1) 157 : audit [INF] from='client.? 172.21.15.59:0/1779683912' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:43:00.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:00 smithi139 bash[43301]: audit 2022-04-28T21:42:59.139303+0000 mon.a (mon.0) 3688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]: dispatch 2022-04-28T21:43:01.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[61152]: audit 2022-04-28T21:43:00.146979+0000 mon.a (mon.0) 3689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:43:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[61152]: cluster 2022-04-28T21:43:00.147098+0000 mon.a (mon.0) 3690 : cluster [DBG] osdmap e1298: 8 total, 8 up, 8 in 2022-04-28T21:43:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[61152]: audit 2022-04-28T21:43:00.159045+0000 mon.a (mon.0) 3691 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[61152]: audit 2022-04-28T21:43:00.160152+0000 mon.a (mon.0) 3692 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[61152]: audit 2022-04-28T21:43:00.160620+0000 mon.a (mon.0) 3693 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-43", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:01.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[61152]: cluster 2022-04-28T21:43:00.260210+0000 mgr.y (mgr.24961) 3704 : cluster [DBG] pgmap v4362: 169 pgs: 169 active+clean; 24 KiB data, 884 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:01.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[59745]: audit 2022-04-28T21:43:00.146979+0000 mon.a (mon.0) 3689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:43:01.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[59745]: cluster 2022-04-28T21:43:00.147098+0000 mon.a (mon.0) 3690 : cluster [DBG] osdmap e1298: 8 total, 8 up, 8 in 2022-04-28T21:43:01.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[59745]: audit 2022-04-28T21:43:00.159045+0000 mon.a (mon.0) 3691 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:01.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[59745]: audit 2022-04-28T21:43:00.160152+0000 mon.a (mon.0) 3692 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:01.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[59745]: audit 2022-04-28T21:43:00.160620+0000 mon.a (mon.0) 3693 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-43", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:01.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:01 smithi059 bash[59745]: cluster 2022-04-28T21:43:00.260210+0000 mgr.y (mgr.24961) 3704 : cluster [DBG] pgmap v4362: 169 pgs: 169 active+clean; 24 KiB data, 884 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:01 smithi139 bash[43301]: audit 2022-04-28T21:43:00.146979+0000 mon.a (mon.0) 3689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-42"}]': finished 2022-04-28T21:43:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:01 smithi139 bash[43301]: cluster 2022-04-28T21:43:00.147098+0000 mon.a (mon.0) 3690 : cluster [DBG] osdmap e1298: 8 total, 8 up, 8 in 2022-04-28T21:43:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:01 smithi139 bash[43301]: audit 2022-04-28T21:43:00.159045+0000 mon.a (mon.0) 3691 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:01 smithi139 bash[43301]: audit 2022-04-28T21:43:00.160152+0000 mon.a (mon.0) 3692 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:01.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:01 smithi139 bash[43301]: audit 2022-04-28T21:43:00.160620+0000 mon.a (mon.0) 3693 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-43", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:01.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:01 smithi139 bash[43301]: cluster 2022-04-28T21:43:00.260210+0000 mgr.y (mgr.24961) 3704 : cluster [DBG] pgmap v4362: 169 pgs: 169 active+clean; 24 KiB data, 884 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:02.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[61152]: audit 2022-04-28T21:43:01.154809+0000 mon.a (mon.0) 3694 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-43", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[61152]: cluster 2022-04-28T21:43:01.155043+0000 mon.a (mon.0) 3695 : cluster [DBG] osdmap e1299: 8 total, 8 up, 8 in 2022-04-28T21:43:03.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[61152]: audit 2022-04-28T21:43:01.159783+0000 mon.a (mon.0) 3696 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-43", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[59745]: audit 2022-04-28T21:43:01.154809+0000 mon.a (mon.0) 3694 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-43", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:03.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[59745]: cluster 2022-04-28T21:43:01.155043+0000 mon.a (mon.0) 3695 : cluster [DBG] osdmap e1299: 8 total, 8 up, 8 in 2022-04-28T21:43:03.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:02 smithi059 bash[59745]: audit 2022-04-28T21:43:01.159783+0000 mon.a (mon.0) 3696 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-43", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:02 smithi139 bash[43301]: audit 2022-04-28T21:43:01.154809+0000 mon.a (mon.0) 3694 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-43", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:02 smithi139 bash[43301]: cluster 2022-04-28T21:43:01.155043+0000 mon.a (mon.0) 3695 : cluster [DBG] osdmap e1299: 8 total, 8 up, 8 in 2022-04-28T21:43:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:02 smithi139 bash[43301]: audit 2022-04-28T21:43:01.159783+0000 mon.a (mon.0) 3696 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-43", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:03.956 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[61152]: cluster 2022-04-28T21:43:02.260798+0000 mgr.y (mgr.24961) 3705 : cluster [DBG] pgmap v4365: 169 pgs: 169 active+clean; 24 KiB data, 884 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:03.956 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[61152]: cluster 2022-04-28T21:43:02.633945+0000 mon.a (mon.0) 3697 : cluster [DBG] osdmap e1300: 8 total, 8 up, 8 in 2022-04-28T21:43:03.956 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[61152]: audit 2022-04-28T21:43:03.169248+0000 mon.a (mon.0) 3698 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-43", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:03.957 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[61152]: cluster 2022-04-28T21:43:03.169379+0000 mon.a (mon.0) 3699 : cluster [DBG] osdmap e1301: 8 total, 8 up, 8 in 2022-04-28T21:43:03.957 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:03.955Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:03.958 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[59745]: cluster 2022-04-28T21:43:02.260798+0000 mgr.y (mgr.24961) 3705 : cluster [DBG] pgmap v4365: 169 pgs: 169 active+clean; 24 KiB data, 884 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:03.958 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[59745]: cluster 2022-04-28T21:43:02.633945+0000 mon.a (mon.0) 3697 : cluster [DBG] osdmap e1300: 8 total, 8 up, 8 in 2022-04-28T21:43:03.958 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[59745]: audit 2022-04-28T21:43:03.169248+0000 mon.a (mon.0) 3698 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-43", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:03.958 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:03 smithi059 bash[59745]: cluster 2022-04-28T21:43:03.169379+0000 mon.a (mon.0) 3699 : cluster [DBG] osdmap e1301: 8 total, 8 up, 8 in 2022-04-28T21:43:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:03 smithi139 bash[43301]: cluster 2022-04-28T21:43:02.260798+0000 mgr.y (mgr.24961) 3705 : cluster [DBG] pgmap v4365: 169 pgs: 169 active+clean; 24 KiB data, 884 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:03 smithi139 bash[43301]: cluster 2022-04-28T21:43:02.633945+0000 mon.a (mon.0) 3697 : cluster [DBG] osdmap e1300: 8 total, 8 up, 8 in 2022-04-28T21:43:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:03 smithi139 bash[43301]: audit 2022-04-28T21:43:03.169248+0000 mon.a (mon.0) 3698 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-43", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:03 smithi139 bash[43301]: cluster 2022-04-28T21:43:03.169379+0000 mon.a (mon.0) 3699 : cluster [DBG] osdmap e1301: 8 total, 8 up, 8 in 2022-04-28T21:43:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:05 smithi059 bash[61152]: cluster 2022-04-28T21:43:04.171927+0000 mon.a (mon.0) 3700 : cluster [DBG] osdmap e1302: 8 total, 8 up, 8 in 2022-04-28T21:43:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:05 smithi059 bash[61152]: cluster 2022-04-28T21:43:04.261565+0000 mgr.y (mgr.24961) 3706 : cluster [DBG] pgmap v4368: 177 pgs: 3 creating+peering, 5 unknown, 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:05 smithi059 bash[59745]: cluster 2022-04-28T21:43:04.171927+0000 mon.a (mon.0) 3700 : cluster [DBG] osdmap e1302: 8 total, 8 up, 8 in 2022-04-28T21:43:05.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:05 smithi059 bash[59745]: cluster 2022-04-28T21:43:04.261565+0000 mgr.y (mgr.24961) 3706 : cluster [DBG] pgmap v4368: 177 pgs: 3 creating+peering, 5 unknown, 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:05 smithi139 bash[43301]: cluster 2022-04-28T21:43:04.171927+0000 mon.a (mon.0) 3700 : cluster [DBG] osdmap e1302: 8 total, 8 up, 8 in 2022-04-28T21:43:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:05 smithi139 bash[43301]: cluster 2022-04-28T21:43:04.261565+0000 mgr.y (mgr.24961) 3706 : cluster [DBG] pgmap v4368: 177 pgs: 3 creating+peering, 5 unknown, 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:06 smithi059 bash[59745]: cluster 2022-04-28T21:43:05.173731+0000 mon.a (mon.0) 3701 : cluster [DBG] osdmap e1303: 8 total, 8 up, 8 in 2022-04-28T21:43:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:06 smithi059 bash[59745]: audit 2022-04-28T21:43:05.183560+0000 mon.a (mon.0) 3702 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:06 smithi059 bash[61152]: cluster 2022-04-28T21:43:05.173731+0000 mon.a (mon.0) 3701 : cluster [DBG] osdmap e1303: 8 total, 8 up, 8 in 2022-04-28T21:43:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:06 smithi059 bash[61152]: audit 2022-04-28T21:43:05.183560+0000 mon.a (mon.0) 3702 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:06 smithi139 bash[43301]: cluster 2022-04-28T21:43:05.173731+0000 mon.a (mon.0) 3701 : cluster [DBG] osdmap e1303: 8 total, 8 up, 8 in 2022-04-28T21:43:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:06 smithi139 bash[43301]: audit 2022-04-28T21:43:05.183560+0000 mon.a (mon.0) 3702 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[61152]: audit 2022-04-28T21:43:06.189531+0000 mon.a (mon.0) 3703 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[61152]: cluster 2022-04-28T21:43:06.189690+0000 mon.a (mon.0) 3704 : cluster [DBG] osdmap e1304: 8 total, 8 up, 8 in 2022-04-28T21:43:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[61152]: audit 2022-04-28T21:43:06.190599+0000 mon.a (mon.0) 3705 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[61152]: cluster 2022-04-28T21:43:06.262163+0000 mgr.y (mgr.24961) 3707 : cluster [DBG] pgmap v4371: 169 pgs: 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[61152]: audit 2022-04-28T21:43:07.191025+0000 mon.a (mon.0) 3706 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[61152]: cluster 2022-04-28T21:43:07.191119+0000 mon.a (mon.0) 3707 : cluster [DBG] osdmap e1305: 8 total, 8 up, 8 in 2022-04-28T21:43:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[59745]: audit 2022-04-28T21:43:06.189531+0000 mon.a (mon.0) 3703 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:07.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[59745]: cluster 2022-04-28T21:43:06.189690+0000 mon.a (mon.0) 3704 : cluster [DBG] osdmap e1304: 8 total, 8 up, 8 in 2022-04-28T21:43:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[59745]: audit 2022-04-28T21:43:06.190599+0000 mon.a (mon.0) 3705 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[59745]: cluster 2022-04-28T21:43:06.262163+0000 mgr.y (mgr.24961) 3707 : cluster [DBG] pgmap v4371: 169 pgs: 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[59745]: audit 2022-04-28T21:43:07.191025+0000 mon.a (mon.0) 3706 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:07 smithi059 bash[59745]: cluster 2022-04-28T21:43:07.191119+0000 mon.a (mon.0) 3707 : cluster [DBG] osdmap e1305: 8 total, 8 up, 8 in 2022-04-28T21:43:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:07 smithi139 bash[43301]: audit 2022-04-28T21:43:06.189531+0000 mon.a (mon.0) 3703 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:07 smithi139 bash[43301]: cluster 2022-04-28T21:43:06.189690+0000 mon.a (mon.0) 3704 : cluster [DBG] osdmap e1304: 8 total, 8 up, 8 in 2022-04-28T21:43:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:07 smithi139 bash[43301]: audit 2022-04-28T21:43:06.190599+0000 mon.a (mon.0) 3705 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]: dispatch 2022-04-28T21:43:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:07 smithi139 bash[43301]: cluster 2022-04-28T21:43:06.262163+0000 mgr.y (mgr.24961) 3707 : cluster [DBG] pgmap v4371: 169 pgs: 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:07.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:07 smithi139 bash[43301]: audit 2022-04-28T21:43:07.191025+0000 mon.a (mon.0) 3706 : audit [INF] from='client.? 172.21.15.59:0/4046206449' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-43"}]': finished 2022-04-28T21:43:07.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:07 smithi139 bash[43301]: cluster 2022-04-28T21:43:07.191119+0000 mon.a (mon.0) 3707 : cluster [DBG] osdmap e1305: 8 total, 8 up, 8 in 2022-04-28T21:43:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[61152]: audit 2022-04-28T21:43:07.207555+0000 mon.c (mon.1) 158 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[61152]: audit 2022-04-28T21:43:07.207960+0000 mon.a (mon.0) 3708 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[61152]: audit 2022-04-28T21:43:07.208687+0000 mon.c (mon.1) 159 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[61152]: audit 2022-04-28T21:43:07.209022+0000 mon.a (mon.0) 3709 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[61152]: audit 2022-04-28T21:43:07.209500+0000 mon.c (mon.1) 160 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:08.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[61152]: audit 2022-04-28T21:43:07.209800+0000 mon.a (mon.0) 3710 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:08.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[59745]: audit 2022-04-28T21:43:07.207555+0000 mon.c (mon.1) 158 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[59745]: audit 2022-04-28T21:43:07.207960+0000 mon.a (mon.0) 3708 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[59745]: audit 2022-04-28T21:43:07.208687+0000 mon.c (mon.1) 159 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[59745]: audit 2022-04-28T21:43:07.209022+0000 mon.a (mon.0) 3709 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[59745]: audit 2022-04-28T21:43:07.209500+0000 mon.c (mon.1) 160 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:08.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:08 smithi059 bash[59745]: audit 2022-04-28T21:43:07.209800+0000 mon.a (mon.0) 3710 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:08.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:08 smithi139 bash[43301]: audit 2022-04-28T21:43:07.207555+0000 mon.c (mon.1) 158 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:08 smithi139 bash[43301]: audit 2022-04-28T21:43:07.207960+0000 mon.a (mon.0) 3708 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:08 smithi139 bash[43301]: audit 2022-04-28T21:43:07.208687+0000 mon.c (mon.1) 159 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:08 smithi139 bash[43301]: audit 2022-04-28T21:43:07.209022+0000 mon.a (mon.0) 3709 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:08.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:08 smithi139 bash[43301]: audit 2022-04-28T21:43:07.209500+0000 mon.c (mon.1) 160 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:08.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:08 smithi139 bash[43301]: audit 2022-04-28T21:43:07.209800+0000 mon.a (mon.0) 3710 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[59745]: audit 2022-04-28T21:43:08.218205+0000 mon.a (mon.0) 3711 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:09.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[59745]: cluster 2022-04-28T21:43:08.218312+0000 mon.a (mon.0) 3712 : cluster [DBG] osdmap e1306: 8 total, 8 up, 8 in 2022-04-28T21:43:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[59745]: audit 2022-04-28T21:43:08.222062+0000 mon.c (mon.1) 161 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:09.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[59745]: audit 2022-04-28T21:43:08.222616+0000 mon.a (mon.0) 3713 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:09.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[59745]: cluster 2022-04-28T21:43:08.262865+0000 mgr.y (mgr.24961) 3708 : cluster [DBG] pgmap v4374: 169 pgs: 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[61152]: audit 2022-04-28T21:43:08.218205+0000 mon.a (mon.0) 3711 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[61152]: cluster 2022-04-28T21:43:08.218312+0000 mon.a (mon.0) 3712 : cluster [DBG] osdmap e1306: 8 total, 8 up, 8 in 2022-04-28T21:43:09.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[61152]: audit 2022-04-28T21:43:08.222062+0000 mon.c (mon.1) 161 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:09.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[61152]: audit 2022-04-28T21:43:08.222616+0000 mon.a (mon.0) 3713 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:09.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:09 smithi059 bash[61152]: cluster 2022-04-28T21:43:08.262865+0000 mgr.y (mgr.24961) 3708 : cluster [DBG] pgmap v4374: 169 pgs: 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:09 smithi139 bash[43301]: audit 2022-04-28T21:43:08.218205+0000 mon.a (mon.0) 3711 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-44", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:09 smithi139 bash[43301]: cluster 2022-04-28T21:43:08.218312+0000 mon.a (mon.0) 3712 : cluster [DBG] osdmap e1306: 8 total, 8 up, 8 in 2022-04-28T21:43:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:09 smithi139 bash[43301]: audit 2022-04-28T21:43:08.222062+0000 mon.c (mon.1) 161 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:09.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:09 smithi139 bash[43301]: audit 2022-04-28T21:43:08.222616+0000 mon.a (mon.0) 3713 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:09.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:09 smithi139 bash[43301]: cluster 2022-04-28T21:43:08.262865+0000 mgr.y (mgr.24961) 3708 : cluster [DBG] pgmap v4374: 169 pgs: 169 active+clean; 24 KiB data, 885 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:10 smithi059 bash[61152]: cluster 2022-04-28T21:43:09.697635+0000 mon.a (mon.0) 3714 : cluster [DBG] osdmap e1307: 8 total, 8 up, 8 in 2022-04-28T21:43:10.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:10 smithi059 bash[59745]: cluster 2022-04-28T21:43:09.697635+0000 mon.a (mon.0) 3714 : cluster [DBG] osdmap e1307: 8 total, 8 up, 8 in 2022-04-28T21:43:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:10 smithi139 bash[43301]: cluster 2022-04-28T21:43:09.697635+0000 mon.a (mon.0) 3714 : cluster [DBG] osdmap e1307: 8 total, 8 up, 8 in 2022-04-28T21:43:11.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:11 smithi059 bash[61152]: audit 2022-04-28T21:43:10.243360+0000 mon.a (mon.0) 3715 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:11.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:11 smithi059 bash[61152]: cluster 2022-04-28T21:43:10.243472+0000 mon.a (mon.0) 3716 : cluster [DBG] osdmap e1308: 8 total, 8 up, 8 in 2022-04-28T21:43:11.596 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:11 smithi059 bash[61152]: cluster 2022-04-28T21:43:10.263523+0000 mgr.y (mgr.24961) 3709 : cluster [DBG] pgmap v4377: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:11.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:11 smithi059 bash[59745]: audit 2022-04-28T21:43:10.243360+0000 mon.a (mon.0) 3715 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:11.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:11 smithi059 bash[59745]: cluster 2022-04-28T21:43:10.243472+0000 mon.a (mon.0) 3716 : cluster [DBG] osdmap e1308: 8 total, 8 up, 8 in 2022-04-28T21:43:11.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:11 smithi059 bash[59745]: cluster 2022-04-28T21:43:10.263523+0000 mgr.y (mgr.24961) 3709 : cluster [DBG] pgmap v4377: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:11 smithi139 bash[43301]: audit 2022-04-28T21:43:10.243360+0000 mon.a (mon.0) 3715 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-44", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:11 smithi139 bash[43301]: cluster 2022-04-28T21:43:10.243472+0000 mon.a (mon.0) 3716 : cluster [DBG] osdmap e1308: 8 total, 8 up, 8 in 2022-04-28T21:43:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:11 smithi139 bash[43301]: cluster 2022-04-28T21:43:10.263523+0000 mgr.y (mgr.24961) 3709 : cluster [DBG] pgmap v4377: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:12.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[61152]: cluster 2022-04-28T21:43:11.254796+0000 mon.a (mon.0) 3717 : cluster [DBG] osdmap e1309: 8 total, 8 up, 8 in 2022-04-28T21:43:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[61152]: audit 2022-04-28T21:43:11.379117+0000 mon.a (mon.0) 3718 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:43:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[61152]: audit 2022-04-28T21:43:11.382204+0000 mon.a (mon.0) 3719 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:43:12.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[61152]: audit 2022-04-28T21:43:11.384303+0000 mon.a (mon.0) 3720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:43:12.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[61152]: cephadm 2022-04-28T21:43:11.386213+0000 mgr.y (mgr.24961) 3710 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:43:12.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[61152]: audit 2022-04-28T21:43:11.708639+0000 mon.a (mon.0) 3721 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:43:12.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[59745]: cluster 2022-04-28T21:43:11.254796+0000 mon.a (mon.0) 3717 : cluster [DBG] osdmap e1309: 8 total, 8 up, 8 in 2022-04-28T21:43:12.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[59745]: audit 2022-04-28T21:43:11.379117+0000 mon.a (mon.0) 3718 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:43:12.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[59745]: audit 2022-04-28T21:43:11.382204+0000 mon.a (mon.0) 3719 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:43:12.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[59745]: audit 2022-04-28T21:43:11.384303+0000 mon.a (mon.0) 3720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:43:12.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[59745]: cephadm 2022-04-28T21:43:11.386213+0000 mgr.y (mgr.24961) 3710 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:43:12.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:12 smithi059 bash[59745]: audit 2022-04-28T21:43:11.708639+0000 mon.a (mon.0) 3721 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:43:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:12 smithi139 bash[43301]: cluster 2022-04-28T21:43:11.254796+0000 mon.a (mon.0) 3717 : cluster [DBG] osdmap e1309: 8 total, 8 up, 8 in 2022-04-28T21:43:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:12 smithi139 bash[43301]: audit 2022-04-28T21:43:11.379117+0000 mon.a (mon.0) 3718 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:43:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:12 smithi139 bash[43301]: audit 2022-04-28T21:43:11.382204+0000 mon.a (mon.0) 3719 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:43:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:12 smithi139 bash[43301]: audit 2022-04-28T21:43:11.384303+0000 mon.a (mon.0) 3720 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:43:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:12 smithi139 bash[43301]: cephadm 2022-04-28T21:43:11.386213+0000 mgr.y (mgr.24961) 3710 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:43:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:12 smithi139 bash[43301]: audit 2022-04-28T21:43:11.708639+0000 mon.a (mon.0) 3721 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:43:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[61152]: cluster 2022-04-28T21:43:12.257271+0000 mon.a (mon.0) 3722 : cluster [DBG] osdmap e1310: 8 total, 8 up, 8 in 2022-04-28T21:43:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[61152]: audit 2022-04-28T21:43:12.261294+0000 mon.c (mon.1) 162 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[61152]: cluster 2022-04-28T21:43:12.264251+0000 mgr.y (mgr.24961) 3711 : cluster [DBG] pgmap v4380: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[61152]: audit 2022-04-28T21:43:12.269692+0000 mon.a (mon.0) 3723 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[59745]: cluster 2022-04-28T21:43:12.257271+0000 mon.a (mon.0) 3722 : cluster [DBG] osdmap e1310: 8 total, 8 up, 8 in 2022-04-28T21:43:13.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[59745]: audit 2022-04-28T21:43:12.261294+0000 mon.c (mon.1) 162 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[59745]: cluster 2022-04-28T21:43:12.264251+0000 mgr.y (mgr.24961) 3711 : cluster [DBG] pgmap v4380: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[59745]: audit 2022-04-28T21:43:12.269692+0000 mon.a (mon.0) 3723 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:13 smithi139 bash[43301]: cluster 2022-04-28T21:43:12.257271+0000 mon.a (mon.0) 3722 : cluster [DBG] osdmap e1310: 8 total, 8 up, 8 in 2022-04-28T21:43:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:13 smithi139 bash[43301]: audit 2022-04-28T21:43:12.261294+0000 mon.c (mon.1) 162 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:13 smithi139 bash[43301]: cluster 2022-04-28T21:43:12.264251+0000 mgr.y (mgr.24961) 3711 : cluster [DBG] pgmap v4380: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:13 smithi139 bash[43301]: audit 2022-04-28T21:43:12.269692+0000 mon.a (mon.0) 3723 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:14.287 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:13.956Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:14.287 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[61152]: audit 2022-04-28T21:43:13.277919+0000 mon.a (mon.0) 3724 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:14.287 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[61152]: cluster 2022-04-28T21:43:13.278029+0000 mon.a (mon.0) 3725 : cluster [DBG] osdmap e1311: 8 total, 8 up, 8 in 2022-04-28T21:43:14.288 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[61152]: audit 2022-04-28T21:43:13.282546+0000 mon.c (mon.1) 163 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:14.288 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[61152]: audit 2022-04-28T21:43:13.282893+0000 mon.a (mon.0) 3726 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[59745]: audit 2022-04-28T21:43:13.277919+0000 mon.a (mon.0) 3724 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[59745]: cluster 2022-04-28T21:43:13.278029+0000 mon.a (mon.0) 3725 : cluster [DBG] osdmap e1311: 8 total, 8 up, 8 in 2022-04-28T21:43:14.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[59745]: audit 2022-04-28T21:43:13.282546+0000 mon.c (mon.1) 163 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:14.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:14 smithi059 bash[59745]: audit 2022-04-28T21:43:13.282893+0000 mon.a (mon.0) 3726 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:14 smithi139 bash[43301]: audit 2022-04-28T21:43:13.277919+0000 mon.a (mon.0) 3724 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:14.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:14 smithi139 bash[43301]: cluster 2022-04-28T21:43:13.278029+0000 mon.a (mon.0) 3725 : cluster [DBG] osdmap e1311: 8 total, 8 up, 8 in 2022-04-28T21:43:14.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:14 smithi139 bash[43301]: audit 2022-04-28T21:43:13.282546+0000 mon.c (mon.1) 163 : audit [INF] from='client.? 172.21.15.59:0/1136696927' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:14.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:14 smithi139 bash[43301]: audit 2022-04-28T21:43:13.282893+0000 mon.a (mon.0) 3726 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]: dispatch 2022-04-28T21:43:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: cluster 2022-04-28T21:43:14.265429+0000 mgr.y (mgr.24961) 3712 : cluster [DBG] pgmap v4382: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.279895+0000 mon.a (mon.0) 3727 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: cluster 2022-04-28T21:43:14.280004+0000 mon.a (mon.0) 3728 : cluster [DBG] osdmap e1312: 8 total, 8 up, 8 in 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.299305+0000 mon.c (mon.1) 164 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.299742+0000 mon.a (mon.0) 3729 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.300472+0000 mon.c (mon.1) 165 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.300779+0000 mon.a (mon.0) 3730 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.301313+0000 mon.c (mon.1) 166 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:15.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[59745]: audit 2022-04-28T21:43:14.301644+0000 mon.a (mon.0) 3731 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:15.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: cluster 2022-04-28T21:43:14.265429+0000 mgr.y (mgr.24961) 3712 : cluster [DBG] pgmap v4382: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:15.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.279895+0000 mon.a (mon.0) 3727 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:15.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: cluster 2022-04-28T21:43:14.280004+0000 mon.a (mon.0) 3728 : cluster [DBG] osdmap e1312: 8 total, 8 up, 8 in 2022-04-28T21:43:15.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.299305+0000 mon.c (mon.1) 164 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.299742+0000 mon.a (mon.0) 3729 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.300472+0000 mon.c (mon.1) 165 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.300779+0000 mon.a (mon.0) 3730 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.301313+0000 mon.c (mon.1) 166 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:15.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:15 smithi059 bash[61152]: audit 2022-04-28T21:43:14.301644+0000 mon.a (mon.0) 3731 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: cluster 2022-04-28T21:43:14.265429+0000 mgr.y (mgr.24961) 3712 : cluster [DBG] pgmap v4382: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.279895+0000 mon.a (mon.0) 3727 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-44"}]': finished 2022-04-28T21:43:15.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: cluster 2022-04-28T21:43:14.280004+0000 mon.a (mon.0) 3728 : cluster [DBG] osdmap e1312: 8 total, 8 up, 8 in 2022-04-28T21:43:15.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.299305+0000 mon.c (mon.1) 164 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.299742+0000 mon.a (mon.0) 3729 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.300472+0000 mon.c (mon.1) 165 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.300779+0000 mon.a (mon.0) 3730 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:15.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.301313+0000 mon.c (mon.1) 166 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:15.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:15 smithi139 bash[43301]: audit 2022-04-28T21:43:14.301644+0000 mon.a (mon.0) 3731 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[61152]: audit 2022-04-28T21:43:15.304441+0000 mon.a (mon.0) 3732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[61152]: cluster 2022-04-28T21:43:15.304571+0000 mon.a (mon.0) 3733 : cluster [DBG] osdmap e1313: 8 total, 8 up, 8 in 2022-04-28T21:43:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[61152]: audit 2022-04-28T21:43:15.305343+0000 mon.c (mon.1) 167 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:17.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[61152]: audit 2022-04-28T21:43:15.306890+0000 mon.a (mon.0) 3734 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[59745]: audit 2022-04-28T21:43:15.304441+0000 mon.a (mon.0) 3732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[59745]: cluster 2022-04-28T21:43:15.304571+0000 mon.a (mon.0) 3733 : cluster [DBG] osdmap e1313: 8 total, 8 up, 8 in 2022-04-28T21:43:17.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[59745]: audit 2022-04-28T21:43:15.305343+0000 mon.c (mon.1) 167 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:17.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:16 smithi059 bash[59745]: audit 2022-04-28T21:43:15.306890+0000 mon.a (mon.0) 3734 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:16 smithi139 bash[43301]: audit 2022-04-28T21:43:15.304441+0000 mon.a (mon.0) 3732 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-45", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:16 smithi139 bash[43301]: cluster 2022-04-28T21:43:15.304571+0000 mon.a (mon.0) 3733 : cluster [DBG] osdmap e1313: 8 total, 8 up, 8 in 2022-04-28T21:43:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:16 smithi139 bash[43301]: audit 2022-04-28T21:43:15.305343+0000 mon.c (mon.1) 167 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:16 smithi139 bash[43301]: audit 2022-04-28T21:43:15.306890+0000 mon.a (mon.0) 3734 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[61152]: cluster 2022-04-28T21:43:16.266486+0000 mgr.y (mgr.24961) 3713 : cluster [DBG] pgmap v4385: 169 pgs: 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[61152]: cluster 2022-04-28T21:43:16.773348+0000 mon.a (mon.0) 3735 : cluster [DBG] osdmap e1314: 8 total, 8 up, 8 in 2022-04-28T21:43:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[61152]: audit 2022-04-28T21:43:17.315144+0000 mon.a (mon.0) 3736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[61152]: cluster 2022-04-28T21:43:17.315294+0000 mon.a (mon.0) 3737 : cluster [DBG] osdmap e1315: 8 total, 8 up, 8 in 2022-04-28T21:43:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[59745]: cluster 2022-04-28T21:43:16.266486+0000 mgr.y (mgr.24961) 3713 : cluster [DBG] pgmap v4385: 169 pgs: 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[59745]: cluster 2022-04-28T21:43:16.773348+0000 mon.a (mon.0) 3735 : cluster [DBG] osdmap e1314: 8 total, 8 up, 8 in 2022-04-28T21:43:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[59745]: audit 2022-04-28T21:43:17.315144+0000 mon.a (mon.0) 3736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:18.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:17 smithi059 bash[59745]: cluster 2022-04-28T21:43:17.315294+0000 mon.a (mon.0) 3737 : cluster [DBG] osdmap e1315: 8 total, 8 up, 8 in 2022-04-28T21:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:17 smithi139 bash[43301]: cluster 2022-04-28T21:43:16.266486+0000 mgr.y (mgr.24961) 3713 : cluster [DBG] pgmap v4385: 169 pgs: 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:17 smithi139 bash[43301]: cluster 2022-04-28T21:43:16.773348+0000 mon.a (mon.0) 3735 : cluster [DBG] osdmap e1314: 8 total, 8 up, 8 in 2022-04-28T21:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:17 smithi139 bash[43301]: audit 2022-04-28T21:43:17.315144+0000 mon.a (mon.0) 3736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-45", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:17 smithi139 bash[43301]: cluster 2022-04-28T21:43:17.315294+0000 mon.a (mon.0) 3737 : cluster [DBG] osdmap e1315: 8 total, 8 up, 8 in 2022-04-28T21:43:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:19 smithi059 bash[61152]: cluster 2022-04-28T21:43:18.267273+0000 mgr.y (mgr.24961) 3714 : cluster [DBG] pgmap v4388: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:19.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:19 smithi059 bash[61152]: cluster 2022-04-28T21:43:18.315110+0000 mon.a (mon.0) 3738 : cluster [DBG] osdmap e1316: 8 total, 8 up, 8 in 2022-04-28T21:43:19.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:19 smithi059 bash[59745]: cluster 2022-04-28T21:43:18.267273+0000 mgr.y (mgr.24961) 3714 : cluster [DBG] pgmap v4388: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:19.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:19 smithi059 bash[59745]: cluster 2022-04-28T21:43:18.315110+0000 mon.a (mon.0) 3738 : cluster [DBG] osdmap e1316: 8 total, 8 up, 8 in 2022-04-28T21:43:19.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:19 smithi139 bash[43301]: cluster 2022-04-28T21:43:18.267273+0000 mgr.y (mgr.24961) 3714 : cluster [DBG] pgmap v4388: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:19.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:19 smithi139 bash[43301]: cluster 2022-04-28T21:43:18.315110+0000 mon.a (mon.0) 3738 : cluster [DBG] osdmap e1316: 8 total, 8 up, 8 in 2022-04-28T21:43:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: cluster 2022-04-28T21:43:19.320491+0000 mon.a (mon.0) 3739 : cluster [DBG] osdmap e1317: 8 total, 8 up, 8 in 2022-04-28T21:43:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.322960+0000 mon.c (mon.1) 168 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:20.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.330470+0000 mon.a (mon.0) 3740 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.347676+0000 mon.a (mon.0) 3741 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:43:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.356914+0000 mon.a (mon.0) 3742 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:43:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.370734+0000 mon.a (mon.0) 3743 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:43:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.371414+0000 mon.a (mon.0) 3744 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:43:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.371988+0000 mon.a (mon.0) 3745 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:43:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[61152]: audit 2022-04-28T21:43:19.372515+0000 mon.a (mon.0) 3746 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.d", "id": [4, 2]}]: dispatch 2022-04-28T21:43:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: cluster 2022-04-28T21:43:19.320491+0000 mon.a (mon.0) 3739 : cluster [DBG] osdmap e1317: 8 total, 8 up, 8 in 2022-04-28T21:43:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.322960+0000 mon.c (mon.1) 168 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.330470+0000 mon.a (mon.0) 3740 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.347676+0000 mon.a (mon.0) 3741 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:43:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.356914+0000 mon.a (mon.0) 3742 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:43:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.370734+0000 mon.a (mon.0) 3743 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:43:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.371414+0000 mon.a (mon.0) 3744 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:43:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.371988+0000 mon.a (mon.0) 3745 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:43:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:20 smithi059 bash[59745]: audit 2022-04-28T21:43:19.372515+0000 mon.a (mon.0) 3746 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.d", "id": [4, 2]}]: dispatch 2022-04-28T21:43:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: cluster 2022-04-28T21:43:19.320491+0000 mon.a (mon.0) 3739 : cluster [DBG] osdmap e1317: 8 total, 8 up, 8 in 2022-04-28T21:43:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.322960+0000 mon.c (mon.1) 168 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.330470+0000 mon.a (mon.0) 3740 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.347676+0000 mon.a (mon.0) 3741 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:43:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.356914+0000 mon.a (mon.0) 3742 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:43:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.370734+0000 mon.a (mon.0) 3743 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:43:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.371414+0000 mon.a (mon.0) 3744 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:43:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.371988+0000 mon.a (mon.0) 3745 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:43:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:20 smithi139 bash[43301]: audit 2022-04-28T21:43:19.372515+0000 mon.a (mon.0) 3746 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.d", "id": [4, 2]}]: dispatch 2022-04-28T21:43:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:21 smithi139 bash[43301]: cluster 2022-04-28T21:43:20.268350+0000 mgr.y (mgr.24961) 3715 : cluster [DBG] pgmap v4391: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:21 smithi139 bash[43301]: audit 2022-04-28T21:43:20.330453+0000 mon.a (mon.0) 3747 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:21.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:21 smithi139 bash[43301]: cluster 2022-04-28T21:43:20.330569+0000 mon.a (mon.0) 3748 : cluster [DBG] osdmap e1318: 8 total, 8 up, 8 in 2022-04-28T21:43:21.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:21 smithi139 bash[43301]: audit 2022-04-28T21:43:20.331575+0000 mon.c (mon.1) 169 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:21.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:21 smithi139 bash[43301]: audit 2022-04-28T21:43:20.335870+0000 mon.a (mon.0) 3749 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:21.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[59745]: cluster 2022-04-28T21:43:20.268350+0000 mgr.y (mgr.24961) 3715 : cluster [DBG] pgmap v4391: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[59745]: audit 2022-04-28T21:43:20.330453+0000 mon.a (mon.0) 3747 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[59745]: cluster 2022-04-28T21:43:20.330569+0000 mon.a (mon.0) 3748 : cluster [DBG] osdmap e1318: 8 total, 8 up, 8 in 2022-04-28T21:43:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[59745]: audit 2022-04-28T21:43:20.331575+0000 mon.c (mon.1) 169 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[59745]: audit 2022-04-28T21:43:20.335870+0000 mon.a (mon.0) 3749 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:21.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[61152]: cluster 2022-04-28T21:43:20.268350+0000 mgr.y (mgr.24961) 3715 : cluster [DBG] pgmap v4391: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:21.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[61152]: audit 2022-04-28T21:43:20.330453+0000 mon.a (mon.0) 3747 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:21.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[61152]: cluster 2022-04-28T21:43:20.330569+0000 mon.a (mon.0) 3748 : cluster [DBG] osdmap e1318: 8 total, 8 up, 8 in 2022-04-28T21:43:21.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[61152]: audit 2022-04-28T21:43:20.331575+0000 mon.c (mon.1) 169 : audit [INF] from='client.? 172.21.15.59:0/2187430128' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:21.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:21 smithi059 bash[61152]: audit 2022-04-28T21:43:20.335870+0000 mon.a (mon.0) 3749 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]: dispatch 2022-04-28T21:43:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:22.066Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:22 smithi139 bash[43301]: audit 2022-04-28T21:43:21.338964+0000 mon.a (mon.0) 3750 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:22 smithi139 bash[43301]: cluster 2022-04-28T21:43:21.339219+0000 mon.a (mon.0) 3751 : cluster [DBG] osdmap e1319: 8 total, 8 up, 8 in 2022-04-28T21:43:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:22 smithi139 bash[43301]: audit 2022-04-28T21:43:21.352000+0000 mon.a (mon.0) 3752 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:22.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:22 smithi139 bash[43301]: audit 2022-04-28T21:43:21.352837+0000 mon.a (mon.0) 3753 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:22.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:22 smithi139 bash[43301]: audit 2022-04-28T21:43:21.353253+0000 mon.a (mon.0) 3754 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-46", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:22.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[59745]: audit 2022-04-28T21:43:21.338964+0000 mon.a (mon.0) 3750 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:22.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[59745]: cluster 2022-04-28T21:43:21.339219+0000 mon.a (mon.0) 3751 : cluster [DBG] osdmap e1319: 8 total, 8 up, 8 in 2022-04-28T21:43:22.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[59745]: audit 2022-04-28T21:43:21.352000+0000 mon.a (mon.0) 3752 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:22.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[59745]: audit 2022-04-28T21:43:21.352837+0000 mon.a (mon.0) 3753 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:22.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[59745]: audit 2022-04-28T21:43:21.353253+0000 mon.a (mon.0) 3754 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-46", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:22.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[61152]: audit 2022-04-28T21:43:21.338964+0000 mon.a (mon.0) 3750 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-45"}]': finished 2022-04-28T21:43:22.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[61152]: cluster 2022-04-28T21:43:21.339219+0000 mon.a (mon.0) 3751 : cluster [DBG] osdmap e1319: 8 total, 8 up, 8 in 2022-04-28T21:43:22.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[61152]: audit 2022-04-28T21:43:21.352000+0000 mon.a (mon.0) 3752 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:22.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[61152]: audit 2022-04-28T21:43:21.352837+0000 mon.a (mon.0) 3753 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:22.849 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:22 smithi059 bash[61152]: audit 2022-04-28T21:43:21.353253+0000 mon.a (mon.0) 3754 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-46", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:24.091 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:23.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:24.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[61152]: cluster 2022-04-28T21:43:22.269101+0000 mgr.y (mgr.24961) 3716 : cluster [DBG] pgmap v4394: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[61152]: audit 2022-04-28T21:43:22.359255+0000 mon.a (mon.0) 3755 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-46", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[61152]: cluster 2022-04-28T21:43:22.359444+0000 mon.a (mon.0) 3756 : cluster [DBG] osdmap e1320: 8 total, 8 up, 8 in 2022-04-28T21:43:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[61152]: audit 2022-04-28T21:43:22.360682+0000 mon.a (mon.0) 3757 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-46", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[59745]: cluster 2022-04-28T21:43:22.269101+0000 mgr.y (mgr.24961) 3716 : cluster [DBG] pgmap v4394: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:24.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[59745]: audit 2022-04-28T21:43:22.359255+0000 mon.a (mon.0) 3755 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-46", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[59745]: cluster 2022-04-28T21:43:22.359444+0000 mon.a (mon.0) 3756 : cluster [DBG] osdmap e1320: 8 total, 8 up, 8 in 2022-04-28T21:43:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:23 smithi059 bash[59745]: audit 2022-04-28T21:43:22.360682+0000 mon.a (mon.0) 3757 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-46", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:23 smithi139 bash[43301]: cluster 2022-04-28T21:43:22.269101+0000 mgr.y (mgr.24961) 3716 : cluster [DBG] pgmap v4394: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:23 smithi139 bash[43301]: audit 2022-04-28T21:43:22.359255+0000 mon.a (mon.0) 3755 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-46", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:23 smithi139 bash[43301]: cluster 2022-04-28T21:43:22.359444+0000 mon.a (mon.0) 3756 : cluster [DBG] osdmap e1320: 8 total, 8 up, 8 in 2022-04-28T21:43:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:23 smithi139 bash[43301]: audit 2022-04-28T21:43:22.360682+0000 mon.a (mon.0) 3757 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-46", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:25.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:24 smithi139 bash[43301]: cluster 2022-04-28T21:43:23.834097+0000 mon.a (mon.0) 3758 : cluster [DBG] osdmap e1321: 8 total, 8 up, 8 in 2022-04-28T21:43:25.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:24 smithi139 bash[43301]: cluster 2022-04-28T21:43:24.270382+0000 mgr.y (mgr.24961) 3717 : cluster [DBG] pgmap v4397: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:25.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:24 smithi139 bash[43301]: audit 2022-04-28T21:43:24.365250+0000 mon.a (mon.0) 3759 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-46", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:25.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:24 smithi139 bash[43301]: cluster 2022-04-28T21:43:24.365561+0000 mon.a (mon.0) 3760 : cluster [DBG] osdmap e1322: 8 total, 8 up, 8 in 2022-04-28T21:43:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[61152]: cluster 2022-04-28T21:43:23.834097+0000 mon.a (mon.0) 3758 : cluster [DBG] osdmap e1321: 8 total, 8 up, 8 in 2022-04-28T21:43:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[61152]: cluster 2022-04-28T21:43:24.270382+0000 mgr.y (mgr.24961) 3717 : cluster [DBG] pgmap v4397: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[61152]: audit 2022-04-28T21:43:24.365250+0000 mon.a (mon.0) 3759 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-46", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:25.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[61152]: cluster 2022-04-28T21:43:24.365561+0000 mon.a (mon.0) 3760 : cluster [DBG] osdmap e1322: 8 total, 8 up, 8 in 2022-04-28T21:43:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[59745]: cluster 2022-04-28T21:43:23.834097+0000 mon.a (mon.0) 3758 : cluster [DBG] osdmap e1321: 8 total, 8 up, 8 in 2022-04-28T21:43:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[59745]: cluster 2022-04-28T21:43:24.270382+0000 mgr.y (mgr.24961) 3717 : cluster [DBG] pgmap v4397: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[59745]: audit 2022-04-28T21:43:24.365250+0000 mon.a (mon.0) 3759 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-46", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:25.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:24 smithi059 bash[59745]: cluster 2022-04-28T21:43:24.365561+0000 mon.a (mon.0) 3760 : cluster [DBG] osdmap e1322: 8 total, 8 up, 8 in 2022-04-28T21:43:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:26 smithi139 bash[43301]: cluster 2022-04-28T21:43:25.370743+0000 mon.a (mon.0) 3761 : cluster [DBG] osdmap e1323: 8 total, 8 up, 8 in 2022-04-28T21:43:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:26 smithi059 bash[59745]: cluster 2022-04-28T21:43:25.370743+0000 mon.a (mon.0) 3761 : cluster [DBG] osdmap e1323: 8 total, 8 up, 8 in 2022-04-28T21:43:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:26 smithi059 bash[61152]: cluster 2022-04-28T21:43:25.370743+0000 mon.a (mon.0) 3761 : cluster [DBG] osdmap e1323: 8 total, 8 up, 8 in 2022-04-28T21:43:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[61152]: cluster 2022-04-28T21:43:26.271695+0000 mgr.y (mgr.24961) 3718 : cluster [DBG] pgmap v4400: 177 pgs: 3 creating+peering, 2 unknown, 172 active+clean; 25 KiB data, 890 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:43:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[61152]: cluster 2022-04-28T21:43:26.364334+0000 mon.a (mon.0) 3762 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2022-04-28T21:43:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[61152]: cluster 2022-04-28T21:43:26.377150+0000 mon.a (mon.0) 3763 : cluster [DBG] osdmap e1324: 8 total, 8 up, 8 in 2022-04-28T21:43:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[61152]: audit 2022-04-28T21:43:26.378806+0000 mon.a (mon.0) 3764 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[59745]: cluster 2022-04-28T21:43:26.271695+0000 mgr.y (mgr.24961) 3718 : cluster [DBG] pgmap v4400: 177 pgs: 3 creating+peering, 2 unknown, 172 active+clean; 25 KiB data, 890 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:43:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[59745]: cluster 2022-04-28T21:43:26.364334+0000 mon.a (mon.0) 3762 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2022-04-28T21:43:27.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[59745]: cluster 2022-04-28T21:43:26.377150+0000 mon.a (mon.0) 3763 : cluster [DBG] osdmap e1324: 8 total, 8 up, 8 in 2022-04-28T21:43:27.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:27 smithi059 bash[59745]: audit 2022-04-28T21:43:26.378806+0000 mon.a (mon.0) 3764 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:27 smithi139 bash[43301]: cluster 2022-04-28T21:43:26.271695+0000 mgr.y (mgr.24961) 3718 : cluster [DBG] pgmap v4400: 177 pgs: 3 creating+peering, 2 unknown, 172 active+clean; 25 KiB data, 890 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2022-04-28T21:43:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:27 smithi139 bash[43301]: cluster 2022-04-28T21:43:26.364334+0000 mon.a (mon.0) 3762 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2022-04-28T21:43:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:27 smithi139 bash[43301]: cluster 2022-04-28T21:43:26.377150+0000 mon.a (mon.0) 3763 : cluster [DBG] osdmap e1324: 8 total, 8 up, 8 in 2022-04-28T21:43:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:27 smithi139 bash[43301]: audit 2022-04-28T21:43:26.378806+0000 mon.a (mon.0) 3764 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:28 smithi059 bash[61152]: audit 2022-04-28T21:43:27.385154+0000 mon.a (mon.0) 3765 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:28 smithi059 bash[61152]: cluster 2022-04-28T21:43:27.385326+0000 mon.a (mon.0) 3766 : cluster [DBG] osdmap e1325: 8 total, 8 up, 8 in 2022-04-28T21:43:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:28 smithi059 bash[61152]: audit 2022-04-28T21:43:27.387434+0000 mon.a (mon.0) 3767 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:28 smithi059 bash[59745]: audit 2022-04-28T21:43:27.385154+0000 mon.a (mon.0) 3765 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:28 smithi059 bash[59745]: cluster 2022-04-28T21:43:27.385326+0000 mon.a (mon.0) 3766 : cluster [DBG] osdmap e1325: 8 total, 8 up, 8 in 2022-04-28T21:43:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:28 smithi059 bash[59745]: audit 2022-04-28T21:43:27.387434+0000 mon.a (mon.0) 3767 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:28 smithi139 bash[43301]: audit 2022-04-28T21:43:27.385154+0000 mon.a (mon.0) 3765 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:28 smithi139 bash[43301]: cluster 2022-04-28T21:43:27.385326+0000 mon.a (mon.0) 3766 : cluster [DBG] osdmap e1325: 8 total, 8 up, 8 in 2022-04-28T21:43:28.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:28 smithi139 bash[43301]: audit 2022-04-28T21:43:27.387434+0000 mon.a (mon.0) 3767 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]: dispatch 2022-04-28T21:43:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[59745]: cluster 2022-04-28T21:43:28.272554+0000 mgr.y (mgr.24961) 3719 : cluster [DBG] pgmap v4403: 169 pgs: 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[59745]: audit 2022-04-28T21:43:28.390889+0000 mon.a (mon.0) 3768 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[59745]: cluster 2022-04-28T21:43:28.391064+0000 mon.a (mon.0) 3769 : cluster [DBG] osdmap e1326: 8 total, 8 up, 8 in 2022-04-28T21:43:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[59745]: cluster 2022-04-28T21:43:28.882911+0000 mon.a (mon.0) 3770 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2022-04-28T21:43:29.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[59745]: cluster 2022-04-28T21:43:28.883011+0000 mon.a (mon.0) 3771 : cluster [INF] Cluster is now healthy 2022-04-28T21:43:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[61152]: cluster 2022-04-28T21:43:28.272554+0000 mgr.y (mgr.24961) 3719 : cluster [DBG] pgmap v4403: 169 pgs: 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[61152]: audit 2022-04-28T21:43:28.390889+0000 mon.a (mon.0) 3768 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:29.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[61152]: cluster 2022-04-28T21:43:28.391064+0000 mon.a (mon.0) 3769 : cluster [DBG] osdmap e1326: 8 total, 8 up, 8 in 2022-04-28T21:43:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[61152]: cluster 2022-04-28T21:43:28.882911+0000 mon.a (mon.0) 3770 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2022-04-28T21:43:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:29 smithi059 bash[61152]: cluster 2022-04-28T21:43:28.883011+0000 mon.a (mon.0) 3771 : cluster [INF] Cluster is now healthy 2022-04-28T21:43:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:29 smithi139 bash[43301]: cluster 2022-04-28T21:43:28.272554+0000 mgr.y (mgr.24961) 3719 : cluster [DBG] pgmap v4403: 169 pgs: 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:29.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:29 smithi139 bash[43301]: audit 2022-04-28T21:43:28.390889+0000 mon.a (mon.0) 3768 : audit [INF] from='client.? 172.21.15.59:0/3783646430' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-46"}]': finished 2022-04-28T21:43:29.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:29 smithi139 bash[43301]: cluster 2022-04-28T21:43:28.391064+0000 mon.a (mon.0) 3769 : cluster [DBG] osdmap e1326: 8 total, 8 up, 8 in 2022-04-28T21:43:29.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:29 smithi139 bash[43301]: cluster 2022-04-28T21:43:28.882911+0000 mon.a (mon.0) 3770 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2022-04-28T21:43:29.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:29 smithi139 bash[43301]: cluster 2022-04-28T21:43:28.883011+0000 mon.a (mon.0) 3771 : cluster [INF] Cluster is now healthy 2022-04-28T21:43:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:30 smithi059 bash[61152]: cluster 2022-04-28T21:43:29.414038+0000 mon.a (mon.0) 3772 : cluster [DBG] osdmap e1327: 8 total, 8 up, 8 in 2022-04-28T21:43:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:30 smithi059 bash[61152]: audit 2022-04-28T21:43:29.417421+0000 mon.a (mon.0) 3773 : audit [INF] from='client.? 172.21.15.59:0/1842650' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-47","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:43:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:30 smithi059 bash[59745]: cluster 2022-04-28T21:43:29.414038+0000 mon.a (mon.0) 3772 : cluster [DBG] osdmap e1327: 8 total, 8 up, 8 in 2022-04-28T21:43:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:30 smithi059 bash[59745]: audit 2022-04-28T21:43:29.417421+0000 mon.a (mon.0) 3773 : audit [INF] from='client.? 172.21.15.59:0/1842650' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-47","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:43:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:30 smithi139 bash[43301]: cluster 2022-04-28T21:43:29.414038+0000 mon.a (mon.0) 3772 : cluster [DBG] osdmap e1327: 8 total, 8 up, 8 in 2022-04-28T21:43:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:30 smithi139 bash[43301]: audit 2022-04-28T21:43:29.417421+0000 mon.a (mon.0) 3773 : audit [INF] from='client.? 172.21.15.59:0/1842650' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-47","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:43:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:31 smithi059 bash[59745]: cluster 2022-04-28T21:43:30.274000+0000 mgr.y (mgr.24961) 3720 : cluster [DBG] pgmap v4406: 201 pgs: 11 creating+peering, 21 unknown, 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:31 smithi059 bash[59745]: audit 2022-04-28T21:43:30.424711+0000 mon.a (mon.0) 3774 : audit [INF] from='client.? 172.21.15.59:0/1842650' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-47","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:43:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:31 smithi059 bash[59745]: cluster 2022-04-28T21:43:30.425084+0000 mon.a (mon.0) 3775 : cluster [DBG] osdmap e1328: 8 total, 8 up, 8 in 2022-04-28T21:43:31.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:31 smithi059 bash[61152]: cluster 2022-04-28T21:43:30.274000+0000 mgr.y (mgr.24961) 3720 : cluster [DBG] pgmap v4406: 201 pgs: 11 creating+peering, 21 unknown, 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:31.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:31 smithi059 bash[61152]: audit 2022-04-28T21:43:30.424711+0000 mon.a (mon.0) 3774 : audit [INF] from='client.? 172.21.15.59:0/1842650' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-47","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:43:31.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:31 smithi059 bash[61152]: cluster 2022-04-28T21:43:30.425084+0000 mon.a (mon.0) 3775 : cluster [DBG] osdmap e1328: 8 total, 8 up, 8 in 2022-04-28T21:43:32.020 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:31 smithi139 bash[43301]: cluster 2022-04-28T21:43:30.274000+0000 mgr.y (mgr.24961) 3720 : cluster [DBG] pgmap v4406: 201 pgs: 11 creating+peering, 21 unknown, 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:32.021 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:31 smithi139 bash[43301]: audit 2022-04-28T21:43:30.424711+0000 mon.a (mon.0) 3774 : audit [INF] from='client.? 172.21.15.59:0/1842650' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-47","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:43:32.021 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:31 smithi139 bash[43301]: cluster 2022-04-28T21:43:30.425084+0000 mon.a (mon.0) 3775 : cluster [DBG] osdmap e1328: 8 total, 8 up, 8 in 2022-04-28T21:43:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:32.067Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:32.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: cluster 2022-04-28T21:43:31.429492+0000 mon.a (mon.0) 3776 : cluster [DBG] osdmap e1329: 8 total, 8 up, 8 in 2022-04-28T21:43:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: audit 2022-04-28T21:43:31.451544+0000 mon.c (mon.1) 170 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: audit 2022-04-28T21:43:31.452100+0000 mon.a (mon.0) 3777 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: audit 2022-04-28T21:43:31.452762+0000 mon.c (mon.1) 171 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: audit 2022-04-28T21:43:31.453132+0000 mon.a (mon.0) 3778 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: audit 2022-04-28T21:43:31.453686+0000 mon.c (mon.1) 172 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:32.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[59745]: audit 2022-04-28T21:43:31.454078+0000 mon.a (mon.0) 3779 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:32.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: cluster 2022-04-28T21:43:31.429492+0000 mon.a (mon.0) 3776 : cluster [DBG] osdmap e1329: 8 total, 8 up, 8 in 2022-04-28T21:43:32.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: audit 2022-04-28T21:43:31.451544+0000 mon.c (mon.1) 170 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: audit 2022-04-28T21:43:31.452100+0000 mon.a (mon.0) 3777 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: audit 2022-04-28T21:43:31.452762+0000 mon.c (mon.1) 171 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: audit 2022-04-28T21:43:31.453132+0000 mon.a (mon.0) 3778 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: audit 2022-04-28T21:43:31.453686+0000 mon.c (mon.1) 172 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:32.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:32 smithi059 bash[61152]: audit 2022-04-28T21:43:31.454078+0000 mon.a (mon.0) 3779 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: cluster 2022-04-28T21:43:31.429492+0000 mon.a (mon.0) 3776 : cluster [DBG] osdmap e1329: 8 total, 8 up, 8 in 2022-04-28T21:43:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: audit 2022-04-28T21:43:31.451544+0000 mon.c (mon.1) 170 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: audit 2022-04-28T21:43:31.452100+0000 mon.a (mon.0) 3777 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: audit 2022-04-28T21:43:31.452762+0000 mon.c (mon.1) 171 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: audit 2022-04-28T21:43:31.453132+0000 mon.a (mon.0) 3778 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:32.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: audit 2022-04-28T21:43:31.453686+0000 mon.c (mon.1) 172 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:32.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:32 smithi139 bash[43301]: audit 2022-04-28T21:43:31.454078+0000 mon.a (mon.0) 3779 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:33.957Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[61152]: cluster 2022-04-28T21:43:32.274791+0000 mgr.y (mgr.24961) 3721 : cluster [DBG] pgmap v4409: 169 pgs: 169 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[61152]: audit 2022-04-28T21:43:32.455559+0000 mon.a (mon.0) 3780 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[61152]: cluster 2022-04-28T21:43:32.455699+0000 mon.a (mon.0) 3781 : cluster [DBG] osdmap e1330: 8 total, 8 up, 8 in 2022-04-28T21:43:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[61152]: audit 2022-04-28T21:43:32.456542+0000 mon.c (mon.1) 173 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:34.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[61152]: audit 2022-04-28T21:43:32.457124+0000 mon.a (mon.0) 3782 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:34.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[59745]: cluster 2022-04-28T21:43:32.274791+0000 mgr.y (mgr.24961) 3721 : cluster [DBG] pgmap v4409: 169 pgs: 169 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:34.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[59745]: audit 2022-04-28T21:43:32.455559+0000 mon.a (mon.0) 3780 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:34.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[59745]: cluster 2022-04-28T21:43:32.455699+0000 mon.a (mon.0) 3781 : cluster [DBG] osdmap e1330: 8 total, 8 up, 8 in 2022-04-28T21:43:34.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[59745]: audit 2022-04-28T21:43:32.456542+0000 mon.c (mon.1) 173 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:34.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:33 smithi059 bash[59745]: audit 2022-04-28T21:43:32.457124+0000 mon.a (mon.0) 3782 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:33 smithi139 bash[43301]: cluster 2022-04-28T21:43:32.274791+0000 mgr.y (mgr.24961) 3721 : cluster [DBG] pgmap v4409: 169 pgs: 169 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:33 smithi139 bash[43301]: audit 2022-04-28T21:43:32.455559+0000 mon.a (mon.0) 3780 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-48", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:33 smithi139 bash[43301]: cluster 2022-04-28T21:43:32.455699+0000 mon.a (mon.0) 3781 : cluster [DBG] osdmap e1330: 8 total, 8 up, 8 in 2022-04-28T21:43:34.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:33 smithi139 bash[43301]: audit 2022-04-28T21:43:32.456542+0000 mon.c (mon.1) 173 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:34.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:33 smithi139 bash[43301]: audit 2022-04-28T21:43:32.457124+0000 mon.a (mon.0) 3782 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[61152]: cluster 2022-04-28T21:43:33.923648+0000 mon.a (mon.0) 3783 : cluster [DBG] osdmap e1331: 8 total, 8 up, 8 in 2022-04-28T21:43:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[61152]: cluster 2022-04-28T21:43:34.276017+0000 mgr.y (mgr.24961) 3722 : cluster [DBG] pgmap v4412: 169 pgs: 169 active+clean; 24 KiB data, 900 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[61152]: audit 2022-04-28T21:43:34.462395+0000 mon.a (mon.0) 3784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:35.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[61152]: cluster 2022-04-28T21:43:34.462763+0000 mon.a (mon.0) 3785 : cluster [DBG] osdmap e1332: 8 total, 8 up, 8 in 2022-04-28T21:43:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[59745]: cluster 2022-04-28T21:43:33.923648+0000 mon.a (mon.0) 3783 : cluster [DBG] osdmap e1331: 8 total, 8 up, 8 in 2022-04-28T21:43:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[59745]: cluster 2022-04-28T21:43:34.276017+0000 mgr.y (mgr.24961) 3722 : cluster [DBG] pgmap v4412: 169 pgs: 169 active+clean; 24 KiB data, 900 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:35.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[59745]: audit 2022-04-28T21:43:34.462395+0000 mon.a (mon.0) 3784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:35.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:34 smithi059 bash[59745]: cluster 2022-04-28T21:43:34.462763+0000 mon.a (mon.0) 3785 : cluster [DBG] osdmap e1332: 8 total, 8 up, 8 in 2022-04-28T21:43:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:34 smithi139 bash[43301]: cluster 2022-04-28T21:43:33.923648+0000 mon.a (mon.0) 3783 : cluster [DBG] osdmap e1331: 8 total, 8 up, 8 in 2022-04-28T21:43:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:34 smithi139 bash[43301]: cluster 2022-04-28T21:43:34.276017+0000 mgr.y (mgr.24961) 3722 : cluster [DBG] pgmap v4412: 169 pgs: 169 active+clean; 24 KiB data, 900 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:34 smithi139 bash[43301]: audit 2022-04-28T21:43:34.462395+0000 mon.a (mon.0) 3784 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-48", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:34 smithi139 bash[43301]: cluster 2022-04-28T21:43:34.462763+0000 mon.a (mon.0) 3785 : cluster [DBG] osdmap e1332: 8 total, 8 up, 8 in 2022-04-28T21:43:36.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:36 smithi059 bash[61152]: cluster 2022-04-28T21:43:35.462301+0000 mon.a (mon.0) 3786 : cluster [DBG] osdmap e1333: 8 total, 8 up, 8 in 2022-04-28T21:43:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:36 smithi059 bash[59745]: cluster 2022-04-28T21:43:35.462301+0000 mon.a (mon.0) 3786 : cluster [DBG] osdmap e1333: 8 total, 8 up, 8 in 2022-04-28T21:43:36.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:36 smithi139 bash[43301]: cluster 2022-04-28T21:43:35.462301+0000 mon.a (mon.0) 3786 : cluster [DBG] osdmap e1333: 8 total, 8 up, 8 in 2022-04-28T21:43:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[61152]: cluster 2022-04-28T21:43:36.277120+0000 mgr.y (mgr.24961) 3723 : cluster [DBG] pgmap v4415: 177 pgs: 4 creating+peering, 1 unknown, 172 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[61152]: cluster 2022-04-28T21:43:36.476934+0000 mon.a (mon.0) 3787 : cluster [DBG] osdmap e1334: 8 total, 8 up, 8 in 2022-04-28T21:43:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[61152]: audit 2022-04-28T21:43:36.478283+0000 mon.c (mon.1) 174 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:37.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[61152]: audit 2022-04-28T21:43:36.479081+0000 mon.a (mon.0) 3788 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:37.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[59745]: cluster 2022-04-28T21:43:36.277120+0000 mgr.y (mgr.24961) 3723 : cluster [DBG] pgmap v4415: 177 pgs: 4 creating+peering, 1 unknown, 172 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:37.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[59745]: cluster 2022-04-28T21:43:36.476934+0000 mon.a (mon.0) 3787 : cluster [DBG] osdmap e1334: 8 total, 8 up, 8 in 2022-04-28T21:43:37.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[59745]: audit 2022-04-28T21:43:36.478283+0000 mon.c (mon.1) 174 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:37.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:37 smithi059 bash[59745]: audit 2022-04-28T21:43:36.479081+0000 mon.a (mon.0) 3788 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:37.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:37 smithi139 bash[43301]: cluster 2022-04-28T21:43:36.277120+0000 mgr.y (mgr.24961) 3723 : cluster [DBG] pgmap v4415: 177 pgs: 4 creating+peering, 1 unknown, 172 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:37.882 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:37 smithi139 bash[43301]: cluster 2022-04-28T21:43:36.476934+0000 mon.a (mon.0) 3787 : cluster [DBG] osdmap e1334: 8 total, 8 up, 8 in 2022-04-28T21:43:37.883 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:37 smithi139 bash[43301]: audit 2022-04-28T21:43:36.478283+0000 mon.c (mon.1) 174 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:37.883 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:37 smithi139 bash[43301]: audit 2022-04-28T21:43:36.479081+0000 mon.a (mon.0) 3788 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[59745]: audit 2022-04-28T21:43:37.474025+0000 mon.a (mon.0) 3789 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[59745]: cluster 2022-04-28T21:43:37.474119+0000 mon.a (mon.0) 3790 : cluster [DBG] osdmap e1335: 8 total, 8 up, 8 in 2022-04-28T21:43:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[59745]: audit 2022-04-28T21:43:37.475051+0000 mon.c (mon.1) 175 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:38.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[59745]: audit 2022-04-28T21:43:37.475751+0000 mon.a (mon.0) 3791 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:38.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[61152]: audit 2022-04-28T21:43:37.474025+0000 mon.a (mon.0) 3789 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:38.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[61152]: cluster 2022-04-28T21:43:37.474119+0000 mon.a (mon.0) 3790 : cluster [DBG] osdmap e1335: 8 total, 8 up, 8 in 2022-04-28T21:43:38.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[61152]: audit 2022-04-28T21:43:37.475051+0000 mon.c (mon.1) 175 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:38.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:38 smithi059 bash[61152]: audit 2022-04-28T21:43:37.475751+0000 mon.a (mon.0) 3791 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:38 smithi139 bash[43301]: audit 2022-04-28T21:43:37.474025+0000 mon.a (mon.0) 3789 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:38 smithi139 bash[43301]: cluster 2022-04-28T21:43:37.474119+0000 mon.a (mon.0) 3790 : cluster [DBG] osdmap e1335: 8 total, 8 up, 8 in 2022-04-28T21:43:38.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:38 smithi139 bash[43301]: audit 2022-04-28T21:43:37.475051+0000 mon.c (mon.1) 175 : audit [INF] from='client.? 172.21.15.59:0/526769345' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:38.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:38 smithi139 bash[43301]: audit 2022-04-28T21:43:37.475751+0000 mon.a (mon.0) 3791 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]: dispatch 2022-04-28T21:43:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: cluster 2022-04-28T21:43:38.278044+0000 mgr.y (mgr.24961) 3724 : cluster [DBG] pgmap v4418: 169 pgs: 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.478186+0000 mon.a (mon.0) 3792 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: cluster 2022-04-28T21:43:38.478352+0000 mon.a (mon.0) 3793 : cluster [DBG] osdmap e1336: 8 total, 8 up, 8 in 2022-04-28T21:43:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.492864+0000 mon.c (mon.1) 176 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.493264+0000 mon.a (mon.0) 3794 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.494014+0000 mon.c (mon.1) 177 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.494348+0000 mon.a (mon.0) 3795 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.494895+0000 mon.c (mon.1) 178 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:39.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[61152]: audit 2022-04-28T21:43:38.495202+0000 mon.a (mon.0) 3796 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:39.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: cluster 2022-04-28T21:43:38.278044+0000 mgr.y (mgr.24961) 3724 : cluster [DBG] pgmap v4418: 169 pgs: 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:39.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.478186+0000 mon.a (mon.0) 3792 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:39.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: cluster 2022-04-28T21:43:38.478352+0000 mon.a (mon.0) 3793 : cluster [DBG] osdmap e1336: 8 total, 8 up, 8 in 2022-04-28T21:43:39.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.492864+0000 mon.c (mon.1) 176 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.493264+0000 mon.a (mon.0) 3794 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.494014+0000 mon.c (mon.1) 177 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.494348+0000 mon.a (mon.0) 3795 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.494895+0000 mon.c (mon.1) 178 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:39.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:39 smithi059 bash[59745]: audit 2022-04-28T21:43:38.495202+0000 mon.a (mon.0) 3796 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: cluster 2022-04-28T21:43:38.278044+0000 mgr.y (mgr.24961) 3724 : cluster [DBG] pgmap v4418: 169 pgs: 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.478186+0000 mon.a (mon.0) 3792 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-48"}]': finished 2022-04-28T21:43:39.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: cluster 2022-04-28T21:43:38.478352+0000 mon.a (mon.0) 3793 : cluster [DBG] osdmap e1336: 8 total, 8 up, 8 in 2022-04-28T21:43:39.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.492864+0000 mon.c (mon.1) 176 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.493264+0000 mon.a (mon.0) 3794 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.494014+0000 mon.c (mon.1) 177 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.494348+0000 mon.a (mon.0) 3795 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:39.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.494895+0000 mon.c (mon.1) 178 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:39.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:39 smithi139 bash[43301]: audit 2022-04-28T21:43:38.495202+0000 mon.a (mon.0) 3796 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[61152]: audit 2022-04-28T21:43:39.502905+0000 mon.a (mon.0) 3797 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[61152]: cluster 2022-04-28T21:43:39.503028+0000 mon.a (mon.0) 3798 : cluster [DBG] osdmap e1337: 8 total, 8 up, 8 in 2022-04-28T21:43:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[61152]: audit 2022-04-28T21:43:39.504524+0000 mon.c (mon.1) 179 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[61152]: audit 2022-04-28T21:43:39.506321+0000 mon.a (mon.0) 3799 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[59745]: audit 2022-04-28T21:43:39.502905+0000 mon.a (mon.0) 3797 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[59745]: cluster 2022-04-28T21:43:39.503028+0000 mon.a (mon.0) 3798 : cluster [DBG] osdmap e1337: 8 total, 8 up, 8 in 2022-04-28T21:43:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[59745]: audit 2022-04-28T21:43:39.504524+0000 mon.c (mon.1) 179 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:40.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:40 smithi059 bash[59745]: audit 2022-04-28T21:43:39.506321+0000 mon.a (mon.0) 3799 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:40.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:40 smithi139 bash[43301]: audit 2022-04-28T21:43:39.502905+0000 mon.a (mon.0) 3797 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-49", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:40.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:40 smithi139 bash[43301]: cluster 2022-04-28T21:43:39.503028+0000 mon.a (mon.0) 3798 : cluster [DBG] osdmap e1337: 8 total, 8 up, 8 in 2022-04-28T21:43:40.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:40 smithi139 bash[43301]: audit 2022-04-28T21:43:39.504524+0000 mon.c (mon.1) 179 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:40.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:40 smithi139 bash[43301]: audit 2022-04-28T21:43:39.506321+0000 mon.a (mon.0) 3799 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:41 smithi059 bash[61152]: cluster 2022-04-28T21:43:40.279269+0000 mgr.y (mgr.24961) 3725 : cluster [DBG] pgmap v4421: 169 pgs: 169 active+clean; 24 KiB data, 902 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:41 smithi059 bash[61152]: cluster 2022-04-28T21:43:40.979449+0000 mon.a (mon.0) 3800 : cluster [DBG] osdmap e1338: 8 total, 8 up, 8 in 2022-04-28T21:43:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:41 smithi059 bash[59745]: cluster 2022-04-28T21:43:40.279269+0000 mgr.y (mgr.24961) 3725 : cluster [DBG] pgmap v4421: 169 pgs: 169 active+clean; 24 KiB data, 902 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:41 smithi059 bash[59745]: cluster 2022-04-28T21:43:40.979449+0000 mon.a (mon.0) 3800 : cluster [DBG] osdmap e1338: 8 total, 8 up, 8 in 2022-04-28T21:43:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:41 smithi139 bash[43301]: cluster 2022-04-28T21:43:40.279269+0000 mgr.y (mgr.24961) 3725 : cluster [DBG] pgmap v4421: 169 pgs: 169 active+clean; 24 KiB data, 902 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:41 smithi139 bash[43301]: cluster 2022-04-28T21:43:40.979449+0000 mon.a (mon.0) 3800 : cluster [DBG] osdmap e1338: 8 total, 8 up, 8 in 2022-04-28T21:43:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:42.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:42 smithi059 bash[61152]: audit 2022-04-28T21:43:41.519508+0000 mon.a (mon.0) 3801 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:42 smithi059 bash[61152]: cluster 2022-04-28T21:43:41.519587+0000 mon.a (mon.0) 3802 : cluster [DBG] osdmap e1339: 8 total, 8 up, 8 in 2022-04-28T21:43:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:42 smithi059 bash[59745]: audit 2022-04-28T21:43:41.519508+0000 mon.a (mon.0) 3801 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:42 smithi059 bash[59745]: cluster 2022-04-28T21:43:41.519587+0000 mon.a (mon.0) 3802 : cluster [DBG] osdmap e1339: 8 total, 8 up, 8 in 2022-04-28T21:43:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:42 smithi139 bash[43301]: audit 2022-04-28T21:43:41.519508+0000 mon.a (mon.0) 3801 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-49", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:42 smithi139 bash[43301]: cluster 2022-04-28T21:43:41.519587+0000 mon.a (mon.0) 3802 : cluster [DBG] osdmap e1339: 8 total, 8 up, 8 in 2022-04-28T21:43:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:43 smithi059 bash[61152]: cluster 2022-04-28T21:43:42.280060+0000 mgr.y (mgr.24961) 3726 : cluster [DBG] pgmap v4424: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:43.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:43 smithi059 bash[61152]: cluster 2022-04-28T21:43:42.521862+0000 mon.a (mon.0) 3803 : cluster [DBG] osdmap e1340: 8 total, 8 up, 8 in 2022-04-28T21:43:43.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:43 smithi059 bash[59745]: cluster 2022-04-28T21:43:42.280060+0000 mgr.y (mgr.24961) 3726 : cluster [DBG] pgmap v4424: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:43.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:43 smithi059 bash[59745]: cluster 2022-04-28T21:43:42.521862+0000 mon.a (mon.0) 3803 : cluster [DBG] osdmap e1340: 8 total, 8 up, 8 in 2022-04-28T21:43:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:43 smithi139 bash[43301]: cluster 2022-04-28T21:43:42.280060+0000 mgr.y (mgr.24961) 3726 : cluster [DBG] pgmap v4424: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:43 smithi139 bash[43301]: cluster 2022-04-28T21:43:42.521862+0000 mon.a (mon.0) 3803 : cluster [DBG] osdmap e1340: 8 total, 8 up, 8 in 2022-04-28T21:43:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:43.959Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:44 smithi059 bash[61152]: cluster 2022-04-28T21:43:43.546752+0000 mon.a (mon.0) 3804 : cluster [DBG] osdmap e1341: 8 total, 8 up, 8 in 2022-04-28T21:43:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:44 smithi059 bash[61152]: audit 2022-04-28T21:43:43.549508+0000 mon.c (mon.1) 180 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:44 smithi059 bash[61152]: audit 2022-04-28T21:43:43.550231+0000 mon.a (mon.0) 3805 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:44 smithi059 bash[59745]: cluster 2022-04-28T21:43:43.546752+0000 mon.a (mon.0) 3804 : cluster [DBG] osdmap e1341: 8 total, 8 up, 8 in 2022-04-28T21:43:44.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:44 smithi059 bash[59745]: audit 2022-04-28T21:43:43.549508+0000 mon.c (mon.1) 180 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:44.849 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:44 smithi059 bash[59745]: audit 2022-04-28T21:43:43.550231+0000 mon.a (mon.0) 3805 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:44 smithi139 bash[43301]: cluster 2022-04-28T21:43:43.546752+0000 mon.a (mon.0) 3804 : cluster [DBG] osdmap e1341: 8 total, 8 up, 8 in 2022-04-28T21:43:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:44 smithi139 bash[43301]: audit 2022-04-28T21:43:43.549508+0000 mon.c (mon.1) 180 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:44.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:44 smithi139 bash[43301]: audit 2022-04-28T21:43:43.550231+0000 mon.a (mon.0) 3805 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: cluster 2022-04-28T21:43:44.281267+0000 mgr.y (mgr.24961) 3727 : cluster [DBG] pgmap v4427: 169 pgs: 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: audit 2022-04-28T21:43:44.549774+0000 mon.a (mon.0) 3806 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: cluster 2022-04-28T21:43:44.550047+0000 mon.a (mon.0) 3807 : cluster [DBG] osdmap e1342: 8 total, 8 up, 8 in 2022-04-28T21:43:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: audit 2022-04-28T21:43:44.552543+0000 mon.c (mon.1) 181 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: audit 2022-04-28T21:43:44.553349+0000 mon.a (mon.0) 3808 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: audit 2022-04-28T21:43:45.556029+0000 mon.a (mon.0) 3809 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:45.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[61152]: cluster 2022-04-28T21:43:45.556193+0000 mon.a (mon.0) 3810 : cluster [DBG] osdmap e1343: 8 total, 8 up, 8 in 2022-04-28T21:43:45.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: cluster 2022-04-28T21:43:44.281267+0000 mgr.y (mgr.24961) 3727 : cluster [DBG] pgmap v4427: 169 pgs: 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:45.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: audit 2022-04-28T21:43:44.549774+0000 mon.a (mon.0) 3806 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:45.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: cluster 2022-04-28T21:43:44.550047+0000 mon.a (mon.0) 3807 : cluster [DBG] osdmap e1342: 8 total, 8 up, 8 in 2022-04-28T21:43:45.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: audit 2022-04-28T21:43:44.552543+0000 mon.c (mon.1) 181 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: audit 2022-04-28T21:43:44.553349+0000 mon.a (mon.0) 3808 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: audit 2022-04-28T21:43:45.556029+0000 mon.a (mon.0) 3809 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:45.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:45 smithi059 bash[59745]: cluster 2022-04-28T21:43:45.556193+0000 mon.a (mon.0) 3810 : cluster [DBG] osdmap e1343: 8 total, 8 up, 8 in 2022-04-28T21:43:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: cluster 2022-04-28T21:43:44.281267+0000 mgr.y (mgr.24961) 3727 : cluster [DBG] pgmap v4427: 169 pgs: 169 active+clean; 24 KiB data, 886 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: audit 2022-04-28T21:43:44.549774+0000 mon.a (mon.0) 3806 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:45.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: cluster 2022-04-28T21:43:44.550047+0000 mon.a (mon.0) 3807 : cluster [DBG] osdmap e1342: 8 total, 8 up, 8 in 2022-04-28T21:43:45.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: audit 2022-04-28T21:43:44.552543+0000 mon.c (mon.1) 181 : audit [INF] from='client.? 172.21.15.59:0/1825416926' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: audit 2022-04-28T21:43:44.553349+0000 mon.a (mon.0) 3808 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]: dispatch 2022-04-28T21:43:45.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: audit 2022-04-28T21:43:45.556029+0000 mon.a (mon.0) 3809 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-49"}]': finished 2022-04-28T21:43:45.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:45 smithi139 bash[43301]: cluster 2022-04-28T21:43:45.556193+0000 mon.a (mon.0) 3810 : cluster [DBG] osdmap e1343: 8 total, 8 up, 8 in 2022-04-28T21:43:46.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:46 smithi059 bash[61152]: audit 2022-04-28T21:43:45.569300+0000 mon.a (mon.0) 3811 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:46.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:46 smithi059 bash[61152]: audit 2022-04-28T21:43:45.570642+0000 mon.a (mon.0) 3812 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:46.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:46 smithi059 bash[61152]: audit 2022-04-28T21:43:45.571094+0000 mon.a (mon.0) 3813 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-50", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:46.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:46 smithi059 bash[59745]: audit 2022-04-28T21:43:45.569300+0000 mon.a (mon.0) 3811 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:46.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:46 smithi059 bash[59745]: audit 2022-04-28T21:43:45.570642+0000 mon.a (mon.0) 3812 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:46.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:46 smithi059 bash[59745]: audit 2022-04-28T21:43:45.571094+0000 mon.a (mon.0) 3813 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-50", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:46 smithi139 bash[43301]: audit 2022-04-28T21:43:45.569300+0000 mon.a (mon.0) 3811 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:46 smithi139 bash[43301]: audit 2022-04-28T21:43:45.570642+0000 mon.a (mon.0) 3812 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:46 smithi139 bash[43301]: audit 2022-04-28T21:43:45.571094+0000 mon.a (mon.0) 3813 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-50", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[61152]: cluster 2022-04-28T21:43:46.282430+0000 mgr.y (mgr.24961) 3728 : cluster [DBG] pgmap v4430: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[61152]: audit 2022-04-28T21:43:46.576101+0000 mon.a (mon.0) 3814 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-50", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[61152]: cluster 2022-04-28T21:43:46.576256+0000 mon.a (mon.0) 3815 : cluster [DBG] osdmap e1344: 8 total, 8 up, 8 in 2022-04-28T21:43:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[61152]: audit 2022-04-28T21:43:46.577274+0000 mon.a (mon.0) 3816 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-50", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[59745]: cluster 2022-04-28T21:43:46.282430+0000 mgr.y (mgr.24961) 3728 : cluster [DBG] pgmap v4430: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[59745]: audit 2022-04-28T21:43:46.576101+0000 mon.a (mon.0) 3814 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-50", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[59745]: cluster 2022-04-28T21:43:46.576256+0000 mon.a (mon.0) 3815 : cluster [DBG] osdmap e1344: 8 total, 8 up, 8 in 2022-04-28T21:43:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:47 smithi059 bash[59745]: audit 2022-04-28T21:43:46.577274+0000 mon.a (mon.0) 3816 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-50", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:47 smithi139 bash[43301]: cluster 2022-04-28T21:43:46.282430+0000 mgr.y (mgr.24961) 3728 : cluster [DBG] pgmap v4430: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:47 smithi139 bash[43301]: audit 2022-04-28T21:43:46.576101+0000 mon.a (mon.0) 3814 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-50", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:47 smithi139 bash[43301]: cluster 2022-04-28T21:43:46.576256+0000 mon.a (mon.0) 3815 : cluster [DBG] osdmap e1344: 8 total, 8 up, 8 in 2022-04-28T21:43:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:47 smithi139 bash[43301]: audit 2022-04-28T21:43:46.577274+0000 mon.a (mon.0) 3816 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-50", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:48 smithi059 bash[61152]: cluster 2022-04-28T21:43:48.050058+0000 mon.a (mon.0) 3817 : cluster [DBG] osdmap e1345: 8 total, 8 up, 8 in 2022-04-28T21:43:48.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:48 smithi059 bash[59745]: cluster 2022-04-28T21:43:48.050058+0000 mon.a (mon.0) 3817 : cluster [DBG] osdmap e1345: 8 total, 8 up, 8 in 2022-04-28T21:43:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:48 smithi139 bash[43301]: cluster 2022-04-28T21:43:48.050058+0000 mon.a (mon.0) 3817 : cluster [DBG] osdmap e1345: 8 total, 8 up, 8 in 2022-04-28T21:43:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:49 smithi139 bash[43301]: cluster 2022-04-28T21:43:48.283149+0000 mgr.y (mgr.24961) 3729 : cluster [DBG] pgmap v4433: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:49 smithi139 bash[43301]: audit 2022-04-28T21:43:48.598454+0000 mon.a (mon.0) 3818 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-50", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:49 smithi139 bash[43301]: cluster 2022-04-28T21:43:48.598666+0000 mon.a (mon.0) 3819 : cluster [DBG] osdmap e1346: 8 total, 8 up, 8 in 2022-04-28T21:43:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:49 smithi059 bash[61152]: cluster 2022-04-28T21:43:48.283149+0000 mgr.y (mgr.24961) 3729 : cluster [DBG] pgmap v4433: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:50.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:49 smithi059 bash[61152]: audit 2022-04-28T21:43:48.598454+0000 mon.a (mon.0) 3818 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-50", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:50.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:49 smithi059 bash[61152]: cluster 2022-04-28T21:43:48.598666+0000 mon.a (mon.0) 3819 : cluster [DBG] osdmap e1346: 8 total, 8 up, 8 in 2022-04-28T21:43:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:49 smithi059 bash[59745]: cluster 2022-04-28T21:43:48.283149+0000 mgr.y (mgr.24961) 3729 : cluster [DBG] pgmap v4433: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:49 smithi059 bash[59745]: audit 2022-04-28T21:43:48.598454+0000 mon.a (mon.0) 3818 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-50", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:50.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:49 smithi059 bash[59745]: cluster 2022-04-28T21:43:48.598666+0000 mon.a (mon.0) 3819 : cluster [DBG] osdmap e1346: 8 total, 8 up, 8 in 2022-04-28T21:43:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:50 smithi139 bash[43301]: cluster 2022-04-28T21:43:49.602927+0000 mon.a (mon.0) 3820 : cluster [DBG] osdmap e1347: 8 total, 8 up, 8 in 2022-04-28T21:43:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:50 smithi059 bash[61152]: cluster 2022-04-28T21:43:49.602927+0000 mon.a (mon.0) 3820 : cluster [DBG] osdmap e1347: 8 total, 8 up, 8 in 2022-04-28T21:43:51.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:50 smithi059 bash[59745]: cluster 2022-04-28T21:43:49.602927+0000 mon.a (mon.0) 3820 : cluster [DBG] osdmap e1347: 8 total, 8 up, 8 in 2022-04-28T21:43:52.071 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:51 smithi059 bash[61152]: cluster 2022-04-28T21:43:50.284779+0000 mgr.y (mgr.24961) 3730 : cluster [DBG] pgmap v4436: 177 pgs: 5 creating+peering, 1 unknown, 171 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:52.072 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:51 smithi059 bash[61152]: cluster 2022-04-28T21:43:50.620936+0000 mon.a (mon.0) 3821 : cluster [DBG] osdmap e1348: 8 total, 8 up, 8 in 2022-04-28T21:43:52.072 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:51 smithi059 bash[61152]: audit 2022-04-28T21:43:50.630675+0000 mon.a (mon.0) 3822 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:52.072 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:51 smithi059 bash[59745]: cluster 2022-04-28T21:43:50.284779+0000 mgr.y (mgr.24961) 3730 : cluster [DBG] pgmap v4436: 177 pgs: 5 creating+peering, 1 unknown, 171 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:52.073 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:51 smithi059 bash[59745]: cluster 2022-04-28T21:43:50.620936+0000 mon.a (mon.0) 3821 : cluster [DBG] osdmap e1348: 8 total, 8 up, 8 in 2022-04-28T21:43:52.074 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:51 smithi059 bash[59745]: audit 2022-04-28T21:43:50.630675+0000 mon.a (mon.0) 3822 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:51 smithi139 bash[43301]: cluster 2022-04-28T21:43:50.284779+0000 mgr.y (mgr.24961) 3730 : cluster [DBG] pgmap v4436: 177 pgs: 5 creating+peering, 1 unknown, 171 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:51 smithi139 bash[43301]: cluster 2022-04-28T21:43:50.620936+0000 mon.a (mon.0) 3821 : cluster [DBG] osdmap e1348: 8 total, 8 up, 8 in 2022-04-28T21:43:52.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:51 smithi139 bash[43301]: audit 2022-04-28T21:43:50.630675+0000 mon.a (mon.0) 3822 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:52.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:53.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[59745]: audit 2022-04-28T21:43:51.624094+0000 mon.a (mon.0) 3823 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:53.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[59745]: cluster 2022-04-28T21:43:51.624264+0000 mon.a (mon.0) 3824 : cluster [DBG] osdmap e1349: 8 total, 8 up, 8 in 2022-04-28T21:43:53.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[59745]: audit 2022-04-28T21:43:51.626796+0000 mon.a (mon.0) 3825 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[61152]: audit 2022-04-28T21:43:51.624094+0000 mon.a (mon.0) 3823 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[61152]: cluster 2022-04-28T21:43:51.624264+0000 mon.a (mon.0) 3824 : cluster [DBG] osdmap e1349: 8 total, 8 up, 8 in 2022-04-28T21:43:53.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:52 smithi059 bash[61152]: audit 2022-04-28T21:43:51.626796+0000 mon.a (mon.0) 3825 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:53.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:52 smithi139 bash[43301]: audit 2022-04-28T21:43:51.624094+0000 mon.a (mon.0) 3823 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:53.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:52 smithi139 bash[43301]: cluster 2022-04-28T21:43:51.624264+0000 mon.a (mon.0) 3824 : cluster [DBG] osdmap e1349: 8 total, 8 up, 8 in 2022-04-28T21:43:53.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:52 smithi139 bash[43301]: audit 2022-04-28T21:43:51.626796+0000 mon.a (mon.0) 3825 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]: dispatch 2022-04-28T21:43:53.962 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[61152]: cluster 2022-04-28T21:43:52.285573+0000 mgr.y (mgr.24961) 3731 : cluster [DBG] pgmap v4439: 169 pgs: 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:53.962 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[61152]: audit 2022-04-28T21:43:52.627995+0000 mon.a (mon.0) 3826 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:53.963 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[61152]: cluster 2022-04-28T21:43:52.628238+0000 mon.a (mon.0) 3827 : cluster [DBG] osdmap e1350: 8 total, 8 up, 8 in 2022-04-28T21:43:53.963 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[61152]: audit 2022-04-28T21:43:52.644891+0000 mon.a (mon.0) 3828 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:53.963 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[61152]: audit 2022-04-28T21:43:52.645717+0000 mon.a (mon.0) 3829 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:53.963 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[61152]: audit 2022-04-28T21:43:52.646095+0000 mon.a (mon.0) 3830 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-51", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:53.964 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:43:53.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:43:53.964 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[59745]: cluster 2022-04-28T21:43:52.285573+0000 mgr.y (mgr.24961) 3731 : cluster [DBG] pgmap v4439: 169 pgs: 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:53.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[59745]: audit 2022-04-28T21:43:52.627995+0000 mon.a (mon.0) 3826 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:53.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[59745]: cluster 2022-04-28T21:43:52.628238+0000 mon.a (mon.0) 3827 : cluster [DBG] osdmap e1350: 8 total, 8 up, 8 in 2022-04-28T21:43:53.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[59745]: audit 2022-04-28T21:43:52.644891+0000 mon.a (mon.0) 3828 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:53.965 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[59745]: audit 2022-04-28T21:43:52.645717+0000 mon.a (mon.0) 3829 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:53.966 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:53 smithi059 bash[59745]: audit 2022-04-28T21:43:52.646095+0000 mon.a (mon.0) 3830 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-51", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:53 smithi139 bash[43301]: cluster 2022-04-28T21:43:52.285573+0000 mgr.y (mgr.24961) 3731 : cluster [DBG] pgmap v4439: 169 pgs: 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:53 smithi139 bash[43301]: audit 2022-04-28T21:43:52.627995+0000 mon.a (mon.0) 3826 : audit [INF] from='client.? 172.21.15.59:0/2104565091' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-50"}]': finished 2022-04-28T21:43:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:53 smithi139 bash[43301]: cluster 2022-04-28T21:43:52.628238+0000 mon.a (mon.0) 3827 : cluster [DBG] osdmap e1350: 8 total, 8 up, 8 in 2022-04-28T21:43:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:53 smithi139 bash[43301]: audit 2022-04-28T21:43:52.644891+0000 mon.a (mon.0) 3828 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:53 smithi139 bash[43301]: audit 2022-04-28T21:43:52.645717+0000 mon.a (mon.0) 3829 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:54.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:53 smithi139 bash[43301]: audit 2022-04-28T21:43:52.646095+0000 mon.a (mon.0) 3830 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-51", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:43:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:54 smithi059 bash[61152]: audit 2022-04-28T21:43:53.654943+0000 mon.a (mon.0) 3831 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-51", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:54 smithi059 bash[61152]: cluster 2022-04-28T21:43:53.655159+0000 mon.a (mon.0) 3832 : cluster [DBG] osdmap e1351: 8 total, 8 up, 8 in 2022-04-28T21:43:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:54 smithi059 bash[61152]: audit 2022-04-28T21:43:53.655996+0000 mon.a (mon.0) 3833 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-51", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:54 smithi059 bash[59745]: audit 2022-04-28T21:43:53.654943+0000 mon.a (mon.0) 3831 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-51", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:55.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:54 smithi059 bash[59745]: cluster 2022-04-28T21:43:53.655159+0000 mon.a (mon.0) 3832 : cluster [DBG] osdmap e1351: 8 total, 8 up, 8 in 2022-04-28T21:43:55.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:54 smithi059 bash[59745]: audit 2022-04-28T21:43:53.655996+0000 mon.a (mon.0) 3833 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-51", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:55.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:54 smithi139 bash[43301]: audit 2022-04-28T21:43:53.654943+0000 mon.a (mon.0) 3831 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-51", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:43:55.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:54 smithi139 bash[43301]: cluster 2022-04-28T21:43:53.655159+0000 mon.a (mon.0) 3832 : cluster [DBG] osdmap e1351: 8 total, 8 up, 8 in 2022-04-28T21:43:55.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:54 smithi139 bash[43301]: audit 2022-04-28T21:43:53.655996+0000 mon.a (mon.0) 3833 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-51", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:55 smithi059 bash[59745]: cluster 2022-04-28T21:43:54.286451+0000 mgr.y (mgr.24961) 3732 : cluster [DBG] pgmap v4442: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:56.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:55 smithi059 bash[59745]: cluster 2022-04-28T21:43:55.138391+0000 mon.a (mon.0) 3834 : cluster [DBG] osdmap e1352: 8 total, 8 up, 8 in 2022-04-28T21:43:56.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:55 smithi059 bash[61152]: cluster 2022-04-28T21:43:54.286451+0000 mgr.y (mgr.24961) 3732 : cluster [DBG] pgmap v4442: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:56.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:55 smithi059 bash[61152]: cluster 2022-04-28T21:43:55.138391+0000 mon.a (mon.0) 3834 : cluster [DBG] osdmap e1352: 8 total, 8 up, 8 in 2022-04-28T21:43:56.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:55 smithi139 bash[43301]: cluster 2022-04-28T21:43:54.286451+0000 mgr.y (mgr.24961) 3732 : cluster [DBG] pgmap v4442: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:56.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:55 smithi139 bash[43301]: cluster 2022-04-28T21:43:55.138391+0000 mon.a (mon.0) 3834 : cluster [DBG] osdmap e1352: 8 total, 8 up, 8 in 2022-04-28T21:43:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:56 smithi059 bash[59745]: audit 2022-04-28T21:43:55.679759+0000 mon.a (mon.0) 3835 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-51", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:43:57.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:56 smithi059 bash[59745]: cluster 2022-04-28T21:43:55.679850+0000 mon.a (mon.0) 3836 : cluster [DBG] osdmap e1353: 8 total, 8 up, 8 in 2022-04-28T21:43:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:56 smithi059 bash[61152]: audit 2022-04-28T21:43:55.679759+0000 mon.a (mon.0) 3835 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-51", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:43:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:56 smithi059 bash[61152]: cluster 2022-04-28T21:43:55.679850+0000 mon.a (mon.0) 3836 : cluster [DBG] osdmap e1353: 8 total, 8 up, 8 in 2022-04-28T21:43:57.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:56 smithi139 bash[43301]: audit 2022-04-28T21:43:55.679759+0000 mon.a (mon.0) 3835 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-51", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:43:57.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:56 smithi139 bash[43301]: cluster 2022-04-28T21:43:55.679850+0000 mon.a (mon.0) 3836 : cluster [DBG] osdmap e1353: 8 total, 8 up, 8 in 2022-04-28T21:43:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:57 smithi059 bash[61152]: cluster 2022-04-28T21:43:56.286993+0000 mgr.y (mgr.24961) 3733 : cluster [DBG] pgmap v4445: 177 pgs: 1 creating+peering, 7 unknown, 169 active+clean; 24 KiB data, 893 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:57 smithi059 bash[61152]: cluster 2022-04-28T21:43:56.689322+0000 mon.a (mon.0) 3837 : cluster [DBG] osdmap e1354: 8 total, 8 up, 8 in 2022-04-28T21:43:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:57 smithi059 bash[59745]: cluster 2022-04-28T21:43:56.286993+0000 mgr.y (mgr.24961) 3733 : cluster [DBG] pgmap v4445: 177 pgs: 1 creating+peering, 7 unknown, 169 active+clean; 24 KiB data, 893 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:57 smithi059 bash[59745]: cluster 2022-04-28T21:43:56.689322+0000 mon.a (mon.0) 3837 : cluster [DBG] osdmap e1354: 8 total, 8 up, 8 in 2022-04-28T21:43:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:57 smithi139 bash[43301]: cluster 2022-04-28T21:43:56.286993+0000 mgr.y (mgr.24961) 3733 : cluster [DBG] pgmap v4445: 177 pgs: 1 creating+peering, 7 unknown, 169 active+clean; 24 KiB data, 893 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:57 smithi139 bash[43301]: cluster 2022-04-28T21:43:56.689322+0000 mon.a (mon.0) 3837 : cluster [DBG] osdmap e1354: 8 total, 8 up, 8 in 2022-04-28T21:43:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[61152]: cluster 2022-04-28T21:43:57.692879+0000 mon.a (mon.0) 3838 : cluster [DBG] osdmap e1355: 8 total, 8 up, 8 in 2022-04-28T21:43:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[61152]: audit 2022-04-28T21:43:57.695799+0000 mon.a (mon.0) 3839 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:59.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[61152]: cluster 2022-04-28T21:43:58.287683+0000 mgr.y (mgr.24961) 3734 : cluster [DBG] pgmap v4448: 169 pgs: 169 active+clean; 24 KiB data, 893 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:59.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[61152]: audit 2022-04-28T21:43:58.692710+0000 mon.a (mon.0) 3840 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:43:59.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[61152]: cluster 2022-04-28T21:43:58.692808+0000 mon.a (mon.0) 3841 : cluster [DBG] osdmap e1356: 8 total, 8 up, 8 in 2022-04-28T21:43:59.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[61152]: audit 2022-04-28T21:43:58.693939+0000 mon.a (mon.0) 3842 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:59.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[59745]: cluster 2022-04-28T21:43:57.692879+0000 mon.a (mon.0) 3838 : cluster [DBG] osdmap e1355: 8 total, 8 up, 8 in 2022-04-28T21:43:59.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[59745]: audit 2022-04-28T21:43:57.695799+0000 mon.a (mon.0) 3839 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:59.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[59745]: cluster 2022-04-28T21:43:58.287683+0000 mgr.y (mgr.24961) 3734 : cluster [DBG] pgmap v4448: 169 pgs: 169 active+clean; 24 KiB data, 893 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:59.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[59745]: audit 2022-04-28T21:43:58.692710+0000 mon.a (mon.0) 3840 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:43:59.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[59745]: cluster 2022-04-28T21:43:58.692808+0000 mon.a (mon.0) 3841 : cluster [DBG] osdmap e1356: 8 total, 8 up, 8 in 2022-04-28T21:43:59.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:43:58 smithi059 bash[59745]: audit 2022-04-28T21:43:58.693939+0000 mon.a (mon.0) 3842 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:58 smithi139 bash[43301]: cluster 2022-04-28T21:43:57.692879+0000 mon.a (mon.0) 3838 : cluster [DBG] osdmap e1355: 8 total, 8 up, 8 in 2022-04-28T21:43:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:58 smithi139 bash[43301]: audit 2022-04-28T21:43:57.695799+0000 mon.a (mon.0) 3839 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:43:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:58 smithi139 bash[43301]: cluster 2022-04-28T21:43:58.287683+0000 mgr.y (mgr.24961) 3734 : cluster [DBG] pgmap v4448: 169 pgs: 169 active+clean; 24 KiB data, 893 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:43:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:58 smithi139 bash[43301]: audit 2022-04-28T21:43:58.692710+0000 mon.a (mon.0) 3840 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:43:59.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:58 smithi139 bash[43301]: cluster 2022-04-28T21:43:58.692808+0000 mon.a (mon.0) 3841 : cluster [DBG] osdmap e1356: 8 total, 8 up, 8 in 2022-04-28T21:43:59.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:43:58 smithi139 bash[43301]: audit 2022-04-28T21:43:58.693939+0000 mon.a (mon.0) 3842 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]: dispatch 2022-04-28T21:44:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[59745]: audit 2022-04-28T21:43:59.697663+0000 mon.a (mon.0) 3843 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:44:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[59745]: cluster 2022-04-28T21:43:59.697982+0000 mon.a (mon.0) 3844 : cluster [DBG] osdmap e1357: 8 total, 8 up, 8 in 2022-04-28T21:44:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[59745]: audit 2022-04-28T21:43:59.711238+0000 mon.a (mon.0) 3845 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[59745]: audit 2022-04-28T21:43:59.712597+0000 mon.a (mon.0) 3846 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[59745]: audit 2022-04-28T21:43:59.713024+0000 mon.a (mon.0) 3847 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-52", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:01.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[59745]: cluster 2022-04-28T21:44:00.288865+0000 mgr.y (mgr.24961) 3735 : cluster [DBG] pgmap v4451: 169 pgs: 169 active+clean; 24 KiB data, 894 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:01.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[61152]: audit 2022-04-28T21:43:59.697663+0000 mon.a (mon.0) 3843 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:44:01.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[61152]: cluster 2022-04-28T21:43:59.697982+0000 mon.a (mon.0) 3844 : cluster [DBG] osdmap e1357: 8 total, 8 up, 8 in 2022-04-28T21:44:01.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[61152]: audit 2022-04-28T21:43:59.711238+0000 mon.a (mon.0) 3845 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:01.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[61152]: audit 2022-04-28T21:43:59.712597+0000 mon.a (mon.0) 3846 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:01.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[61152]: audit 2022-04-28T21:43:59.713024+0000 mon.a (mon.0) 3847 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-52", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:01.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:00 smithi059 bash[61152]: cluster 2022-04-28T21:44:00.288865+0000 mgr.y (mgr.24961) 3735 : cluster [DBG] pgmap v4451: 169 pgs: 169 active+clean; 24 KiB data, 894 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:00 smithi139 bash[43301]: audit 2022-04-28T21:43:59.697663+0000 mon.a (mon.0) 3843 : audit [INF] from='client.? 172.21.15.59:0/992019440' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-51"}]': finished 2022-04-28T21:44:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:00 smithi139 bash[43301]: cluster 2022-04-28T21:43:59.697982+0000 mon.a (mon.0) 3844 : cluster [DBG] osdmap e1357: 8 total, 8 up, 8 in 2022-04-28T21:44:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:00 smithi139 bash[43301]: audit 2022-04-28T21:43:59.711238+0000 mon.a (mon.0) 3845 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:00 smithi139 bash[43301]: audit 2022-04-28T21:43:59.712597+0000 mon.a (mon.0) 3846 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:01.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:00 smithi139 bash[43301]: audit 2022-04-28T21:43:59.713024+0000 mon.a (mon.0) 3847 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-52", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:01.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:00 smithi139 bash[43301]: cluster 2022-04-28T21:44:00.288865+0000 mgr.y (mgr.24961) 3735 : cluster [DBG] pgmap v4451: 169 pgs: 169 active+clean; 24 KiB data, 894 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:02.073 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:01 smithi059 bash[61152]: audit 2022-04-28T21:44:00.703504+0000 mon.a (mon.0) 3848 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-52", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:02.075 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:01 smithi059 bash[61152]: cluster 2022-04-28T21:44:00.703615+0000 mon.a (mon.0) 3849 : cluster [DBG] osdmap e1358: 8 total, 8 up, 8 in 2022-04-28T21:44:02.075 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:01 smithi059 bash[61152]: audit 2022-04-28T21:44:00.704530+0000 mon.a (mon.0) 3850 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-52", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:02.076 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:01 smithi059 bash[59745]: audit 2022-04-28T21:44:00.703504+0000 mon.a (mon.0) 3848 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-52", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:02.077 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:01 smithi059 bash[59745]: cluster 2022-04-28T21:44:00.703615+0000 mon.a (mon.0) 3849 : cluster [DBG] osdmap e1358: 8 total, 8 up, 8 in 2022-04-28T21:44:02.077 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:01 smithi059 bash[59745]: audit 2022-04-28T21:44:00.704530+0000 mon.a (mon.0) 3850 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-52", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:02.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:01 smithi139 bash[43301]: audit 2022-04-28T21:44:00.703504+0000 mon.a (mon.0) 3848 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-52", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:02.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:01 smithi139 bash[43301]: cluster 2022-04-28T21:44:00.703615+0000 mon.a (mon.0) 3849 : cluster [DBG] osdmap e1358: 8 total, 8 up, 8 in 2022-04-28T21:44:02.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:01 smithi139 bash[43301]: audit 2022-04-28T21:44:00.704530+0000 mon.a (mon.0) 3850 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-52", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:02.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:02.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:02 smithi059 bash[61152]: cluster 2022-04-28T21:44:02.189170+0000 mon.a (mon.0) 3851 : cluster [DBG] osdmap e1359: 8 total, 8 up, 8 in 2022-04-28T21:44:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:02 smithi059 bash[61152]: cluster 2022-04-28T21:44:02.289616+0000 mgr.y (mgr.24961) 3736 : cluster [DBG] pgmap v4454: 169 pgs: 169 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:02 smithi059 bash[59745]: cluster 2022-04-28T21:44:02.189170+0000 mon.a (mon.0) 3851 : cluster [DBG] osdmap e1359: 8 total, 8 up, 8 in 2022-04-28T21:44:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:02 smithi059 bash[59745]: cluster 2022-04-28T21:44:02.289616+0000 mgr.y (mgr.24961) 3736 : cluster [DBG] pgmap v4454: 169 pgs: 169 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:02 smithi139 bash[43301]: cluster 2022-04-28T21:44:02.189170+0000 mon.a (mon.0) 3851 : cluster [DBG] osdmap e1359: 8 total, 8 up, 8 in 2022-04-28T21:44:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:02 smithi139 bash[43301]: cluster 2022-04-28T21:44:02.289616+0000 mgr.y (mgr.24961) 3736 : cluster [DBG] pgmap v4454: 169 pgs: 169 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:03 smithi059 bash[61152]: audit 2022-04-28T21:44:02.726205+0000 mon.a (mon.0) 3852 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-52", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:03 smithi059 bash[61152]: cluster 2022-04-28T21:44:02.726326+0000 mon.a (mon.0) 3853 : cluster [DBG] osdmap e1360: 8 total, 8 up, 8 in 2022-04-28T21:44:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:03.960Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:03 smithi059 bash[59745]: audit 2022-04-28T21:44:02.726205+0000 mon.a (mon.0) 3852 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-52", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:03 smithi059 bash[59745]: cluster 2022-04-28T21:44:02.726326+0000 mon.a (mon.0) 3853 : cluster [DBG] osdmap e1360: 8 total, 8 up, 8 in 2022-04-28T21:44:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:03 smithi139 bash[43301]: audit 2022-04-28T21:44:02.726205+0000 mon.a (mon.0) 3852 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-52", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:03 smithi139 bash[43301]: cluster 2022-04-28T21:44:02.726326+0000 mon.a (mon.0) 3853 : cluster [DBG] osdmap e1360: 8 total, 8 up, 8 in 2022-04-28T21:44:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:04 smithi059 bash[61152]: cluster 2022-04-28T21:44:03.730663+0000 mon.a (mon.0) 3854 : cluster [DBG] osdmap e1361: 8 total, 8 up, 8 in 2022-04-28T21:44:05.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:04 smithi059 bash[61152]: cluster 2022-04-28T21:44:04.290903+0000 mgr.y (mgr.24961) 3737 : cluster [DBG] pgmap v4457: 177 pgs: 4 creating+activating, 3 unknown, 170 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:04 smithi059 bash[59745]: cluster 2022-04-28T21:44:03.730663+0000 mon.a (mon.0) 3854 : cluster [DBG] osdmap e1361: 8 total, 8 up, 8 in 2022-04-28T21:44:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:04 smithi059 bash[59745]: cluster 2022-04-28T21:44:04.290903+0000 mgr.y (mgr.24961) 3737 : cluster [DBG] pgmap v4457: 177 pgs: 4 creating+activating, 3 unknown, 170 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:05.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:04 smithi139 bash[43301]: cluster 2022-04-28T21:44:03.730663+0000 mon.a (mon.0) 3854 : cluster [DBG] osdmap e1361: 8 total, 8 up, 8 in 2022-04-28T21:44:05.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:04 smithi139 bash[43301]: cluster 2022-04-28T21:44:04.290903+0000 mgr.y (mgr.24961) 3737 : cluster [DBG] pgmap v4457: 177 pgs: 4 creating+activating, 3 unknown, 170 active+clean; 24 KiB data, 895 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[61152]: cluster 2022-04-28T21:44:04.746832+0000 mon.a (mon.0) 3855 : cluster [DBG] osdmap e1362: 8 total, 8 up, 8 in 2022-04-28T21:44:06.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[61152]: audit 2022-04-28T21:44:04.748646+0000 mon.a (mon.0) 3856 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:06.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[61152]: audit 2022-04-28T21:44:05.750418+0000 mon.a (mon.0) 3857 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:06.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[61152]: cluster 2022-04-28T21:44:05.750562+0000 mon.a (mon.0) 3858 : cluster [DBG] osdmap e1363: 8 total, 8 up, 8 in 2022-04-28T21:44:06.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[61152]: audit 2022-04-28T21:44:05.751294+0000 mon.a (mon.0) 3859 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:06.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[59745]: cluster 2022-04-28T21:44:04.746832+0000 mon.a (mon.0) 3855 : cluster [DBG] osdmap e1362: 8 total, 8 up, 8 in 2022-04-28T21:44:06.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[59745]: audit 2022-04-28T21:44:04.748646+0000 mon.a (mon.0) 3856 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:06.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[59745]: audit 2022-04-28T21:44:05.750418+0000 mon.a (mon.0) 3857 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:06.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[59745]: cluster 2022-04-28T21:44:05.750562+0000 mon.a (mon.0) 3858 : cluster [DBG] osdmap e1363: 8 total, 8 up, 8 in 2022-04-28T21:44:06.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:05 smithi059 bash[59745]: audit 2022-04-28T21:44:05.751294+0000 mon.a (mon.0) 3859 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:06.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:05 smithi139 bash[43301]: cluster 2022-04-28T21:44:04.746832+0000 mon.a (mon.0) 3855 : cluster [DBG] osdmap e1362: 8 total, 8 up, 8 in 2022-04-28T21:44:06.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:05 smithi139 bash[43301]: audit 2022-04-28T21:44:04.748646+0000 mon.a (mon.0) 3856 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:06.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:05 smithi139 bash[43301]: audit 2022-04-28T21:44:05.750418+0000 mon.a (mon.0) 3857 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:06.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:05 smithi139 bash[43301]: cluster 2022-04-28T21:44:05.750562+0000 mon.a (mon.0) 3858 : cluster [DBG] osdmap e1363: 8 total, 8 up, 8 in 2022-04-28T21:44:06.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:05 smithi139 bash[43301]: audit 2022-04-28T21:44:05.751294+0000 mon.a (mon.0) 3859 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]: dispatch 2022-04-28T21:44:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:06 smithi059 bash[59745]: cluster 2022-04-28T21:44:06.291965+0000 mgr.y (mgr.24961) 3738 : cluster [DBG] pgmap v4460: 169 pgs: 169 active+clean; 24 KiB data, 896 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:07.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:06 smithi059 bash[59745]: audit 2022-04-28T21:44:06.754016+0000 mon.a (mon.0) 3860 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:07.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:06 smithi059 bash[59745]: cluster 2022-04-28T21:44:06.754149+0000 mon.a (mon.0) 3861 : cluster [DBG] osdmap e1364: 8 total, 8 up, 8 in 2022-04-28T21:44:07.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:06 smithi059 bash[61152]: cluster 2022-04-28T21:44:06.291965+0000 mgr.y (mgr.24961) 3738 : cluster [DBG] pgmap v4460: 169 pgs: 169 active+clean; 24 KiB data, 896 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:07.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:06 smithi059 bash[61152]: audit 2022-04-28T21:44:06.754016+0000 mon.a (mon.0) 3860 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:07.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:06 smithi059 bash[61152]: cluster 2022-04-28T21:44:06.754149+0000 mon.a (mon.0) 3861 : cluster [DBG] osdmap e1364: 8 total, 8 up, 8 in 2022-04-28T21:44:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:06 smithi139 bash[43301]: cluster 2022-04-28T21:44:06.291965+0000 mgr.y (mgr.24961) 3738 : cluster [DBG] pgmap v4460: 169 pgs: 169 active+clean; 24 KiB data, 896 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:06 smithi139 bash[43301]: audit 2022-04-28T21:44:06.754016+0000 mon.a (mon.0) 3860 : audit [INF] from='client.? 172.21.15.59:0/664113427' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-52"}]': finished 2022-04-28T21:44:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:06 smithi139 bash[43301]: cluster 2022-04-28T21:44:06.754149+0000 mon.a (mon.0) 3861 : cluster [DBG] osdmap e1364: 8 total, 8 up, 8 in 2022-04-28T21:44:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:07 smithi059 bash[61152]: audit 2022-04-28T21:44:06.768189+0000 mon.a (mon.0) 3862 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:07 smithi059 bash[61152]: audit 2022-04-28T21:44:06.769028+0000 mon.a (mon.0) 3863 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:07 smithi059 bash[61152]: audit 2022-04-28T21:44:06.769480+0000 mon.a (mon.0) 3864 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-53", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:07 smithi059 bash[59745]: audit 2022-04-28T21:44:06.768189+0000 mon.a (mon.0) 3862 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:07 smithi059 bash[59745]: audit 2022-04-28T21:44:06.769028+0000 mon.a (mon.0) 3863 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:07 smithi059 bash[59745]: audit 2022-04-28T21:44:06.769480+0000 mon.a (mon.0) 3864 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-53", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:07 smithi139 bash[43301]: audit 2022-04-28T21:44:06.768189+0000 mon.a (mon.0) 3862 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:07 smithi139 bash[43301]: audit 2022-04-28T21:44:06.769028+0000 mon.a (mon.0) 3863 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:08.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:07 smithi139 bash[43301]: audit 2022-04-28T21:44:06.769480+0000 mon.a (mon.0) 3864 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-53", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:09.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[61152]: audit 2022-04-28T21:44:07.776911+0000 mon.a (mon.0) 3865 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-53", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[61152]: cluster 2022-04-28T21:44:07.777096+0000 mon.a (mon.0) 3866 : cluster [DBG] osdmap e1365: 8 total, 8 up, 8 in 2022-04-28T21:44:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[61152]: audit 2022-04-28T21:44:07.778114+0000 mon.a (mon.0) 3867 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-53", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:09.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[61152]: cluster 2022-04-28T21:44:08.292731+0000 mgr.y (mgr.24961) 3739 : cluster [DBG] pgmap v4463: 169 pgs: 169 active+clean; 24 KiB data, 896 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[59745]: audit 2022-04-28T21:44:07.776911+0000 mon.a (mon.0) 3865 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-53", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[59745]: cluster 2022-04-28T21:44:07.777096+0000 mon.a (mon.0) 3866 : cluster [DBG] osdmap e1365: 8 total, 8 up, 8 in 2022-04-28T21:44:09.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[59745]: audit 2022-04-28T21:44:07.778114+0000 mon.a (mon.0) 3867 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-53", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:09.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:09 smithi059 bash[59745]: cluster 2022-04-28T21:44:08.292731+0000 mgr.y (mgr.24961) 3739 : cluster [DBG] pgmap v4463: 169 pgs: 169 active+clean; 24 KiB data, 896 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:09 smithi139 bash[43301]: audit 2022-04-28T21:44:07.776911+0000 mon.a (mon.0) 3865 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-53", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:09 smithi139 bash[43301]: cluster 2022-04-28T21:44:07.777096+0000 mon.a (mon.0) 3866 : cluster [DBG] osdmap e1365: 8 total, 8 up, 8 in 2022-04-28T21:44:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:09 smithi139 bash[43301]: audit 2022-04-28T21:44:07.778114+0000 mon.a (mon.0) 3867 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-53", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:09.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:09 smithi139 bash[43301]: cluster 2022-04-28T21:44:08.292731+0000 mgr.y (mgr.24961) 3739 : cluster [DBG] pgmap v4463: 169 pgs: 169 active+clean; 24 KiB data, 896 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:10 smithi059 bash[61152]: cluster 2022-04-28T21:44:09.243235+0000 mon.a (mon.0) 3868 : cluster [DBG] osdmap e1366: 8 total, 8 up, 8 in 2022-04-28T21:44:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:10 smithi059 bash[61152]: audit 2022-04-28T21:44:09.781032+0000 mon.a (mon.0) 3869 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-53", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:10 smithi059 bash[61152]: cluster 2022-04-28T21:44:09.781134+0000 mon.a (mon.0) 3870 : cluster [DBG] osdmap e1367: 8 total, 8 up, 8 in 2022-04-28T21:44:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:10 smithi059 bash[59745]: cluster 2022-04-28T21:44:09.243235+0000 mon.a (mon.0) 3868 : cluster [DBG] osdmap e1366: 8 total, 8 up, 8 in 2022-04-28T21:44:10.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:10 smithi059 bash[59745]: audit 2022-04-28T21:44:09.781032+0000 mon.a (mon.0) 3869 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-53", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:10.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:10 smithi059 bash[59745]: cluster 2022-04-28T21:44:09.781134+0000 mon.a (mon.0) 3870 : cluster [DBG] osdmap e1367: 8 total, 8 up, 8 in 2022-04-28T21:44:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:10 smithi139 bash[43301]: cluster 2022-04-28T21:44:09.243235+0000 mon.a (mon.0) 3868 : cluster [DBG] osdmap e1366: 8 total, 8 up, 8 in 2022-04-28T21:44:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:10 smithi139 bash[43301]: audit 2022-04-28T21:44:09.781032+0000 mon.a (mon.0) 3869 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-53", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:10 smithi139 bash[43301]: cluster 2022-04-28T21:44:09.781134+0000 mon.a (mon.0) 3870 : cluster [DBG] osdmap e1367: 8 total, 8 up, 8 in 2022-04-28T21:44:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:11 smithi059 bash[61152]: cluster 2022-04-28T21:44:10.293718+0000 mgr.y (mgr.24961) 3740 : cluster [DBG] pgmap v4466: 177 pgs: 3 creating+peering, 5 unknown, 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:11 smithi059 bash[61152]: cluster 2022-04-28T21:44:10.782865+0000 mon.a (mon.0) 3871 : cluster [DBG] osdmap e1368: 8 total, 8 up, 8 in 2022-04-28T21:44:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:11 smithi059 bash[59745]: cluster 2022-04-28T21:44:10.293718+0000 mgr.y (mgr.24961) 3740 : cluster [DBG] pgmap v4466: 177 pgs: 3 creating+peering, 5 unknown, 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:11 smithi059 bash[59745]: cluster 2022-04-28T21:44:10.782865+0000 mon.a (mon.0) 3871 : cluster [DBG] osdmap e1368: 8 total, 8 up, 8 in 2022-04-28T21:44:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:11 smithi139 bash[43301]: cluster 2022-04-28T21:44:10.293718+0000 mgr.y (mgr.24961) 3740 : cluster [DBG] pgmap v4466: 177 pgs: 3 creating+peering, 5 unknown, 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:11 smithi139 bash[43301]: cluster 2022-04-28T21:44:10.782865+0000 mon.a (mon.0) 3871 : cluster [DBG] osdmap e1368: 8 total, 8 up, 8 in 2022-04-28T21:44:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[59745]: audit 2022-04-28T21:44:11.715547+0000 mon.a (mon.0) 3872 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:44:12.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[59745]: audit 2022-04-28T21:44:11.718727+0000 mon.a (mon.0) 3873 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:44:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[59745]: audit 2022-04-28T21:44:11.720943+0000 mon.a (mon.0) 3874 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:44:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[59745]: cluster 2022-04-28T21:44:11.785716+0000 mon.a (mon.0) 3875 : cluster [DBG] osdmap e1369: 8 total, 8 up, 8 in 2022-04-28T21:44:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[59745]: audit 2022-04-28T21:44:11.787680+0000 mon.a (mon.0) 3876 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:12.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[59745]: audit 2022-04-28T21:44:12.047538+0000 mon.a (mon.0) 3877 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:44:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[61152]: audit 2022-04-28T21:44:11.715547+0000 mon.a (mon.0) 3872 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:44:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[61152]: audit 2022-04-28T21:44:11.718727+0000 mon.a (mon.0) 3873 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:44:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[61152]: audit 2022-04-28T21:44:11.720943+0000 mon.a (mon.0) 3874 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:44:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[61152]: cluster 2022-04-28T21:44:11.785716+0000 mon.a (mon.0) 3875 : cluster [DBG] osdmap e1369: 8 total, 8 up, 8 in 2022-04-28T21:44:12.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[61152]: audit 2022-04-28T21:44:11.787680+0000 mon.a (mon.0) 3876 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:12.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[61152]: audit 2022-04-28T21:44:12.047538+0000 mon.a (mon.0) 3877 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:44:12.344 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:12.070Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:12 smithi139 bash[43301]: audit 2022-04-28T21:44:11.715547+0000 mon.a (mon.0) 3872 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:44:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:12 smithi139 bash[43301]: audit 2022-04-28T21:44:11.718727+0000 mon.a (mon.0) 3873 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:44:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:12 smithi139 bash[43301]: audit 2022-04-28T21:44:11.720943+0000 mon.a (mon.0) 3874 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:44:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:12 smithi139 bash[43301]: cluster 2022-04-28T21:44:11.785716+0000 mon.a (mon.0) 3875 : cluster [DBG] osdmap e1369: 8 total, 8 up, 8 in 2022-04-28T21:44:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:12 smithi139 bash[43301]: audit 2022-04-28T21:44:11.787680+0000 mon.a (mon.0) 3876 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:12 smithi139 bash[43301]: audit 2022-04-28T21:44:12.047538+0000 mon.a (mon.0) 3877 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:44:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[61152]: cephadm 2022-04-28T21:44:11.722936+0000 mgr.y (mgr.24961) 3741 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:44:13.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[61152]: cluster 2022-04-28T21:44:12.294292+0000 mgr.y (mgr.24961) 3742 : cluster [DBG] pgmap v4469: 169 pgs: 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[61152]: audit 2022-04-28T21:44:12.789241+0000 mon.a (mon.0) 3878 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[61152]: cluster 2022-04-28T21:44:12.789837+0000 mon.a (mon.0) 3879 : cluster [DBG] osdmap e1370: 8 total, 8 up, 8 in 2022-04-28T21:44:13.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[61152]: audit 2022-04-28T21:44:12.791090+0000 mon.a (mon.0) 3880 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[59745]: cephadm 2022-04-28T21:44:11.722936+0000 mgr.y (mgr.24961) 3741 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:44:13.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[59745]: cluster 2022-04-28T21:44:12.294292+0000 mgr.y (mgr.24961) 3742 : cluster [DBG] pgmap v4469: 169 pgs: 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[59745]: audit 2022-04-28T21:44:12.789241+0000 mon.a (mon.0) 3878 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[59745]: cluster 2022-04-28T21:44:12.789837+0000 mon.a (mon.0) 3879 : cluster [DBG] osdmap e1370: 8 total, 8 up, 8 in 2022-04-28T21:44:13.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[59745]: audit 2022-04-28T21:44:12.791090+0000 mon.a (mon.0) 3880 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:13 smithi139 bash[43301]: cephadm 2022-04-28T21:44:11.722936+0000 mgr.y (mgr.24961) 3741 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:44:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:13 smithi139 bash[43301]: cluster 2022-04-28T21:44:12.294292+0000 mgr.y (mgr.24961) 3742 : cluster [DBG] pgmap v4469: 169 pgs: 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:13.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:13 smithi139 bash[43301]: audit 2022-04-28T21:44:12.789241+0000 mon.a (mon.0) 3878 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:13 smithi139 bash[43301]: cluster 2022-04-28T21:44:12.789837+0000 mon.a (mon.0) 3879 : cluster [DBG] osdmap e1370: 8 total, 8 up, 8 in 2022-04-28T21:44:13.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:13 smithi139 bash[43301]: audit 2022-04-28T21:44:12.791090+0000 mon.a (mon.0) 3880 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]: dispatch 2022-04-28T21:44:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:13.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:15.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.791628+0000 mon.a (mon.0) 3881 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:15.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: cluster 2022-04-28T21:44:13.791909+0000 mon.a (mon.0) 3882 : cluster [DBG] osdmap e1371: 8 total, 8 up, 8 in 2022-04-28T21:44:15.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.804333+0000 mon.c (mon.1) 182 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.805131+0000 mon.a (mon.0) 3883 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.805875+0000 mon.c (mon.1) 183 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.806265+0000 mon.a (mon.0) 3884 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.806887+0000 mon.c (mon.1) 184 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:15.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: audit 2022-04-28T21:44:13.807266+0000 mon.a (mon.0) 3885 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:15.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[59745]: cluster 2022-04-28T21:44:14.295587+0000 mgr.y (mgr.24961) 3743 : cluster [DBG] pgmap v4472: 169 pgs: 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:15.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.791628+0000 mon.a (mon.0) 3881 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:15.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: cluster 2022-04-28T21:44:13.791909+0000 mon.a (mon.0) 3882 : cluster [DBG] osdmap e1371: 8 total, 8 up, 8 in 2022-04-28T21:44:15.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.804333+0000 mon.c (mon.1) 182 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.805131+0000 mon.a (mon.0) 3883 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.805875+0000 mon.c (mon.1) 183 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.806265+0000 mon.a (mon.0) 3884 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.806887+0000 mon.c (mon.1) 184 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:15.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: audit 2022-04-28T21:44:13.807266+0000 mon.a (mon.0) 3885 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:15.097 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:14 smithi059 bash[61152]: cluster 2022-04-28T21:44:14.295587+0000 mgr.y (mgr.24961) 3743 : cluster [DBG] pgmap v4472: 169 pgs: 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.791628+0000 mon.a (mon.0) 3881 : audit [INF] from='client.? 172.21.15.59:0/1098950565' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-53"}]': finished 2022-04-28T21:44:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: cluster 2022-04-28T21:44:13.791909+0000 mon.a (mon.0) 3882 : cluster [DBG] osdmap e1371: 8 total, 8 up, 8 in 2022-04-28T21:44:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.804333+0000 mon.c (mon.1) 182 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.805131+0000 mon.a (mon.0) 3883 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.805875+0000 mon.c (mon.1) 183 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.806265+0000 mon.a (mon.0) 3884 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:15.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.806887+0000 mon.c (mon.1) 184 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:15.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: audit 2022-04-28T21:44:13.807266+0000 mon.a (mon.0) 3885 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:15.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:14 smithi139 bash[43301]: cluster 2022-04-28T21:44:14.295587+0000 mgr.y (mgr.24961) 3743 : cluster [DBG] pgmap v4472: 169 pgs: 169 active+clean; 24 KiB data, 897 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[61152]: audit 2022-04-28T21:44:14.800026+0000 mon.a (mon.0) 3886 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[61152]: cluster 2022-04-28T21:44:14.800118+0000 mon.a (mon.0) 3887 : cluster [DBG] osdmap e1372: 8 total, 8 up, 8 in 2022-04-28T21:44:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[61152]: audit 2022-04-28T21:44:14.803025+0000 mon.c (mon.1) 185 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[61152]: audit 2022-04-28T21:44:14.803441+0000 mon.a (mon.0) 3888 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[59745]: audit 2022-04-28T21:44:14.800026+0000 mon.a (mon.0) 3886 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[59745]: cluster 2022-04-28T21:44:14.800118+0000 mon.a (mon.0) 3887 : cluster [DBG] osdmap e1372: 8 total, 8 up, 8 in 2022-04-28T21:44:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[59745]: audit 2022-04-28T21:44:14.803025+0000 mon.c (mon.1) 185 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:16.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:16 smithi059 bash[59745]: audit 2022-04-28T21:44:14.803441+0000 mon.a (mon.0) 3888 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:16 smithi139 bash[43301]: audit 2022-04-28T21:44:14.800026+0000 mon.a (mon.0) 3886 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-54", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:16 smithi139 bash[43301]: cluster 2022-04-28T21:44:14.800118+0000 mon.a (mon.0) 3887 : cluster [DBG] osdmap e1372: 8 total, 8 up, 8 in 2022-04-28T21:44:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:16 smithi139 bash[43301]: audit 2022-04-28T21:44:14.803025+0000 mon.c (mon.1) 185 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:16 smithi139 bash[43301]: audit 2022-04-28T21:44:14.803441+0000 mon.a (mon.0) 3888 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[61152]: cluster 2022-04-28T21:44:16.269058+0000 mon.a (mon.0) 3889 : cluster [DBG] osdmap e1373: 8 total, 8 up, 8 in 2022-04-28T21:44:17.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[61152]: cluster 2022-04-28T21:44:16.296450+0000 mgr.y (mgr.24961) 3744 : cluster [DBG] pgmap v4475: 169 pgs: 169 active+clean; 24 KiB data, 898 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[61152]: audit 2022-04-28T21:44:16.808379+0000 mon.a (mon.0) 3890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:17.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[61152]: cluster 2022-04-28T21:44:16.808695+0000 mon.a (mon.0) 3891 : cluster [DBG] osdmap e1374: 8 total, 8 up, 8 in 2022-04-28T21:44:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[59745]: cluster 2022-04-28T21:44:16.269058+0000 mon.a (mon.0) 3889 : cluster [DBG] osdmap e1373: 8 total, 8 up, 8 in 2022-04-28T21:44:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[59745]: cluster 2022-04-28T21:44:16.296450+0000 mgr.y (mgr.24961) 3744 : cluster [DBG] pgmap v4475: 169 pgs: 169 active+clean; 24 KiB data, 898 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:17.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[59745]: audit 2022-04-28T21:44:16.808379+0000 mon.a (mon.0) 3890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:17.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:17 smithi059 bash[59745]: cluster 2022-04-28T21:44:16.808695+0000 mon.a (mon.0) 3891 : cluster [DBG] osdmap e1374: 8 total, 8 up, 8 in 2022-04-28T21:44:17.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:17 smithi139 bash[43301]: cluster 2022-04-28T21:44:16.269058+0000 mon.a (mon.0) 3889 : cluster [DBG] osdmap e1373: 8 total, 8 up, 8 in 2022-04-28T21:44:17.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:17 smithi139 bash[43301]: cluster 2022-04-28T21:44:16.296450+0000 mgr.y (mgr.24961) 3744 : cluster [DBG] pgmap v4475: 169 pgs: 169 active+clean; 24 KiB data, 898 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:17.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:17 smithi139 bash[43301]: audit 2022-04-28T21:44:16.808379+0000 mon.a (mon.0) 3890 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-54", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:17.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:17 smithi139 bash[43301]: cluster 2022-04-28T21:44:16.808695+0000 mon.a (mon.0) 3891 : cluster [DBG] osdmap e1374: 8 total, 8 up, 8 in 2022-04-28T21:44:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:18 smithi059 bash[59745]: cluster 2022-04-28T21:44:17.809006+0000 mon.a (mon.0) 3892 : cluster [DBG] osdmap e1375: 8 total, 8 up, 8 in 2022-04-28T21:44:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:18 smithi059 bash[59745]: cluster 2022-04-28T21:44:18.297026+0000 mgr.y (mgr.24961) 3745 : cluster [DBG] pgmap v4478: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 898 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:18 smithi059 bash[61152]: cluster 2022-04-28T21:44:17.809006+0000 mon.a (mon.0) 3892 : cluster [DBG] osdmap e1375: 8 total, 8 up, 8 in 2022-04-28T21:44:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:18 smithi059 bash[61152]: cluster 2022-04-28T21:44:18.297026+0000 mgr.y (mgr.24961) 3745 : cluster [DBG] pgmap v4478: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 898 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:19.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:18 smithi139 bash[43301]: cluster 2022-04-28T21:44:17.809006+0000 mon.a (mon.0) 3892 : cluster [DBG] osdmap e1375: 8 total, 8 up, 8 in 2022-04-28T21:44:19.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:18 smithi139 bash[43301]: cluster 2022-04-28T21:44:18.297026+0000 mgr.y (mgr.24961) 3745 : cluster [DBG] pgmap v4478: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 898 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: cluster 2022-04-28T21:44:18.813348+0000 mon.a (mon.0) 3893 : cluster [DBG] osdmap e1376: 8 total, 8 up, 8 in 2022-04-28T21:44:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:18.817700+0000 mon.c (mon.1) 186 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:18.824987+0000 mon.a (mon.0) 3894 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:19.353986+0000 mon.a (mon.0) 3895 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:44:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:19.361957+0000 mon.a (mon.0) 3896 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:44:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:19.380175+0000 mon.a (mon.0) 3897 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:44:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:19.380770+0000 mon.a (mon.0) 3898 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:44:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:19.381365+0000 mon.a (mon.0) 3899 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:44:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[61152]: audit 2022-04-28T21:44:19.381916+0000 mon.a (mon.0) 3900 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:44:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: cluster 2022-04-28T21:44:18.813348+0000 mon.a (mon.0) 3893 : cluster [DBG] osdmap e1376: 8 total, 8 up, 8 in 2022-04-28T21:44:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:18.817700+0000 mon.c (mon.1) 186 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:18.824987+0000 mon.a (mon.0) 3894 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:19.353986+0000 mon.a (mon.0) 3895 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:44:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:19.361957+0000 mon.a (mon.0) 3896 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:44:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:19.380175+0000 mon.a (mon.0) 3897 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:44:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:19.380770+0000 mon.a (mon.0) 3898 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:44:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:19.381365+0000 mon.a (mon.0) 3899 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:44:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:19 smithi059 bash[59745]: audit 2022-04-28T21:44:19.381916+0000 mon.a (mon.0) 3900 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:44:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: cluster 2022-04-28T21:44:18.813348+0000 mon.a (mon.0) 3893 : cluster [DBG] osdmap e1376: 8 total, 8 up, 8 in 2022-04-28T21:44:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:18.817700+0000 mon.c (mon.1) 186 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:18.824987+0000 mon.a (mon.0) 3894 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:19.353986+0000 mon.a (mon.0) 3895 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:44:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:19.361957+0000 mon.a (mon.0) 3896 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:44:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:19.380175+0000 mon.a (mon.0) 3897 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:44:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:19.380770+0000 mon.a (mon.0) 3898 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T21:44:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:19.381365+0000 mon.a (mon.0) 3899 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:44:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:19 smithi139 bash[43301]: audit 2022-04-28T21:44:19.381916+0000 mon.a (mon.0) 3900 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:44:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: audit 2022-04-28T21:44:19.827211+0000 mon.a (mon.0) 3901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: cluster 2022-04-28T21:44:19.827332+0000 mon.a (mon.0) 3902 : cluster [DBG] osdmap e1377: 8 total, 8 up, 8 in 2022-04-28T21:44:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: audit 2022-04-28T21:44:19.829011+0000 mon.c (mon.1) 187 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: audit 2022-04-28T21:44:19.829739+0000 mon.a (mon.0) 3903 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: cluster 2022-04-28T21:44:20.298103+0000 mgr.y (mgr.24961) 3746 : cluster [DBG] pgmap v4481: 169 pgs: 169 active+clean; 24 KiB data, 903 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: audit 2022-04-28T21:44:20.830136+0000 mon.a (mon.0) 3904 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:21.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[59745]: cluster 2022-04-28T21:44:20.830228+0000 mon.a (mon.0) 3905 : cluster [DBG] osdmap e1378: 8 total, 8 up, 8 in 2022-04-28T21:44:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: audit 2022-04-28T21:44:19.827211+0000 mon.a (mon.0) 3901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: cluster 2022-04-28T21:44:19.827332+0000 mon.a (mon.0) 3902 : cluster [DBG] osdmap e1377: 8 total, 8 up, 8 in 2022-04-28T21:44:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: audit 2022-04-28T21:44:19.829011+0000 mon.c (mon.1) 187 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: audit 2022-04-28T21:44:19.829739+0000 mon.a (mon.0) 3903 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:21.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: cluster 2022-04-28T21:44:20.298103+0000 mgr.y (mgr.24961) 3746 : cluster [DBG] pgmap v4481: 169 pgs: 169 active+clean; 24 KiB data, 903 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:21.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: audit 2022-04-28T21:44:20.830136+0000 mon.a (mon.0) 3904 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:21.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:20 smithi059 bash[61152]: cluster 2022-04-28T21:44:20.830228+0000 mon.a (mon.0) 3905 : cluster [DBG] osdmap e1378: 8 total, 8 up, 8 in 2022-04-28T21:44:21.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: audit 2022-04-28T21:44:19.827211+0000 mon.a (mon.0) 3901 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:21.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: cluster 2022-04-28T21:44:19.827332+0000 mon.a (mon.0) 3902 : cluster [DBG] osdmap e1377: 8 total, 8 up, 8 in 2022-04-28T21:44:21.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: audit 2022-04-28T21:44:19.829011+0000 mon.c (mon.1) 187 : audit [INF] from='client.? 172.21.15.59:0/2339204099' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:21.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: audit 2022-04-28T21:44:19.829739+0000 mon.a (mon.0) 3903 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]: dispatch 2022-04-28T21:44:21.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: cluster 2022-04-28T21:44:20.298103+0000 mgr.y (mgr.24961) 3746 : cluster [DBG] pgmap v4481: 169 pgs: 169 active+clean; 24 KiB data, 903 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:21.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: audit 2022-04-28T21:44:20.830136+0000 mon.a (mon.0) 3904 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-54"}]': finished 2022-04-28T21:44:21.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:20 smithi139 bash[43301]: cluster 2022-04-28T21:44:20.830228+0000 mon.a (mon.0) 3905 : cluster [DBG] osdmap e1378: 8 total, 8 up, 8 in 2022-04-28T21:44:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:21 smithi139 bash[43301]: audit 2022-04-28T21:44:20.849617+0000 mon.c (mon.1) 188 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:21 smithi139 bash[43301]: audit 2022-04-28T21:44:20.850046+0000 mon.a (mon.0) 3906 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:21 smithi139 bash[43301]: audit 2022-04-28T21:44:20.850696+0000 mon.c (mon.1) 189 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:21 smithi139 bash[43301]: audit 2022-04-28T21:44:20.851046+0000 mon.a (mon.0) 3907 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:21 smithi139 bash[43301]: audit 2022-04-28T21:44:20.851565+0000 mon.c (mon.1) 190 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:22.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:21 smithi139 bash[43301]: audit 2022-04-28T21:44:20.851960+0000 mon.a (mon.0) 3908 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:22.071Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[61152]: audit 2022-04-28T21:44:20.849617+0000 mon.c (mon.1) 188 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[61152]: audit 2022-04-28T21:44:20.850046+0000 mon.a (mon.0) 3906 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[61152]: audit 2022-04-28T21:44:20.850696+0000 mon.c (mon.1) 189 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[61152]: audit 2022-04-28T21:44:20.851046+0000 mon.a (mon.0) 3907 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[61152]: audit 2022-04-28T21:44:20.851565+0000 mon.c (mon.1) 190 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:22.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[61152]: audit 2022-04-28T21:44:20.851960+0000 mon.a (mon.0) 3908 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[59745]: audit 2022-04-28T21:44:20.849617+0000 mon.c (mon.1) 188 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[59745]: audit 2022-04-28T21:44:20.850046+0000 mon.a (mon.0) 3906 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[59745]: audit 2022-04-28T21:44:20.850696+0000 mon.c (mon.1) 189 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[59745]: audit 2022-04-28T21:44:20.851046+0000 mon.a (mon.0) 3907 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:22.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[59745]: audit 2022-04-28T21:44:20.851565+0000 mon.c (mon.1) 190 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:22.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:21 smithi059 bash[59745]: audit 2022-04-28T21:44:20.851960+0000 mon.a (mon.0) 3908 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:23.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[61152]: audit 2022-04-28T21:44:21.852334+0000 mon.a (mon.0) 3909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[61152]: cluster 2022-04-28T21:44:21.852619+0000 mon.a (mon.0) 3910 : cluster [DBG] osdmap e1379: 8 total, 8 up, 8 in 2022-04-28T21:44:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[61152]: audit 2022-04-28T21:44:21.854803+0000 mon.c (mon.1) 191 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[61152]: audit 2022-04-28T21:44:21.855653+0000 mon.a (mon.0) 3911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:23.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[61152]: cluster 2022-04-28T21:44:22.298933+0000 mgr.y (mgr.24961) 3747 : cluster [DBG] pgmap v4484: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[59745]: audit 2022-04-28T21:44:21.852334+0000 mon.a (mon.0) 3909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[59745]: cluster 2022-04-28T21:44:21.852619+0000 mon.a (mon.0) 3910 : cluster [DBG] osdmap e1379: 8 total, 8 up, 8 in 2022-04-28T21:44:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[59745]: audit 2022-04-28T21:44:21.854803+0000 mon.c (mon.1) 191 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[59745]: audit 2022-04-28T21:44:21.855653+0000 mon.a (mon.0) 3911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:23.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[59745]: cluster 2022-04-28T21:44:22.298933+0000 mgr.y (mgr.24961) 3747 : cluster [DBG] pgmap v4484: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:23 smithi139 bash[43301]: audit 2022-04-28T21:44:21.852334+0000 mon.a (mon.0) 3909 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-55", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:23 smithi139 bash[43301]: cluster 2022-04-28T21:44:21.852619+0000 mon.a (mon.0) 3910 : cluster [DBG] osdmap e1379: 8 total, 8 up, 8 in 2022-04-28T21:44:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:23 smithi139 bash[43301]: audit 2022-04-28T21:44:21.854803+0000 mon.c (mon.1) 191 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:23.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:23 smithi139 bash[43301]: audit 2022-04-28T21:44:21.855653+0000 mon.a (mon.0) 3911 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:23.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:23 smithi139 bash[43301]: cluster 2022-04-28T21:44:22.298933+0000 mgr.y (mgr.24961) 3747 : cluster [DBG] pgmap v4484: 169 pgs: 169 active+clean; 24 KiB data, 887 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:24.327 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:23.961Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:24.327 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:24 smithi059 bash[59745]: cluster 2022-04-28T21:44:23.314880+0000 mon.a (mon.0) 3912 : cluster [DBG] osdmap e1380: 8 total, 8 up, 8 in 2022-04-28T21:44:24.328 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:24 smithi059 bash[59745]: audit 2022-04-28T21:44:23.857535+0000 mon.a (mon.0) 3913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:24.328 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:24 smithi059 bash[59745]: cluster 2022-04-28T21:44:23.857864+0000 mon.a (mon.0) 3914 : cluster [DBG] osdmap e1381: 8 total, 8 up, 8 in 2022-04-28T21:44:24.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:24 smithi059 bash[61152]: cluster 2022-04-28T21:44:23.314880+0000 mon.a (mon.0) 3912 : cluster [DBG] osdmap e1380: 8 total, 8 up, 8 in 2022-04-28T21:44:24.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:24 smithi059 bash[61152]: audit 2022-04-28T21:44:23.857535+0000 mon.a (mon.0) 3913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:24.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:24 smithi059 bash[61152]: cluster 2022-04-28T21:44:23.857864+0000 mon.a (mon.0) 3914 : cluster [DBG] osdmap e1381: 8 total, 8 up, 8 in 2022-04-28T21:44:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:24 smithi139 bash[43301]: cluster 2022-04-28T21:44:23.314880+0000 mon.a (mon.0) 3912 : cluster [DBG] osdmap e1380: 8 total, 8 up, 8 in 2022-04-28T21:44:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:24 smithi139 bash[43301]: audit 2022-04-28T21:44:23.857535+0000 mon.a (mon.0) 3913 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-55", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:24 smithi139 bash[43301]: cluster 2022-04-28T21:44:23.857864+0000 mon.a (mon.0) 3914 : cluster [DBG] osdmap e1381: 8 total, 8 up, 8 in 2022-04-28T21:44:25.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:25 smithi059 bash[61152]: cluster 2022-04-28T21:44:24.300206+0000 mgr.y (mgr.24961) 3748 : cluster [DBG] pgmap v4487: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:25.691 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:25 smithi059 bash[61152]: cluster 2022-04-28T21:44:24.862517+0000 mon.a (mon.0) 3915 : cluster [DBG] osdmap e1382: 8 total, 8 up, 8 in 2022-04-28T21:44:25.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:25 smithi059 bash[59745]: cluster 2022-04-28T21:44:24.300206+0000 mgr.y (mgr.24961) 3748 : cluster [DBG] pgmap v4487: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:25.691 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:25 smithi059 bash[59745]: cluster 2022-04-28T21:44:24.862517+0000 mon.a (mon.0) 3915 : cluster [DBG] osdmap e1382: 8 total, 8 up, 8 in 2022-04-28T21:44:25.692 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:25 smithi139 bash[43301]: cluster 2022-04-28T21:44:24.300206+0000 mgr.y (mgr.24961) 3748 : cluster [DBG] pgmap v4487: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 888 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:25.692 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:25 smithi139 bash[43301]: cluster 2022-04-28T21:44:24.862517+0000 mon.a (mon.0) 3915 : cluster [DBG] osdmap e1382: 8 total, 8 up, 8 in 2022-04-28T21:44:27.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[59745]: cluster 2022-04-28T21:44:25.876012+0000 mon.a (mon.0) 3916 : cluster [DBG] osdmap e1383: 8 total, 8 up, 8 in 2022-04-28T21:44:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[59745]: audit 2022-04-28T21:44:25.878681+0000 mon.c (mon.1) 192 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[59745]: audit 2022-04-28T21:44:25.879261+0000 mon.a (mon.0) 3917 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[59745]: cluster 2022-04-28T21:44:26.301213+0000 mgr.y (mgr.24961) 3749 : cluster [DBG] pgmap v4490: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[61152]: cluster 2022-04-28T21:44:25.876012+0000 mon.a (mon.0) 3916 : cluster [DBG] osdmap e1383: 8 total, 8 up, 8 in 2022-04-28T21:44:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[61152]: audit 2022-04-28T21:44:25.878681+0000 mon.c (mon.1) 192 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[61152]: audit 2022-04-28T21:44:25.879261+0000 mon.a (mon.0) 3917 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:27.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:26 smithi059 bash[61152]: cluster 2022-04-28T21:44:26.301213+0000 mgr.y (mgr.24961) 3749 : cluster [DBG] pgmap v4490: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:26 smithi139 bash[43301]: cluster 2022-04-28T21:44:25.876012+0000 mon.a (mon.0) 3916 : cluster [DBG] osdmap e1383: 8 total, 8 up, 8 in 2022-04-28T21:44:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:26 smithi139 bash[43301]: audit 2022-04-28T21:44:25.878681+0000 mon.c (mon.1) 192 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:26 smithi139 bash[43301]: audit 2022-04-28T21:44:25.879261+0000 mon.a (mon.0) 3917 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:27.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:26 smithi139 bash[43301]: cluster 2022-04-28T21:44:26.301213+0000 mgr.y (mgr.24961) 3749 : cluster [DBG] pgmap v4490: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:28.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[59745]: audit 2022-04-28T21:44:26.885854+0000 mon.a (mon.0) 3918 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[59745]: cluster 2022-04-28T21:44:26.885999+0000 mon.a (mon.0) 3919 : cluster [DBG] osdmap e1384: 8 total, 8 up, 8 in 2022-04-28T21:44:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[59745]: audit 2022-04-28T21:44:26.888952+0000 mon.c (mon.1) 193 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:28.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[59745]: audit 2022-04-28T21:44:26.889410+0000 mon.a (mon.0) 3920 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:28.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[61152]: audit 2022-04-28T21:44:26.885854+0000 mon.a (mon.0) 3918 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:28.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[61152]: cluster 2022-04-28T21:44:26.885999+0000 mon.a (mon.0) 3919 : cluster [DBG] osdmap e1384: 8 total, 8 up, 8 in 2022-04-28T21:44:28.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[61152]: audit 2022-04-28T21:44:26.888952+0000 mon.c (mon.1) 193 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:28.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:27 smithi059 bash[61152]: audit 2022-04-28T21:44:26.889410+0000 mon.a (mon.0) 3920 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:27 smithi139 bash[43301]: audit 2022-04-28T21:44:26.885854+0000 mon.a (mon.0) 3918 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:27 smithi139 bash[43301]: cluster 2022-04-28T21:44:26.885999+0000 mon.a (mon.0) 3919 : cluster [DBG] osdmap e1384: 8 total, 8 up, 8 in 2022-04-28T21:44:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:27 smithi139 bash[43301]: audit 2022-04-28T21:44:26.888952+0000 mon.c (mon.1) 193 : audit [INF] from='client.? 172.21.15.59:0/3254503186' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:28.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:27 smithi139 bash[43301]: audit 2022-04-28T21:44:26.889410+0000 mon.a (mon.0) 3920 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]: dispatch 2022-04-28T21:44:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:28 smithi059 bash[61152]: audit 2022-04-28T21:44:27.889195+0000 mon.a (mon.0) 3921 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:29.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:28 smithi059 bash[61152]: cluster 2022-04-28T21:44:27.889465+0000 mon.a (mon.0) 3922 : cluster [DBG] osdmap e1385: 8 total, 8 up, 8 in 2022-04-28T21:44:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:28 smithi059 bash[61152]: cluster 2022-04-28T21:44:28.301901+0000 mgr.y (mgr.24961) 3750 : cluster [DBG] pgmap v4493: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:28 smithi059 bash[59745]: audit 2022-04-28T21:44:27.889195+0000 mon.a (mon.0) 3921 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:28 smithi059 bash[59745]: cluster 2022-04-28T21:44:27.889465+0000 mon.a (mon.0) 3922 : cluster [DBG] osdmap e1385: 8 total, 8 up, 8 in 2022-04-28T21:44:29.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:28 smithi059 bash[59745]: cluster 2022-04-28T21:44:28.301901+0000 mgr.y (mgr.24961) 3750 : cluster [DBG] pgmap v4493: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:28 smithi139 bash[43301]: audit 2022-04-28T21:44:27.889195+0000 mon.a (mon.0) 3921 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-55"}]': finished 2022-04-28T21:44:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:28 smithi139 bash[43301]: cluster 2022-04-28T21:44:27.889465+0000 mon.a (mon.0) 3922 : cluster [DBG] osdmap e1385: 8 total, 8 up, 8 in 2022-04-28T21:44:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:28 smithi139 bash[43301]: cluster 2022-04-28T21:44:28.301901+0000 mgr.y (mgr.24961) 3750 : cluster [DBG] pgmap v4493: 169 pgs: 169 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:30.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:29 smithi059 bash[61152]: cluster 2022-04-28T21:44:28.908595+0000 mon.a (mon.0) 3923 : cluster [DBG] osdmap e1386: 8 total, 8 up, 8 in 2022-04-28T21:44:30.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:29 smithi059 bash[61152]: audit 2022-04-28T21:44:28.910800+0000 mon.c (mon.1) 194 : audit [INF] from='client.? 172.21.15.59:0/4083663883' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:44:30.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:29 smithi059 bash[61152]: audit 2022-04-28T21:44:28.913689+0000 mon.a (mon.0) 3924 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:44:30.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:29 smithi059 bash[59745]: cluster 2022-04-28T21:44:28.908595+0000 mon.a (mon.0) 3923 : cluster [DBG] osdmap e1386: 8 total, 8 up, 8 in 2022-04-28T21:44:30.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:29 smithi059 bash[59745]: audit 2022-04-28T21:44:28.910800+0000 mon.c (mon.1) 194 : audit [INF] from='client.? 172.21.15.59:0/4083663883' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:44:30.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:29 smithi059 bash[59745]: audit 2022-04-28T21:44:28.913689+0000 mon.a (mon.0) 3924 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:44:30.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:29 smithi139 bash[43301]: cluster 2022-04-28T21:44:28.908595+0000 mon.a (mon.0) 3923 : cluster [DBG] osdmap e1386: 8 total, 8 up, 8 in 2022-04-28T21:44:30.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:29 smithi139 bash[43301]: audit 2022-04-28T21:44:28.910800+0000 mon.c (mon.1) 194 : audit [INF] from='client.? 172.21.15.59:0/4083663883' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:44:30.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:29 smithi139 bash[43301]: audit 2022-04-28T21:44:28.913689+0000 mon.a (mon.0) 3924 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:44:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:30 smithi059 bash[59745]: audit 2022-04-28T21:44:29.914156+0000 mon.a (mon.0) 3925 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:44:31.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:30 smithi059 bash[59745]: cluster 2022-04-28T21:44:29.914966+0000 mon.a (mon.0) 3926 : cluster [DBG] osdmap e1387: 8 total, 8 up, 8 in 2022-04-28T21:44:31.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:30 smithi059 bash[59745]: cluster 2022-04-28T21:44:30.303049+0000 mgr.y (mgr.24961) 3751 : cluster [DBG] pgmap v4496: 201 pgs: 9 creating+peering, 11 unknown, 181 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:30 smithi059 bash[61152]: audit 2022-04-28T21:44:29.914156+0000 mon.a (mon.0) 3925 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:44:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:30 smithi059 bash[61152]: cluster 2022-04-28T21:44:29.914966+0000 mon.a (mon.0) 3926 : cluster [DBG] osdmap e1387: 8 total, 8 up, 8 in 2022-04-28T21:44:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:30 smithi059 bash[61152]: cluster 2022-04-28T21:44:30.303049+0000 mgr.y (mgr.24961) 3751 : cluster [DBG] pgmap v4496: 201 pgs: 9 creating+peering, 11 unknown, 181 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:30 smithi139 bash[43301]: audit 2022-04-28T21:44:29.914156+0000 mon.a (mon.0) 3925 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-84622-56","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:44:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:30 smithi139 bash[43301]: cluster 2022-04-28T21:44:29.914966+0000 mon.a (mon.0) 3926 : cluster [DBG] osdmap e1387: 8 total, 8 up, 8 in 2022-04-28T21:44:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:30 smithi139 bash[43301]: cluster 2022-04-28T21:44:30.303049+0000 mgr.y (mgr.24961) 3751 : cluster [DBG] pgmap v4496: 201 pgs: 9 creating+peering, 11 unknown, 181 active+clean; 24 KiB data, 889 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:32.072Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: cluster 2022-04-28T21:44:30.927958+0000 mon.a (mon.0) 3927 : cluster [DBG] osdmap e1388: 8 total, 8 up, 8 in 2022-04-28T21:44:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: audit 2022-04-28T21:44:30.952933+0000 mon.c (mon.1) 195 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: audit 2022-04-28T21:44:30.953360+0000 mon.a (mon.0) 3928 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: audit 2022-04-28T21:44:30.954556+0000 mon.c (mon.1) 196 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: audit 2022-04-28T21:44:30.955007+0000 mon.a (mon.0) 3929 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: audit 2022-04-28T21:44:30.955532+0000 mon.c (mon.1) 197 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:32.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[61152]: audit 2022-04-28T21:44:30.955862+0000 mon.a (mon.0) 3930 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: cluster 2022-04-28T21:44:30.927958+0000 mon.a (mon.0) 3927 : cluster [DBG] osdmap e1388: 8 total, 8 up, 8 in 2022-04-28T21:44:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: audit 2022-04-28T21:44:30.952933+0000 mon.c (mon.1) 195 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: audit 2022-04-28T21:44:30.953360+0000 mon.a (mon.0) 3928 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: audit 2022-04-28T21:44:30.954556+0000 mon.c (mon.1) 196 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: audit 2022-04-28T21:44:30.955007+0000 mon.a (mon.0) 3929 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: audit 2022-04-28T21:44:30.955532+0000 mon.c (mon.1) 197 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:32.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:31 smithi059 bash[59745]: audit 2022-04-28T21:44:30.955862+0000 mon.a (mon.0) 3930 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: cluster 2022-04-28T21:44:30.927958+0000 mon.a (mon.0) 3927 : cluster [DBG] osdmap e1388: 8 total, 8 up, 8 in 2022-04-28T21:44:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: audit 2022-04-28T21:44:30.952933+0000 mon.c (mon.1) 195 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: audit 2022-04-28T21:44:30.953360+0000 mon.a (mon.0) 3928 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: audit 2022-04-28T21:44:30.954556+0000 mon.c (mon.1) 196 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: audit 2022-04-28T21:44:30.955007+0000 mon.a (mon.0) 3929 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:32.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: audit 2022-04-28T21:44:30.955532+0000 mon.c (mon.1) 197 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:32.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:31 smithi139 bash[43301]: audit 2022-04-28T21:44:30.955862+0000 mon.a (mon.0) 3930 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[61152]: audit 2022-04-28T21:44:31.947637+0000 mon.a (mon.0) 3931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:33.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[61152]: cluster 2022-04-28T21:44:31.947738+0000 mon.a (mon.0) 3932 : cluster [DBG] osdmap e1389: 8 total, 8 up, 8 in 2022-04-28T21:44:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[61152]: audit 2022-04-28T21:44:31.950769+0000 mon.c (mon.1) 198 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[61152]: audit 2022-04-28T21:44:31.951405+0000 mon.a (mon.0) 3933 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[61152]: cluster 2022-04-28T21:44:32.303867+0000 mgr.y (mgr.24961) 3752 : cluster [DBG] pgmap v4499: 169 pgs: 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:33.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[59745]: audit 2022-04-28T21:44:31.947637+0000 mon.a (mon.0) 3931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:33.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[59745]: cluster 2022-04-28T21:44:31.947738+0000 mon.a (mon.0) 3932 : cluster [DBG] osdmap e1389: 8 total, 8 up, 8 in 2022-04-28T21:44:33.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[59745]: audit 2022-04-28T21:44:31.950769+0000 mon.c (mon.1) 198 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:33.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[59745]: audit 2022-04-28T21:44:31.951405+0000 mon.a (mon.0) 3933 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:33.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:32 smithi059 bash[59745]: cluster 2022-04-28T21:44:32.303867+0000 mgr.y (mgr.24961) 3752 : cluster [DBG] pgmap v4499: 169 pgs: 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:32 smithi139 bash[43301]: audit 2022-04-28T21:44:31.947637+0000 mon.a (mon.0) 3931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-57", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:32 smithi139 bash[43301]: cluster 2022-04-28T21:44:31.947738+0000 mon.a (mon.0) 3932 : cluster [DBG] osdmap e1389: 8 total, 8 up, 8 in 2022-04-28T21:44:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:32 smithi139 bash[43301]: audit 2022-04-28T21:44:31.950769+0000 mon.c (mon.1) 198 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:33.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:32 smithi139 bash[43301]: audit 2022-04-28T21:44:31.951405+0000 mon.a (mon.0) 3933 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:32 smithi139 bash[43301]: cluster 2022-04-28T21:44:32.303867+0000 mgr.y (mgr.24961) 3752 : cluster [DBG] pgmap v4499: 169 pgs: 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:33 smithi059 bash[61152]: cluster 2022-04-28T21:44:33.429122+0000 mon.a (mon.0) 3934 : cluster [DBG] osdmap e1390: 8 total, 8 up, 8 in 2022-04-28T21:44:34.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:33 smithi059 bash[59745]: cluster 2022-04-28T21:44:33.429122+0000 mon.a (mon.0) 3934 : cluster [DBG] osdmap e1390: 8 total, 8 up, 8 in 2022-04-28T21:44:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:33.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:34.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:33 smithi139 bash[43301]: cluster 2022-04-28T21:44:33.429122+0000 mon.a (mon.0) 3934 : cluster [DBG] osdmap e1390: 8 total, 8 up, 8 in 2022-04-28T21:44:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:34 smithi059 bash[61152]: audit 2022-04-28T21:44:33.971991+0000 mon.a (mon.0) 3935 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:34 smithi059 bash[61152]: cluster 2022-04-28T21:44:33.972101+0000 mon.a (mon.0) 3936 : cluster [DBG] osdmap e1391: 8 total, 8 up, 8 in 2022-04-28T21:44:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:34 smithi059 bash[61152]: cluster 2022-04-28T21:44:34.305107+0000 mgr.y (mgr.24961) 3753 : cluster [DBG] pgmap v4502: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:34 smithi059 bash[59745]: audit 2022-04-28T21:44:33.971991+0000 mon.a (mon.0) 3935 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:34 smithi059 bash[59745]: cluster 2022-04-28T21:44:33.972101+0000 mon.a (mon.0) 3936 : cluster [DBG] osdmap e1391: 8 total, 8 up, 8 in 2022-04-28T21:44:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:34 smithi059 bash[59745]: cluster 2022-04-28T21:44:34.305107+0000 mgr.y (mgr.24961) 3753 : cluster [DBG] pgmap v4502: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:34 smithi139 bash[43301]: audit 2022-04-28T21:44:33.971991+0000 mon.a (mon.0) 3935 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-57", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:34 smithi139 bash[43301]: cluster 2022-04-28T21:44:33.972101+0000 mon.a (mon.0) 3936 : cluster [DBG] osdmap e1391: 8 total, 8 up, 8 in 2022-04-28T21:44:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:34 smithi139 bash[43301]: cluster 2022-04-28T21:44:34.305107+0000 mgr.y (mgr.24961) 3753 : cluster [DBG] pgmap v4502: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 890 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:36.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:35 smithi059 bash[61152]: cluster 2022-04-28T21:44:34.983657+0000 mon.a (mon.0) 3937 : cluster [DBG] osdmap e1392: 8 total, 8 up, 8 in 2022-04-28T21:44:36.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:35 smithi059 bash[59745]: cluster 2022-04-28T21:44:34.983657+0000 mon.a (mon.0) 3937 : cluster [DBG] osdmap e1392: 8 total, 8 up, 8 in 2022-04-28T21:44:36.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:35 smithi139 bash[43301]: cluster 2022-04-28T21:44:34.983657+0000 mon.a (mon.0) 3937 : cluster [DBG] osdmap e1392: 8 total, 8 up, 8 in 2022-04-28T21:44:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[61152]: cluster 2022-04-28T21:44:35.985610+0000 mon.a (mon.0) 3938 : cluster [DBG] osdmap e1393: 8 total, 8 up, 8 in 2022-04-28T21:44:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[61152]: audit 2022-04-28T21:44:35.989571+0000 mon.c (mon.1) 199 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[61152]: audit 2022-04-28T21:44:35.999731+0000 mon.a (mon.0) 3939 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[61152]: cluster 2022-04-28T21:44:36.306263+0000 mgr.y (mgr.24961) 3754 : cluster [DBG] pgmap v4505: 169 pgs: 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[59745]: cluster 2022-04-28T21:44:35.985610+0000 mon.a (mon.0) 3938 : cluster [DBG] osdmap e1393: 8 total, 8 up, 8 in 2022-04-28T21:44:37.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[59745]: audit 2022-04-28T21:44:35.989571+0000 mon.c (mon.1) 199 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:37.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[59745]: audit 2022-04-28T21:44:35.999731+0000 mon.a (mon.0) 3939 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:37.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:36 smithi059 bash[59745]: cluster 2022-04-28T21:44:36.306263+0000 mgr.y (mgr.24961) 3754 : cluster [DBG] pgmap v4505: 169 pgs: 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:36 smithi139 bash[43301]: cluster 2022-04-28T21:44:35.985610+0000 mon.a (mon.0) 3938 : cluster [DBG] osdmap e1393: 8 total, 8 up, 8 in 2022-04-28T21:44:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:36 smithi139 bash[43301]: audit 2022-04-28T21:44:35.989571+0000 mon.c (mon.1) 199 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:36 smithi139 bash[43301]: audit 2022-04-28T21:44:35.999731+0000 mon.a (mon.0) 3939 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:37.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:36 smithi139 bash[43301]: cluster 2022-04-28T21:44:36.306263+0000 mgr.y (mgr.24961) 3754 : cluster [DBG] pgmap v4505: 169 pgs: 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[61152]: audit 2022-04-28T21:44:37.008881+0000 mon.a (mon.0) 3940 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:38.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[61152]: cluster 2022-04-28T21:44:37.008963+0000 mon.a (mon.0) 3941 : cluster [DBG] osdmap e1394: 8 total, 8 up, 8 in 2022-04-28T21:44:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[61152]: audit 2022-04-28T21:44:37.010397+0000 mon.c (mon.1) 200 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:38.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[61152]: audit 2022-04-28T21:44:37.010952+0000 mon.a (mon.0) 3942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[59745]: audit 2022-04-28T21:44:37.008881+0000 mon.a (mon.0) 3940 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[59745]: cluster 2022-04-28T21:44:37.008963+0000 mon.a (mon.0) 3941 : cluster [DBG] osdmap e1394: 8 total, 8 up, 8 in 2022-04-28T21:44:38.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[59745]: audit 2022-04-28T21:44:37.010397+0000 mon.c (mon.1) 200 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:38.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:38 smithi059 bash[59745]: audit 2022-04-28T21:44:37.010952+0000 mon.a (mon.0) 3942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:38 smithi139 bash[43301]: audit 2022-04-28T21:44:37.008881+0000 mon.a (mon.0) 3940 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:38 smithi139 bash[43301]: cluster 2022-04-28T21:44:37.008963+0000 mon.a (mon.0) 3941 : cluster [DBG] osdmap e1394: 8 total, 8 up, 8 in 2022-04-28T21:44:38.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:38 smithi139 bash[43301]: audit 2022-04-28T21:44:37.010397+0000 mon.c (mon.1) 200 : audit [INF] from='client.? 172.21.15.59:0/3296825952' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:38.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:38 smithi139 bash[43301]: audit 2022-04-28T21:44:37.010952+0000 mon.a (mon.0) 3942 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]: dispatch 2022-04-28T21:44:39.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.018655+0000 mon.a (mon.0) 3943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: cluster 2022-04-28T21:44:38.018755+0000 mon.a (mon.0) 3944 : cluster [DBG] osdmap e1395: 8 total, 8 up, 8 in 2022-04-28T21:44:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.032519+0000 mon.c (mon.1) 201 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.033026+0000 mon.a (mon.0) 3945 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.034016+0000 mon.c (mon.1) 202 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.034367+0000 mon.a (mon.0) 3946 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.034964+0000 mon.c (mon.1) 203 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:39.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: audit 2022-04-28T21:44:38.035328+0000 mon.a (mon.0) 3947 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:39.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[59745]: cluster 2022-04-28T21:44:38.307016+0000 mgr.y (mgr.24961) 3755 : cluster [DBG] pgmap v4508: 169 pgs: 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.018655+0000 mon.a (mon.0) 3943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:39.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: cluster 2022-04-28T21:44:38.018755+0000 mon.a (mon.0) 3944 : cluster [DBG] osdmap e1395: 8 total, 8 up, 8 in 2022-04-28T21:44:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.032519+0000 mon.c (mon.1) 201 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.033026+0000 mon.a (mon.0) 3945 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.034016+0000 mon.c (mon.1) 202 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.034367+0000 mon.a (mon.0) 3946 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.034964+0000 mon.c (mon.1) 203 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: audit 2022-04-28T21:44:38.035328+0000 mon.a (mon.0) 3947 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:39.345 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:39 smithi059 bash[61152]: cluster 2022-04-28T21:44:38.307016+0000 mgr.y (mgr.24961) 3755 : cluster [DBG] pgmap v4508: 169 pgs: 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.018655+0000 mon.a (mon.0) 3943 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-57"}]': finished 2022-04-28T21:44:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: cluster 2022-04-28T21:44:38.018755+0000 mon.a (mon.0) 3944 : cluster [DBG] osdmap e1395: 8 total, 8 up, 8 in 2022-04-28T21:44:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.032519+0000 mon.c (mon.1) 201 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.033026+0000 mon.a (mon.0) 3945 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.034016+0000 mon.c (mon.1) 202 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.034367+0000 mon.a (mon.0) 3946 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.034964+0000 mon.c (mon.1) 203 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:39.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: audit 2022-04-28T21:44:38.035328+0000 mon.a (mon.0) 3947 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:39.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:39 smithi139 bash[43301]: cluster 2022-04-28T21:44:38.307016+0000 mgr.y (mgr.24961) 3755 : cluster [DBG] pgmap v4508: 169 pgs: 169 active+clean; 24 KiB data, 891 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:40.032 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:40 smithi139 bash[43301]: audit 2022-04-28T21:44:39.032161+0000 mon.a (mon.0) 3948 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:40.032 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:40 smithi139 bash[43301]: cluster 2022-04-28T21:44:39.032278+0000 mon.a (mon.0) 3949 : cluster [DBG] osdmap e1396: 8 total, 8 up, 8 in 2022-04-28T21:44:40.032 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:40 smithi139 bash[43301]: audit 2022-04-28T21:44:39.033955+0000 mon.c (mon.1) 204 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[61152]: audit 2022-04-28T21:44:39.032161+0000 mon.a (mon.0) 3948 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[61152]: cluster 2022-04-28T21:44:39.032278+0000 mon.a (mon.0) 3949 : cluster [DBG] osdmap e1396: 8 total, 8 up, 8 in 2022-04-28T21:44:40.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[61152]: audit 2022-04-28T21:44:39.033955+0000 mon.c (mon.1) 204 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:40.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[61152]: audit 2022-04-28T21:44:39.034377+0000 mon.a (mon.0) 3950 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[59745]: audit 2022-04-28T21:44:39.032161+0000 mon.a (mon.0) 3948 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-58", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[59745]: cluster 2022-04-28T21:44:39.032278+0000 mon.a (mon.0) 3949 : cluster [DBG] osdmap e1396: 8 total, 8 up, 8 in 2022-04-28T21:44:40.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[59745]: audit 2022-04-28T21:44:39.033955+0000 mon.c (mon.1) 204 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:40.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:40 smithi059 bash[59745]: audit 2022-04-28T21:44:39.034377+0000 mon.a (mon.0) 3950 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:41 smithi059 bash[61152]: cluster 2022-04-28T21:44:40.307993+0000 mgr.y (mgr.24961) 3756 : cluster [DBG] pgmap v4511: 169 pgs: 169 active+clean; 24 KiB data, 900 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:41.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:41 smithi059 bash[61152]: cluster 2022-04-28T21:44:40.509665+0000 mon.a (mon.0) 3951 : cluster [DBG] osdmap e1397: 8 total, 8 up, 8 in 2022-04-28T21:44:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:41 smithi059 bash[59745]: cluster 2022-04-28T21:44:40.307993+0000 mgr.y (mgr.24961) 3756 : cluster [DBG] pgmap v4511: 169 pgs: 169 active+clean; 24 KiB data, 900 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:41 smithi059 bash[59745]: cluster 2022-04-28T21:44:40.509665+0000 mon.a (mon.0) 3951 : cluster [DBG] osdmap e1397: 8 total, 8 up, 8 in 2022-04-28T21:44:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:40 smithi139 bash[43301]: audit 2022-04-28T21:44:39.034377+0000 mon.a (mon.0) 3950 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:41 smithi139 bash[43301]: cluster 2022-04-28T21:44:40.307993+0000 mgr.y (mgr.24961) 3756 : cluster [DBG] pgmap v4511: 169 pgs: 169 active+clean; 24 KiB data, 900 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:41 smithi139 bash[43301]: cluster 2022-04-28T21:44:40.509665+0000 mon.a (mon.0) 3951 : cluster [DBG] osdmap e1397: 8 total, 8 up, 8 in 2022-04-28T21:44:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:42 smithi059 bash[61152]: audit 2022-04-28T21:44:41.055421+0000 mon.a (mon.0) 3952 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:42.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:42 smithi059 bash[61152]: cluster 2022-04-28T21:44:41.055763+0000 mon.a (mon.0) 3953 : cluster [DBG] osdmap e1398: 8 total, 8 up, 8 in 2022-04-28T21:44:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:42.069Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:42.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:42 smithi059 bash[59745]: audit 2022-04-28T21:44:41.055421+0000 mon.a (mon.0) 3952 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:42.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:42 smithi059 bash[59745]: cluster 2022-04-28T21:44:41.055763+0000 mon.a (mon.0) 3953 : cluster [DBG] osdmap e1398: 8 total, 8 up, 8 in 2022-04-28T21:44:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:42 smithi139 bash[43301]: audit 2022-04-28T21:44:41.055421+0000 mon.a (mon.0) 3952 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-58", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:42 smithi139 bash[43301]: cluster 2022-04-28T21:44:41.055763+0000 mon.a (mon.0) 3953 : cluster [DBG] osdmap e1398: 8 total, 8 up, 8 in 2022-04-28T21:44:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:43 smithi059 bash[61152]: cluster 2022-04-28T21:44:42.061819+0000 mon.a (mon.0) 3954 : cluster [DBG] osdmap e1399: 8 total, 8 up, 8 in 2022-04-28T21:44:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:43 smithi059 bash[61152]: cluster 2022-04-28T21:44:42.308662+0000 mgr.y (mgr.24961) 3757 : cluster [DBG] pgmap v4514: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:43 smithi059 bash[59745]: cluster 2022-04-28T21:44:42.061819+0000 mon.a (mon.0) 3954 : cluster [DBG] osdmap e1399: 8 total, 8 up, 8 in 2022-04-28T21:44:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:43 smithi059 bash[59745]: cluster 2022-04-28T21:44:42.308662+0000 mgr.y (mgr.24961) 3757 : cluster [DBG] pgmap v4514: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:43 smithi139 bash[43301]: cluster 2022-04-28T21:44:42.061819+0000 mon.a (mon.0) 3954 : cluster [DBG] osdmap e1399: 8 total, 8 up, 8 in 2022-04-28T21:44:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:43 smithi139 bash[43301]: cluster 2022-04-28T21:44:42.308662+0000 mgr.y (mgr.24961) 3757 : cluster [DBG] pgmap v4514: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:43.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:44 smithi059 bash[61152]: cluster 2022-04-28T21:44:43.068003+0000 mon.a (mon.0) 3955 : cluster [DBG] osdmap e1400: 8 total, 8 up, 8 in 2022-04-28T21:44:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:44 smithi059 bash[61152]: audit 2022-04-28T21:44:43.071265+0000 mon.c (mon.1) 205 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:44.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:44 smithi059 bash[61152]: audit 2022-04-28T21:44:43.079212+0000 mon.a (mon.0) 3956 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:44 smithi059 bash[59745]: cluster 2022-04-28T21:44:43.068003+0000 mon.a (mon.0) 3955 : cluster [DBG] osdmap e1400: 8 total, 8 up, 8 in 2022-04-28T21:44:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:44 smithi059 bash[59745]: audit 2022-04-28T21:44:43.071265+0000 mon.c (mon.1) 205 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:44.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:44 smithi059 bash[59745]: audit 2022-04-28T21:44:43.079212+0000 mon.a (mon.0) 3956 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:44 smithi139 bash[43301]: cluster 2022-04-28T21:44:43.068003+0000 mon.a (mon.0) 3955 : cluster [DBG] osdmap e1400: 8 total, 8 up, 8 in 2022-04-28T21:44:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:44 smithi139 bash[43301]: audit 2022-04-28T21:44:43.071265+0000 mon.c (mon.1) 205 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:44 smithi139 bash[43301]: audit 2022-04-28T21:44:43.079212+0000 mon.a (mon.0) 3956 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:45 smithi139 bash[43301]: audit 2022-04-28T21:44:44.082496+0000 mon.a (mon.0) 3957 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:45 smithi139 bash[43301]: cluster 2022-04-28T21:44:44.082713+0000 mon.a (mon.0) 3958 : cluster [DBG] osdmap e1401: 8 total, 8 up, 8 in 2022-04-28T21:44:45.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:45 smithi139 bash[43301]: audit 2022-04-28T21:44:44.085119+0000 mon.c (mon.1) 206 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:45 smithi139 bash[43301]: audit 2022-04-28T21:44:44.090022+0000 mon.a (mon.0) 3959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:45 smithi139 bash[43301]: cluster 2022-04-28T21:44:44.309688+0000 mgr.y (mgr.24961) 3758 : cluster [DBG] pgmap v4517: 169 pgs: 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[59745]: audit 2022-04-28T21:44:44.082496+0000 mon.a (mon.0) 3957 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:45.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[59745]: cluster 2022-04-28T21:44:44.082713+0000 mon.a (mon.0) 3958 : cluster [DBG] osdmap e1401: 8 total, 8 up, 8 in 2022-04-28T21:44:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[59745]: audit 2022-04-28T21:44:44.085119+0000 mon.c (mon.1) 206 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[59745]: audit 2022-04-28T21:44:44.090022+0000 mon.a (mon.0) 3959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[59745]: cluster 2022-04-28T21:44:44.309688+0000 mgr.y (mgr.24961) 3758 : cluster [DBG] pgmap v4517: 169 pgs: 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[61152]: audit 2022-04-28T21:44:44.082496+0000 mon.a (mon.0) 3957 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:45.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[61152]: cluster 2022-04-28T21:44:44.082713+0000 mon.a (mon.0) 3958 : cluster [DBG] osdmap e1401: 8 total, 8 up, 8 in 2022-04-28T21:44:45.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[61152]: audit 2022-04-28T21:44:44.085119+0000 mon.c (mon.1) 206 : audit [INF] from='client.? 172.21.15.59:0/1291704397' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[61152]: audit 2022-04-28T21:44:44.090022+0000 mon.a (mon.0) 3959 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]: dispatch 2022-04-28T21:44:45.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:45 smithi059 bash[61152]: cluster 2022-04-28T21:44:44.309688+0000 mgr.y (mgr.24961) 3758 : cluster [DBG] pgmap v4517: 169 pgs: 169 active+clean; 24 KiB data, 901 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:46 smithi139 bash[43301]: audit 2022-04-28T21:44:45.096290+0000 mon.a (mon.0) 3960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:46 smithi139 bash[43301]: cluster 2022-04-28T21:44:45.096494+0000 mon.a (mon.0) 3961 : cluster [DBG] osdmap e1402: 8 total, 8 up, 8 in 2022-04-28T21:44:46.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:46 smithi139 bash[43301]: audit 2022-04-28T21:44:45.111020+0000 mon.a (mon.0) 3962 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:46.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:46 smithi139 bash[43301]: audit 2022-04-28T21:44:45.112005+0000 mon.a (mon.0) 3963 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:46.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:46 smithi139 bash[43301]: audit 2022-04-28T21:44:45.112452+0000 mon.a (mon.0) 3964 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-59", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[61152]: audit 2022-04-28T21:44:45.096290+0000 mon.a (mon.0) 3960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[61152]: cluster 2022-04-28T21:44:45.096494+0000 mon.a (mon.0) 3961 : cluster [DBG] osdmap e1402: 8 total, 8 up, 8 in 2022-04-28T21:44:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[61152]: audit 2022-04-28T21:44:45.111020+0000 mon.a (mon.0) 3962 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:46.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[61152]: audit 2022-04-28T21:44:45.112005+0000 mon.a (mon.0) 3963 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:46.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[61152]: audit 2022-04-28T21:44:45.112452+0000 mon.a (mon.0) 3964 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-59", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:46.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[59745]: audit 2022-04-28T21:44:45.096290+0000 mon.a (mon.0) 3960 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-58"}]': finished 2022-04-28T21:44:46.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[59745]: cluster 2022-04-28T21:44:45.096494+0000 mon.a (mon.0) 3961 : cluster [DBG] osdmap e1402: 8 total, 8 up, 8 in 2022-04-28T21:44:46.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[59745]: audit 2022-04-28T21:44:45.111020+0000 mon.a (mon.0) 3962 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:46.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[59745]: audit 2022-04-28T21:44:45.112005+0000 mon.a (mon.0) 3963 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:46.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:46 smithi059 bash[59745]: audit 2022-04-28T21:44:45.112452+0000 mon.a (mon.0) 3964 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-59", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[61152]: audit 2022-04-28T21:44:46.116519+0000 mon.a (mon.0) 3965 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-59", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[61152]: cluster 2022-04-28T21:44:46.116607+0000 mon.a (mon.0) 3966 : cluster [DBG] osdmap e1403: 8 total, 8 up, 8 in 2022-04-28T21:44:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[61152]: audit 2022-04-28T21:44:46.117594+0000 mon.a (mon.0) 3967 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-59", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:47.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[61152]: cluster 2022-04-28T21:44:46.310460+0000 mgr.y (mgr.24961) 3759 : cluster [DBG] pgmap v4520: 169 pgs: 169 active+clean; 24 KiB data, 905 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[59745]: audit 2022-04-28T21:44:46.116519+0000 mon.a (mon.0) 3965 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-59", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[59745]: cluster 2022-04-28T21:44:46.116607+0000 mon.a (mon.0) 3966 : cluster [DBG] osdmap e1403: 8 total, 8 up, 8 in 2022-04-28T21:44:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[59745]: audit 2022-04-28T21:44:46.117594+0000 mon.a (mon.0) 3967 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-59", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:47.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:47 smithi059 bash[59745]: cluster 2022-04-28T21:44:46.310460+0000 mgr.y (mgr.24961) 3759 : cluster [DBG] pgmap v4520: 169 pgs: 169 active+clean; 24 KiB data, 905 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:47 smithi139 bash[43301]: audit 2022-04-28T21:44:46.116519+0000 mon.a (mon.0) 3965 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-59", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:47 smithi139 bash[43301]: cluster 2022-04-28T21:44:46.116607+0000 mon.a (mon.0) 3966 : cluster [DBG] osdmap e1403: 8 total, 8 up, 8 in 2022-04-28T21:44:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:47 smithi139 bash[43301]: audit 2022-04-28T21:44:46.117594+0000 mon.a (mon.0) 3967 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-59", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:47 smithi139 bash[43301]: cluster 2022-04-28T21:44:46.310460+0000 mgr.y (mgr.24961) 3759 : cluster [DBG] pgmap v4520: 169 pgs: 169 active+clean; 24 KiB data, 905 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:48 smithi139 bash[43301]: cluster 2022-04-28T21:44:47.587178+0000 mon.a (mon.0) 3968 : cluster [DBG] osdmap e1404: 8 total, 8 up, 8 in 2022-04-28T21:44:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:48 smithi139 bash[43301]: audit 2022-04-28T21:44:48.124775+0000 mon.a (mon.0) 3969 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-59", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:48 smithi139 bash[43301]: cluster 2022-04-28T21:44:48.124897+0000 mon.a (mon.0) 3970 : cluster [DBG] osdmap e1405: 8 total, 8 up, 8 in 2022-04-28T21:44:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:48 smithi059 bash[61152]: cluster 2022-04-28T21:44:47.587178+0000 mon.a (mon.0) 3968 : cluster [DBG] osdmap e1404: 8 total, 8 up, 8 in 2022-04-28T21:44:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:48 smithi059 bash[61152]: audit 2022-04-28T21:44:48.124775+0000 mon.a (mon.0) 3969 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-59", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:49.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:48 smithi059 bash[61152]: cluster 2022-04-28T21:44:48.124897+0000 mon.a (mon.0) 3970 : cluster [DBG] osdmap e1405: 8 total, 8 up, 8 in 2022-04-28T21:44:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:48 smithi059 bash[59745]: cluster 2022-04-28T21:44:47.587178+0000 mon.a (mon.0) 3968 : cluster [DBG] osdmap e1404: 8 total, 8 up, 8 in 2022-04-28T21:44:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:48 smithi059 bash[59745]: audit 2022-04-28T21:44:48.124775+0000 mon.a (mon.0) 3969 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-59", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:48 smithi059 bash[59745]: cluster 2022-04-28T21:44:48.124897+0000 mon.a (mon.0) 3970 : cluster [DBG] osdmap e1405: 8 total, 8 up, 8 in 2022-04-28T21:44:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:49 smithi139 bash[43301]: cluster 2022-04-28T21:44:48.311230+0000 mgr.y (mgr.24961) 3760 : cluster [DBG] pgmap v4523: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 905 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:49 smithi139 bash[43301]: cluster 2022-04-28T21:44:49.128439+0000 mon.a (mon.0) 3971 : cluster [DBG] osdmap e1406: 8 total, 8 up, 8 in 2022-04-28T21:44:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:49 smithi059 bash[61152]: cluster 2022-04-28T21:44:48.311230+0000 mgr.y (mgr.24961) 3760 : cluster [DBG] pgmap v4523: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 905 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:49 smithi059 bash[61152]: cluster 2022-04-28T21:44:49.128439+0000 mon.a (mon.0) 3971 : cluster [DBG] osdmap e1406: 8 total, 8 up, 8 in 2022-04-28T21:44:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:49 smithi059 bash[59745]: cluster 2022-04-28T21:44:48.311230+0000 mgr.y (mgr.24961) 3760 : cluster [DBG] pgmap v4523: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 905 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:49 smithi059 bash[59745]: cluster 2022-04-28T21:44:49.128439+0000 mon.a (mon.0) 3971 : cluster [DBG] osdmap e1406: 8 total, 8 up, 8 in 2022-04-28T21:44:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:51 smithi059 bash[61152]: cluster 2022-04-28T21:44:50.143068+0000 mon.a (mon.0) 3972 : cluster [DBG] osdmap e1407: 8 total, 8 up, 8 in 2022-04-28T21:44:51.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:51 smithi059 bash[61152]: audit 2022-04-28T21:44:50.145532+0000 mon.a (mon.0) 3973 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:51.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:51 smithi059 bash[61152]: cluster 2022-04-28T21:44:50.312216+0000 mgr.y (mgr.24961) 3761 : cluster [DBG] pgmap v4526: 169 pgs: 169 active+clean; 24 KiB data, 906 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:51 smithi059 bash[59745]: cluster 2022-04-28T21:44:50.143068+0000 mon.a (mon.0) 3972 : cluster [DBG] osdmap e1407: 8 total, 8 up, 8 in 2022-04-28T21:44:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:51 smithi059 bash[59745]: audit 2022-04-28T21:44:50.145532+0000 mon.a (mon.0) 3973 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:51.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:51 smithi059 bash[59745]: cluster 2022-04-28T21:44:50.312216+0000 mgr.y (mgr.24961) 3761 : cluster [DBG] pgmap v4526: 169 pgs: 169 active+clean; 24 KiB data, 906 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:51.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:51 smithi139 bash[43301]: cluster 2022-04-28T21:44:50.143068+0000 mon.a (mon.0) 3972 : cluster [DBG] osdmap e1407: 8 total, 8 up, 8 in 2022-04-28T21:44:51.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:51 smithi139 bash[43301]: audit 2022-04-28T21:44:50.145532+0000 mon.a (mon.0) 3973 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:51.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:51 smithi139 bash[43301]: cluster 2022-04-28T21:44:50.312216+0000 mgr.y (mgr.24961) 3761 : cluster [DBG] pgmap v4526: 169 pgs: 169 active+clean; 24 KiB data, 906 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:52.074Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[59745]: audit 2022-04-28T21:44:51.144153+0000 mon.a (mon.0) 3974 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[59745]: cluster 2022-04-28T21:44:51.144353+0000 mon.a (mon.0) 3975 : cluster [DBG] osdmap e1408: 8 total, 8 up, 8 in 2022-04-28T21:44:52.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[59745]: audit 2022-04-28T21:44:51.147562+0000 mon.a (mon.0) 3976 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[61152]: audit 2022-04-28T21:44:51.144153+0000 mon.a (mon.0) 3974 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[61152]: cluster 2022-04-28T21:44:51.144353+0000 mon.a (mon.0) 3975 : cluster [DBG] osdmap e1408: 8 total, 8 up, 8 in 2022-04-28T21:44:52.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:52 smithi059 bash[61152]: audit 2022-04-28T21:44:51.147562+0000 mon.a (mon.0) 3976 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:52 smithi139 bash[43301]: audit 2022-04-28T21:44:51.144153+0000 mon.a (mon.0) 3974 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:52 smithi139 bash[43301]: cluster 2022-04-28T21:44:51.144353+0000 mon.a (mon.0) 3975 : cluster [DBG] osdmap e1408: 8 total, 8 up, 8 in 2022-04-28T21:44:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:52 smithi139 bash[43301]: audit 2022-04-28T21:44:51.147562+0000 mon.a (mon.0) 3976 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]: dispatch 2022-04-28T21:44:53.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.149915+0000 mon.a (mon.0) 3977 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: cluster 2022-04-28T21:44:52.150132+0000 mon.a (mon.0) 3978 : cluster [DBG] osdmap e1409: 8 total, 8 up, 8 in 2022-04-28T21:44:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.164851+0000 mon.c (mon.1) 207 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.165200+0000 mon.a (mon.0) 3979 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.165845+0000 mon.c (mon.1) 208 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.166147+0000 mon.a (mon.0) 3980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.166681+0000 mon.c (mon.1) 209 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:53.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: audit 2022-04-28T21:44:52.167033+0000 mon.a (mon.0) 3981 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:53.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[61152]: cluster 2022-04-28T21:44:52.312958+0000 mgr.y (mgr.24961) 3762 : cluster [DBG] pgmap v4529: 169 pgs: 169 active+clean; 24 KiB data, 907 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:53.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.149915+0000 mon.a (mon.0) 3977 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:53.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: cluster 2022-04-28T21:44:52.150132+0000 mon.a (mon.0) 3978 : cluster [DBG] osdmap e1409: 8 total, 8 up, 8 in 2022-04-28T21:44:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.164851+0000 mon.c (mon.1) 207 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.165200+0000 mon.a (mon.0) 3979 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.165845+0000 mon.c (mon.1) 208 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.166147+0000 mon.a (mon.0) 3980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.166681+0000 mon.c (mon.1) 209 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: audit 2022-04-28T21:44:52.167033+0000 mon.a (mon.0) 3981 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:53.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[59745]: cluster 2022-04-28T21:44:52.312958+0000 mgr.y (mgr.24961) 3762 : cluster [DBG] pgmap v4529: 169 pgs: 169 active+clean; 24 KiB data, 907 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.149915+0000 mon.a (mon.0) 3977 : audit [INF] from='client.? 172.21.15.59:0/2600221056' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-59"}]': finished 2022-04-28T21:44:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: cluster 2022-04-28T21:44:52.150132+0000 mon.a (mon.0) 3978 : cluster [DBG] osdmap e1409: 8 total, 8 up, 8 in 2022-04-28T21:44:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.164851+0000 mon.c (mon.1) 207 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.165200+0000 mon.a (mon.0) 3979 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.165845+0000 mon.c (mon.1) 208 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.166147+0000 mon.a (mon.0) 3980 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.166681+0000 mon.c (mon.1) 209 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: audit 2022-04-28T21:44:52.167033+0000 mon.a (mon.0) 3981 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:44:53.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:53 smithi139 bash[43301]: cluster 2022-04-28T21:44:52.312958+0000 mgr.y (mgr.24961) 3762 : cluster [DBG] pgmap v4529: 169 pgs: 169 active+clean; 24 KiB data, 907 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:44:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:44:53.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:44:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[61152]: audit 2022-04-28T21:44:53.173236+0000 mon.a (mon.0) 3982 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[61152]: cluster 2022-04-28T21:44:53.173366+0000 mon.a (mon.0) 3983 : cluster [DBG] osdmap e1410: 8 total, 8 up, 8 in 2022-04-28T21:44:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[61152]: audit 2022-04-28T21:44:53.177646+0000 mon.c (mon.1) 210 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:55.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[61152]: audit 2022-04-28T21:44:53.178623+0000 mon.a (mon.0) 3984 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:55.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[59745]: audit 2022-04-28T21:44:53.173236+0000 mon.a (mon.0) 3982 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:55.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[59745]: cluster 2022-04-28T21:44:53.173366+0000 mon.a (mon.0) 3983 : cluster [DBG] osdmap e1410: 8 total, 8 up, 8 in 2022-04-28T21:44:55.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[59745]: audit 2022-04-28T21:44:53.177646+0000 mon.c (mon.1) 210 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:55.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:54 smithi059 bash[59745]: audit 2022-04-28T21:44:53.178623+0000 mon.a (mon.0) 3984 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:55.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:54 smithi139 bash[43301]: audit 2022-04-28T21:44:53.173236+0000 mon.a (mon.0) 3982 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-60", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:44:55.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:54 smithi139 bash[43301]: cluster 2022-04-28T21:44:53.173366+0000 mon.a (mon.0) 3983 : cluster [DBG] osdmap e1410: 8 total, 8 up, 8 in 2022-04-28T21:44:55.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:54 smithi139 bash[43301]: audit 2022-04-28T21:44:53.177646+0000 mon.c (mon.1) 210 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:55.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:54 smithi139 bash[43301]: audit 2022-04-28T21:44:53.178623+0000 mon.a (mon.0) 3984 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:56.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[61152]: cluster 2022-04-28T21:44:54.313754+0000 mgr.y (mgr.24961) 3763 : cluster [DBG] pgmap v4532: 169 pgs: 169 active+clean; 24 KiB data, 907 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:56.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[61152]: cluster 2022-04-28T21:44:54.648490+0000 mon.a (mon.0) 3985 : cluster [DBG] osdmap e1411: 8 total, 8 up, 8 in 2022-04-28T21:44:56.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[61152]: audit 2022-04-28T21:44:55.185751+0000 mon.a (mon.0) 3986 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:44:56.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[61152]: cluster 2022-04-28T21:44:55.185887+0000 mon.a (mon.0) 3987 : cluster [DBG] osdmap e1412: 8 total, 8 up, 8 in 2022-04-28T21:44:56.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[59745]: cluster 2022-04-28T21:44:54.313754+0000 mgr.y (mgr.24961) 3763 : cluster [DBG] pgmap v4532: 169 pgs: 169 active+clean; 24 KiB data, 907 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:56.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[59745]: cluster 2022-04-28T21:44:54.648490+0000 mon.a (mon.0) 3985 : cluster [DBG] osdmap e1411: 8 total, 8 up, 8 in 2022-04-28T21:44:56.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[59745]: audit 2022-04-28T21:44:55.185751+0000 mon.a (mon.0) 3986 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:44:56.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:55 smithi059 bash[59745]: cluster 2022-04-28T21:44:55.185887+0000 mon.a (mon.0) 3987 : cluster [DBG] osdmap e1412: 8 total, 8 up, 8 in 2022-04-28T21:44:56.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:55 smithi139 bash[43301]: cluster 2022-04-28T21:44:54.313754+0000 mgr.y (mgr.24961) 3763 : cluster [DBG] pgmap v4532: 169 pgs: 169 active+clean; 24 KiB data, 907 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:56.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:55 smithi139 bash[43301]: cluster 2022-04-28T21:44:54.648490+0000 mon.a (mon.0) 3985 : cluster [DBG] osdmap e1411: 8 total, 8 up, 8 in 2022-04-28T21:44:56.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:55 smithi139 bash[43301]: audit 2022-04-28T21:44:55.185751+0000 mon.a (mon.0) 3986 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-60", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:44:56.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:55 smithi139 bash[43301]: cluster 2022-04-28T21:44:55.185887+0000 mon.a (mon.0) 3987 : cluster [DBG] osdmap e1412: 8 total, 8 up, 8 in 2022-04-28T21:44:57.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:57 smithi059 bash[59745]: cluster 2022-04-28T21:44:56.189446+0000 mon.a (mon.0) 3988 : cluster [DBG] osdmap e1413: 8 total, 8 up, 8 in 2022-04-28T21:44:57.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:57 smithi059 bash[59745]: cluster 2022-04-28T21:44:56.314536+0000 mgr.y (mgr.24961) 3764 : cluster [DBG] pgmap v4535: 177 pgs: 2 creating+peering, 6 unknown, 169 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:57 smithi059 bash[61152]: cluster 2022-04-28T21:44:56.189446+0000 mon.a (mon.0) 3988 : cluster [DBG] osdmap e1413: 8 total, 8 up, 8 in 2022-04-28T21:44:57.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:57 smithi059 bash[61152]: cluster 2022-04-28T21:44:56.314536+0000 mgr.y (mgr.24961) 3764 : cluster [DBG] pgmap v4535: 177 pgs: 2 creating+peering, 6 unknown, 169 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:57.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:57 smithi139 bash[43301]: cluster 2022-04-28T21:44:56.189446+0000 mon.a (mon.0) 3988 : cluster [DBG] osdmap e1413: 8 total, 8 up, 8 in 2022-04-28T21:44:57.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:57 smithi139 bash[43301]: cluster 2022-04-28T21:44:56.314536+0000 mgr.y (mgr.24961) 3764 : cluster [DBG] pgmap v4535: 177 pgs: 2 creating+peering, 6 unknown, 169 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:58 smithi059 bash[61152]: cluster 2022-04-28T21:44:57.190478+0000 mon.a (mon.0) 3989 : cluster [DBG] osdmap e1414: 8 total, 8 up, 8 in 2022-04-28T21:44:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:58 smithi059 bash[61152]: audit 2022-04-28T21:44:57.195084+0000 mon.c (mon.1) 211 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:58.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:58 smithi059 bash[61152]: audit 2022-04-28T21:44:57.202611+0000 mon.a (mon.0) 3990 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:58 smithi059 bash[59745]: cluster 2022-04-28T21:44:57.190478+0000 mon.a (mon.0) 3989 : cluster [DBG] osdmap e1414: 8 total, 8 up, 8 in 2022-04-28T21:44:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:58 smithi059 bash[59745]: audit 2022-04-28T21:44:57.195084+0000 mon.c (mon.1) 211 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:58 smithi059 bash[59745]: audit 2022-04-28T21:44:57.202611+0000 mon.a (mon.0) 3990 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:58.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:58 smithi139 bash[43301]: cluster 2022-04-28T21:44:57.190478+0000 mon.a (mon.0) 3989 : cluster [DBG] osdmap e1414: 8 total, 8 up, 8 in 2022-04-28T21:44:58.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:58 smithi139 bash[43301]: audit 2022-04-28T21:44:57.195084+0000 mon.c (mon.1) 211 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:58.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:58 smithi139 bash[43301]: audit 2022-04-28T21:44:57.202611+0000 mon.a (mon.0) 3990 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[59745]: audit 2022-04-28T21:44:58.212763+0000 mon.a (mon.0) 3991 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:44:59.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[59745]: cluster 2022-04-28T21:44:58.212843+0000 mon.a (mon.0) 3992 : cluster [DBG] osdmap e1415: 8 total, 8 up, 8 in 2022-04-28T21:44:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[59745]: audit 2022-04-28T21:44:58.214716+0000 mon.c (mon.1) 212 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[59745]: audit 2022-04-28T21:44:58.215221+0000 mon.a (mon.0) 3993 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[59745]: cluster 2022-04-28T21:44:58.315342+0000 mgr.y (mgr.24961) 3765 : cluster [DBG] pgmap v4538: 169 pgs: 169 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[61152]: audit 2022-04-28T21:44:58.212763+0000 mon.a (mon.0) 3991 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:44:59.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[61152]: cluster 2022-04-28T21:44:58.212843+0000 mon.a (mon.0) 3992 : cluster [DBG] osdmap e1415: 8 total, 8 up, 8 in 2022-04-28T21:44:59.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[61152]: audit 2022-04-28T21:44:58.214716+0000 mon.c (mon.1) 212 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[61152]: audit 2022-04-28T21:44:58.215221+0000 mon.a (mon.0) 3993 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:44:59 smithi059 bash[61152]: cluster 2022-04-28T21:44:58.315342+0000 mgr.y (mgr.24961) 3765 : cluster [DBG] pgmap v4538: 169 pgs: 169 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:44:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:59 smithi139 bash[43301]: audit 2022-04-28T21:44:58.212763+0000 mon.a (mon.0) 3991 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:44:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:59 smithi139 bash[43301]: cluster 2022-04-28T21:44:58.212843+0000 mon.a (mon.0) 3992 : cluster [DBG] osdmap e1415: 8 total, 8 up, 8 in 2022-04-28T21:44:59.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:59 smithi139 bash[43301]: audit 2022-04-28T21:44:58.214716+0000 mon.c (mon.1) 212 : audit [INF] from='client.? 172.21.15.59:0/2229196031' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:59 smithi139 bash[43301]: audit 2022-04-28T21:44:58.215221+0000 mon.a (mon.0) 3993 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]: dispatch 2022-04-28T21:44:59.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:44:59 smithi139 bash[43301]: cluster 2022-04-28T21:44:58.315342+0000 mgr.y (mgr.24961) 3765 : cluster [DBG] pgmap v4538: 169 pgs: 169 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[61152]: audit 2022-04-28T21:44:59.216052+0000 mon.a (mon.0) 3994 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:45:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[61152]: cluster 2022-04-28T21:44:59.216231+0000 mon.a (mon.0) 3995 : cluster [DBG] osdmap e1416: 8 total, 8 up, 8 in 2022-04-28T21:45:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[61152]: audit 2022-04-28T21:44:59.234212+0000 mon.a (mon.0) 3996 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[61152]: audit 2022-04-28T21:44:59.235340+0000 mon.a (mon.0) 3997 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[61152]: audit 2022-04-28T21:44:59.235841+0000 mon.a (mon.0) 3998 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-61", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:45:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[59745]: audit 2022-04-28T21:44:59.216052+0000 mon.a (mon.0) 3994 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:45:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[59745]: cluster 2022-04-28T21:44:59.216231+0000 mon.a (mon.0) 3995 : cluster [DBG] osdmap e1416: 8 total, 8 up, 8 in 2022-04-28T21:45:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[59745]: audit 2022-04-28T21:44:59.234212+0000 mon.a (mon.0) 3996 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[59745]: audit 2022-04-28T21:44:59.235340+0000 mon.a (mon.0) 3997 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:00.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:00 smithi059 bash[59745]: audit 2022-04-28T21:44:59.235841+0000 mon.a (mon.0) 3998 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-61", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:45:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:00 smithi139 bash[43301]: audit 2022-04-28T21:44:59.216052+0000 mon.a (mon.0) 3994 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-60"}]': finished 2022-04-28T21:45:00.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:00 smithi139 bash[43301]: cluster 2022-04-28T21:44:59.216231+0000 mon.a (mon.0) 3995 : cluster [DBG] osdmap e1416: 8 total, 8 up, 8 in 2022-04-28T21:45:00.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:00 smithi139 bash[43301]: audit 2022-04-28T21:44:59.234212+0000 mon.a (mon.0) 3996 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:00.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:00 smithi139 bash[43301]: audit 2022-04-28T21:44:59.235340+0000 mon.a (mon.0) 3997 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:00.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:00 smithi139 bash[43301]: audit 2022-04-28T21:44:59.235841+0000 mon.a (mon.0) 3998 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-61", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:45:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[61152]: audit 2022-04-28T21:45:00.236177+0000 mon.a (mon.0) 3999 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-61", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:45:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[61152]: cluster 2022-04-28T21:45:00.236281+0000 mon.a (mon.0) 4000 : cluster [DBG] osdmap e1417: 8 total, 8 up, 8 in 2022-04-28T21:45:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[61152]: audit 2022-04-28T21:45:00.237424+0000 mon.a (mon.0) 4001 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-61", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:01.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[61152]: cluster 2022-04-28T21:45:00.316119+0000 mgr.y (mgr.24961) 3766 : cluster [DBG] pgmap v4541: 169 pgs: 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:01.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[59745]: audit 2022-04-28T21:45:00.236177+0000 mon.a (mon.0) 3999 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-61", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:45:01.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[59745]: cluster 2022-04-28T21:45:00.236281+0000 mon.a (mon.0) 4000 : cluster [DBG] osdmap e1417: 8 total, 8 up, 8 in 2022-04-28T21:45:01.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[59745]: audit 2022-04-28T21:45:00.237424+0000 mon.a (mon.0) 4001 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-61", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:01.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:01 smithi059 bash[59745]: cluster 2022-04-28T21:45:00.316119+0000 mgr.y (mgr.24961) 3766 : cluster [DBG] pgmap v4541: 169 pgs: 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:01 smithi139 bash[43301]: audit 2022-04-28T21:45:00.236177+0000 mon.a (mon.0) 3999 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-61", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:45:01.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:01 smithi139 bash[43301]: cluster 2022-04-28T21:45:00.236281+0000 mon.a (mon.0) 4000 : cluster [DBG] osdmap e1417: 8 total, 8 up, 8 in 2022-04-28T21:45:01.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:01 smithi139 bash[43301]: audit 2022-04-28T21:45:00.237424+0000 mon.a (mon.0) 4001 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-61", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:01.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:01 smithi139 bash[43301]: cluster 2022-04-28T21:45:00.316119+0000 mgr.y (mgr.24961) 3766 : cluster [DBG] pgmap v4541: 169 pgs: 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:02.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:02 smithi059 bash[61152]: cluster 2022-04-28T21:45:01.707210+0000 mon.a (mon.0) 4002 : cluster [DBG] osdmap e1418: 8 total, 8 up, 8 in 2022-04-28T21:45:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:02.073Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:02.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:02 smithi059 bash[59745]: cluster 2022-04-28T21:45:01.707210+0000 mon.a (mon.0) 4002 : cluster [DBG] osdmap e1418: 8 total, 8 up, 8 in 2022-04-28T21:45:02.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:02 smithi139 bash[43301]: cluster 2022-04-28T21:45:01.707210+0000 mon.a (mon.0) 4002 : cluster [DBG] osdmap e1418: 8 total, 8 up, 8 in 2022-04-28T21:45:03.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[61152]: audit 2022-04-28T21:45:02.254075+0000 mon.a (mon.0) 4003 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-61", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:03.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[61152]: cluster 2022-04-28T21:45:02.254178+0000 mon.a (mon.0) 4004 : cluster [DBG] osdmap e1419: 8 total, 8 up, 8 in 2022-04-28T21:45:03.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[61152]: cluster 2022-04-28T21:45:02.316861+0000 mgr.y (mgr.24961) 3767 : cluster [DBG] pgmap v4544: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:03.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[59745]: audit 2022-04-28T21:45:02.254075+0000 mon.a (mon.0) 4003 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-61", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:03.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[59745]: cluster 2022-04-28T21:45:02.254178+0000 mon.a (mon.0) 4004 : cluster [DBG] osdmap e1419: 8 total, 8 up, 8 in 2022-04-28T21:45:03.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[59745]: cluster 2022-04-28T21:45:02.316861+0000 mgr.y (mgr.24961) 3767 : cluster [DBG] pgmap v4544: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:03.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:03 smithi139 bash[43301]: audit 2022-04-28T21:45:02.254075+0000 mon.a (mon.0) 4003 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-61", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:03.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:03 smithi139 bash[43301]: cluster 2022-04-28T21:45:02.254178+0000 mon.a (mon.0) 4004 : cluster [DBG] osdmap e1419: 8 total, 8 up, 8 in 2022-04-28T21:45:03.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:03 smithi139 bash[43301]: cluster 2022-04-28T21:45:02.316861+0000 mgr.y (mgr.24961) 3767 : cluster [DBG] pgmap v4544: 177 pgs: 8 unknown, 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:04.267 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:03.962Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:04.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:04 smithi059 bash[61152]: cluster 2022-04-28T21:45:03.272699+0000 mon.a (mon.0) 4005 : cluster [DBG] osdmap e1420: 8 total, 8 up, 8 in 2022-04-28T21:45:04.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:04 smithi059 bash[59745]: cluster 2022-04-28T21:45:03.272699+0000 mon.a (mon.0) 4005 : cluster [DBG] osdmap e1420: 8 total, 8 up, 8 in 2022-04-28T21:45:04.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:04 smithi139 bash[43301]: cluster 2022-04-28T21:45:03.272699+0000 mon.a (mon.0) 4005 : cluster [DBG] osdmap e1420: 8 total, 8 up, 8 in 2022-04-28T21:45:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:05 smithi059 bash[61152]: cluster 2022-04-28T21:45:04.279090+0000 mon.a (mon.0) 4006 : cluster [DBG] osdmap e1421: 8 total, 8 up, 8 in 2022-04-28T21:45:05.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:05 smithi059 bash[61152]: audit 2022-04-28T21:45:04.280115+0000 mon.a (mon.0) 4007 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:05.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:05 smithi059 bash[61152]: cluster 2022-04-28T21:45:04.317461+0000 mgr.y (mgr.24961) 3768 : cluster [DBG] pgmap v4547: 169 pgs: 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:05 smithi059 bash[59745]: cluster 2022-04-28T21:45:04.279090+0000 mon.a (mon.0) 4006 : cluster [DBG] osdmap e1421: 8 total, 8 up, 8 in 2022-04-28T21:45:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:05 smithi059 bash[59745]: audit 2022-04-28T21:45:04.280115+0000 mon.a (mon.0) 4007 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:05.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:05 smithi059 bash[59745]: cluster 2022-04-28T21:45:04.317461+0000 mgr.y (mgr.24961) 3768 : cluster [DBG] pgmap v4547: 169 pgs: 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:05 smithi139 bash[43301]: cluster 2022-04-28T21:45:04.279090+0000 mon.a (mon.0) 4006 : cluster [DBG] osdmap e1421: 8 total, 8 up, 8 in 2022-04-28T21:45:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:05 smithi139 bash[43301]: audit 2022-04-28T21:45:04.280115+0000 mon.a (mon.0) 4007 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:05.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:05 smithi139 bash[43301]: cluster 2022-04-28T21:45:04.317461+0000 mgr.y (mgr.24961) 3768 : cluster [DBG] pgmap v4547: 169 pgs: 169 active+clean; 24 KiB data, 909 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:06 smithi059 bash[59745]: audit 2022-04-28T21:45:05.292477+0000 mon.a (mon.0) 4008 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:06.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:06 smithi059 bash[59745]: cluster 2022-04-28T21:45:05.292669+0000 mon.a (mon.0) 4009 : cluster [DBG] osdmap e1422: 8 total, 8 up, 8 in 2022-04-28T21:45:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:06 smithi059 bash[59745]: audit 2022-04-28T21:45:05.293771+0000 mon.a (mon.0) 4010 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:06 smithi059 bash[61152]: audit 2022-04-28T21:45:05.292477+0000 mon.a (mon.0) 4008 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:06 smithi059 bash[61152]: cluster 2022-04-28T21:45:05.292669+0000 mon.a (mon.0) 4009 : cluster [DBG] osdmap e1422: 8 total, 8 up, 8 in 2022-04-28T21:45:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:06 smithi059 bash[61152]: audit 2022-04-28T21:45:05.293771+0000 mon.a (mon.0) 4010 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:06 smithi139 bash[43301]: audit 2022-04-28T21:45:05.292477+0000 mon.a (mon.0) 4008 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:06 smithi139 bash[43301]: cluster 2022-04-28T21:45:05.292669+0000 mon.a (mon.0) 4009 : cluster [DBG] osdmap e1422: 8 total, 8 up, 8 in 2022-04-28T21:45:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:06 smithi139 bash[43301]: audit 2022-04-28T21:45:05.293771+0000 mon.a (mon.0) 4010 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]: dispatch 2022-04-28T21:45:07.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[61152]: audit 2022-04-28T21:45:06.297140+0000 mon.a (mon.0) 4011 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[61152]: cluster 2022-04-28T21:45:06.297222+0000 mon.a (mon.0) 4012 : cluster [DBG] osdmap e1423: 8 total, 8 up, 8 in 2022-04-28T21:45:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[61152]: audit 2022-04-28T21:45:06.308255+0000 mon.a (mon.0) 4013 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[61152]: audit 2022-04-28T21:45:06.309688+0000 mon.a (mon.0) 4014 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:07.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[61152]: audit 2022-04-28T21:45:06.310104+0000 mon.a (mon.0) 4015 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-62", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:45:07.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[61152]: cluster 2022-04-28T21:45:06.318142+0000 mgr.y (mgr.24961) 3769 : cluster [DBG] pgmap v4550: 169 pgs: 169 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[59745]: audit 2022-04-28T21:45:06.297140+0000 mon.a (mon.0) 4011 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[59745]: cluster 2022-04-28T21:45:06.297222+0000 mon.a (mon.0) 4012 : cluster [DBG] osdmap e1423: 8 total, 8 up, 8 in 2022-04-28T21:45:07.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[59745]: audit 2022-04-28T21:45:06.308255+0000 mon.a (mon.0) 4013 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:07.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[59745]: audit 2022-04-28T21:45:06.309688+0000 mon.a (mon.0) 4014 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:07.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[59745]: audit 2022-04-28T21:45:06.310104+0000 mon.a (mon.0) 4015 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-62", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:45:07.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:07 smithi059 bash[59745]: cluster 2022-04-28T21:45:06.318142+0000 mgr.y (mgr.24961) 3769 : cluster [DBG] pgmap v4550: 169 pgs: 169 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:07 smithi139 bash[43301]: audit 2022-04-28T21:45:06.297140+0000 mon.a (mon.0) 4011 : audit [INF] from='client.? 172.21.15.59:0/1374707770' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-61"}]': finished 2022-04-28T21:45:07.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:07 smithi139 bash[43301]: cluster 2022-04-28T21:45:06.297222+0000 mon.a (mon.0) 4012 : cluster [DBG] osdmap e1423: 8 total, 8 up, 8 in 2022-04-28T21:45:07.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:07 smithi139 bash[43301]: audit 2022-04-28T21:45:06.308255+0000 mon.a (mon.0) 4013 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:07.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:07 smithi139 bash[43301]: audit 2022-04-28T21:45:06.309688+0000 mon.a (mon.0) 4014 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:07.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:07 smithi139 bash[43301]: audit 2022-04-28T21:45:06.310104+0000 mon.a (mon.0) 4015 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-62", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2022-04-28T21:45:07.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:07 smithi139 bash[43301]: cluster 2022-04-28T21:45:06.318142+0000 mgr.y (mgr.24961) 3769 : cluster [DBG] pgmap v4550: 169 pgs: 169 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:08 smithi059 bash[61152]: audit 2022-04-28T21:45:07.308558+0000 mon.a (mon.0) 4016 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-62", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:45:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:08 smithi059 bash[61152]: cluster 2022-04-28T21:45:07.308731+0000 mon.a (mon.0) 4017 : cluster [DBG] osdmap e1424: 8 total, 8 up, 8 in 2022-04-28T21:45:09.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:08 smithi059 bash[61152]: audit 2022-04-28T21:45:07.309876+0000 mon.a (mon.0) 4018 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-62", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:08 smithi059 bash[59745]: audit 2022-04-28T21:45:07.308558+0000 mon.a (mon.0) 4016 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-62", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:45:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:08 smithi059 bash[59745]: cluster 2022-04-28T21:45:07.308731+0000 mon.a (mon.0) 4017 : cluster [DBG] osdmap e1424: 8 total, 8 up, 8 in 2022-04-28T21:45:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:08 smithi059 bash[59745]: audit 2022-04-28T21:45:07.309876+0000 mon.a (mon.0) 4018 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-62", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:08 smithi139 bash[43301]: audit 2022-04-28T21:45:07.308558+0000 mon.a (mon.0) 4016 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "testprofile-test-rados-api-smithi059-84622-62", "profile": [ "k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2022-04-28T21:45:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:08 smithi139 bash[43301]: cluster 2022-04-28T21:45:07.308731+0000 mon.a (mon.0) 4017 : cluster [DBG] osdmap e1424: 8 total, 8 up, 8 in 2022-04-28T21:45:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:08 smithi139 bash[43301]: audit 2022-04-28T21:45:07.309876+0000 mon.a (mon.0) 4018 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-62", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[61152]: cluster 2022-04-28T21:45:08.318616+0000 mgr.y (mgr.24961) 3770 : cluster [DBG] pgmap v4553: 169 pgs: 169 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[61152]: cluster 2022-04-28T21:45:08.785119+0000 mon.a (mon.0) 4019 : cluster [DBG] osdmap e1425: 8 total, 8 up, 8 in 2022-04-28T21:45:10.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[61152]: audit 2022-04-28T21:45:09.320531+0000 mon.a (mon.0) 4020 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-62", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:10.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[61152]: cluster 2022-04-28T21:45:09.320808+0000 mon.a (mon.0) 4021 : cluster [DBG] osdmap e1426: 8 total, 8 up, 8 in 2022-04-28T21:45:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[59745]: cluster 2022-04-28T21:45:08.318616+0000 mgr.y (mgr.24961) 3770 : cluster [DBG] pgmap v4553: 169 pgs: 169 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[59745]: cluster 2022-04-28T21:45:08.785119+0000 mon.a (mon.0) 4019 : cluster [DBG] osdmap e1425: 8 total, 8 up, 8 in 2022-04-28T21:45:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[59745]: audit 2022-04-28T21:45:09.320531+0000 mon.a (mon.0) 4020 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-62", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:10.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:09 smithi059 bash[59745]: cluster 2022-04-28T21:45:09.320808+0000 mon.a (mon.0) 4021 : cluster [DBG] osdmap e1426: 8 total, 8 up, 8 in 2022-04-28T21:45:10.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:09 smithi139 bash[43301]: cluster 2022-04-28T21:45:08.318616+0000 mgr.y (mgr.24961) 3770 : cluster [DBG] pgmap v4553: 169 pgs: 169 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:10.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:09 smithi139 bash[43301]: cluster 2022-04-28T21:45:08.785119+0000 mon.a (mon.0) 4019 : cluster [DBG] osdmap e1425: 8 total, 8 up, 8 in 2022-04-28T21:45:10.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:09 smithi139 bash[43301]: audit 2022-04-28T21:45:09.320531+0000 mon.a (mon.0) 4020 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "test-rados-api-smithi059-84622-62", "pool_type":"erasure", "pg_num":8, "pgp_num":8, "erasure_code_profile":"testprofile-test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:10.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:09 smithi139 bash[43301]: cluster 2022-04-28T21:45:09.320808+0000 mon.a (mon.0) 4021 : cluster [DBG] osdmap e1426: 8 total, 8 up, 8 in 2022-04-28T21:45:11.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:11 smithi059 bash[61152]: cluster 2022-04-28T21:45:10.320481+0000 mgr.y (mgr.24961) 3771 : cluster [DBG] pgmap v4556: 177 pgs: 7 creating+peering, 1 unknown, 169 active+clean; 24 KiB data, 911 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:11.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:11 smithi059 bash[61152]: cluster 2022-04-28T21:45:10.325181+0000 mon.a (mon.0) 4022 : cluster [DBG] osdmap e1427: 8 total, 8 up, 8 in 2022-04-28T21:45:11.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:11 smithi059 bash[59745]: cluster 2022-04-28T21:45:10.320481+0000 mgr.y (mgr.24961) 3771 : cluster [DBG] pgmap v4556: 177 pgs: 7 creating+peering, 1 unknown, 169 active+clean; 24 KiB data, 911 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:11.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:11 smithi059 bash[59745]: cluster 2022-04-28T21:45:10.325181+0000 mon.a (mon.0) 4022 : cluster [DBG] osdmap e1427: 8 total, 8 up, 8 in 2022-04-28T21:45:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:11 smithi139 bash[43301]: cluster 2022-04-28T21:45:10.320481+0000 mgr.y (mgr.24961) 3771 : cluster [DBG] pgmap v4556: 177 pgs: 7 creating+peering, 1 unknown, 169 active+clean; 24 KiB data, 911 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:11.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:11 smithi139 bash[43301]: cluster 2022-04-28T21:45:10.325181+0000 mon.a (mon.0) 4022 : cluster [DBG] osdmap e1427: 8 total, 8 up, 8 in 2022-04-28T21:45:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:12.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: cluster 2022-04-28T21:45:11.325537+0000 mon.a (mon.0) 4023 : cluster [DBG] osdmap e1428: 8 total, 8 up, 8 in 2022-04-28T21:45:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: audit 2022-04-28T21:45:11.327338+0000 mon.a (mon.0) 4024 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: audit 2022-04-28T21:45:12.055006+0000 mon.a (mon.0) 4025 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:45:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: audit 2022-04-28T21:45:12.057963+0000 mon.a (mon.0) 4026 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:45:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: audit 2022-04-28T21:45:12.059965+0000 mon.a (mon.0) 4027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:45:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: audit 2022-04-28T21:45:12.329807+0000 mon.a (mon.0) 4028 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: cluster 2022-04-28T21:45:12.330201+0000 mon.a (mon.0) 4029 : cluster [DBG] osdmap e1429: 8 total, 8 up, 8 in 2022-04-28T21:45:12.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:12 smithi139 bash[43301]: audit 2022-04-28T21:45:12.331198+0000 mon.a (mon.0) 4030 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:12.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: cluster 2022-04-28T21:45:11.325537+0000 mon.a (mon.0) 4023 : cluster [DBG] osdmap e1428: 8 total, 8 up, 8 in 2022-04-28T21:45:12.855 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: audit 2022-04-28T21:45:11.327338+0000 mon.a (mon.0) 4024 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:12.856 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: audit 2022-04-28T21:45:12.055006+0000 mon.a (mon.0) 4025 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:45:12.856 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: audit 2022-04-28T21:45:12.057963+0000 mon.a (mon.0) 4026 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:45:12.856 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: audit 2022-04-28T21:45:12.059965+0000 mon.a (mon.0) 4027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:45:12.856 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: audit 2022-04-28T21:45:12.329807+0000 mon.a (mon.0) 4028 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:12.857 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: cluster 2022-04-28T21:45:12.330201+0000 mon.a (mon.0) 4029 : cluster [DBG] osdmap e1429: 8 total, 8 up, 8 in 2022-04-28T21:45:12.857 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[59745]: audit 2022-04-28T21:45:12.331198+0000 mon.a (mon.0) 4030 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:12.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: cluster 2022-04-28T21:45:11.325537+0000 mon.a (mon.0) 4023 : cluster [DBG] osdmap e1428: 8 total, 8 up, 8 in 2022-04-28T21:45:12.857 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: audit 2022-04-28T21:45:11.327338+0000 mon.a (mon.0) 4024 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:12.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: audit 2022-04-28T21:45:12.055006+0000 mon.a (mon.0) 4025 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:45:12.858 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: audit 2022-04-28T21:45:12.057963+0000 mon.a (mon.0) 4026 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:45:12.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: audit 2022-04-28T21:45:12.059965+0000 mon.a (mon.0) 4027 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:45:12.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: audit 2022-04-28T21:45:12.329807+0000 mon.a (mon.0) 4028 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile rm", "name": "testprofile-test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:12.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: cluster 2022-04-28T21:45:12.330201+0000 mon.a (mon.0) 4029 : cluster [DBG] osdmap e1429: 8 total, 8 up, 8 in 2022-04-28T21:45:12.859 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:12 smithi059 bash[61152]: audit 2022-04-28T21:45:12.331198+0000 mon.a (mon.0) 4030 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd=[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]: dispatch 2022-04-28T21:45:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[61152]: cephadm 2022-04-28T21:45:12.061689+0000 mgr.y (mgr.24961) 3772 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:45:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[61152]: cluster 2022-04-28T21:45:12.321321+0000 mgr.y (mgr.24961) 3773 : cluster [DBG] pgmap v4558: 169 pgs: 169 active+clean; 24 KiB data, 911 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[61152]: audit 2022-04-28T21:45:12.388964+0000 mon.a (mon.0) 4031 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:45:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[61152]: audit 2022-04-28T21:45:13.333945+0000 mon.a (mon.0) 4032 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[61152]: cluster 2022-04-28T21:45:13.334158+0000 mon.a (mon.0) 4033 : cluster [DBG] osdmap e1430: 8 total, 8 up, 8 in 2022-04-28T21:45:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[59745]: cephadm 2022-04-28T21:45:12.061689+0000 mgr.y (mgr.24961) 3772 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:45:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[59745]: cluster 2022-04-28T21:45:12.321321+0000 mgr.y (mgr.24961) 3773 : cluster [DBG] pgmap v4558: 169 pgs: 169 active+clean; 24 KiB data, 911 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[59745]: audit 2022-04-28T21:45:12.388964+0000 mon.a (mon.0) 4031 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:45:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[59745]: audit 2022-04-28T21:45:13.333945+0000 mon.a (mon.0) 4032 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[59745]: cluster 2022-04-28T21:45:13.334158+0000 mon.a (mon.0) 4033 : cluster [DBG] osdmap e1430: 8 total, 8 up, 8 in 2022-04-28T21:45:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:13 smithi139 bash[43301]: cephadm 2022-04-28T21:45:12.061689+0000 mgr.y (mgr.24961) 3772 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:45:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:13 smithi139 bash[43301]: cluster 2022-04-28T21:45:12.321321+0000 mgr.y (mgr.24961) 3773 : cluster [DBG] pgmap v4558: 169 pgs: 169 active+clean; 24 KiB data, 911 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:13 smithi139 bash[43301]: audit 2022-04-28T21:45:12.388964+0000 mon.a (mon.0) 4031 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:45:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:13 smithi139 bash[43301]: audit 2022-04-28T21:45:13.333945+0000 mon.a (mon.0) 4032 : audit [INF] from='client.? 172.21.15.59:0/2299540364' entity='client.admin' cmd='[{"prefix": "osd crush rule rm", "name":"test-rados-api-smithi059-84622-62"}]': finished 2022-04-28T21:45:13.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:13 smithi139 bash[43301]: cluster 2022-04-28T21:45:13.334158+0000 mon.a (mon.0) 4033 : cluster [DBG] osdmap e1430: 8 total, 8 up, 8 in 2022-04-28T21:45:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:13.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:14 smithi059 bash[61152]: cluster 2022-04-28T21:45:14.322608+0000 mgr.y (mgr.24961) 3774 : cluster [DBG] pgmap v4561: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:15.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:14 smithi059 bash[59745]: cluster 2022-04-28T21:45:14.322608+0000 mgr.y (mgr.24961) 3774 : cluster [DBG] pgmap v4561: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:14 smithi139 bash[43301]: cluster 2022-04-28T21:45:14.322608+0000 mgr.y (mgr.24961) 3774 : cluster [DBG] pgmap v4561: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:17 smithi059 bash[61152]: cluster 2022-04-28T21:45:16.324057+0000 mgr.y (mgr.24961) 3775 : cluster [DBG] pgmap v4562: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:17 smithi059 bash[59745]: cluster 2022-04-28T21:45:16.324057+0000 mgr.y (mgr.24961) 3775 : cluster [DBG] pgmap v4562: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:17 smithi139 bash[43301]: cluster 2022-04-28T21:45:16.324057+0000 mgr.y (mgr.24961) 3775 : cluster [DBG] pgmap v4562: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:19 smithi059 bash[61152]: cluster 2022-04-28T21:45:18.324781+0000 mgr.y (mgr.24961) 3776 : cluster [DBG] pgmap v4563: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:19 smithi059 bash[61152]: audit 2022-04-28T21:45:19.361478+0000 mon.a (mon.0) 4034 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:45:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:19 smithi059 bash[61152]: audit 2022-04-28T21:45:19.369339+0000 mon.a (mon.0) 4035 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:45:19.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:19 smithi059 bash[59745]: cluster 2022-04-28T21:45:18.324781+0000 mgr.y (mgr.24961) 3776 : cluster [DBG] pgmap v4563: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:19.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:19 smithi059 bash[59745]: audit 2022-04-28T21:45:19.361478+0000 mon.a (mon.0) 4034 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:45:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:19 smithi059 bash[59745]: audit 2022-04-28T21:45:19.369339+0000 mon.a (mon.0) 4035 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:45:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:19 smithi139 bash[43301]: cluster 2022-04-28T21:45:18.324781+0000 mgr.y (mgr.24961) 3776 : cluster [DBG] pgmap v4563: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:19 smithi139 bash[43301]: audit 2022-04-28T21:45:19.361478+0000 mon.a (mon.0) 4034 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:45:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:19 smithi139 bash[43301]: audit 2022-04-28T21:45:19.369339+0000 mon.a (mon.0) 4035 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:45:20.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[61152]: audit 2022-04-28T21:45:19.391010+0000 mon.a (mon.0) 4036 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:45:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[61152]: audit 2022-04-28T21:45:19.391983+0000 mon.a (mon.0) 4037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:45:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[61152]: audit 2022-04-28T21:45:19.392771+0000 mon.a (mon.0) 4038 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:45:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[61152]: audit 2022-04-28T21:45:19.393491+0000 mon.a (mon.0) 4039 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:45:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[59745]: audit 2022-04-28T21:45:19.391010+0000 mon.a (mon.0) 4036 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:45:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[59745]: audit 2022-04-28T21:45:19.391983+0000 mon.a (mon.0) 4037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:45:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[59745]: audit 2022-04-28T21:45:19.392771+0000 mon.a (mon.0) 4038 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:45:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:20 smithi059 bash[59745]: audit 2022-04-28T21:45:19.393491+0000 mon.a (mon.0) 4039 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:45:20.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:20 smithi139 bash[43301]: audit 2022-04-28T21:45:19.391010+0000 mon.a (mon.0) 4036 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:45:20.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:20 smithi139 bash[43301]: audit 2022-04-28T21:45:19.391983+0000 mon.a (mon.0) 4037 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2022-04-28T21:45:20.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:20 smithi139 bash[43301]: audit 2022-04-28T21:45:19.392771+0000 mon.a (mon.0) 4038 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:45:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:20 smithi139 bash[43301]: audit 2022-04-28T21:45:19.393491+0000 mon.a (mon.0) 4039 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T21:45:21.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:21 smithi059 bash[59745]: cluster 2022-04-28T21:45:20.326324+0000 mgr.y (mgr.24961) 3777 : cluster [DBG] pgmap v4564: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:21.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:21 smithi059 bash[61152]: cluster 2022-04-28T21:45:20.326324+0000 mgr.y (mgr.24961) 3777 : cluster [DBG] pgmap v4564: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:21 smithi139 bash[43301]: cluster 2022-04-28T21:45:20.326324+0000 mgr.y (mgr.24961) 3777 : cluster [DBG] pgmap v4564: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:22.075Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:23 smithi059 bash[61152]: cluster 2022-04-28T21:45:22.327426+0000 mgr.y (mgr.24961) 3778 : cluster [DBG] pgmap v4565: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:23 smithi059 bash[59745]: cluster 2022-04-28T21:45:22.327426+0000 mgr.y (mgr.24961) 3778 : cluster [DBG] pgmap v4565: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:23 smithi139 bash[43301]: cluster 2022-04-28T21:45:22.327426+0000 mgr.y (mgr.24961) 3778 : cluster [DBG] pgmap v4565: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:23.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:24 smithi059 bash[61152]: cluster 2022-04-28T21:45:24.328774+0000 mgr.y (mgr.24961) 3779 : cluster [DBG] pgmap v4566: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:25.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:24 smithi059 bash[59745]: cluster 2022-04-28T21:45:24.328774+0000 mgr.y (mgr.24961) 3779 : cluster [DBG] pgmap v4566: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:24 smithi139 bash[43301]: cluster 2022-04-28T21:45:24.328774+0000 mgr.y (mgr.24961) 3779 : cluster [DBG] pgmap v4566: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:27 smithi059 bash[61152]: cluster 2022-04-28T21:45:26.330218+0000 mgr.y (mgr.24961) 3780 : cluster [DBG] pgmap v4567: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:27 smithi059 bash[59745]: cluster 2022-04-28T21:45:26.330218+0000 mgr.y (mgr.24961) 3780 : cluster [DBG] pgmap v4567: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:27 smithi139 bash[43301]: cluster 2022-04-28T21:45:26.330218+0000 mgr.y (mgr.24961) 3780 : cluster [DBG] pgmap v4567: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:29 smithi059 bash[61152]: cluster 2022-04-28T21:45:28.330966+0000 mgr.y (mgr.24961) 3781 : cluster [DBG] pgmap v4568: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:29 smithi059 bash[59745]: cluster 2022-04-28T21:45:28.330966+0000 mgr.y (mgr.24961) 3781 : cluster [DBG] pgmap v4568: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:29 smithi139 bash[43301]: cluster 2022-04-28T21:45:28.330966+0000 mgr.y (mgr.24961) 3781 : cluster [DBG] pgmap v4568: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:31 smithi059 bash[61152]: cluster 2022-04-28T21:45:30.332535+0000 mgr.y (mgr.24961) 3782 : cluster [DBG] pgmap v4569: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:31.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:31 smithi059 bash[59745]: cluster 2022-04-28T21:45:30.332535+0000 mgr.y (mgr.24961) 3782 : cluster [DBG] pgmap v4569: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:31 smithi139 bash[43301]: cluster 2022-04-28T21:45:30.332535+0000 mgr.y (mgr.24961) 3782 : cluster [DBG] pgmap v4569: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:32.076Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[59745]: cluster 2022-04-28T21:45:32.333567+0000 mgr.y (mgr.24961) 3783 : cluster [DBG] pgmap v4570: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[59745]: cluster 2022-04-28T21:45:32.413633+0000 mon.a (mon.0) 4040 : cluster [DBG] osdmap e1431: 8 total, 8 up, 8 in 2022-04-28T21:45:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[59745]: audit 2022-04-28T21:45:32.414902+0000 mon.a (mon.0) 4041 : audit [INF] from='client.? 172.21.15.59:0/4174862152' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86137-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:45:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[61152]: cluster 2022-04-28T21:45:32.333567+0000 mgr.y (mgr.24961) 3783 : cluster [DBG] pgmap v4570: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[61152]: cluster 2022-04-28T21:45:32.413633+0000 mon.a (mon.0) 4040 : cluster [DBG] osdmap e1431: 8 total, 8 up, 8 in 2022-04-28T21:45:33.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[61152]: audit 2022-04-28T21:45:32.414902+0000 mon.a (mon.0) 4041 : audit [INF] from='client.? 172.21.15.59:0/4174862152' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86137-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:45:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:33 smithi139 bash[43301]: cluster 2022-04-28T21:45:32.333567+0000 mgr.y (mgr.24961) 3783 : cluster [DBG] pgmap v4570: 169 pgs: 169 active+clean; 24 KiB data, 912 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:33 smithi139 bash[43301]: cluster 2022-04-28T21:45:32.413633+0000 mon.a (mon.0) 4040 : cluster [DBG] osdmap e1431: 8 total, 8 up, 8 in 2022-04-28T21:45:33.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:33 smithi139 bash[43301]: audit 2022-04-28T21:45:32.414902+0000 mon.a (mon.0) 4041 : audit [INF] from='client.? 172.21.15.59:0/4174862152' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86137-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:45:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:33.965Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:34 smithi059 bash[61152]: audit 2022-04-28T21:45:33.420250+0000 mon.a (mon.0) 4042 : audit [INF] from='client.? 172.21.15.59:0/4174862152' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86137-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:45:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:34 smithi059 bash[61152]: cluster 2022-04-28T21:45:33.420963+0000 mon.a (mon.0) 4043 : cluster [DBG] osdmap e1432: 8 total, 8 up, 8 in 2022-04-28T21:45:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:34 smithi059 bash[61152]: cluster 2022-04-28T21:45:34.424478+0000 mon.a (mon.0) 4044 : cluster [DBG] osdmap e1433: 8 total, 8 up, 8 in 2022-04-28T21:45:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:34 smithi059 bash[59745]: audit 2022-04-28T21:45:33.420250+0000 mon.a (mon.0) 4042 : audit [INF] from='client.? 172.21.15.59:0/4174862152' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86137-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:45:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:34 smithi059 bash[59745]: cluster 2022-04-28T21:45:33.420963+0000 mon.a (mon.0) 4043 : cluster [DBG] osdmap e1432: 8 total, 8 up, 8 in 2022-04-28T21:45:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:34 smithi059 bash[59745]: cluster 2022-04-28T21:45:34.424478+0000 mon.a (mon.0) 4044 : cluster [DBG] osdmap e1433: 8 total, 8 up, 8 in 2022-04-28T21:45:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:34 smithi139 bash[43301]: audit 2022-04-28T21:45:33.420250+0000 mon.a (mon.0) 4042 : audit [INF] from='client.? 172.21.15.59:0/4174862152' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86137-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:45:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:34 smithi139 bash[43301]: cluster 2022-04-28T21:45:33.420963+0000 mon.a (mon.0) 4043 : cluster [DBG] osdmap e1432: 8 total, 8 up, 8 in 2022-04-28T21:45:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:34 smithi139 bash[43301]: cluster 2022-04-28T21:45:34.424478+0000 mon.a (mon.0) 4044 : cluster [DBG] osdmap e1433: 8 total, 8 up, 8 in 2022-04-28T21:45:35.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:35 smithi059 bash[61152]: cluster 2022-04-28T21:45:34.334817+0000 mgr.y (mgr.24961) 3784 : cluster [DBG] pgmap v4573: 201 pgs: 25 unknown, 176 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:35.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:35 smithi059 bash[59745]: cluster 2022-04-28T21:45:34.334817+0000 mgr.y (mgr.24961) 3784 : cluster [DBG] pgmap v4573: 201 pgs: 25 unknown, 176 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:35.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:35 smithi139 bash[43301]: cluster 2022-04-28T21:45:34.334817+0000 mgr.y (mgr.24961) 3784 : cluster [DBG] pgmap v4573: 201 pgs: 25 unknown, 176 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:37.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:37 smithi059 bash[61152]: cluster 2022-04-28T21:45:36.336380+0000 mgr.y (mgr.24961) 3785 : cluster [DBG] pgmap v4575: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T21:45:37.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:37 smithi059 bash[59745]: cluster 2022-04-28T21:45:36.336380+0000 mgr.y (mgr.24961) 3785 : cluster [DBG] pgmap v4575: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T21:45:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:37 smithi139 bash[43301]: cluster 2022-04-28T21:45:36.336380+0000 mgr.y (mgr.24961) 3785 : cluster [DBG] pgmap v4575: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T21:45:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:39 smithi059 bash[59745]: cluster 2022-04-28T21:45:38.337127+0000 mgr.y (mgr.24961) 3786 : cluster [DBG] pgmap v4576: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T21:45:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:39 smithi059 bash[61152]: cluster 2022-04-28T21:45:38.337127+0000 mgr.y (mgr.24961) 3786 : cluster [DBG] pgmap v4576: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T21:45:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:39 smithi139 bash[43301]: cluster 2022-04-28T21:45:38.337127+0000 mgr.y (mgr.24961) 3786 : cluster [DBG] pgmap v4576: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T21:45:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:41 smithi059 bash[61152]: cluster 2022-04-28T21:45:40.338954+0000 mgr.y (mgr.24961) 3787 : cluster [DBG] pgmap v4577: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 645 B/s rd, 129 B/s wr, 0 op/s 2022-04-28T21:45:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:41 smithi059 bash[59745]: cluster 2022-04-28T21:45:40.338954+0000 mgr.y (mgr.24961) 3787 : cluster [DBG] pgmap v4577: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 645 B/s rd, 129 B/s wr, 0 op/s 2022-04-28T21:45:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:41 smithi139 bash[43301]: cluster 2022-04-28T21:45:40.338954+0000 mgr.y (mgr.24961) 3787 : cluster [DBG] pgmap v4577: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 645 B/s rd, 129 B/s wr, 0 op/s 2022-04-28T21:45:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:42.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:43 smithi059 bash[61152]: cluster 2022-04-28T21:45:42.340221+0000 mgr.y (mgr.24961) 3788 : cluster [DBG] pgmap v4578: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 573 B/s rd, 114 B/s wr, 0 op/s 2022-04-28T21:45:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:43 smithi059 bash[59745]: cluster 2022-04-28T21:45:42.340221+0000 mgr.y (mgr.24961) 3788 : cluster [DBG] pgmap v4578: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 573 B/s rd, 114 B/s wr, 0 op/s 2022-04-28T21:45:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:43 smithi139 bash[43301]: cluster 2022-04-28T21:45:42.340221+0000 mgr.y (mgr.24961) 3788 : cluster [DBG] pgmap v4578: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 573 B/s rd, 114 B/s wr, 0 op/s 2022-04-28T21:45:44.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:43.966Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:44 smithi059 bash[61152]: cluster 2022-04-28T21:45:44.341648+0000 mgr.y (mgr.24961) 3789 : cluster [DBG] pgmap v4579: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 0 op/s 2022-04-28T21:45:45.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:44 smithi059 bash[59745]: cluster 2022-04-28T21:45:44.341648+0000 mgr.y (mgr.24961) 3789 : cluster [DBG] pgmap v4579: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 0 op/s 2022-04-28T21:45:45.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:44 smithi139 bash[43301]: cluster 2022-04-28T21:45:44.341648+0000 mgr.y (mgr.24961) 3789 : cluster [DBG] pgmap v4579: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 0 op/s 2022-04-28T21:45:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:47 smithi059 bash[61152]: cluster 2022-04-28T21:45:46.343312+0000 mgr.y (mgr.24961) 3790 : cluster [DBG] pgmap v4580: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 429 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:45:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:47 smithi059 bash[59745]: cluster 2022-04-28T21:45:46.343312+0000 mgr.y (mgr.24961) 3790 : cluster [DBG] pgmap v4580: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 429 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:45:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:47 smithi139 bash[43301]: cluster 2022-04-28T21:45:46.343312+0000 mgr.y (mgr.24961) 3790 : cluster [DBG] pgmap v4580: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail; 429 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:45:49.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:49 smithi059 bash[61152]: cluster 2022-04-28T21:45:48.344072+0000 mgr.y (mgr.24961) 3791 : cluster [DBG] pgmap v4581: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:49.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:49 smithi059 bash[59745]: cluster 2022-04-28T21:45:48.344072+0000 mgr.y (mgr.24961) 3791 : cluster [DBG] pgmap v4581: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:49 smithi139 bash[43301]: cluster 2022-04-28T21:45:48.344072+0000 mgr.y (mgr.24961) 3791 : cluster [DBG] pgmap v4581: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:51 smithi059 bash[61152]: cluster 2022-04-28T21:45:50.345786+0000 mgr.y (mgr.24961) 3792 : cluster [DBG] pgmap v4582: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:51 smithi059 bash[59745]: cluster 2022-04-28T21:45:50.345786+0000 mgr.y (mgr.24961) 3792 : cluster [DBG] pgmap v4582: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:51 smithi139 bash[43301]: cluster 2022-04-28T21:45:50.345786+0000 mgr.y (mgr.24961) 3792 : cluster [DBG] pgmap v4582: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:52.077Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:53.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:53 smithi059 bash[59745]: cluster 2022-04-28T21:45:52.347021+0000 mgr.y (mgr.24961) 3793 : cluster [DBG] pgmap v4583: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:53.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:53 smithi059 bash[61152]: cluster 2022-04-28T21:45:52.347021+0000 mgr.y (mgr.24961) 3793 : cluster [DBG] pgmap v4583: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:53.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:53 smithi139 bash[43301]: cluster 2022-04-28T21:45:52.347021+0000 mgr.y (mgr.24961) 3793 : cluster [DBG] pgmap v4583: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:45:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:45:53.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:45:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:54 smithi059 bash[61152]: cluster 2022-04-28T21:45:54.348335+0000 mgr.y (mgr.24961) 3794 : cluster [DBG] pgmap v4584: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:55.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:54 smithi059 bash[59745]: cluster 2022-04-28T21:45:54.348335+0000 mgr.y (mgr.24961) 3794 : cluster [DBG] pgmap v4584: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:55.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:54 smithi139 bash[43301]: cluster 2022-04-28T21:45:54.348335+0000 mgr.y (mgr.24961) 3794 : cluster [DBG] pgmap v4584: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:57 smithi059 bash[61152]: cluster 2022-04-28T21:45:56.349930+0000 mgr.y (mgr.24961) 3795 : cluster [DBG] pgmap v4585: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:57 smithi059 bash[59745]: cluster 2022-04-28T21:45:56.349930+0000 mgr.y (mgr.24961) 3795 : cluster [DBG] pgmap v4585: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:57 smithi139 bash[43301]: cluster 2022-04-28T21:45:56.349930+0000 mgr.y (mgr.24961) 3795 : cluster [DBG] pgmap v4585: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:45:59 smithi059 bash[61152]: cluster 2022-04-28T21:45:58.350690+0000 mgr.y (mgr.24961) 3796 : cluster [DBG] pgmap v4586: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:59.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:45:59 smithi059 bash[59745]: cluster 2022-04-28T21:45:58.350690+0000 mgr.y (mgr.24961) 3796 : cluster [DBG] pgmap v4586: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:45:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:45:59 smithi139 bash[43301]: cluster 2022-04-28T21:45:58.350690+0000 mgr.y (mgr.24961) 3796 : cluster [DBG] pgmap v4586: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:01 smithi059 bash[61152]: cluster 2022-04-28T21:46:00.352402+0000 mgr.y (mgr.24961) 3797 : cluster [DBG] pgmap v4587: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:01 smithi059 bash[59745]: cluster 2022-04-28T21:46:00.352402+0000 mgr.y (mgr.24961) 3797 : cluster [DBG] pgmap v4587: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:01 smithi139 bash[43301]: cluster 2022-04-28T21:46:00.352402+0000 mgr.y (mgr.24961) 3797 : cluster [DBG] pgmap v4587: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:02.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:03 smithi059 bash[61152]: cluster 2022-04-28T21:46:02.353610+0000 mgr.y (mgr.24961) 3798 : cluster [DBG] pgmap v4588: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:03.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:03 smithi059 bash[59745]: cluster 2022-04-28T21:46:02.353610+0000 mgr.y (mgr.24961) 3798 : cluster [DBG] pgmap v4588: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:03 smithi139 bash[43301]: cluster 2022-04-28T21:46:02.353610+0000 mgr.y (mgr.24961) 3798 : cluster [DBG] pgmap v4588: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:03.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:05.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:04 smithi059 bash[61152]: cluster 2022-04-28T21:46:04.354890+0000 mgr.y (mgr.24961) 3799 : cluster [DBG] pgmap v4589: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:05.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:04 smithi059 bash[59745]: cluster 2022-04-28T21:46:04.354890+0000 mgr.y (mgr.24961) 3799 : cluster [DBG] pgmap v4589: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:05.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:04 smithi139 bash[43301]: cluster 2022-04-28T21:46:04.354890+0000 mgr.y (mgr.24961) 3799 : cluster [DBG] pgmap v4589: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:07 smithi059 bash[61152]: cluster 2022-04-28T21:46:06.356556+0000 mgr.y (mgr.24961) 3800 : cluster [DBG] pgmap v4590: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:07 smithi059 bash[59745]: cluster 2022-04-28T21:46:06.356556+0000 mgr.y (mgr.24961) 3800 : cluster [DBG] pgmap v4590: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:07 smithi139 bash[43301]: cluster 2022-04-28T21:46:06.356556+0000 mgr.y (mgr.24961) 3800 : cluster [DBG] pgmap v4590: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:09 smithi059 bash[61152]: cluster 2022-04-28T21:46:08.357317+0000 mgr.y (mgr.24961) 3801 : cluster [DBG] pgmap v4591: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:09 smithi059 bash[59745]: cluster 2022-04-28T21:46:08.357317+0000 mgr.y (mgr.24961) 3801 : cluster [DBG] pgmap v4591: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:09 smithi139 bash[43301]: cluster 2022-04-28T21:46:08.357317+0000 mgr.y (mgr.24961) 3801 : cluster [DBG] pgmap v4591: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:11 smithi059 bash[61152]: cluster 2022-04-28T21:46:10.359133+0000 mgr.y (mgr.24961) 3802 : cluster [DBG] pgmap v4592: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:11 smithi059 bash[59745]: cluster 2022-04-28T21:46:10.359133+0000 mgr.y (mgr.24961) 3802 : cluster [DBG] pgmap v4592: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:11 smithi139 bash[43301]: cluster 2022-04-28T21:46:10.359133+0000 mgr.y (mgr.24961) 3802 : cluster [DBG] pgmap v4592: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:12.079Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:12.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[59745]: audit 2022-04-28T21:46:12.394391+0000 mon.a (mon.0) 4045 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:46:12.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[59745]: audit 2022-04-28T21:46:12.397135+0000 mon.a (mon.0) 4046 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:46:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[59745]: audit 2022-04-28T21:46:12.398833+0000 mon.a (mon.0) 4047 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:46:12.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[61152]: audit 2022-04-28T21:46:12.394391+0000 mon.a (mon.0) 4045 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:46:12.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[61152]: audit 2022-04-28T21:46:12.397135+0000 mon.a (mon.0) 4046 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:46:12.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:12 smithi059 bash[61152]: audit 2022-04-28T21:46:12.398833+0000 mon.a (mon.0) 4047 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:46:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:12 smithi139 bash[43301]: audit 2022-04-28T21:46:12.394391+0000 mon.a (mon.0) 4045 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:46:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:12 smithi139 bash[43301]: audit 2022-04-28T21:46:12.397135+0000 mon.a (mon.0) 4046 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:46:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:12 smithi139 bash[43301]: audit 2022-04-28T21:46:12.398833+0000 mon.a (mon.0) 4047 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:46:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[61152]: cluster 2022-04-28T21:46:12.360381+0000 mgr.y (mgr.24961) 3803 : cluster [DBG] pgmap v4593: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[61152]: cephadm 2022-04-28T21:46:12.400298+0000 mgr.y (mgr.24961) 3804 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:46:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[61152]: audit 2022-04-28T21:46:12.718880+0000 mon.a (mon.0) 4048 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:46:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[59745]: cluster 2022-04-28T21:46:12.360381+0000 mgr.y (mgr.24961) 3803 : cluster [DBG] pgmap v4593: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[59745]: cephadm 2022-04-28T21:46:12.400298+0000 mgr.y (mgr.24961) 3804 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:46:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[59745]: audit 2022-04-28T21:46:12.718880+0000 mon.a (mon.0) 4048 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:46:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:13 smithi139 bash[43301]: cluster 2022-04-28T21:46:12.360381+0000 mgr.y (mgr.24961) 3803 : cluster [DBG] pgmap v4593: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:13 smithi139 bash[43301]: cephadm 2022-04-28T21:46:12.400298+0000 mgr.y (mgr.24961) 3804 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:46:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:13 smithi139 bash[43301]: audit 2022-04-28T21:46:12.718880+0000 mon.a (mon.0) 4048 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:46:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:13.968Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:15.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:14 smithi059 bash[61152]: cluster 2022-04-28T21:46:14.361663+0000 mgr.y (mgr.24961) 3805 : cluster [DBG] pgmap v4594: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:14 smithi059 bash[59745]: cluster 2022-04-28T21:46:14.361663+0000 mgr.y (mgr.24961) 3805 : cluster [DBG] pgmap v4594: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:14 smithi139 bash[43301]: cluster 2022-04-28T21:46:14.361663+0000 mgr.y (mgr.24961) 3805 : cluster [DBG] pgmap v4594: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:17 smithi059 bash[61152]: cluster 2022-04-28T21:46:16.363232+0000 mgr.y (mgr.24961) 3806 : cluster [DBG] pgmap v4595: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:17 smithi059 bash[59745]: cluster 2022-04-28T21:46:16.363232+0000 mgr.y (mgr.24961) 3806 : cluster [DBG] pgmap v4595: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:17 smithi139 bash[43301]: cluster 2022-04-28T21:46:16.363232+0000 mgr.y (mgr.24961) 3806 : cluster [DBG] pgmap v4595: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: cluster 2022-04-28T21:46:18.364021+0000 mgr.y (mgr.24961) 3807 : cluster [DBG] pgmap v4596: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: audit 2022-04-28T21:46:19.369502+0000 mon.a (mon.0) 4049 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:46:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: audit 2022-04-28T21:46:19.376152+0000 mon.a (mon.0) 4050 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:46:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: audit 2022-04-28T21:46:19.400984+0000 mon.a (mon.0) 4051 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:46:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: audit 2022-04-28T21:46:19.401875+0000 mon.a (mon.0) 4052 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:46:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: audit 2022-04-28T21:46:19.402617+0000 mon.a (mon.0) 4053 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T21:46:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[61152]: audit 2022-04-28T21:46:19.403288+0000 mon.a (mon.0) 4054 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:46:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: cluster 2022-04-28T21:46:18.364021+0000 mgr.y (mgr.24961) 3807 : cluster [DBG] pgmap v4596: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: audit 2022-04-28T21:46:19.369502+0000 mon.a (mon.0) 4049 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:46:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: audit 2022-04-28T21:46:19.376152+0000 mon.a (mon.0) 4050 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:46:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: audit 2022-04-28T21:46:19.400984+0000 mon.a (mon.0) 4051 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:46:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: audit 2022-04-28T21:46:19.401875+0000 mon.a (mon.0) 4052 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:46:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: audit 2022-04-28T21:46:19.402617+0000 mon.a (mon.0) 4053 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T21:46:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:19 smithi059 bash[59745]: audit 2022-04-28T21:46:19.403288+0000 mon.a (mon.0) 4054 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:46:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: cluster 2022-04-28T21:46:18.364021+0000 mgr.y (mgr.24961) 3807 : cluster [DBG] pgmap v4596: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: audit 2022-04-28T21:46:19.369502+0000 mon.a (mon.0) 4049 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:46:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: audit 2022-04-28T21:46:19.376152+0000 mon.a (mon.0) 4050 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:46:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: audit 2022-04-28T21:46:19.400984+0000 mon.a (mon.0) 4051 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:46:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: audit 2022-04-28T21:46:19.401875+0000 mon.a (mon.0) 4052 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:46:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: audit 2022-04-28T21:46:19.402617+0000 mon.a (mon.0) 4053 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T21:46:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:19 smithi139 bash[43301]: audit 2022-04-28T21:46:19.403288+0000 mon.a (mon.0) 4054 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:46:21.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:21 smithi059 bash[61152]: cluster 2022-04-28T21:46:20.365736+0000 mgr.y (mgr.24961) 3808 : cluster [DBG] pgmap v4597: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:21 smithi059 bash[59745]: cluster 2022-04-28T21:46:20.365736+0000 mgr.y (mgr.24961) 3808 : cluster [DBG] pgmap v4597: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:21 smithi139 bash[43301]: cluster 2022-04-28T21:46:20.365736+0000 mgr.y (mgr.24961) 3808 : cluster [DBG] pgmap v4597: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:22.078Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:23 smithi059 bash[61152]: cluster 2022-04-28T21:46:22.366954+0000 mgr.y (mgr.24961) 3809 : cluster [DBG] pgmap v4598: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:23 smithi059 bash[59745]: cluster 2022-04-28T21:46:22.366954+0000 mgr.y (mgr.24961) 3809 : cluster [DBG] pgmap v4598: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:23 smithi139 bash[43301]: cluster 2022-04-28T21:46:22.366954+0000 mgr.y (mgr.24961) 3809 : cluster [DBG] pgmap v4598: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:23.967Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:25.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:24 smithi059 bash[61152]: cluster 2022-04-28T21:46:24.368275+0000 mgr.y (mgr.24961) 3810 : cluster [DBG] pgmap v4599: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:25.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:24 smithi059 bash[59745]: cluster 2022-04-28T21:46:24.368275+0000 mgr.y (mgr.24961) 3810 : cluster [DBG] pgmap v4599: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:24 smithi139 bash[43301]: cluster 2022-04-28T21:46:24.368275+0000 mgr.y (mgr.24961) 3810 : cluster [DBG] pgmap v4599: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:27 smithi059 bash[61152]: cluster 2022-04-28T21:46:26.369973+0000 mgr.y (mgr.24961) 3811 : cluster [DBG] pgmap v4600: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:27 smithi059 bash[59745]: cluster 2022-04-28T21:46:26.369973+0000 mgr.y (mgr.24961) 3811 : cluster [DBG] pgmap v4600: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:27.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:27 smithi139 bash[43301]: cluster 2022-04-28T21:46:26.369973+0000 mgr.y (mgr.24961) 3811 : cluster [DBG] pgmap v4600: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:29 smithi059 bash[61152]: cluster 2022-04-28T21:46:28.370773+0000 mgr.y (mgr.24961) 3812 : cluster [DBG] pgmap v4601: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:29.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:29 smithi059 bash[59745]: cluster 2022-04-28T21:46:28.370773+0000 mgr.y (mgr.24961) 3812 : cluster [DBG] pgmap v4601: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:29 smithi139 bash[43301]: cluster 2022-04-28T21:46:28.370773+0000 mgr.y (mgr.24961) 3812 : cluster [DBG] pgmap v4601: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:31.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:31 smithi059 bash[61152]: cluster 2022-04-28T21:46:30.372500+0000 mgr.y (mgr.24961) 3813 : cluster [DBG] pgmap v4602: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:31.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:31 smithi059 bash[59745]: cluster 2022-04-28T21:46:30.372500+0000 mgr.y (mgr.24961) 3813 : cluster [DBG] pgmap v4602: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:31 smithi139 bash[43301]: cluster 2022-04-28T21:46:30.372500+0000 mgr.y (mgr.24961) 3813 : cluster [DBG] pgmap v4602: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:32.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:33 smithi059 bash[61152]: cluster 2022-04-28T21:46:32.373657+0000 mgr.y (mgr.24961) 3814 : cluster [DBG] pgmap v4603: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:33 smithi059 bash[59745]: cluster 2022-04-28T21:46:32.373657+0000 mgr.y (mgr.24961) 3814 : cluster [DBG] pgmap v4603: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:33 smithi139 bash[43301]: cluster 2022-04-28T21:46:32.373657+0000 mgr.y (mgr.24961) 3814 : cluster [DBG] pgmap v4603: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:33.970Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:35.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:34 smithi059 bash[61152]: cluster 2022-04-28T21:46:34.374933+0000 mgr.y (mgr.24961) 3815 : cluster [DBG] pgmap v4604: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:35.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:34 smithi059 bash[59745]: cluster 2022-04-28T21:46:34.374933+0000 mgr.y (mgr.24961) 3815 : cluster [DBG] pgmap v4604: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:35.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:34 smithi139 bash[43301]: cluster 2022-04-28T21:46:34.374933+0000 mgr.y (mgr.24961) 3815 : cluster [DBG] pgmap v4604: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:37 smithi059 bash[61152]: cluster 2022-04-28T21:46:36.376615+0000 mgr.y (mgr.24961) 3816 : cluster [DBG] pgmap v4605: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:37 smithi059 bash[59745]: cluster 2022-04-28T21:46:36.376615+0000 mgr.y (mgr.24961) 3816 : cluster [DBG] pgmap v4605: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:37 smithi139 bash[43301]: cluster 2022-04-28T21:46:36.376615+0000 mgr.y (mgr.24961) 3816 : cluster [DBG] pgmap v4605: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:39 smithi059 bash[61152]: cluster 2022-04-28T21:46:38.377398+0000 mgr.y (mgr.24961) 3817 : cluster [DBG] pgmap v4606: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:39.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:39 smithi059 bash[59745]: cluster 2022-04-28T21:46:38.377398+0000 mgr.y (mgr.24961) 3817 : cluster [DBG] pgmap v4606: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:39 smithi139 bash[43301]: cluster 2022-04-28T21:46:38.377398+0000 mgr.y (mgr.24961) 3817 : cluster [DBG] pgmap v4606: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:41 smithi059 bash[61152]: cluster 2022-04-28T21:46:40.379183+0000 mgr.y (mgr.24961) 3818 : cluster [DBG] pgmap v4607: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:41 smithi059 bash[59745]: cluster 2022-04-28T21:46:40.379183+0000 mgr.y (mgr.24961) 3818 : cluster [DBG] pgmap v4607: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:41 smithi139 bash[43301]: cluster 2022-04-28T21:46:40.379183+0000 mgr.y (mgr.24961) 3818 : cluster [DBG] pgmap v4607: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:42.080Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:43.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:43 smithi059 bash[59745]: cluster 2022-04-28T21:46:42.380425+0000 mgr.y (mgr.24961) 3819 : cluster [DBG] pgmap v4608: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:43.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:43 smithi059 bash[61152]: cluster 2022-04-28T21:46:42.380425+0000 mgr.y (mgr.24961) 3819 : cluster [DBG] pgmap v4608: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:43 smithi139 bash[43301]: cluster 2022-04-28T21:46:42.380425+0000 mgr.y (mgr.24961) 3819 : cluster [DBG] pgmap v4608: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:43.969Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:45.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:44 smithi059 bash[61152]: cluster 2022-04-28T21:46:44.381671+0000 mgr.y (mgr.24961) 3820 : cluster [DBG] pgmap v4609: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:45.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:44 smithi059 bash[59745]: cluster 2022-04-28T21:46:44.381671+0000 mgr.y (mgr.24961) 3820 : cluster [DBG] pgmap v4609: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:45.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:44 smithi139 bash[43301]: cluster 2022-04-28T21:46:44.381671+0000 mgr.y (mgr.24961) 3820 : cluster [DBG] pgmap v4609: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:47 smithi059 bash[61152]: cluster 2022-04-28T21:46:46.383226+0000 mgr.y (mgr.24961) 3821 : cluster [DBG] pgmap v4610: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:47 smithi059 bash[59745]: cluster 2022-04-28T21:46:46.383226+0000 mgr.y (mgr.24961) 3821 : cluster [DBG] pgmap v4610: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:47 smithi139 bash[43301]: cluster 2022-04-28T21:46:46.383226+0000 mgr.y (mgr.24961) 3821 : cluster [DBG] pgmap v4610: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:49 smithi059 bash[61152]: cluster 2022-04-28T21:46:48.384070+0000 mgr.y (mgr.24961) 3822 : cluster [DBG] pgmap v4611: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:49 smithi059 bash[59745]: cluster 2022-04-28T21:46:48.384070+0000 mgr.y (mgr.24961) 3822 : cluster [DBG] pgmap v4611: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:49 smithi139 bash[43301]: cluster 2022-04-28T21:46:48.384070+0000 mgr.y (mgr.24961) 3822 : cluster [DBG] pgmap v4611: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:51 smithi059 bash[61152]: cluster 2022-04-28T21:46:50.385783+0000 mgr.y (mgr.24961) 3823 : cluster [DBG] pgmap v4612: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:51 smithi059 bash[59745]: cluster 2022-04-28T21:46:50.385783+0000 mgr.y (mgr.24961) 3823 : cluster [DBG] pgmap v4612: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:51 smithi139 bash[43301]: cluster 2022-04-28T21:46:50.385783+0000 mgr.y (mgr.24961) 3823 : cluster [DBG] pgmap v4612: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:52.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:53 smithi059 bash[61152]: cluster 2022-04-28T21:46:52.386988+0000 mgr.y (mgr.24961) 3824 : cluster [DBG] pgmap v4613: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:53 smithi059 bash[59745]: cluster 2022-04-28T21:46:52.386988+0000 mgr.y (mgr.24961) 3824 : cluster [DBG] pgmap v4613: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:53 smithi139 bash[43301]: cluster 2022-04-28T21:46:52.386988+0000 mgr.y (mgr.24961) 3824 : cluster [DBG] pgmap v4613: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:46:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:46:53.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:46:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:54 smithi059 bash[61152]: cluster 2022-04-28T21:46:54.388301+0000 mgr.y (mgr.24961) 3825 : cluster [DBG] pgmap v4614: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:54 smithi059 bash[59745]: cluster 2022-04-28T21:46:54.388301+0000 mgr.y (mgr.24961) 3825 : cluster [DBG] pgmap v4614: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:55.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:54 smithi139 bash[43301]: cluster 2022-04-28T21:46:54.388301+0000 mgr.y (mgr.24961) 3825 : cluster [DBG] pgmap v4614: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:57 smithi059 bash[61152]: cluster 2022-04-28T21:46:56.389884+0000 mgr.y (mgr.24961) 3826 : cluster [DBG] pgmap v4615: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:57 smithi059 bash[59745]: cluster 2022-04-28T21:46:56.389884+0000 mgr.y (mgr.24961) 3826 : cluster [DBG] pgmap v4615: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:57 smithi139 bash[43301]: cluster 2022-04-28T21:46:56.389884+0000 mgr.y (mgr.24961) 3826 : cluster [DBG] pgmap v4615: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:46:59 smithi059 bash[61152]: cluster 2022-04-28T21:46:58.390765+0000 mgr.y (mgr.24961) 3827 : cluster [DBG] pgmap v4616: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:46:59 smithi059 bash[59745]: cluster 2022-04-28T21:46:58.390765+0000 mgr.y (mgr.24961) 3827 : cluster [DBG] pgmap v4616: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:46:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:46:59 smithi139 bash[43301]: cluster 2022-04-28T21:46:58.390765+0000 mgr.y (mgr.24961) 3827 : cluster [DBG] pgmap v4616: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:01 smithi059 bash[61152]: cluster 2022-04-28T21:47:00.393004+0000 mgr.y (mgr.24961) 3828 : cluster [DBG] pgmap v4617: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:01 smithi059 bash[59745]: cluster 2022-04-28T21:47:00.393004+0000 mgr.y (mgr.24961) 3828 : cluster [DBG] pgmap v4617: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:01 smithi139 bash[43301]: cluster 2022-04-28T21:47:00.393004+0000 mgr.y (mgr.24961) 3828 : cluster [DBG] pgmap v4617: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:02.081Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:03 smithi059 bash[61152]: cluster 2022-04-28T21:47:02.394198+0000 mgr.y (mgr.24961) 3829 : cluster [DBG] pgmap v4618: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:03 smithi059 bash[59745]: cluster 2022-04-28T21:47:02.394198+0000 mgr.y (mgr.24961) 3829 : cluster [DBG] pgmap v4618: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:03.869 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:03 smithi139 bash[43301]: cluster 2022-04-28T21:47:02.394198+0000 mgr.y (mgr.24961) 3829 : cluster [DBG] pgmap v4618: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:03.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:05.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:04 smithi139 bash[43301]: cluster 2022-04-28T21:47:04.395466+0000 mgr.y (mgr.24961) 3830 : cluster [DBG] pgmap v4619: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:04 smithi059 bash[61152]: cluster 2022-04-28T21:47:04.395466+0000 mgr.y (mgr.24961) 3830 : cluster [DBG] pgmap v4619: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:04 smithi059 bash[59745]: cluster 2022-04-28T21:47:04.395466+0000 mgr.y (mgr.24961) 3830 : cluster [DBG] pgmap v4619: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:07 smithi059 bash[61152]: cluster 2022-04-28T21:47:06.397069+0000 mgr.y (mgr.24961) 3831 : cluster [DBG] pgmap v4620: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:07 smithi059 bash[59745]: cluster 2022-04-28T21:47:06.397069+0000 mgr.y (mgr.24961) 3831 : cluster [DBG] pgmap v4620: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:07 smithi139 bash[43301]: cluster 2022-04-28T21:47:06.397069+0000 mgr.y (mgr.24961) 3831 : cluster [DBG] pgmap v4620: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:09 smithi059 bash[61152]: cluster 2022-04-28T21:47:08.397781+0000 mgr.y (mgr.24961) 3832 : cluster [DBG] pgmap v4621: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:09 smithi059 bash[59745]: cluster 2022-04-28T21:47:08.397781+0000 mgr.y (mgr.24961) 3832 : cluster [DBG] pgmap v4621: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:09 smithi139 bash[43301]: cluster 2022-04-28T21:47:08.397781+0000 mgr.y (mgr.24961) 3832 : cluster [DBG] pgmap v4621: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:11 smithi059 bash[61152]: cluster 2022-04-28T21:47:10.399563+0000 mgr.y (mgr.24961) 3833 : cluster [DBG] pgmap v4622: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:11 smithi059 bash[59745]: cluster 2022-04-28T21:47:10.399563+0000 mgr.y (mgr.24961) 3833 : cluster [DBG] pgmap v4622: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:11 smithi139 bash[43301]: cluster 2022-04-28T21:47:10.399563+0000 mgr.y (mgr.24961) 3833 : cluster [DBG] pgmap v4622: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:12.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[61152]: cluster 2022-04-28T21:47:12.400728+0000 mgr.y (mgr.24961) 3834 : cluster [DBG] pgmap v4623: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[61152]: audit 2022-04-28T21:47:12.727611+0000 mon.a (mon.0) 4055 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:47:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[61152]: audit 2022-04-28T21:47:12.730559+0000 mon.a (mon.0) 4056 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:47:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[61152]: audit 2022-04-28T21:47:12.732633+0000 mon.a (mon.0) 4057 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:47:13.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[61152]: audit 2022-04-28T21:47:13.036953+0000 mon.a (mon.0) 4058 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:47:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[59745]: cluster 2022-04-28T21:47:12.400728+0000 mgr.y (mgr.24961) 3834 : cluster [DBG] pgmap v4623: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[59745]: audit 2022-04-28T21:47:12.727611+0000 mon.a (mon.0) 4055 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:47:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[59745]: audit 2022-04-28T21:47:12.730559+0000 mon.a (mon.0) 4056 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:47:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[59745]: audit 2022-04-28T21:47:12.732633+0000 mon.a (mon.0) 4057 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:47:13.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[59745]: audit 2022-04-28T21:47:13.036953+0000 mon.a (mon.0) 4058 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:47:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:13 smithi139 bash[43301]: cluster 2022-04-28T21:47:12.400728+0000 mgr.y (mgr.24961) 3834 : cluster [DBG] pgmap v4623: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:13 smithi139 bash[43301]: audit 2022-04-28T21:47:12.727611+0000 mon.a (mon.0) 4055 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:47:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:13 smithi139 bash[43301]: audit 2022-04-28T21:47:12.730559+0000 mon.a (mon.0) 4056 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:47:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:13 smithi139 bash[43301]: audit 2022-04-28T21:47:12.732633+0000 mon.a (mon.0) 4057 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:47:13.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:13 smithi139 bash[43301]: audit 2022-04-28T21:47:13.036953+0000 mon.a (mon.0) 4058 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:47:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:13.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:14.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:14 smithi059 bash[61152]: cephadm 2022-04-28T21:47:12.734355+0000 mgr.y (mgr.24961) 3835 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:47:14.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:14 smithi059 bash[59745]: cephadm 2022-04-28T21:47:12.734355+0000 mgr.y (mgr.24961) 3835 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:47:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:14 smithi139 bash[43301]: cephadm 2022-04-28T21:47:12.734355+0000 mgr.y (mgr.24961) 3835 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:47:15.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:15 smithi059 bash[61152]: cluster 2022-04-28T21:47:14.402008+0000 mgr.y (mgr.24961) 3836 : cluster [DBG] pgmap v4624: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:15.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:15 smithi059 bash[59745]: cluster 2022-04-28T21:47:14.402008+0000 mgr.y (mgr.24961) 3836 : cluster [DBG] pgmap v4624: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:15 smithi139 bash[43301]: cluster 2022-04-28T21:47:14.402008+0000 mgr.y (mgr.24961) 3836 : cluster [DBG] pgmap v4624: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:17 smithi059 bash[59745]: cluster 2022-04-28T21:47:16.403594+0000 mgr.y (mgr.24961) 3837 : cluster [DBG] pgmap v4625: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:17 smithi059 bash[61152]: cluster 2022-04-28T21:47:16.403594+0000 mgr.y (mgr.24961) 3837 : cluster [DBG] pgmap v4625: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:17 smithi139 bash[43301]: cluster 2022-04-28T21:47:16.403594+0000 mgr.y (mgr.24961) 3837 : cluster [DBG] pgmap v4625: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:19.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: cluster 2022-04-28T21:47:18.404343+0000 mgr.y (mgr.24961) 3838 : cluster [DBG] pgmap v4626: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:19.865 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: audit 2022-04-28T21:47:19.374629+0000 mon.a (mon.0) 4059 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:47:19.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: audit 2022-04-28T21:47:19.382549+0000 mon.a (mon.0) 4060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:47:19.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: audit 2022-04-28T21:47:19.411558+0000 mon.a (mon.0) 4061 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:47:19.866 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: audit 2022-04-28T21:47:19.412566+0000 mon.a (mon.0) 4062 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T21:47:19.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: audit 2022-04-28T21:47:19.413077+0000 mon.a (mon.0) 4063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:47:19.867 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[61152]: audit 2022-04-28T21:47:19.413647+0000 mon.a (mon.0) 4064 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:47:19.867 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: cluster 2022-04-28T21:47:18.404343+0000 mgr.y (mgr.24961) 3838 : cluster [DBG] pgmap v4626: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:19.868 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: audit 2022-04-28T21:47:19.374629+0000 mon.a (mon.0) 4059 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:47:19.868 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: audit 2022-04-28T21:47:19.382549+0000 mon.a (mon.0) 4060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:47:19.868 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: audit 2022-04-28T21:47:19.411558+0000 mon.a (mon.0) 4061 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:47:19.868 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: audit 2022-04-28T21:47:19.412566+0000 mon.a (mon.0) 4062 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T21:47:19.868 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: audit 2022-04-28T21:47:19.413077+0000 mon.a (mon.0) 4063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:47:19.869 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:19 smithi059 bash[59745]: audit 2022-04-28T21:47:19.413647+0000 mon.a (mon.0) 4064 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:47:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: cluster 2022-04-28T21:47:18.404343+0000 mgr.y (mgr.24961) 3838 : cluster [DBG] pgmap v4626: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: audit 2022-04-28T21:47:19.374629+0000 mon.a (mon.0) 4059 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:47:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: audit 2022-04-28T21:47:19.382549+0000 mon.a (mon.0) 4060 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:47:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: audit 2022-04-28T21:47:19.411558+0000 mon.a (mon.0) 4061 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:47:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: audit 2022-04-28T21:47:19.412566+0000 mon.a (mon.0) 4062 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T21:47:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: audit 2022-04-28T21:47:19.413077+0000 mon.a (mon.0) 4063 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:47:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:19 smithi139 bash[43301]: audit 2022-04-28T21:47:19.413647+0000 mon.a (mon.0) 4064 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T21:47:21.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:21 smithi059 bash[61152]: cluster 2022-04-28T21:47:20.406121+0000 mgr.y (mgr.24961) 3839 : cluster [DBG] pgmap v4627: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:21 smithi059 bash[59745]: cluster 2022-04-28T21:47:20.406121+0000 mgr.y (mgr.24961) 3839 : cluster [DBG] pgmap v4627: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:21 smithi139 bash[43301]: cluster 2022-04-28T21:47:20.406121+0000 mgr.y (mgr.24961) 3839 : cluster [DBG] pgmap v4627: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:22.082Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:23 smithi059 bash[61152]: cluster 2022-04-28T21:47:22.407299+0000 mgr.y (mgr.24961) 3840 : cluster [DBG] pgmap v4628: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:23 smithi059 bash[59745]: cluster 2022-04-28T21:47:22.407299+0000 mgr.y (mgr.24961) 3840 : cluster [DBG] pgmap v4628: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:23 smithi139 bash[43301]: cluster 2022-04-28T21:47:22.407299+0000 mgr.y (mgr.24961) 3840 : cluster [DBG] pgmap v4628: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:23.972Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:25.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:24 smithi139 bash[43301]: cluster 2022-04-28T21:47:24.408557+0000 mgr.y (mgr.24961) 3841 : cluster [DBG] pgmap v4629: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:25.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:24 smithi059 bash[59745]: cluster 2022-04-28T21:47:24.408557+0000 mgr.y (mgr.24961) 3841 : cluster [DBG] pgmap v4629: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:25.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:24 smithi059 bash[61152]: cluster 2022-04-28T21:47:24.408557+0000 mgr.y (mgr.24961) 3841 : cluster [DBG] pgmap v4629: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:27 smithi059 bash[61152]: cluster 2022-04-28T21:47:26.410163+0000 mgr.y (mgr.24961) 3842 : cluster [DBG] pgmap v4630: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:27 smithi059 bash[59745]: cluster 2022-04-28T21:47:26.410163+0000 mgr.y (mgr.24961) 3842 : cluster [DBG] pgmap v4630: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:27 smithi139 bash[43301]: cluster 2022-04-28T21:47:26.410163+0000 mgr.y (mgr.24961) 3842 : cluster [DBG] pgmap v4630: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:29 smithi059 bash[61152]: cluster 2022-04-28T21:47:28.410937+0000 mgr.y (mgr.24961) 3843 : cluster [DBG] pgmap v4631: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:29 smithi059 bash[59745]: cluster 2022-04-28T21:47:28.410937+0000 mgr.y (mgr.24961) 3843 : cluster [DBG] pgmap v4631: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:29 smithi139 bash[43301]: cluster 2022-04-28T21:47:28.410937+0000 mgr.y (mgr.24961) 3843 : cluster [DBG] pgmap v4631: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:31 smithi059 bash[61152]: cluster 2022-04-28T21:47:30.412692+0000 mgr.y (mgr.24961) 3844 : cluster [DBG] pgmap v4632: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:31.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:31 smithi059 bash[59745]: cluster 2022-04-28T21:47:30.412692+0000 mgr.y (mgr.24961) 3844 : cluster [DBG] pgmap v4632: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:31 smithi139 bash[43301]: cluster 2022-04-28T21:47:30.412692+0000 mgr.y (mgr.24961) 3844 : cluster [DBG] pgmap v4632: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:32.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:33 smithi059 bash[61152]: cluster 2022-04-28T21:47:32.413794+0000 mgr.y (mgr.24961) 3845 : cluster [DBG] pgmap v4633: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:33 smithi059 bash[59745]: cluster 2022-04-28T21:47:32.413794+0000 mgr.y (mgr.24961) 3845 : cluster [DBG] pgmap v4633: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:33 smithi139 bash[43301]: cluster 2022-04-28T21:47:32.413794+0000 mgr.y (mgr.24961) 3845 : cluster [DBG] pgmap v4633: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:33.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:35.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:34 smithi139 bash[43301]: cluster 2022-04-28T21:47:34.415005+0000 mgr.y (mgr.24961) 3846 : cluster [DBG] pgmap v4634: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:34 smithi059 bash[61152]: cluster 2022-04-28T21:47:34.415005+0000 mgr.y (mgr.24961) 3846 : cluster [DBG] pgmap v4634: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:34 smithi059 bash[59745]: cluster 2022-04-28T21:47:34.415005+0000 mgr.y (mgr.24961) 3846 : cluster [DBG] pgmap v4634: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:37 smithi059 bash[61152]: cluster 2022-04-28T21:47:36.416589+0000 mgr.y (mgr.24961) 3847 : cluster [DBG] pgmap v4635: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:37 smithi059 bash[59745]: cluster 2022-04-28T21:47:36.416589+0000 mgr.y (mgr.24961) 3847 : cluster [DBG] pgmap v4635: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:37 smithi139 bash[43301]: cluster 2022-04-28T21:47:36.416589+0000 mgr.y (mgr.24961) 3847 : cluster [DBG] pgmap v4635: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:39 smithi059 bash[61152]: cluster 2022-04-28T21:47:38.417326+0000 mgr.y (mgr.24961) 3848 : cluster [DBG] pgmap v4636: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:39.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:39 smithi059 bash[59745]: cluster 2022-04-28T21:47:38.417326+0000 mgr.y (mgr.24961) 3848 : cluster [DBG] pgmap v4636: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:39 smithi139 bash[43301]: cluster 2022-04-28T21:47:38.417326+0000 mgr.y (mgr.24961) 3848 : cluster [DBG] pgmap v4636: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:41 smithi059 bash[61152]: cluster 2022-04-28T21:47:40.418987+0000 mgr.y (mgr.24961) 3849 : cluster [DBG] pgmap v4637: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:41 smithi059 bash[59745]: cluster 2022-04-28T21:47:40.418987+0000 mgr.y (mgr.24961) 3849 : cluster [DBG] pgmap v4637: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:41 smithi139 bash[43301]: cluster 2022-04-28T21:47:40.418987+0000 mgr.y (mgr.24961) 3849 : cluster [DBG] pgmap v4637: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:42.084Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:43 smithi059 bash[61152]: cluster 2022-04-28T21:47:42.420183+0000 mgr.y (mgr.24961) 3850 : cluster [DBG] pgmap v4638: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:43 smithi059 bash[59745]: cluster 2022-04-28T21:47:42.420183+0000 mgr.y (mgr.24961) 3850 : cluster [DBG] pgmap v4638: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:43 smithi139 bash[43301]: cluster 2022-04-28T21:47:42.420183+0000 mgr.y (mgr.24961) 3850 : cluster [DBG] pgmap v4638: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:43.973Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:44 smithi059 bash[61152]: cluster 2022-04-28T21:47:44.421441+0000 mgr.y (mgr.24961) 3851 : cluster [DBG] pgmap v4639: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:44 smithi059 bash[59745]: cluster 2022-04-28T21:47:44.421441+0000 mgr.y (mgr.24961) 3851 : cluster [DBG] pgmap v4639: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:44 smithi139 bash[43301]: cluster 2022-04-28T21:47:44.421441+0000 mgr.y (mgr.24961) 3851 : cluster [DBG] pgmap v4639: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:47 smithi059 bash[61152]: cluster 2022-04-28T21:47:46.422983+0000 mgr.y (mgr.24961) 3852 : cluster [DBG] pgmap v4640: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:47 smithi059 bash[59745]: cluster 2022-04-28T21:47:46.422983+0000 mgr.y (mgr.24961) 3852 : cluster [DBG] pgmap v4640: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:47 smithi139 bash[43301]: cluster 2022-04-28T21:47:46.422983+0000 mgr.y (mgr.24961) 3852 : cluster [DBG] pgmap v4640: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:49 smithi059 bash[61152]: cluster 2022-04-28T21:47:48.423761+0000 mgr.y (mgr.24961) 3853 : cluster [DBG] pgmap v4641: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:49 smithi059 bash[59745]: cluster 2022-04-28T21:47:48.423761+0000 mgr.y (mgr.24961) 3853 : cluster [DBG] pgmap v4641: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:49 smithi139 bash[43301]: cluster 2022-04-28T21:47:48.423761+0000 mgr.y (mgr.24961) 3853 : cluster [DBG] pgmap v4641: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:51 smithi059 bash[61152]: cluster 2022-04-28T21:47:50.425484+0000 mgr.y (mgr.24961) 3854 : cluster [DBG] pgmap v4642: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:51 smithi059 bash[59745]: cluster 2022-04-28T21:47:50.425484+0000 mgr.y (mgr.24961) 3854 : cluster [DBG] pgmap v4642: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:51 smithi139 bash[43301]: cluster 2022-04-28T21:47:50.425484+0000 mgr.y (mgr.24961) 3854 : cluster [DBG] pgmap v4642: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:52.085Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:53 smithi059 bash[61152]: cluster 2022-04-28T21:47:52.426677+0000 mgr.y (mgr.24961) 3855 : cluster [DBG] pgmap v4643: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:53 smithi059 bash[59745]: cluster 2022-04-28T21:47:52.426677+0000 mgr.y (mgr.24961) 3855 : cluster [DBG] pgmap v4643: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:53.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:53 smithi139 bash[43301]: cluster 2022-04-28T21:47:52.426677+0000 mgr.y (mgr.24961) 3855 : cluster [DBG] pgmap v4643: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:47:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:47:53.974Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:47:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:54 smithi059 bash[61152]: cluster 2022-04-28T21:47:54.428010+0000 mgr.y (mgr.24961) 3856 : cluster [DBG] pgmap v4644: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:54 smithi059 bash[59745]: cluster 2022-04-28T21:47:54.428010+0000 mgr.y (mgr.24961) 3856 : cluster [DBG] pgmap v4644: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:54 smithi139 bash[43301]: cluster 2022-04-28T21:47:54.428010+0000 mgr.y (mgr.24961) 3856 : cluster [DBG] pgmap v4644: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:57 smithi059 bash[61152]: cluster 2022-04-28T21:47:56.429563+0000 mgr.y (mgr.24961) 3857 : cluster [DBG] pgmap v4645: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:57 smithi059 bash[59745]: cluster 2022-04-28T21:47:56.429563+0000 mgr.y (mgr.24961) 3857 : cluster [DBG] pgmap v4645: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:57 smithi139 bash[43301]: cluster 2022-04-28T21:47:56.429563+0000 mgr.y (mgr.24961) 3857 : cluster [DBG] pgmap v4645: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:47:59 smithi059 bash[61152]: cluster 2022-04-28T21:47:58.430342+0000 mgr.y (mgr.24961) 3858 : cluster [DBG] pgmap v4646: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:47:59 smithi059 bash[59745]: cluster 2022-04-28T21:47:58.430342+0000 mgr.y (mgr.24961) 3858 : cluster [DBG] pgmap v4646: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:47:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:47:59 smithi139 bash[43301]: cluster 2022-04-28T21:47:58.430342+0000 mgr.y (mgr.24961) 3858 : cluster [DBG] pgmap v4646: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:01 smithi059 bash[61152]: cluster 2022-04-28T21:48:00.432141+0000 mgr.y (mgr.24961) 3859 : cluster [DBG] pgmap v4647: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:01 smithi059 bash[59745]: cluster 2022-04-28T21:48:00.432141+0000 mgr.y (mgr.24961) 3859 : cluster [DBG] pgmap v4647: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:01 smithi139 bash[43301]: cluster 2022-04-28T21:48:00.432141+0000 mgr.y (mgr.24961) 3859 : cluster [DBG] pgmap v4647: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:02.086Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:03 smithi059 bash[61152]: cluster 2022-04-28T21:48:02.433430+0000 mgr.y (mgr.24961) 3860 : cluster [DBG] pgmap v4648: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:03 smithi059 bash[59745]: cluster 2022-04-28T21:48:02.433430+0000 mgr.y (mgr.24961) 3860 : cluster [DBG] pgmap v4648: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:03 smithi139 bash[43301]: cluster 2022-04-28T21:48:02.433430+0000 mgr.y (mgr.24961) 3860 : cluster [DBG] pgmap v4648: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:03.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:04 smithi059 bash[61152]: cluster 2022-04-28T21:48:04.434761+0000 mgr.y (mgr.24961) 3861 : cluster [DBG] pgmap v4649: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:04 smithi059 bash[59745]: cluster 2022-04-28T21:48:04.434761+0000 mgr.y (mgr.24961) 3861 : cluster [DBG] pgmap v4649: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:04 smithi139 bash[43301]: cluster 2022-04-28T21:48:04.434761+0000 mgr.y (mgr.24961) 3861 : cluster [DBG] pgmap v4649: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:07 smithi059 bash[61152]: cluster 2022-04-28T21:48:06.436486+0000 mgr.y (mgr.24961) 3862 : cluster [DBG] pgmap v4650: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:07 smithi059 bash[59745]: cluster 2022-04-28T21:48:06.436486+0000 mgr.y (mgr.24961) 3862 : cluster [DBG] pgmap v4650: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:07 smithi139 bash[43301]: cluster 2022-04-28T21:48:06.436486+0000 mgr.y (mgr.24961) 3862 : cluster [DBG] pgmap v4650: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:09 smithi059 bash[61152]: cluster 2022-04-28T21:48:08.437392+0000 mgr.y (mgr.24961) 3863 : cluster [DBG] pgmap v4651: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:09 smithi059 bash[59745]: cluster 2022-04-28T21:48:08.437392+0000 mgr.y (mgr.24961) 3863 : cluster [DBG] pgmap v4651: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:09 smithi139 bash[43301]: cluster 2022-04-28T21:48:08.437392+0000 mgr.y (mgr.24961) 3863 : cluster [DBG] pgmap v4651: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:11.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:11 smithi059 bash[59745]: cluster 2022-04-28T21:48:10.439161+0000 mgr.y (mgr.24961) 3864 : cluster [DBG] pgmap v4652: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:11.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:11 smithi059 bash[61152]: cluster 2022-04-28T21:48:10.439161+0000 mgr.y (mgr.24961) 3864 : cluster [DBG] pgmap v4652: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:11 smithi139 bash[43301]: cluster 2022-04-28T21:48:10.439161+0000 mgr.y (mgr.24961) 3864 : cluster [DBG] pgmap v4652: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:12.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:12.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:13.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[59745]: cluster 2022-04-28T21:48:12.440348+0000 mgr.y (mgr.24961) 3865 : cluster [DBG] pgmap v4653: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[59745]: audit 2022-04-28T21:48:13.045183+0000 mon.a (mon.0) 4065 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:48:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[59745]: audit 2022-04-28T21:48:13.048133+0000 mon.a (mon.0) 4066 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:48:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[59745]: audit 2022-04-28T21:48:13.050120+0000 mon.a (mon.0) 4067 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:48:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[59745]: audit 2022-04-28T21:48:13.053464+0000 mon.a (mon.0) 4068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:48:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[59745]: audit 2022-04-28T21:48:13.054164+0000 mon.a (mon.0) 4069 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[61152]: cluster 2022-04-28T21:48:12.440348+0000 mgr.y (mgr.24961) 3865 : cluster [DBG] pgmap v4653: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[61152]: audit 2022-04-28T21:48:13.045183+0000 mon.a (mon.0) 4065 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[61152]: audit 2022-04-28T21:48:13.048133+0000 mon.a (mon.0) 4066 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:48:13.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[61152]: audit 2022-04-28T21:48:13.050120+0000 mon.a (mon.0) 4067 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:48:13.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[61152]: audit 2022-04-28T21:48:13.053464+0000 mon.a (mon.0) 4068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:48:13.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[61152]: audit 2022-04-28T21:48:13.054164+0000 mon.a (mon.0) 4069 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:48:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:13 smithi139 bash[43301]: cluster 2022-04-28T21:48:12.440348+0000 mgr.y (mgr.24961) 3865 : cluster [DBG] pgmap v4653: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:13 smithi139 bash[43301]: audit 2022-04-28T21:48:13.045183+0000 mon.a (mon.0) 4065 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:48:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:13 smithi139 bash[43301]: audit 2022-04-28T21:48:13.048133+0000 mon.a (mon.0) 4066 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:48:13.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:13 smithi139 bash[43301]: audit 2022-04-28T21:48:13.050120+0000 mon.a (mon.0) 4067 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:48:13.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:13 smithi139 bash[43301]: audit 2022-04-28T21:48:13.053464+0000 mon.a (mon.0) 4068 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:48:13.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:13 smithi139 bash[43301]: audit 2022-04-28T21:48:13.054164+0000 mon.a (mon.0) 4069 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:48:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:13.976Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:14.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:14 smithi059 bash[59745]: cephadm 2022-04-28T21:48:13.051939+0000 mgr.y (mgr.24961) 3866 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:48:14.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:14 smithi059 bash[61152]: cephadm 2022-04-28T21:48:13.051939+0000 mgr.y (mgr.24961) 3866 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:48:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:14 smithi139 bash[43301]: cephadm 2022-04-28T21:48:13.051939+0000 mgr.y (mgr.24961) 3866 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:48:15.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:15 smithi059 bash[59745]: cluster 2022-04-28T21:48:14.441708+0000 mgr.y (mgr.24961) 3867 : cluster [DBG] pgmap v4654: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:15.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:15 smithi059 bash[61152]: cluster 2022-04-28T21:48:14.441708+0000 mgr.y (mgr.24961) 3867 : cluster [DBG] pgmap v4654: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:15.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:15 smithi139 bash[43301]: cluster 2022-04-28T21:48:14.441708+0000 mgr.y (mgr.24961) 3867 : cluster [DBG] pgmap v4654: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:17 smithi059 bash[59745]: cluster 2022-04-28T21:48:16.443002+0000 mgr.y (mgr.24961) 3868 : cluster [DBG] pgmap v4655: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:17 smithi059 bash[59745]: audit 2022-04-28T21:48:17.146493+0000 mon.a (mon.0) 4070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:17 smithi059 bash[59745]: audit 2022-04-28T21:48:17.354860+0000 mon.a (mon.0) 4071 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:17 smithi059 bash[61152]: cluster 2022-04-28T21:48:16.443002+0000 mgr.y (mgr.24961) 3868 : cluster [DBG] pgmap v4655: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:17 smithi059 bash[61152]: audit 2022-04-28T21:48:17.146493+0000 mon.a (mon.0) 4070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:17.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:17 smithi059 bash[61152]: audit 2022-04-28T21:48:17.354860+0000 mon.a (mon.0) 4071 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:17 smithi139 bash[43301]: cluster 2022-04-28T21:48:16.443002+0000 mgr.y (mgr.24961) 3868 : cluster [DBG] pgmap v4655: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:17 smithi139 bash[43301]: audit 2022-04-28T21:48:17.146493+0000 mon.a (mon.0) 4070 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:17.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:17 smithi139 bash[43301]: audit 2022-04-28T21:48:17.354860+0000 mon.a (mon.0) 4071 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:19.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:18 smithi059 bash[59745]: audit 2022-04-28T21:48:17.651619+0000 mon.a (mon.0) 4072 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:18 smithi059 bash[61152]: audit 2022-04-28T21:48:17.651619+0000 mon.a (mon.0) 4072 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:18 smithi139 bash[43301]: audit 2022-04-28T21:48:17.651619+0000 mon.a (mon.0) 4072 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:48:20.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: cluster 2022-04-28T21:48:18.443818+0000 mgr.y (mgr.24961) 3869 : cluster [DBG] pgmap v4656: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: audit 2022-04-28T21:48:19.379648+0000 mon.a (mon.0) 4073 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:48:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: audit 2022-04-28T21:48:19.388616+0000 mon.a (mon.0) 4074 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:48:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: audit 2022-04-28T21:48:19.421525+0000 mon.a (mon.0) 4075 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:48:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: audit 2022-04-28T21:48:19.422052+0000 mon.a (mon.0) 4076 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T21:48:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: audit 2022-04-28T21:48:19.422645+0000 mon.a (mon.0) 4077 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:48:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[61152]: audit 2022-04-28T21:48:19.423183+0000 mon.a (mon.0) 4078 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:48:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: cluster 2022-04-28T21:48:18.443818+0000 mgr.y (mgr.24961) 3869 : cluster [DBG] pgmap v4656: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: audit 2022-04-28T21:48:19.379648+0000 mon.a (mon.0) 4073 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:48:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: audit 2022-04-28T21:48:19.388616+0000 mon.a (mon.0) 4074 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:48:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: audit 2022-04-28T21:48:19.421525+0000 mon.a (mon.0) 4075 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:48:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: audit 2022-04-28T21:48:19.422052+0000 mon.a (mon.0) 4076 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T21:48:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: audit 2022-04-28T21:48:19.422645+0000 mon.a (mon.0) 4077 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:48:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:19 smithi059 bash[59745]: audit 2022-04-28T21:48:19.423183+0000 mon.a (mon.0) 4078 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:48:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: cluster 2022-04-28T21:48:18.443818+0000 mgr.y (mgr.24961) 3869 : cluster [DBG] pgmap v4656: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: audit 2022-04-28T21:48:19.379648+0000 mon.a (mon.0) 4073 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:48:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: audit 2022-04-28T21:48:19.388616+0000 mon.a (mon.0) 4074 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:48:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: audit 2022-04-28T21:48:19.421525+0000 mon.a (mon.0) 4075 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:48:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: audit 2022-04-28T21:48:19.422052+0000 mon.a (mon.0) 4076 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 5]}]: dispatch 2022-04-28T21:48:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: audit 2022-04-28T21:48:19.422645+0000 mon.a (mon.0) 4077 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T21:48:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:19 smithi139 bash[43301]: audit 2022-04-28T21:48:19.423183+0000 mon.a (mon.0) 4078 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:48:22.089 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:21 smithi059 bash[61152]: cluster 2022-04-28T21:48:20.445562+0000 mgr.y (mgr.24961) 3870 : cluster [DBG] pgmap v4657: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:22.090 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:21 smithi059 bash[59745]: cluster 2022-04-28T21:48:20.445562+0000 mgr.y (mgr.24961) 3870 : cluster [DBG] pgmap v4657: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:21 smithi139 bash[43301]: cluster 2022-04-28T21:48:20.445562+0000 mgr.y (mgr.24961) 3870 : cluster [DBG] pgmap v4657: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:22.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:22.088Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:23.976 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:23 smithi059 bash[59745]: cluster 2022-04-28T21:48:22.446755+0000 mgr.y (mgr.24961) 3871 : cluster [DBG] pgmap v4658: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:23.978 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:23 smithi059 bash[61152]: cluster 2022-04-28T21:48:22.446755+0000 mgr.y (mgr.24961) 3871 : cluster [DBG] pgmap v4658: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:23 smithi139 bash[43301]: cluster 2022-04-28T21:48:22.446755+0000 mgr.y (mgr.24961) 3871 : cluster [DBG] pgmap v4658: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:23.975Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:24 smithi059 bash[61152]: cluster 2022-04-28T21:48:24.448072+0000 mgr.y (mgr.24961) 3872 : cluster [DBG] pgmap v4659: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:24 smithi059 bash[59745]: cluster 2022-04-28T21:48:24.448072+0000 mgr.y (mgr.24961) 3872 : cluster [DBG] pgmap v4659: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:24 smithi139 bash[43301]: cluster 2022-04-28T21:48:24.448072+0000 mgr.y (mgr.24961) 3872 : cluster [DBG] pgmap v4659: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:27 smithi059 bash[61152]: cluster 2022-04-28T21:48:26.449695+0000 mgr.y (mgr.24961) 3873 : cluster [DBG] pgmap v4660: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:27 smithi059 bash[59745]: cluster 2022-04-28T21:48:26.449695+0000 mgr.y (mgr.24961) 3873 : cluster [DBG] pgmap v4660: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:27 smithi139 bash[43301]: cluster 2022-04-28T21:48:26.449695+0000 mgr.y (mgr.24961) 3873 : cluster [DBG] pgmap v4660: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:29 smithi059 bash[61152]: cluster 2022-04-28T21:48:28.450474+0000 mgr.y (mgr.24961) 3874 : cluster [DBG] pgmap v4661: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:29 smithi059 bash[59745]: cluster 2022-04-28T21:48:28.450474+0000 mgr.y (mgr.24961) 3874 : cluster [DBG] pgmap v4661: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:29 smithi139 bash[43301]: cluster 2022-04-28T21:48:28.450474+0000 mgr.y (mgr.24961) 3874 : cluster [DBG] pgmap v4661: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:31 smithi059 bash[61152]: cluster 2022-04-28T21:48:30.452228+0000 mgr.y (mgr.24961) 3875 : cluster [DBG] pgmap v4662: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:31.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:31 smithi059 bash[59745]: cluster 2022-04-28T21:48:30.452228+0000 mgr.y (mgr.24961) 3875 : cluster [DBG] pgmap v4662: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:31 smithi139 bash[43301]: cluster 2022-04-28T21:48:30.452228+0000 mgr.y (mgr.24961) 3875 : cluster [DBG] pgmap v4662: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:32.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:32.087Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:33 smithi059 bash[61152]: cluster 2022-04-28T21:48:32.453415+0000 mgr.y (mgr.24961) 3876 : cluster [DBG] pgmap v4663: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:33 smithi059 bash[59745]: cluster 2022-04-28T21:48:32.453415+0000 mgr.y (mgr.24961) 3876 : cluster [DBG] pgmap v4663: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:33 smithi139 bash[43301]: cluster 2022-04-28T21:48:32.453415+0000 mgr.y (mgr.24961) 3876 : cluster [DBG] pgmap v4663: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:33.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:34 smithi059 bash[61152]: cluster 2022-04-28T21:48:34.454708+0000 mgr.y (mgr.24961) 3877 : cluster [DBG] pgmap v4664: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:34 smithi059 bash[59745]: cluster 2022-04-28T21:48:34.454708+0000 mgr.y (mgr.24961) 3877 : cluster [DBG] pgmap v4664: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:34 smithi139 bash[43301]: cluster 2022-04-28T21:48:34.454708+0000 mgr.y (mgr.24961) 3877 : cluster [DBG] pgmap v4664: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:37 smithi059 bash[61152]: cluster 2022-04-28T21:48:36.456326+0000 mgr.y (mgr.24961) 3878 : cluster [DBG] pgmap v4665: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:37 smithi059 bash[59745]: cluster 2022-04-28T21:48:36.456326+0000 mgr.y (mgr.24961) 3878 : cluster [DBG] pgmap v4665: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:37 smithi139 bash[43301]: cluster 2022-04-28T21:48:36.456326+0000 mgr.y (mgr.24961) 3878 : cluster [DBG] pgmap v4665: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:39.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:39 smithi059 bash[59745]: cluster 2022-04-28T21:48:38.457114+0000 mgr.y (mgr.24961) 3879 : cluster [DBG] pgmap v4666: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:39 smithi059 bash[61152]: cluster 2022-04-28T21:48:38.457114+0000 mgr.y (mgr.24961) 3879 : cluster [DBG] pgmap v4666: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:39 smithi139 bash[43301]: cluster 2022-04-28T21:48:38.457114+0000 mgr.y (mgr.24961) 3879 : cluster [DBG] pgmap v4666: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:41 smithi059 bash[61152]: cluster 2022-04-28T21:48:40.458825+0000 mgr.y (mgr.24961) 3880 : cluster [DBG] pgmap v4667: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:41 smithi059 bash[59745]: cluster 2022-04-28T21:48:40.458825+0000 mgr.y (mgr.24961) 3880 : cluster [DBG] pgmap v4667: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:41 smithi139 bash[43301]: cluster 2022-04-28T21:48:40.458825+0000 mgr.y (mgr.24961) 3880 : cluster [DBG] pgmap v4667: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:42.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:42.089Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:43 smithi059 bash[61152]: cluster 2022-04-28T21:48:42.460081+0000 mgr.y (mgr.24961) 3881 : cluster [DBG] pgmap v4668: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:43.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:43 smithi059 bash[59745]: cluster 2022-04-28T21:48:42.460081+0000 mgr.y (mgr.24961) 3881 : cluster [DBG] pgmap v4668: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:43 smithi139 bash[43301]: cluster 2022-04-28T21:48:42.460081+0000 mgr.y (mgr.24961) 3881 : cluster [DBG] pgmap v4668: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:43.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:44 smithi059 bash[61152]: cluster 2022-04-28T21:48:44.461387+0000 mgr.y (mgr.24961) 3882 : cluster [DBG] pgmap v4669: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:44 smithi059 bash[59745]: cluster 2022-04-28T21:48:44.461387+0000 mgr.y (mgr.24961) 3882 : cluster [DBG] pgmap v4669: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:44 smithi139 bash[43301]: cluster 2022-04-28T21:48:44.461387+0000 mgr.y (mgr.24961) 3882 : cluster [DBG] pgmap v4669: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:47 smithi059 bash[61152]: cluster 2022-04-28T21:48:46.462921+0000 mgr.y (mgr.24961) 3883 : cluster [DBG] pgmap v4670: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:47 smithi059 bash[59745]: cluster 2022-04-28T21:48:46.462921+0000 mgr.y (mgr.24961) 3883 : cluster [DBG] pgmap v4670: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:47 smithi139 bash[43301]: cluster 2022-04-28T21:48:46.462921+0000 mgr.y (mgr.24961) 3883 : cluster [DBG] pgmap v4670: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:49.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:49 smithi059 bash[61152]: cluster 2022-04-28T21:48:48.463722+0000 mgr.y (mgr.24961) 3884 : cluster [DBG] pgmap v4671: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:49.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:49 smithi059 bash[59745]: cluster 2022-04-28T21:48:48.463722+0000 mgr.y (mgr.24961) 3884 : cluster [DBG] pgmap v4671: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:49 smithi139 bash[43301]: cluster 2022-04-28T21:48:48.463722+0000 mgr.y (mgr.24961) 3884 : cluster [DBG] pgmap v4671: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:51 smithi059 bash[61152]: cluster 2022-04-28T21:48:50.465449+0000 mgr.y (mgr.24961) 3885 : cluster [DBG] pgmap v4672: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:51 smithi059 bash[59745]: cluster 2022-04-28T21:48:50.465449+0000 mgr.y (mgr.24961) 3885 : cluster [DBG] pgmap v4672: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:51 smithi139 bash[43301]: cluster 2022-04-28T21:48:50.465449+0000 mgr.y (mgr.24961) 3885 : cluster [DBG] pgmap v4672: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:52.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:52.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:53 smithi059 bash[61152]: cluster 2022-04-28T21:48:52.466586+0000 mgr.y (mgr.24961) 3886 : cluster [DBG] pgmap v4673: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:53 smithi059 bash[59745]: cluster 2022-04-28T21:48:52.466586+0000 mgr.y (mgr.24961) 3886 : cluster [DBG] pgmap v4673: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:53 smithi139 bash[43301]: cluster 2022-04-28T21:48:52.466586+0000 mgr.y (mgr.24961) 3886 : cluster [DBG] pgmap v4673: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:48:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:48:53.977Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:48:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:54 smithi059 bash[61152]: cluster 2022-04-28T21:48:54.467917+0000 mgr.y (mgr.24961) 3887 : cluster [DBG] pgmap v4674: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:54 smithi059 bash[59745]: cluster 2022-04-28T21:48:54.467917+0000 mgr.y (mgr.24961) 3887 : cluster [DBG] pgmap v4674: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:54 smithi139 bash[43301]: cluster 2022-04-28T21:48:54.467917+0000 mgr.y (mgr.24961) 3887 : cluster [DBG] pgmap v4674: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:57.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:57 smithi059 bash[61152]: cluster 2022-04-28T21:48:56.469470+0000 mgr.y (mgr.24961) 3888 : cluster [DBG] pgmap v4675: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:57 smithi059 bash[59745]: cluster 2022-04-28T21:48:56.469470+0000 mgr.y (mgr.24961) 3888 : cluster [DBG] pgmap v4675: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:57 smithi139 bash[43301]: cluster 2022-04-28T21:48:56.469470+0000 mgr.y (mgr.24961) 3888 : cluster [DBG] pgmap v4675: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:48:59 smithi059 bash[61152]: cluster 2022-04-28T21:48:58.470233+0000 mgr.y (mgr.24961) 3889 : cluster [DBG] pgmap v4676: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:48:59 smithi059 bash[59745]: cluster 2022-04-28T21:48:58.470233+0000 mgr.y (mgr.24961) 3889 : cluster [DBG] pgmap v4676: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:48:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:48:59 smithi139 bash[43301]: cluster 2022-04-28T21:48:58.470233+0000 mgr.y (mgr.24961) 3889 : cluster [DBG] pgmap v4676: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:01.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:01 smithi059 bash[61152]: cluster 2022-04-28T21:49:00.471961+0000 mgr.y (mgr.24961) 3890 : cluster [DBG] pgmap v4677: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:01.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:01 smithi059 bash[59745]: cluster 2022-04-28T21:49:00.471961+0000 mgr.y (mgr.24961) 3890 : cluster [DBG] pgmap v4677: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:01 smithi139 bash[43301]: cluster 2022-04-28T21:49:00.471961+0000 mgr.y (mgr.24961) 3890 : cluster [DBG] pgmap v4677: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:02.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:02.090Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:03 smithi059 bash[61152]: cluster 2022-04-28T21:49:02.473164+0000 mgr.y (mgr.24961) 3891 : cluster [DBG] pgmap v4678: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:03 smithi059 bash[59745]: cluster 2022-04-28T21:49:02.473164+0000 mgr.y (mgr.24961) 3891 : cluster [DBG] pgmap v4678: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:03 smithi139 bash[43301]: cluster 2022-04-28T21:49:02.473164+0000 mgr.y (mgr.24961) 3891 : cluster [DBG] pgmap v4678: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:03.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:04 smithi059 bash[61152]: cluster 2022-04-28T21:49:04.474427+0000 mgr.y (mgr.24961) 3892 : cluster [DBG] pgmap v4679: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:04 smithi059 bash[59745]: cluster 2022-04-28T21:49:04.474427+0000 mgr.y (mgr.24961) 3892 : cluster [DBG] pgmap v4679: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:04 smithi139 bash[43301]: cluster 2022-04-28T21:49:04.474427+0000 mgr.y (mgr.24961) 3892 : cluster [DBG] pgmap v4679: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:07 smithi059 bash[61152]: cluster 2022-04-28T21:49:06.476110+0000 mgr.y (mgr.24961) 3893 : cluster [DBG] pgmap v4680: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:07 smithi059 bash[59745]: cluster 2022-04-28T21:49:06.476110+0000 mgr.y (mgr.24961) 3893 : cluster [DBG] pgmap v4680: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:07 smithi139 bash[43301]: cluster 2022-04-28T21:49:06.476110+0000 mgr.y (mgr.24961) 3893 : cluster [DBG] pgmap v4680: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:09 smithi059 bash[61152]: cluster 2022-04-28T21:49:08.476907+0000 mgr.y (mgr.24961) 3894 : cluster [DBG] pgmap v4681: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:09 smithi059 bash[59745]: cluster 2022-04-28T21:49:08.476907+0000 mgr.y (mgr.24961) 3894 : cluster [DBG] pgmap v4681: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:09 smithi139 bash[43301]: cluster 2022-04-28T21:49:08.476907+0000 mgr.y (mgr.24961) 3894 : cluster [DBG] pgmap v4681: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:11 smithi059 bash[61152]: cluster 2022-04-28T21:49:10.478620+0000 mgr.y (mgr.24961) 3895 : cluster [DBG] pgmap v4682: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:11 smithi059 bash[59745]: cluster 2022-04-28T21:49:10.478620+0000 mgr.y (mgr.24961) 3895 : cluster [DBG] pgmap v4682: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:11 smithi139 bash[43301]: cluster 2022-04-28T21:49:10.478620+0000 mgr.y (mgr.24961) 3895 : cluster [DBG] pgmap v4682: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:12.091Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:13.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:13 smithi059 bash[61152]: cluster 2022-04-28T21:49:12.479936+0000 mgr.y (mgr.24961) 3896 : cluster [DBG] pgmap v4683: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:13.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:13 smithi059 bash[59745]: cluster 2022-04-28T21:49:12.479936+0000 mgr.y (mgr.24961) 3896 : cluster [DBG] pgmap v4683: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:13 smithi139 bash[43301]: cluster 2022-04-28T21:49:12.479936+0000 mgr.y (mgr.24961) 3896 : cluster [DBG] pgmap v4683: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:13.978Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:14 smithi059 bash[61152]: cluster 2022-04-28T21:49:14.481245+0000 mgr.y (mgr.24961) 3897 : cluster [DBG] pgmap v4684: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:14 smithi059 bash[59745]: cluster 2022-04-28T21:49:14.481245+0000 mgr.y (mgr.24961) 3897 : cluster [DBG] pgmap v4684: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:14 smithi139 bash[43301]: cluster 2022-04-28T21:49:14.481245+0000 mgr.y (mgr.24961) 3897 : cluster [DBG] pgmap v4684: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:17.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:17 smithi059 bash[59745]: cluster 2022-04-28T21:49:16.482762+0000 mgr.y (mgr.24961) 3898 : cluster [DBG] pgmap v4685: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:17.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:17 smithi059 bash[61152]: cluster 2022-04-28T21:49:16.482762+0000 mgr.y (mgr.24961) 3898 : cluster [DBG] pgmap v4685: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:17 smithi139 bash[43301]: cluster 2022-04-28T21:49:16.482762+0000 mgr.y (mgr.24961) 3898 : cluster [DBG] pgmap v4685: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[61152]: audit 2022-04-28T21:49:17.661083+0000 mon.a (mon.0) 4079 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:49:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[61152]: audit 2022-04-28T21:49:17.663864+0000 mon.a (mon.0) 4080 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:49:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[61152]: audit 2022-04-28T21:49:17.665700+0000 mon.a (mon.0) 4081 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:49:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[61152]: cephadm 2022-04-28T21:49:17.667436+0000 mgr.y (mgr.24961) 3899 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:49:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[61152]: audit 2022-04-28T21:49:17.972319+0000 mon.a (mon.0) 4082 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:49:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[59745]: audit 2022-04-28T21:49:17.661083+0000 mon.a (mon.0) 4079 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:49:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[59745]: audit 2022-04-28T21:49:17.663864+0000 mon.a (mon.0) 4080 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:49:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[59745]: audit 2022-04-28T21:49:17.665700+0000 mon.a (mon.0) 4081 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:49:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[59745]: cephadm 2022-04-28T21:49:17.667436+0000 mgr.y (mgr.24961) 3899 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:49:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:18 smithi059 bash[59745]: audit 2022-04-28T21:49:17.972319+0000 mon.a (mon.0) 4082 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:49:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:18 smithi139 bash[43301]: audit 2022-04-28T21:49:17.661083+0000 mon.a (mon.0) 4079 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:49:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:18 smithi139 bash[43301]: audit 2022-04-28T21:49:17.663864+0000 mon.a (mon.0) 4080 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:49:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:18 smithi139 bash[43301]: audit 2022-04-28T21:49:17.665700+0000 mon.a (mon.0) 4081 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:49:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:18 smithi139 bash[43301]: cephadm 2022-04-28T21:49:17.667436+0000 mgr.y (mgr.24961) 3899 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:49:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:18 smithi139 bash[43301]: audit 2022-04-28T21:49:17.972319+0000 mon.a (mon.0) 4082 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:49:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: cluster 2022-04-28T21:49:18.483713+0000 mgr.y (mgr.24961) 3900 : cluster [DBG] pgmap v4686: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: audit 2022-04-28T21:49:19.384679+0000 mon.a (mon.0) 4083 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:49:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: audit 2022-04-28T21:49:19.394237+0000 mon.a (mon.0) 4084 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:49:19.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: audit 2022-04-28T21:49:19.431289+0000 mon.a (mon.0) 4085 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:49:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: audit 2022-04-28T21:49:19.432087+0000 mon.a (mon.0) 4086 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:49:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: audit 2022-04-28T21:49:19.432575+0000 mon.a (mon.0) 4087 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:49:19.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[61152]: audit 2022-04-28T21:49:19.433170+0000 mon.a (mon.0) 4088 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:49:19.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: cluster 2022-04-28T21:49:18.483713+0000 mgr.y (mgr.24961) 3900 : cluster [DBG] pgmap v4686: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: audit 2022-04-28T21:49:19.384679+0000 mon.a (mon.0) 4083 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:49:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: audit 2022-04-28T21:49:19.394237+0000 mon.a (mon.0) 4084 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:49:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: audit 2022-04-28T21:49:19.431289+0000 mon.a (mon.0) 4085 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:49:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: audit 2022-04-28T21:49:19.432087+0000 mon.a (mon.0) 4086 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:49:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: audit 2022-04-28T21:49:19.432575+0000 mon.a (mon.0) 4087 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:49:19.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:19 smithi059 bash[59745]: audit 2022-04-28T21:49:19.433170+0000 mon.a (mon.0) 4088 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:49:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: cluster 2022-04-28T21:49:18.483713+0000 mgr.y (mgr.24961) 3900 : cluster [DBG] pgmap v4686: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: audit 2022-04-28T21:49:19.384679+0000 mon.a (mon.0) 4083 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:49:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: audit 2022-04-28T21:49:19.394237+0000 mon.a (mon.0) 4084 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:49:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: audit 2022-04-28T21:49:19.431289+0000 mon.a (mon.0) 4085 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:49:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: audit 2022-04-28T21:49:19.432087+0000 mon.a (mon.0) 4086 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:49:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: audit 2022-04-28T21:49:19.432575+0000 mon.a (mon.0) 4087 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T21:49:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:19 smithi139 bash[43301]: audit 2022-04-28T21:49:19.433170+0000 mon.a (mon.0) 4088 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T21:49:21.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:21 smithi059 bash[61152]: cluster 2022-04-28T21:49:20.485516+0000 mgr.y (mgr.24961) 3901 : cluster [DBG] pgmap v4687: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:21.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:21 smithi059 bash[59745]: cluster 2022-04-28T21:49:20.485516+0000 mgr.y (mgr.24961) 3901 : cluster [DBG] pgmap v4687: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:21 smithi139 bash[43301]: cluster 2022-04-28T21:49:20.485516+0000 mgr.y (mgr.24961) 3901 : cluster [DBG] pgmap v4687: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:22.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:22.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:23.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:23 smithi059 bash[61152]: cluster 2022-04-28T21:49:22.486753+0000 mgr.y (mgr.24961) 3902 : cluster [DBG] pgmap v4688: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:23.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:23 smithi059 bash[59745]: cluster 2022-04-28T21:49:22.486753+0000 mgr.y (mgr.24961) 3902 : cluster [DBG] pgmap v4688: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:23 smithi139 bash[43301]: cluster 2022-04-28T21:49:22.486753+0000 mgr.y (mgr.24961) 3902 : cluster [DBG] pgmap v4688: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:23.979Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:24 smithi059 bash[61152]: cluster 2022-04-28T21:49:24.488071+0000 mgr.y (mgr.24961) 3903 : cluster [DBG] pgmap v4689: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:24 smithi059 bash[59745]: cluster 2022-04-28T21:49:24.488071+0000 mgr.y (mgr.24961) 3903 : cluster [DBG] pgmap v4689: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:24 smithi139 bash[43301]: cluster 2022-04-28T21:49:24.488071+0000 mgr.y (mgr.24961) 3903 : cluster [DBG] pgmap v4689: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:27 smithi059 bash[61152]: cluster 2022-04-28T21:49:26.489622+0000 mgr.y (mgr.24961) 3904 : cluster [DBG] pgmap v4690: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:27 smithi059 bash[59745]: cluster 2022-04-28T21:49:26.489622+0000 mgr.y (mgr.24961) 3904 : cluster [DBG] pgmap v4690: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:27.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:27 smithi139 bash[43301]: cluster 2022-04-28T21:49:26.489622+0000 mgr.y (mgr.24961) 3904 : cluster [DBG] pgmap v4690: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:29 smithi059 bash[61152]: cluster 2022-04-28T21:49:28.490397+0000 mgr.y (mgr.24961) 3905 : cluster [DBG] pgmap v4691: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:29 smithi059 bash[59745]: cluster 2022-04-28T21:49:28.490397+0000 mgr.y (mgr.24961) 3905 : cluster [DBG] pgmap v4691: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:29 smithi139 bash[43301]: cluster 2022-04-28T21:49:28.490397+0000 mgr.y (mgr.24961) 3905 : cluster [DBG] pgmap v4691: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:31.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:31 smithi059 bash[61152]: cluster 2022-04-28T21:49:30.492167+0000 mgr.y (mgr.24961) 3906 : cluster [DBG] pgmap v4692: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:31.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:31 smithi059 bash[59745]: cluster 2022-04-28T21:49:30.492167+0000 mgr.y (mgr.24961) 3906 : cluster [DBG] pgmap v4692: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:31 smithi139 bash[43301]: cluster 2022-04-28T21:49:30.492167+0000 mgr.y (mgr.24961) 3906 : cluster [DBG] pgmap v4692: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:32.092Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:33.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:33 smithi059 bash[59745]: cluster 2022-04-28T21:49:32.493301+0000 mgr.y (mgr.24961) 3907 : cluster [DBG] pgmap v4693: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:33 smithi059 bash[61152]: cluster 2022-04-28T21:49:32.493301+0000 mgr.y (mgr.24961) 3907 : cluster [DBG] pgmap v4693: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:33 smithi139 bash[43301]: cluster 2022-04-28T21:49:32.493301+0000 mgr.y (mgr.24961) 3907 : cluster [DBG] pgmap v4693: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:33.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:34 smithi059 bash[61152]: cluster 2022-04-28T21:49:34.494597+0000 mgr.y (mgr.24961) 3908 : cluster [DBG] pgmap v4694: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:34 smithi059 bash[59745]: cluster 2022-04-28T21:49:34.494597+0000 mgr.y (mgr.24961) 3908 : cluster [DBG] pgmap v4694: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:34 smithi139 bash[43301]: cluster 2022-04-28T21:49:34.494597+0000 mgr.y (mgr.24961) 3908 : cluster [DBG] pgmap v4694: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:37.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:37 smithi059 bash[61152]: cluster 2022-04-28T21:49:36.496254+0000 mgr.y (mgr.24961) 3909 : cluster [DBG] pgmap v4695: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:37 smithi059 bash[59745]: cluster 2022-04-28T21:49:36.496254+0000 mgr.y (mgr.24961) 3909 : cluster [DBG] pgmap v4695: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:37 smithi139 bash[43301]: cluster 2022-04-28T21:49:36.496254+0000 mgr.y (mgr.24961) 3909 : cluster [DBG] pgmap v4695: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:39.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:39 smithi059 bash[61152]: cluster 2022-04-28T21:49:38.497032+0000 mgr.y (mgr.24961) 3910 : cluster [DBG] pgmap v4696: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:39.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:39 smithi059 bash[59745]: cluster 2022-04-28T21:49:38.497032+0000 mgr.y (mgr.24961) 3910 : cluster [DBG] pgmap v4696: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:39 smithi139 bash[43301]: cluster 2022-04-28T21:49:38.497032+0000 mgr.y (mgr.24961) 3910 : cluster [DBG] pgmap v4696: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:41.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:41 smithi059 bash[61152]: cluster 2022-04-28T21:49:40.498792+0000 mgr.y (mgr.24961) 3911 : cluster [DBG] pgmap v4697: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:41.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:41 smithi059 bash[59745]: cluster 2022-04-28T21:49:40.498792+0000 mgr.y (mgr.24961) 3911 : cluster [DBG] pgmap v4697: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:41 smithi139 bash[43301]: cluster 2022-04-28T21:49:40.498792+0000 mgr.y (mgr.24961) 3911 : cluster [DBG] pgmap v4697: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:42.094Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:43.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:43 smithi059 bash[61152]: cluster 2022-04-28T21:49:42.499984+0000 mgr.y (mgr.24961) 3912 : cluster [DBG] pgmap v4698: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:43.879 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:43 smithi059 bash[59745]: cluster 2022-04-28T21:49:42.499984+0000 mgr.y (mgr.24961) 3912 : cluster [DBG] pgmap v4698: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:43.879 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:43 smithi139 bash[43301]: cluster 2022-04-28T21:49:42.499984+0000 mgr.y (mgr.24961) 3912 : cluster [DBG] pgmap v4698: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:43.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:44 smithi059 bash[61152]: cluster 2022-04-28T21:49:44.501242+0000 mgr.y (mgr.24961) 3913 : cluster [DBG] pgmap v4699: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:44 smithi059 bash[59745]: cluster 2022-04-28T21:49:44.501242+0000 mgr.y (mgr.24961) 3913 : cluster [DBG] pgmap v4699: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:44 smithi139 bash[43301]: cluster 2022-04-28T21:49:44.501242+0000 mgr.y (mgr.24961) 3913 : cluster [DBG] pgmap v4699: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:47.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:47 smithi059 bash[61152]: cluster 2022-04-28T21:49:46.502881+0000 mgr.y (mgr.24961) 3914 : cluster [DBG] pgmap v4700: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:47.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:47 smithi059 bash[59745]: cluster 2022-04-28T21:49:46.502881+0000 mgr.y (mgr.24961) 3914 : cluster [DBG] pgmap v4700: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:47 smithi139 bash[43301]: cluster 2022-04-28T21:49:46.502881+0000 mgr.y (mgr.24961) 3914 : cluster [DBG] pgmap v4700: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:49.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:49 smithi059 bash[61152]: cluster 2022-04-28T21:49:48.503657+0000 mgr.y (mgr.24961) 3915 : cluster [DBG] pgmap v4701: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:49.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:49 smithi059 bash[59745]: cluster 2022-04-28T21:49:48.503657+0000 mgr.y (mgr.24961) 3915 : cluster [DBG] pgmap v4701: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:49 smithi139 bash[43301]: cluster 2022-04-28T21:49:48.503657+0000 mgr.y (mgr.24961) 3915 : cluster [DBG] pgmap v4701: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:51.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:51 smithi059 bash[61152]: cluster 2022-04-28T21:49:50.505328+0000 mgr.y (mgr.24961) 3916 : cluster [DBG] pgmap v4702: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:51.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:51 smithi059 bash[59745]: cluster 2022-04-28T21:49:50.505328+0000 mgr.y (mgr.24961) 3916 : cluster [DBG] pgmap v4702: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:51 smithi139 bash[43301]: cluster 2022-04-28T21:49:50.505328+0000 mgr.y (mgr.24961) 3916 : cluster [DBG] pgmap v4702: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:52.095Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:53.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:53 smithi059 bash[61152]: cluster 2022-04-28T21:49:52.506535+0000 mgr.y (mgr.24961) 3917 : cluster [DBG] pgmap v4703: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:53.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:53 smithi059 bash[59745]: cluster 2022-04-28T21:49:52.506535+0000 mgr.y (mgr.24961) 3917 : cluster [DBG] pgmap v4703: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:53 smithi139 bash[43301]: cluster 2022-04-28T21:49:52.506535+0000 mgr.y (mgr.24961) 3917 : cluster [DBG] pgmap v4703: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:49:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:49:53.981Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:49:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:54 smithi059 bash[61152]: cluster 2022-04-28T21:49:54.507786+0000 mgr.y (mgr.24961) 3918 : cluster [DBG] pgmap v4704: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:54 smithi059 bash[59745]: cluster 2022-04-28T21:49:54.507786+0000 mgr.y (mgr.24961) 3918 : cluster [DBG] pgmap v4704: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:54 smithi139 bash[43301]: cluster 2022-04-28T21:49:54.507786+0000 mgr.y (mgr.24961) 3918 : cluster [DBG] pgmap v4704: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:57.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:57 smithi059 bash[61152]: cluster 2022-04-28T21:49:56.509406+0000 mgr.y (mgr.24961) 3919 : cluster [DBG] pgmap v4705: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:57.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:57 smithi059 bash[59745]: cluster 2022-04-28T21:49:56.509406+0000 mgr.y (mgr.24961) 3919 : cluster [DBG] pgmap v4705: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:57 smithi139 bash[43301]: cluster 2022-04-28T21:49:56.509406+0000 mgr.y (mgr.24961) 3919 : cluster [DBG] pgmap v4705: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:59.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:49:59 smithi059 bash[61152]: cluster 2022-04-28T21:49:58.510213+0000 mgr.y (mgr.24961) 3920 : cluster [DBG] pgmap v4706: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:59.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:49:59 smithi059 bash[59745]: cluster 2022-04-28T21:49:58.510213+0000 mgr.y (mgr.24961) 3920 : cluster [DBG] pgmap v4706: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:49:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:49:59 smithi139 bash[43301]: cluster 2022-04-28T21:49:58.510213+0000 mgr.y (mgr.24961) 3920 : cluster [DBG] pgmap v4706: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:00 smithi059 bash[61152]: cluster 2022-04-28T21:50:00.000158+0000 mon.a (mon.0) 4089 : cluster [INF] overall HEALTH_OK 2022-04-28T21:50:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:00 smithi059 bash[59745]: cluster 2022-04-28T21:50:00.000158+0000 mon.a (mon.0) 4089 : cluster [INF] overall HEALTH_OK 2022-04-28T21:50:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:00 smithi139 bash[43301]: cluster 2022-04-28T21:50:00.000158+0000 mon.a (mon.0) 4089 : cluster [INF] overall HEALTH_OK 2022-04-28T21:50:01.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:01 smithi059 bash[61152]: cluster 2022-04-28T21:50:00.511930+0000 mgr.y (mgr.24961) 3921 : cluster [DBG] pgmap v4707: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:01.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:01 smithi059 bash[59745]: cluster 2022-04-28T21:50:00.511930+0000 mgr.y (mgr.24961) 3921 : cluster [DBG] pgmap v4707: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:01 smithi139 bash[43301]: cluster 2022-04-28T21:50:00.511930+0000 mgr.y (mgr.24961) 3921 : cluster [DBG] pgmap v4707: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:02.096Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:03.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:03 smithi059 bash[61152]: cluster 2022-04-28T21:50:02.513131+0000 mgr.y (mgr.24961) 3922 : cluster [DBG] pgmap v4708: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:03.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:03 smithi059 bash[59745]: cluster 2022-04-28T21:50:02.513131+0000 mgr.y (mgr.24961) 3922 : cluster [DBG] pgmap v4708: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:03.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:03 smithi139 bash[43301]: cluster 2022-04-28T21:50:02.513131+0000 mgr.y (mgr.24961) 3922 : cluster [DBG] pgmap v4708: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:03.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:04 smithi059 bash[61152]: cluster 2022-04-28T21:50:04.514435+0000 mgr.y (mgr.24961) 3923 : cluster [DBG] pgmap v4709: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:04 smithi059 bash[59745]: cluster 2022-04-28T21:50:04.514435+0000 mgr.y (mgr.24961) 3923 : cluster [DBG] pgmap v4709: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:04 smithi139 bash[43301]: cluster 2022-04-28T21:50:04.514435+0000 mgr.y (mgr.24961) 3923 : cluster [DBG] pgmap v4709: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:07.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:07 smithi059 bash[61152]: cluster 2022-04-28T21:50:06.516108+0000 mgr.y (mgr.24961) 3924 : cluster [DBG] pgmap v4710: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:07.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:07 smithi059 bash[59745]: cluster 2022-04-28T21:50:06.516108+0000 mgr.y (mgr.24961) 3924 : cluster [DBG] pgmap v4710: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:07 smithi139 bash[43301]: cluster 2022-04-28T21:50:06.516108+0000 mgr.y (mgr.24961) 3924 : cluster [DBG] pgmap v4710: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:09.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:09 smithi059 bash[61152]: cluster 2022-04-28T21:50:08.516894+0000 mgr.y (mgr.24961) 3925 : cluster [DBG] pgmap v4711: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:09.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:09 smithi059 bash[59745]: cluster 2022-04-28T21:50:08.516894+0000 mgr.y (mgr.24961) 3925 : cluster [DBG] pgmap v4711: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:09.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:09 smithi139 bash[43301]: cluster 2022-04-28T21:50:08.516894+0000 mgr.y (mgr.24961) 3925 : cluster [DBG] pgmap v4711: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:11.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:11 smithi059 bash[61152]: cluster 2022-04-28T21:50:10.518565+0000 mgr.y (mgr.24961) 3926 : cluster [DBG] pgmap v4712: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:11.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:11 smithi059 bash[59745]: cluster 2022-04-28T21:50:10.518565+0000 mgr.y (mgr.24961) 3926 : cluster [DBG] pgmap v4712: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:11.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:11 smithi139 bash[43301]: cluster 2022-04-28T21:50:10.518565+0000 mgr.y (mgr.24961) 3926 : cluster [DBG] pgmap v4712: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:12.097Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:13.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:13 smithi139 bash[43301]: cluster 2022-04-28T21:50:12.519766+0000 mgr.y (mgr.24961) 3927 : cluster [DBG] pgmap v4713: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:13.984 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:13 smithi059 bash[61152]: cluster 2022-04-28T21:50:12.519766+0000 mgr.y (mgr.24961) 3927 : cluster [DBG] pgmap v4713: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:13.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:13 smithi059 bash[59745]: cluster 2022-04-28T21:50:12.519766+0000 mgr.y (mgr.24961) 3927 : cluster [DBG] pgmap v4713: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:13.983Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:14 smithi059 bash[61152]: cluster 2022-04-28T21:50:14.521030+0000 mgr.y (mgr.24961) 3928 : cluster [DBG] pgmap v4714: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:14 smithi059 bash[59745]: cluster 2022-04-28T21:50:14.521030+0000 mgr.y (mgr.24961) 3928 : cluster [DBG] pgmap v4714: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:14 smithi139 bash[43301]: cluster 2022-04-28T21:50:14.521030+0000 mgr.y (mgr.24961) 3928 : cluster [DBG] pgmap v4714: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:17.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:17 smithi059 bash[61152]: cluster 2022-04-28T21:50:16.522621+0000 mgr.y (mgr.24961) 3929 : cluster [DBG] pgmap v4715: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:17.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:17 smithi059 bash[59745]: cluster 2022-04-28T21:50:16.522621+0000 mgr.y (mgr.24961) 3929 : cluster [DBG] pgmap v4715: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:17.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:17 smithi139 bash[43301]: cluster 2022-04-28T21:50:16.522621+0000 mgr.y (mgr.24961) 3929 : cluster [DBG] pgmap v4715: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[61152]: audit 2022-04-28T21:50:17.981404+0000 mon.a (mon.0) 4090 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:50:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[61152]: audit 2022-04-28T21:50:17.984570+0000 mon.a (mon.0) 4091 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:50:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[61152]: audit 2022-04-28T21:50:17.986621+0000 mon.a (mon.0) 4092 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:50:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[61152]: audit 2022-04-28T21:50:18.300024+0000 mon.a (mon.0) 4093 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[61152]: audit 2022-04-28T21:50:18.307197+0000 mon.a (mon.0) 4094 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[59745]: audit 2022-04-28T21:50:17.981404+0000 mon.a (mon.0) 4090 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:50:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[59745]: audit 2022-04-28T21:50:17.984570+0000 mon.a (mon.0) 4091 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:50:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[59745]: audit 2022-04-28T21:50:17.986621+0000 mon.a (mon.0) 4092 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:50:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[59745]: audit 2022-04-28T21:50:18.300024+0000 mon.a (mon.0) 4093 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:18 smithi059 bash[59745]: audit 2022-04-28T21:50:18.307197+0000 mon.a (mon.0) 4094 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:18 smithi139 bash[43301]: audit 2022-04-28T21:50:17.981404+0000 mon.a (mon.0) 4090 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:50:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:18 smithi139 bash[43301]: audit 2022-04-28T21:50:17.984570+0000 mon.a (mon.0) 4091 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:50:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:18 smithi139 bash[43301]: audit 2022-04-28T21:50:17.986621+0000 mon.a (mon.0) 4092 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:50:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:18 smithi139 bash[43301]: audit 2022-04-28T21:50:18.300024+0000 mon.a (mon.0) 4093 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:18 smithi139 bash[43301]: audit 2022-04-28T21:50:18.307197+0000 mon.a (mon.0) 4094 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: cephadm 2022-04-28T21:50:17.988502+0000 mgr.y (mgr.24961) 3930 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:50:19.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: cluster 2022-04-28T21:50:18.523509+0000 mgr.y (mgr.24961) 3931 : cluster [DBG] pgmap v4716: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:18.593400+0000 mon.a (mon.0) 4095 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:19.389794+0000 mon.a (mon.0) 4096 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:50:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:19.400738+0000 mon.a (mon.0) 4097 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:50:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:19.443050+0000 mon.a (mon.0) 4098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:50:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:19.444040+0000 mon.a (mon.0) 4099 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:50:19.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:19.444705+0000 mon.a (mon.0) 4100 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:50:19.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:19 smithi139 bash[43301]: audit 2022-04-28T21:50:19.445348+0000 mon.a (mon.0) 4101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:50:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: cephadm 2022-04-28T21:50:17.988502+0000 mgr.y (mgr.24961) 3930 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:50:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: cluster 2022-04-28T21:50:18.523509+0000 mgr.y (mgr.24961) 3931 : cluster [DBG] pgmap v4716: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:18.593400+0000 mon.a (mon.0) 4095 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:19.389794+0000 mon.a (mon.0) 4096 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:50:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:19.400738+0000 mon.a (mon.0) 4097 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:50:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:19.443050+0000 mon.a (mon.0) 4098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:50:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:19.444040+0000 mon.a (mon.0) 4099 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:50:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:19.444705+0000 mon.a (mon.0) 4100 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:50:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[61152]: audit 2022-04-28T21:50:19.445348+0000 mon.a (mon.0) 4101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:50:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: cephadm 2022-04-28T21:50:17.988502+0000 mgr.y (mgr.24961) 3930 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:50:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: cluster 2022-04-28T21:50:18.523509+0000 mgr.y (mgr.24961) 3931 : cluster [DBG] pgmap v4716: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:18.593400+0000 mon.a (mon.0) 4095 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:50:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:19.389794+0000 mon.a (mon.0) 4096 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:50:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:19.400738+0000 mon.a (mon.0) 4097 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:50:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:19.443050+0000 mon.a (mon.0) 4098 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:50:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:19.444040+0000 mon.a (mon.0) 4099 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:50:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:19.444705+0000 mon.a (mon.0) 4100 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:50:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:19 smithi059 bash[59745]: audit 2022-04-28T21:50:19.445348+0000 mon.a (mon.0) 4101 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.9", "id": [4, 2]}]: dispatch 2022-04-28T21:50:21.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:21 smithi139 bash[43301]: cluster 2022-04-28T21:50:20.525200+0000 mgr.y (mgr.24961) 3932 : cluster [DBG] pgmap v4717: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:21 smithi059 bash[61152]: cluster 2022-04-28T21:50:20.525200+0000 mgr.y (mgr.24961) 3932 : cluster [DBG] pgmap v4717: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:21 smithi059 bash[59745]: cluster 2022-04-28T21:50:20.525200+0000 mgr.y (mgr.24961) 3932 : cluster [DBG] pgmap v4717: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:22.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:22.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:23.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:23 smithi139 bash[43301]: cluster 2022-04-28T21:50:22.526375+0000 mgr.y (mgr.24961) 3933 : cluster [DBG] pgmap v4718: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:23.983 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:23 smithi059 bash[61152]: cluster 2022-04-28T21:50:22.526375+0000 mgr.y (mgr.24961) 3933 : cluster [DBG] pgmap v4718: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:23.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:23 smithi059 bash[59745]: cluster 2022-04-28T21:50:22.526375+0000 mgr.y (mgr.24961) 3933 : cluster [DBG] pgmap v4718: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:23.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:24 smithi059 bash[61152]: cluster 2022-04-28T21:50:24.527749+0000 mgr.y (mgr.24961) 3934 : cluster [DBG] pgmap v4719: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:24 smithi059 bash[59745]: cluster 2022-04-28T21:50:24.527749+0000 mgr.y (mgr.24961) 3934 : cluster [DBG] pgmap v4719: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:24 smithi139 bash[43301]: cluster 2022-04-28T21:50:24.527749+0000 mgr.y (mgr.24961) 3934 : cluster [DBG] pgmap v4719: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:27 smithi059 bash[61152]: cluster 2022-04-28T21:50:26.529403+0000 mgr.y (mgr.24961) 3935 : cluster [DBG] pgmap v4720: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:27 smithi059 bash[59745]: cluster 2022-04-28T21:50:26.529403+0000 mgr.y (mgr.24961) 3935 : cluster [DBG] pgmap v4720: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:27 smithi139 bash[43301]: cluster 2022-04-28T21:50:26.529403+0000 mgr.y (mgr.24961) 3935 : cluster [DBG] pgmap v4720: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:29 smithi139 bash[43301]: cluster 2022-04-28T21:50:28.530151+0000 mgr.y (mgr.24961) 3936 : cluster [DBG] pgmap v4721: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:29 smithi059 bash[61152]: cluster 2022-04-28T21:50:28.530151+0000 mgr.y (mgr.24961) 3936 : cluster [DBG] pgmap v4721: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:29 smithi059 bash[59745]: cluster 2022-04-28T21:50:28.530151+0000 mgr.y (mgr.24961) 3936 : cluster [DBG] pgmap v4721: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:31.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:31 smithi139 bash[43301]: cluster 2022-04-28T21:50:30.531881+0000 mgr.y (mgr.24961) 3937 : cluster [DBG] pgmap v4722: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:31 smithi059 bash[61152]: cluster 2022-04-28T21:50:30.531881+0000 mgr.y (mgr.24961) 3937 : cluster [DBG] pgmap v4722: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:31 smithi059 bash[59745]: cluster 2022-04-28T21:50:30.531881+0000 mgr.y (mgr.24961) 3937 : cluster [DBG] pgmap v4722: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:32.098Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:33.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:33 smithi139 bash[43301]: cluster 2022-04-28T21:50:32.533102+0000 mgr.y (mgr.24961) 3938 : cluster [DBG] pgmap v4723: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:33.985 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:33 smithi059 bash[61152]: cluster 2022-04-28T21:50:32.533102+0000 mgr.y (mgr.24961) 3938 : cluster [DBG] pgmap v4723: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:33.985 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:33 smithi059 bash[59745]: cluster 2022-04-28T21:50:32.533102+0000 mgr.y (mgr.24961) 3938 : cluster [DBG] pgmap v4723: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:33.984Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:34 smithi059 bash[61152]: cluster 2022-04-28T21:50:34.534383+0000 mgr.y (mgr.24961) 3939 : cluster [DBG] pgmap v4724: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:34 smithi059 bash[59745]: cluster 2022-04-28T21:50:34.534383+0000 mgr.y (mgr.24961) 3939 : cluster [DBG] pgmap v4724: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:34 smithi139 bash[43301]: cluster 2022-04-28T21:50:34.534383+0000 mgr.y (mgr.24961) 3939 : cluster [DBG] pgmap v4724: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:37 smithi139 bash[43301]: cluster 2022-04-28T21:50:36.536008+0000 mgr.y (mgr.24961) 3940 : cluster [DBG] pgmap v4725: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:37 smithi059 bash[61152]: cluster 2022-04-28T21:50:36.536008+0000 mgr.y (mgr.24961) 3940 : cluster [DBG] pgmap v4725: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:37 smithi059 bash[59745]: cluster 2022-04-28T21:50:36.536008+0000 mgr.y (mgr.24961) 3940 : cluster [DBG] pgmap v4725: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:39 smithi139 bash[43301]: cluster 2022-04-28T21:50:38.536766+0000 mgr.y (mgr.24961) 3941 : cluster [DBG] pgmap v4726: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:39 smithi059 bash[61152]: cluster 2022-04-28T21:50:38.536766+0000 mgr.y (mgr.24961) 3941 : cluster [DBG] pgmap v4726: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:40.290 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:39 smithi059 bash[59745]: cluster 2022-04-28T21:50:38.536766+0000 mgr.y (mgr.24961) 3941 : cluster [DBG] pgmap v4726: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:41.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:41 smithi139 bash[43301]: cluster 2022-04-28T21:50:40.538488+0000 mgr.y (mgr.24961) 3942 : cluster [DBG] pgmap v4727: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:41 smithi059 bash[61152]: cluster 2022-04-28T21:50:40.538488+0000 mgr.y (mgr.24961) 3942 : cluster [DBG] pgmap v4727: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:41 smithi059 bash[59745]: cluster 2022-04-28T21:50:40.538488+0000 mgr.y (mgr.24961) 3942 : cluster [DBG] pgmap v4727: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:42.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:43.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:43 smithi139 bash[43301]: cluster 2022-04-28T21:50:42.539739+0000 mgr.y (mgr.24961) 3943 : cluster [DBG] pgmap v4728: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:43.984 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:43 smithi059 bash[61152]: cluster 2022-04-28T21:50:42.539739+0000 mgr.y (mgr.24961) 3943 : cluster [DBG] pgmap v4728: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:43.984 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:43 smithi059 bash[59745]: cluster 2022-04-28T21:50:42.539739+0000 mgr.y (mgr.24961) 3943 : cluster [DBG] pgmap v4728: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:43.982Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:44 smithi059 bash[61152]: cluster 2022-04-28T21:50:44.541012+0000 mgr.y (mgr.24961) 3944 : cluster [DBG] pgmap v4729: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:44 smithi059 bash[59745]: cluster 2022-04-28T21:50:44.541012+0000 mgr.y (mgr.24961) 3944 : cluster [DBG] pgmap v4729: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:44 smithi139 bash[43301]: cluster 2022-04-28T21:50:44.541012+0000 mgr.y (mgr.24961) 3944 : cluster [DBG] pgmap v4729: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:47.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:47 smithi139 bash[43301]: cluster 2022-04-28T21:50:46.542737+0000 mgr.y (mgr.24961) 3945 : cluster [DBG] pgmap v4730: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:47 smithi059 bash[61152]: cluster 2022-04-28T21:50:46.542737+0000 mgr.y (mgr.24961) 3945 : cluster [DBG] pgmap v4730: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:47 smithi059 bash[59745]: cluster 2022-04-28T21:50:46.542737+0000 mgr.y (mgr.24961) 3945 : cluster [DBG] pgmap v4730: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:49.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:49 smithi139 bash[43301]: cluster 2022-04-28T21:50:48.543555+0000 mgr.y (mgr.24961) 3946 : cluster [DBG] pgmap v4731: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:49 smithi059 bash[61152]: cluster 2022-04-28T21:50:48.543555+0000 mgr.y (mgr.24961) 3946 : cluster [DBG] pgmap v4731: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:49 smithi059 bash[59745]: cluster 2022-04-28T21:50:48.543555+0000 mgr.y (mgr.24961) 3946 : cluster [DBG] pgmap v4731: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:51.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:51 smithi139 bash[43301]: cluster 2022-04-28T21:50:50.545413+0000 mgr.y (mgr.24961) 3947 : cluster [DBG] pgmap v4732: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:51 smithi059 bash[61152]: cluster 2022-04-28T21:50:50.545413+0000 mgr.y (mgr.24961) 3947 : cluster [DBG] pgmap v4732: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:51 smithi059 bash[59745]: cluster 2022-04-28T21:50:50.545413+0000 mgr.y (mgr.24961) 3947 : cluster [DBG] pgmap v4732: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:52.099Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:53.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:53 smithi139 bash[43301]: cluster 2022-04-28T21:50:52.546659+0000 mgr.y (mgr.24961) 3948 : cluster [DBG] pgmap v4733: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:53.988 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:53 smithi059 bash[61152]: cluster 2022-04-28T21:50:52.546659+0000 mgr.y (mgr.24961) 3948 : cluster [DBG] pgmap v4733: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:53.988 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:53 smithi059 bash[59745]: cluster 2022-04-28T21:50:52.546659+0000 mgr.y (mgr.24961) 3948 : cluster [DBG] pgmap v4733: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:50:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:50:53.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:50:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:54 smithi059 bash[61152]: cluster 2022-04-28T21:50:54.547989+0000 mgr.y (mgr.24961) 3949 : cluster [DBG] pgmap v4734: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:54 smithi059 bash[59745]: cluster 2022-04-28T21:50:54.547989+0000 mgr.y (mgr.24961) 3949 : cluster [DBG] pgmap v4734: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:54 smithi139 bash[43301]: cluster 2022-04-28T21:50:54.547989+0000 mgr.y (mgr.24961) 3949 : cluster [DBG] pgmap v4734: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:57.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:57 smithi139 bash[43301]: cluster 2022-04-28T21:50:56.549616+0000 mgr.y (mgr.24961) 3950 : cluster [DBG] pgmap v4735: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:57 smithi059 bash[61152]: cluster 2022-04-28T21:50:56.549616+0000 mgr.y (mgr.24961) 3950 : cluster [DBG] pgmap v4735: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:57 smithi059 bash[59745]: cluster 2022-04-28T21:50:56.549616+0000 mgr.y (mgr.24961) 3950 : cluster [DBG] pgmap v4735: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:50:59.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:50:59 smithi139 bash[43301]: cluster 2022-04-28T21:50:58.550473+0000 mgr.y (mgr.24961) 3951 : cluster [DBG] pgmap v4736: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:50:59 smithi059 bash[59745]: cluster 2022-04-28T21:50:58.550473+0000 mgr.y (mgr.24961) 3951 : cluster [DBG] pgmap v4736: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:00.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:50:59 smithi059 bash[61152]: cluster 2022-04-28T21:50:58.550473+0000 mgr.y (mgr.24961) 3951 : cluster [DBG] pgmap v4736: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:01.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:01 smithi139 bash[43301]: cluster 2022-04-28T21:51:00.552242+0000 mgr.y (mgr.24961) 3952 : cluster [DBG] pgmap v4737: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:01 smithi059 bash[61152]: cluster 2022-04-28T21:51:00.552242+0000 mgr.y (mgr.24961) 3952 : cluster [DBG] pgmap v4737: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:01 smithi059 bash[59745]: cluster 2022-04-28T21:51:00.552242+0000 mgr.y (mgr.24961) 3952 : cluster [DBG] pgmap v4737: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:02.100Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:03.986 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:03 smithi059 bash[61152]: cluster 2022-04-28T21:51:02.553467+0000 mgr.y (mgr.24961) 3953 : cluster [DBG] pgmap v4738: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:03.987 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:03 smithi059 bash[59745]: cluster 2022-04-28T21:51:02.553467+0000 mgr.y (mgr.24961) 3953 : cluster [DBG] pgmap v4738: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:03.987 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:03.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:03 smithi139 bash[43301]: cluster 2022-04-28T21:51:02.553467+0000 mgr.y (mgr.24961) 3953 : cluster [DBG] pgmap v4738: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:04 smithi059 bash[61152]: cluster 2022-04-28T21:51:04.554788+0000 mgr.y (mgr.24961) 3954 : cluster [DBG] pgmap v4739: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:04 smithi059 bash[59745]: cluster 2022-04-28T21:51:04.554788+0000 mgr.y (mgr.24961) 3954 : cluster [DBG] pgmap v4739: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:04 smithi139 bash[43301]: cluster 2022-04-28T21:51:04.554788+0000 mgr.y (mgr.24961) 3954 : cluster [DBG] pgmap v4739: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:07.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:07 smithi139 bash[43301]: cluster 2022-04-28T21:51:06.556427+0000 mgr.y (mgr.24961) 3955 : cluster [DBG] pgmap v4740: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:08.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:07 smithi059 bash[59745]: cluster 2022-04-28T21:51:06.556427+0000 mgr.y (mgr.24961) 3955 : cluster [DBG] pgmap v4740: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:07 smithi059 bash[61152]: cluster 2022-04-28T21:51:06.556427+0000 mgr.y (mgr.24961) 3955 : cluster [DBG] pgmap v4740: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:09 smithi059 bash[61152]: cluster 2022-04-28T21:51:08.557341+0000 mgr.y (mgr.24961) 3956 : cluster [DBG] pgmap v4741: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:09 smithi059 bash[59745]: cluster 2022-04-28T21:51:08.557341+0000 mgr.y (mgr.24961) 3956 : cluster [DBG] pgmap v4741: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:09 smithi139 bash[43301]: cluster 2022-04-28T21:51:08.557341+0000 mgr.y (mgr.24961) 3956 : cluster [DBG] pgmap v4741: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:11 smithi059 bash[61152]: cluster 2022-04-28T21:51:10.559093+0000 mgr.y (mgr.24961) 3957 : cluster [DBG] pgmap v4742: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:11 smithi059 bash[59745]: cluster 2022-04-28T21:51:10.559093+0000 mgr.y (mgr.24961) 3957 : cluster [DBG] pgmap v4742: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:11 smithi139 bash[43301]: cluster 2022-04-28T21:51:10.559093+0000 mgr.y (mgr.24961) 3957 : cluster [DBG] pgmap v4742: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:12.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:13.987 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:13 smithi059 bash[61152]: cluster 2022-04-28T21:51:12.560315+0000 mgr.y (mgr.24961) 3958 : cluster [DBG] pgmap v4743: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:13.988 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:13.985Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:13.989 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:13 smithi059 bash[59745]: cluster 2022-04-28T21:51:12.560315+0000 mgr.y (mgr.24961) 3958 : cluster [DBG] pgmap v4743: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:13 smithi139 bash[43301]: cluster 2022-04-28T21:51:12.560315+0000 mgr.y (mgr.24961) 3958 : cluster [DBG] pgmap v4743: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:14 smithi059 bash[61152]: cluster 2022-04-28T21:51:14.561592+0000 mgr.y (mgr.24961) 3959 : cluster [DBG] pgmap v4744: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:14 smithi059 bash[59745]: cluster 2022-04-28T21:51:14.561592+0000 mgr.y (mgr.24961) 3959 : cluster [DBG] pgmap v4744: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:14 smithi139 bash[43301]: cluster 2022-04-28T21:51:14.561592+0000 mgr.y (mgr.24961) 3959 : cluster [DBG] pgmap v4744: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:17 smithi059 bash[61152]: cluster 2022-04-28T21:51:16.563193+0000 mgr.y (mgr.24961) 3960 : cluster [DBG] pgmap v4745: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:17 smithi059 bash[59745]: cluster 2022-04-28T21:51:16.563193+0000 mgr.y (mgr.24961) 3960 : cluster [DBG] pgmap v4745: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:17 smithi139 bash[43301]: cluster 2022-04-28T21:51:16.563193+0000 mgr.y (mgr.24961) 3960 : cluster [DBG] pgmap v4745: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:18 smithi059 bash[61152]: audit 2022-04-28T21:51:18.598796+0000 mon.a (mon.0) 4102 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:51:19.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:18 smithi059 bash[61152]: audit 2022-04-28T21:51:18.601720+0000 mon.a (mon.0) 4103 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:51:19.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:18 smithi059 bash[61152]: audit 2022-04-28T21:51:18.603719+0000 mon.a (mon.0) 4104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:51:19.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:18 smithi059 bash[59745]: audit 2022-04-28T21:51:18.598796+0000 mon.a (mon.0) 4102 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:51:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:18 smithi059 bash[59745]: audit 2022-04-28T21:51:18.601720+0000 mon.a (mon.0) 4103 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:51:19.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:18 smithi059 bash[59745]: audit 2022-04-28T21:51:18.603719+0000 mon.a (mon.0) 4104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:51:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:18 smithi139 bash[43301]: audit 2022-04-28T21:51:18.598796+0000 mon.a (mon.0) 4102 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:51:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:18 smithi139 bash[43301]: audit 2022-04-28T21:51:18.601720+0000 mon.a (mon.0) 4103 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:51:19.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:18 smithi139 bash[43301]: audit 2022-04-28T21:51:18.603719+0000 mon.a (mon.0) 4104 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:51:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: cluster 2022-04-28T21:51:18.563978+0000 mgr.y (mgr.24961) 3961 : cluster [DBG] pgmap v4746: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: cephadm 2022-04-28T21:51:18.605396+0000 mgr.y (mgr.24961) 3962 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:51:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:18.900791+0000 mon.a (mon.0) 4105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:51:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:19.394908+0000 mon.a (mon.0) 4106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:51:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:19.407028+0000 mon.a (mon.0) 4107 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:51:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:19.453203+0000 mon.a (mon.0) 4108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:51:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:19.453982+0000 mon.a (mon.0) 4109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:51:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:19.454562+0000 mon.a (mon.0) 4110 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:51:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[61152]: audit 2022-04-28T21:51:19.455354+0000 mon.a (mon.0) 4111 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:51:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: cluster 2022-04-28T21:51:18.563978+0000 mgr.y (mgr.24961) 3961 : cluster [DBG] pgmap v4746: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: cephadm 2022-04-28T21:51:18.605396+0000 mgr.y (mgr.24961) 3962 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:51:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:18.900791+0000 mon.a (mon.0) 4105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:51:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:19.394908+0000 mon.a (mon.0) 4106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:51:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:19.407028+0000 mon.a (mon.0) 4107 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:51:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:19.453203+0000 mon.a (mon.0) 4108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:51:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:19.453982+0000 mon.a (mon.0) 4109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:51:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:19.454562+0000 mon.a (mon.0) 4110 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:51:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:19 smithi059 bash[59745]: audit 2022-04-28T21:51:19.455354+0000 mon.a (mon.0) 4111 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:51:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: cluster 2022-04-28T21:51:18.563978+0000 mgr.y (mgr.24961) 3961 : cluster [DBG] pgmap v4746: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: cephadm 2022-04-28T21:51:18.605396+0000 mgr.y (mgr.24961) 3962 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:51:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:18.900791+0000 mon.a (mon.0) 4105 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:51:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:19.394908+0000 mon.a (mon.0) 4106 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:51:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:19.407028+0000 mon.a (mon.0) 4107 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:51:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:19.453203+0000 mon.a (mon.0) 4108 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T21:51:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:19.453982+0000 mon.a (mon.0) 4109 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T21:51:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:19.454562+0000 mon.a (mon.0) 4110 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:51:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:19 smithi139 bash[43301]: audit 2022-04-28T21:51:19.455354+0000 mon.a (mon.0) 4111 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.c", "id": [4, 2]}]: dispatch 2022-04-28T21:51:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:21 smithi059 bash[61152]: cluster 2022-04-28T21:51:20.565816+0000 mgr.y (mgr.24961) 3963 : cluster [DBG] pgmap v4747: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:21 smithi059 bash[59745]: cluster 2022-04-28T21:51:20.565816+0000 mgr.y (mgr.24961) 3963 : cluster [DBG] pgmap v4747: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:21 smithi139 bash[43301]: cluster 2022-04-28T21:51:20.565816+0000 mgr.y (mgr.24961) 3963 : cluster [DBG] pgmap v4747: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:22.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:22.101Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:23.987 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:23 smithi059 bash[61152]: cluster 2022-04-28T21:51:22.566967+0000 mgr.y (mgr.24961) 3964 : cluster [DBG] pgmap v4748: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:23.988 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:23 smithi059 bash[59745]: cluster 2022-04-28T21:51:22.566967+0000 mgr.y (mgr.24961) 3964 : cluster [DBG] pgmap v4748: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:23.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:23.986Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:23 smithi139 bash[43301]: cluster 2022-04-28T21:51:22.566967+0000 mgr.y (mgr.24961) 3964 : cluster [DBG] pgmap v4748: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:24 smithi059 bash[61152]: cluster 2022-04-28T21:51:24.568210+0000 mgr.y (mgr.24961) 3965 : cluster [DBG] pgmap v4749: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:24 smithi059 bash[59745]: cluster 2022-04-28T21:51:24.568210+0000 mgr.y (mgr.24961) 3965 : cluster [DBG] pgmap v4749: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:24 smithi139 bash[43301]: cluster 2022-04-28T21:51:24.568210+0000 mgr.y (mgr.24961) 3965 : cluster [DBG] pgmap v4749: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:27 smithi059 bash[61152]: cluster 2022-04-28T21:51:26.569834+0000 mgr.y (mgr.24961) 3966 : cluster [DBG] pgmap v4750: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:27 smithi059 bash[59745]: cluster 2022-04-28T21:51:26.569834+0000 mgr.y (mgr.24961) 3966 : cluster [DBG] pgmap v4750: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:27 smithi139 bash[43301]: cluster 2022-04-28T21:51:26.569834+0000 mgr.y (mgr.24961) 3966 : cluster [DBG] pgmap v4750: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:29 smithi059 bash[59745]: cluster 2022-04-28T21:51:28.570647+0000 mgr.y (mgr.24961) 3967 : cluster [DBG] pgmap v4751: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:30.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:29 smithi059 bash[61152]: cluster 2022-04-28T21:51:28.570647+0000 mgr.y (mgr.24961) 3967 : cluster [DBG] pgmap v4751: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:29 smithi139 bash[43301]: cluster 2022-04-28T21:51:28.570647+0000 mgr.y (mgr.24961) 3967 : cluster [DBG] pgmap v4751: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:31 smithi059 bash[61152]: cluster 2022-04-28T21:51:30.571664+0000 mgr.y (mgr.24961) 3968 : cluster [DBG] pgmap v4752: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:31 smithi059 bash[59745]: cluster 2022-04-28T21:51:30.571664+0000 mgr.y (mgr.24961) 3968 : cluster [DBG] pgmap v4752: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:31 smithi139 bash[43301]: cluster 2022-04-28T21:51:30.571664+0000 mgr.y (mgr.24961) 3968 : cluster [DBG] pgmap v4752: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:32.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:33.988 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:33 smithi059 bash[61152]: cluster 2022-04-28T21:51:32.572714+0000 mgr.y (mgr.24961) 3969 : cluster [DBG] pgmap v4753: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:33.988 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:33 smithi059 bash[59745]: cluster 2022-04-28T21:51:32.572714+0000 mgr.y (mgr.24961) 3969 : cluster [DBG] pgmap v4753: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:34.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:33 smithi139 bash[43301]: cluster 2022-04-28T21:51:32.572714+0000 mgr.y (mgr.24961) 3969 : cluster [DBG] pgmap v4753: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:33.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:34 smithi059 bash[61152]: cluster 2022-04-28T21:51:34.574004+0000 mgr.y (mgr.24961) 3970 : cluster [DBG] pgmap v4754: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:34 smithi059 bash[59745]: cluster 2022-04-28T21:51:34.574004+0000 mgr.y (mgr.24961) 3970 : cluster [DBG] pgmap v4754: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:34 smithi139 bash[43301]: cluster 2022-04-28T21:51:34.574004+0000 mgr.y (mgr.24961) 3970 : cluster [DBG] pgmap v4754: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:37 smithi059 bash[61152]: cluster 2022-04-28T21:51:36.575653+0000 mgr.y (mgr.24961) 3971 : cluster [DBG] pgmap v4755: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:37 smithi059 bash[59745]: cluster 2022-04-28T21:51:36.575653+0000 mgr.y (mgr.24961) 3971 : cluster [DBG] pgmap v4755: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:37 smithi139 bash[43301]: cluster 2022-04-28T21:51:36.575653+0000 mgr.y (mgr.24961) 3971 : cluster [DBG] pgmap v4755: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:39 smithi059 bash[61152]: cluster 2022-04-28T21:51:38.576454+0000 mgr.y (mgr.24961) 3972 : cluster [DBG] pgmap v4756: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:39 smithi059 bash[59745]: cluster 2022-04-28T21:51:38.576454+0000 mgr.y (mgr.24961) 3972 : cluster [DBG] pgmap v4756: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:40.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:39 smithi139 bash[43301]: cluster 2022-04-28T21:51:38.576454+0000 mgr.y (mgr.24961) 3972 : cluster [DBG] pgmap v4756: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:41 smithi059 bash[61152]: cluster 2022-04-28T21:51:40.578203+0000 mgr.y (mgr.24961) 3973 : cluster [DBG] pgmap v4757: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:41 smithi059 bash[59745]: cluster 2022-04-28T21:51:40.578203+0000 mgr.y (mgr.24961) 3973 : cluster [DBG] pgmap v4757: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:41 smithi139 bash[43301]: cluster 2022-04-28T21:51:40.578203+0000 mgr.y (mgr.24961) 3973 : cluster [DBG] pgmap v4757: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:42.102Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:43.988 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:43 smithi059 bash[61152]: cluster 2022-04-28T21:51:42.579394+0000 mgr.y (mgr.24961) 3974 : cluster [DBG] pgmap v4758: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:43.989 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:43.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:43.989 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:43 smithi059 bash[59745]: cluster 2022-04-28T21:51:42.579394+0000 mgr.y (mgr.24961) 3974 : cluster [DBG] pgmap v4758: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:43 smithi139 bash[43301]: cluster 2022-04-28T21:51:42.579394+0000 mgr.y (mgr.24961) 3974 : cluster [DBG] pgmap v4758: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:44 smithi059 bash[61152]: cluster 2022-04-28T21:51:44.580756+0000 mgr.y (mgr.24961) 3975 : cluster [DBG] pgmap v4759: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:44 smithi059 bash[59745]: cluster 2022-04-28T21:51:44.580756+0000 mgr.y (mgr.24961) 3975 : cluster [DBG] pgmap v4759: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:44 smithi139 bash[43301]: cluster 2022-04-28T21:51:44.580756+0000 mgr.y (mgr.24961) 3975 : cluster [DBG] pgmap v4759: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:47 smithi059 bash[61152]: cluster 2022-04-28T21:51:46.582473+0000 mgr.y (mgr.24961) 3976 : cluster [DBG] pgmap v4760: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:47 smithi059 bash[59745]: cluster 2022-04-28T21:51:46.582473+0000 mgr.y (mgr.24961) 3976 : cluster [DBG] pgmap v4760: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:48.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:47 smithi139 bash[43301]: cluster 2022-04-28T21:51:46.582473+0000 mgr.y (mgr.24961) 3976 : cluster [DBG] pgmap v4760: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:49 smithi059 bash[61152]: cluster 2022-04-28T21:51:48.583274+0000 mgr.y (mgr.24961) 3977 : cluster [DBG] pgmap v4761: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:49 smithi059 bash[59745]: cluster 2022-04-28T21:51:48.583274+0000 mgr.y (mgr.24961) 3977 : cluster [DBG] pgmap v4761: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:50.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:49 smithi139 bash[43301]: cluster 2022-04-28T21:51:48.583274+0000 mgr.y (mgr.24961) 3977 : cluster [DBG] pgmap v4761: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:51 smithi059 bash[61152]: cluster 2022-04-28T21:51:50.585394+0000 mgr.y (mgr.24961) 3978 : cluster [DBG] pgmap v4762: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:51 smithi059 bash[59745]: cluster 2022-04-28T21:51:50.585394+0000 mgr.y (mgr.24961) 3978 : cluster [DBG] pgmap v4762: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:51 smithi139 bash[43301]: cluster 2022-04-28T21:51:50.585394+0000 mgr.y (mgr.24961) 3978 : cluster [DBG] pgmap v4762: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:52.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:53.989 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:53 smithi059 bash[61152]: cluster 2022-04-28T21:51:52.586697+0000 mgr.y (mgr.24961) 3979 : cluster [DBG] pgmap v4763: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:53.990 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:53 smithi059 bash[59745]: cluster 2022-04-28T21:51:52.586697+0000 mgr.y (mgr.24961) 3979 : cluster [DBG] pgmap v4763: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:53 smithi139 bash[43301]: cluster 2022-04-28T21:51:52.586697+0000 mgr.y (mgr.24961) 3979 : cluster [DBG] pgmap v4763: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:51:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:51:53.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:51:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:54 smithi059 bash[61152]: cluster 2022-04-28T21:51:54.588019+0000 mgr.y (mgr.24961) 3980 : cluster [DBG] pgmap v4764: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:54 smithi059 bash[59745]: cluster 2022-04-28T21:51:54.588019+0000 mgr.y (mgr.24961) 3980 : cluster [DBG] pgmap v4764: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:54 smithi139 bash[43301]: cluster 2022-04-28T21:51:54.588019+0000 mgr.y (mgr.24961) 3980 : cluster [DBG] pgmap v4764: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:57 smithi059 bash[61152]: cluster 2022-04-28T21:51:56.589635+0000 mgr.y (mgr.24961) 3981 : cluster [DBG] pgmap v4765: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:57 smithi059 bash[59745]: cluster 2022-04-28T21:51:56.589635+0000 mgr.y (mgr.24961) 3981 : cluster [DBG] pgmap v4765: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:51:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:57 smithi139 bash[43301]: cluster 2022-04-28T21:51:56.589635+0000 mgr.y (mgr.24961) 3981 : cluster [DBG] pgmap v4765: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:00.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:51:59 smithi059 bash[59745]: cluster 2022-04-28T21:51:58.590418+0000 mgr.y (mgr.24961) 3982 : cluster [DBG] pgmap v4766: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:00.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:51:59 smithi059 bash[61152]: cluster 2022-04-28T21:51:58.590418+0000 mgr.y (mgr.24961) 3982 : cluster [DBG] pgmap v4766: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:51:59 smithi139 bash[43301]: cluster 2022-04-28T21:51:58.590418+0000 mgr.y (mgr.24961) 3982 : cluster [DBG] pgmap v4766: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:02.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:01 smithi059 bash[61152]: cluster 2022-04-28T21:52:00.592201+0000 mgr.y (mgr.24961) 3983 : cluster [DBG] pgmap v4767: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:02.261 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:01 smithi059 bash[59745]: cluster 2022-04-28T21:52:00.592201+0000 mgr.y (mgr.24961) 3983 : cluster [DBG] pgmap v4767: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:02.261 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:01 smithi139 bash[43301]: cluster 2022-04-28T21:52:00.592201+0000 mgr.y (mgr.24961) 3983 : cluster [DBG] pgmap v4767: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:02.103Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:03.989 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:03 smithi059 bash[61152]: cluster 2022-04-28T21:52:02.593434+0000 mgr.y (mgr.24961) 3984 : cluster [DBG] pgmap v4768: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:03.990 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:03 smithi059 bash[59745]: cluster 2022-04-28T21:52:02.593434+0000 mgr.y (mgr.24961) 3984 : cluster [DBG] pgmap v4768: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:03 smithi139 bash[43301]: cluster 2022-04-28T21:52:02.593434+0000 mgr.y (mgr.24961) 3984 : cluster [DBG] pgmap v4768: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:03.987Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:04 smithi059 bash[61152]: cluster 2022-04-28T21:52:04.594723+0000 mgr.y (mgr.24961) 3985 : cluster [DBG] pgmap v4769: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:04 smithi059 bash[59745]: cluster 2022-04-28T21:52:04.594723+0000 mgr.y (mgr.24961) 3985 : cluster [DBG] pgmap v4769: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:04 smithi139 bash[43301]: cluster 2022-04-28T21:52:04.594723+0000 mgr.y (mgr.24961) 3985 : cluster [DBG] pgmap v4769: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:07 smithi059 bash[61152]: cluster 2022-04-28T21:52:06.596406+0000 mgr.y (mgr.24961) 3986 : cluster [DBG] pgmap v4770: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:07 smithi059 bash[59745]: cluster 2022-04-28T21:52:06.596406+0000 mgr.y (mgr.24961) 3986 : cluster [DBG] pgmap v4770: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:07 smithi139 bash[43301]: cluster 2022-04-28T21:52:06.596406+0000 mgr.y (mgr.24961) 3986 : cluster [DBG] pgmap v4770: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:10.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:09 smithi059 bash[61152]: cluster 2022-04-28T21:52:08.597169+0000 mgr.y (mgr.24961) 3987 : cluster [DBG] pgmap v4771: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:09 smithi059 bash[59745]: cluster 2022-04-28T21:52:08.597169+0000 mgr.y (mgr.24961) 3987 : cluster [DBG] pgmap v4771: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:09 smithi139 bash[43301]: cluster 2022-04-28T21:52:08.597169+0000 mgr.y (mgr.24961) 3987 : cluster [DBG] pgmap v4771: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:12.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:11 smithi059 bash[59745]: cluster 2022-04-28T21:52:10.598926+0000 mgr.y (mgr.24961) 3988 : cluster [DBG] pgmap v4772: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:11 smithi059 bash[61152]: cluster 2022-04-28T21:52:10.598926+0000 mgr.y (mgr.24961) 3988 : cluster [DBG] pgmap v4772: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:11 smithi139 bash[43301]: cluster 2022-04-28T21:52:10.598926+0000 mgr.y (mgr.24961) 3988 : cluster [DBG] pgmap v4772: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:12.104Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:13.989 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:13 smithi059 bash[61152]: cluster 2022-04-28T21:52:12.600179+0000 mgr.y (mgr.24961) 3989 : cluster [DBG] pgmap v4773: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:13.990 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:13 smithi059 bash[59745]: cluster 2022-04-28T21:52:12.600179+0000 mgr.y (mgr.24961) 3989 : cluster [DBG] pgmap v4773: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:13 smithi139 bash[43301]: cluster 2022-04-28T21:52:12.600179+0000 mgr.y (mgr.24961) 3989 : cluster [DBG] pgmap v4773: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:13.988Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:15.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:14 smithi059 bash[61152]: cluster 2022-04-28T21:52:14.601453+0000 mgr.y (mgr.24961) 3990 : cluster [DBG] pgmap v4774: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:15.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:14 smithi059 bash[59745]: cluster 2022-04-28T21:52:14.601453+0000 mgr.y (mgr.24961) 3990 : cluster [DBG] pgmap v4774: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:14 smithi139 bash[43301]: cluster 2022-04-28T21:52:14.601453+0000 mgr.y (mgr.24961) 3990 : cluster [DBG] pgmap v4774: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:17 smithi059 bash[61152]: cluster 2022-04-28T21:52:16.603046+0000 mgr.y (mgr.24961) 3991 : cluster [DBG] pgmap v4775: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:17 smithi059 bash[59745]: cluster 2022-04-28T21:52:16.603046+0000 mgr.y (mgr.24961) 3991 : cluster [DBG] pgmap v4775: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:17 smithi139 bash[43301]: cluster 2022-04-28T21:52:16.603046+0000 mgr.y (mgr.24961) 3991 : cluster [DBG] pgmap v4775: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: cluster 2022-04-28T21:52:18.603840+0000 mgr.y (mgr.24961) 3992 : cluster [DBG] pgmap v4776: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:18.907269+0000 mon.a (mon.0) 4112 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:52:20.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:18.910089+0000 mon.a (mon.0) 4113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:52:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:18.911945+0000 mon.a (mon.0) 4114 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:52:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.199090+0000 mon.a (mon.0) 4115 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.200008+0000 mon.a (mon.0) 4116 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.200771+0000 mon.a (mon.0) 4117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.201517+0000 mon.a (mon.0) 4118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.202236+0000 mon.a (mon.0) 4119 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.202923+0000 mon.a (mon.0) 4120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.203581+0000 mon.a (mon.0) 4121 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.204230+0000 mon.a (mon.0) 4122 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.204947+0000 mon.a (mon.0) 4123 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:52:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.205715+0000 mon.a (mon.0) 4124 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.216043+0000 mon.a (mon.0) 4125 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.216727+0000 mon.a (mon.0) 4126 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: cluster 2022-04-28T21:52:18.603840+0000 mgr.y (mgr.24961) 3992 : cluster [DBG] pgmap v4776: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:18.907269+0000 mon.a (mon.0) 4112 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:52:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:18.910089+0000 mon.a (mon.0) 4113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:52:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:18.911945+0000 mon.a (mon.0) 4114 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:52:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.199090+0000 mon.a (mon.0) 4115 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.200008+0000 mon.a (mon.0) 4116 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.200771+0000 mon.a (mon.0) 4117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.201517+0000 mon.a (mon.0) 4118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.202236+0000 mon.a (mon.0) 4119 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.202923+0000 mon.a (mon.0) 4120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.203581+0000 mon.a (mon.0) 4121 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.204230+0000 mon.a (mon.0) 4122 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.204947+0000 mon.a (mon.0) 4123 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:52:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.205715+0000 mon.a (mon.0) 4124 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.097 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.216043+0000 mon.a (mon.0) 4125 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.098 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.216727+0000 mon.a (mon.0) 4126 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.217364+0000 mon.a (mon.0) 4127 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.217981+0000 mon.a (mon.0) 4128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.098 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.218614+0000 mon.a (mon.0) 4129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.219216+0000 mon.a (mon.0) 4130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.219854+0000 mon.a (mon.0) 4131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.220464+0000 mon.a (mon.0) 4132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.221111+0000 mon.a (mon.0) 4133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.221814+0000 mon.a (mon.0) 4134 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.230818+0000 mon.a (mon.0) 4135 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:52:20.099 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.400034+0000 mon.a (mon.0) 4136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:52:20.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.413387+0000 mon.a (mon.0) 4137 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:52:20.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.462952+0000 mon.a (mon.0) 4138 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:52:20.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.463424+0000 mon.a (mon.0) 4139 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:52:20.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.463967+0000 mon.a (mon.0) 4140 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:52:20.100 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[61152]: audit 2022-04-28T21:52:19.464482+0000 mon.a (mon.0) 4141 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:52:20.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.217364+0000 mon.a (mon.0) 4127 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.217981+0000 mon.a (mon.0) 4128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.218614+0000 mon.a (mon.0) 4129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.219216+0000 mon.a (mon.0) 4130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.219854+0000 mon.a (mon.0) 4131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.220464+0000 mon.a (mon.0) 4132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.221111+0000 mon.a (mon.0) 4133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:52:20.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.221814+0000 mon.a (mon.0) 4134 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.230818+0000 mon.a (mon.0) 4135 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:52:20.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.400034+0000 mon.a (mon.0) 4136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:52:20.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.413387+0000 mon.a (mon.0) 4137 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:52:20.102 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.462952+0000 mon.a (mon.0) 4138 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:52:20.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.463424+0000 mon.a (mon.0) 4139 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:52:20.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.463967+0000 mon.a (mon.0) 4140 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:52:20.103 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:19 smithi059 bash[59745]: audit 2022-04-28T21:52:19.464482+0000 mon.a (mon.0) 4141 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:52:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: cluster 2022-04-28T21:52:18.603840+0000 mgr.y (mgr.24961) 3992 : cluster [DBG] pgmap v4776: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:18.907269+0000 mon.a (mon.0) 4112 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:52:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:18.910089+0000 mon.a (mon.0) 4113 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:52:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:18.911945+0000 mon.a (mon.0) 4114 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:52:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.199090+0000 mon.a (mon.0) 4115 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.200008+0000 mon.a (mon.0) 4116 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.200771+0000 mon.a (mon.0) 4117 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.201517+0000 mon.a (mon.0) 4118 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.202236+0000 mon.a (mon.0) 4119 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.202923+0000 mon.a (mon.0) 4120 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.203581+0000 mon.a (mon.0) 4121 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.204230+0000 mon.a (mon.0) 4122 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.204947+0000 mon.a (mon.0) 4123 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.205715+0000 mon.a (mon.0) 4124 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.216043+0000 mon.a (mon.0) 4125 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.216727+0000 mon.a (mon.0) 4126 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.217364+0000 mon.a (mon.0) 4127 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.217981+0000 mon.a (mon.0) 4128 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.218614+0000 mon.a (mon.0) 4129 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.219216+0000 mon.a (mon.0) 4130 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.219854+0000 mon.a (mon.0) 4131 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T21:52:20.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.220464+0000 mon.a (mon.0) 4132 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.221111+0000 mon.a (mon.0) 4133 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T21:52:20.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.221814+0000 mon.a (mon.0) 4134 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T21:52:20.124 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.230818+0000 mon.a (mon.0) 4135 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:52:20.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.400034+0000 mon.a (mon.0) 4136 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:52:20.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.413387+0000 mon.a (mon.0) 4137 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:52:20.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.462952+0000 mon.a (mon.0) 4138 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:52:20.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.463424+0000 mon.a (mon.0) 4139 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2022-04-28T21:52:20.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.463967+0000 mon.a (mon.0) 4140 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:52:20.125 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:19 smithi139 bash[43301]: audit 2022-04-28T21:52:19.464482+0000 mon.a (mon.0) 4141 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:52:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:20 smithi059 bash[59745]: cephadm 2022-04-28T21:52:18.913580+0000 mgr.y (mgr.24961) 3993 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:52:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:20 smithi059 bash[61152]: cephadm 2022-04-28T21:52:18.913580+0000 mgr.y (mgr.24961) 3993 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:52:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:20 smithi139 bash[43301]: cephadm 2022-04-28T21:52:18.913580+0000 mgr.y (mgr.24961) 3993 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:52:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:21 smithi059 bash[61152]: cluster 2022-04-28T21:52:20.605574+0000 mgr.y (mgr.24961) 3994 : cluster [DBG] pgmap v4777: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:21 smithi059 bash[59745]: cluster 2022-04-28T21:52:20.605574+0000 mgr.y (mgr.24961) 3994 : cluster [DBG] pgmap v4777: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:21 smithi139 bash[43301]: cluster 2022-04-28T21:52:20.605574+0000 mgr.y (mgr.24961) 3994 : cluster [DBG] pgmap v4777: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:22.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:22.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:23.991 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:23 smithi059 bash[61152]: cluster 2022-04-28T21:52:22.606781+0000 mgr.y (mgr.24961) 3995 : cluster [DBG] pgmap v4778: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:23.992 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:23 smithi059 bash[59745]: cluster 2022-04-28T21:52:22.606781+0000 mgr.y (mgr.24961) 3995 : cluster [DBG] pgmap v4778: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:23 smithi139 bash[43301]: cluster 2022-04-28T21:52:22.606781+0000 mgr.y (mgr.24961) 3995 : cluster [DBG] pgmap v4778: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:23.990Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:24 smithi059 bash[61152]: cluster 2022-04-28T21:52:24.608078+0000 mgr.y (mgr.24961) 3996 : cluster [DBG] pgmap v4779: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:24 smithi059 bash[59745]: cluster 2022-04-28T21:52:24.608078+0000 mgr.y (mgr.24961) 3996 : cluster [DBG] pgmap v4779: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:24 smithi139 bash[43301]: cluster 2022-04-28T21:52:24.608078+0000 mgr.y (mgr.24961) 3996 : cluster [DBG] pgmap v4779: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:27 smithi059 bash[61152]: cluster 2022-04-28T21:52:26.609713+0000 mgr.y (mgr.24961) 3997 : cluster [DBG] pgmap v4780: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:27 smithi059 bash[59745]: cluster 2022-04-28T21:52:26.609713+0000 mgr.y (mgr.24961) 3997 : cluster [DBG] pgmap v4780: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:27 smithi139 bash[43301]: cluster 2022-04-28T21:52:26.609713+0000 mgr.y (mgr.24961) 3997 : cluster [DBG] pgmap v4780: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:29 smithi059 bash[61152]: cluster 2022-04-28T21:52:28.610474+0000 mgr.y (mgr.24961) 3998 : cluster [DBG] pgmap v4781: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:29 smithi059 bash[59745]: cluster 2022-04-28T21:52:28.610474+0000 mgr.y (mgr.24961) 3998 : cluster [DBG] pgmap v4781: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:29 smithi139 bash[43301]: cluster 2022-04-28T21:52:28.610474+0000 mgr.y (mgr.24961) 3998 : cluster [DBG] pgmap v4781: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:32.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:31 smithi059 bash[61152]: cluster 2022-04-28T21:52:30.612244+0000 mgr.y (mgr.24961) 3999 : cluster [DBG] pgmap v4782: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:31 smithi059 bash[59745]: cluster 2022-04-28T21:52:30.612244+0000 mgr.y (mgr.24961) 3999 : cluster [DBG] pgmap v4782: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:31 smithi139 bash[43301]: cluster 2022-04-28T21:52:30.612244+0000 mgr.y (mgr.24961) 3999 : cluster [DBG] pgmap v4782: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:32.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:33.992 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:33 smithi059 bash[59745]: cluster 2022-04-28T21:52:32.613437+0000 mgr.y (mgr.24961) 4000 : cluster [DBG] pgmap v4783: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:33.993 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:33 smithi059 bash[61152]: cluster 2022-04-28T21:52:32.613437+0000 mgr.y (mgr.24961) 4000 : cluster [DBG] pgmap v4783: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:34.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:33 smithi139 bash[43301]: cluster 2022-04-28T21:52:32.613437+0000 mgr.y (mgr.24961) 4000 : cluster [DBG] pgmap v4783: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:33.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:34 smithi059 bash[61152]: cluster 2022-04-28T21:52:34.614734+0000 mgr.y (mgr.24961) 4001 : cluster [DBG] pgmap v4784: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:34 smithi059 bash[59745]: cluster 2022-04-28T21:52:34.614734+0000 mgr.y (mgr.24961) 4001 : cluster [DBG] pgmap v4784: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:34 smithi139 bash[43301]: cluster 2022-04-28T21:52:34.614734+0000 mgr.y (mgr.24961) 4001 : cluster [DBG] pgmap v4784: 201 pgs: 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[59745]: cluster 2022-04-28T21:52:36.499522+0000 mon.a (mon.0) 4142 : cluster [DBG] osdmap e1434: 8 total, 8 up, 8 in 2022-04-28T21:52:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[59745]: audit 2022-04-28T21:52:36.500957+0000 mon.c (mon.1) 213 : audit [INF] from='client.? 172.21.15.59:0/1316027798' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:52:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[59745]: audit 2022-04-28T21:52:36.501439+0000 mon.a (mon.0) 4143 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:52:37.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[59745]: cluster 2022-04-28T21:52:36.615559+0000 mgr.y (mgr.24961) 4002 : cluster [DBG] pgmap v4786: 233 pgs: 32 unknown, 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[61152]: cluster 2022-04-28T21:52:36.499522+0000 mon.a (mon.0) 4142 : cluster [DBG] osdmap e1434: 8 total, 8 up, 8 in 2022-04-28T21:52:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[61152]: audit 2022-04-28T21:52:36.500957+0000 mon.c (mon.1) 213 : audit [INF] from='client.? 172.21.15.59:0/1316027798' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:52:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[61152]: audit 2022-04-28T21:52:36.501439+0000 mon.a (mon.0) 4143 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:52:37.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:37 smithi059 bash[61152]: cluster 2022-04-28T21:52:36.615559+0000 mgr.y (mgr.24961) 4002 : cluster [DBG] pgmap v4786: 233 pgs: 32 unknown, 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:37 smithi139 bash[43301]: cluster 2022-04-28T21:52:36.499522+0000 mon.a (mon.0) 4142 : cluster [DBG] osdmap e1434: 8 total, 8 up, 8 in 2022-04-28T21:52:37.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:37 smithi139 bash[43301]: audit 2022-04-28T21:52:36.500957+0000 mon.c (mon.1) 213 : audit [INF] from='client.? 172.21.15.59:0/1316027798' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:52:37.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:37 smithi139 bash[43301]: audit 2022-04-28T21:52:36.501439+0000 mon.a (mon.0) 4143 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T21:52:37.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:37 smithi139 bash[43301]: cluster 2022-04-28T21:52:36.615559+0000 mgr.y (mgr.24961) 4002 : cluster [DBG] pgmap v4786: 233 pgs: 32 unknown, 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:38 smithi059 bash[61152]: audit 2022-04-28T21:52:37.501146+0000 mon.a (mon.0) 4144 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:52:38.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:38 smithi059 bash[61152]: cluster 2022-04-28T21:52:37.501760+0000 mon.a (mon.0) 4145 : cluster [DBG] osdmap e1435: 8 total, 8 up, 8 in 2022-04-28T21:52:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:38 smithi059 bash[59745]: audit 2022-04-28T21:52:37.501146+0000 mon.a (mon.0) 4144 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:52:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:38 smithi059 bash[59745]: cluster 2022-04-28T21:52:37.501760+0000 mon.a (mon.0) 4145 : cluster [DBG] osdmap e1435: 8 total, 8 up, 8 in 2022-04-28T21:52:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:38 smithi139 bash[43301]: audit 2022-04-28T21:52:37.501146+0000 mon.a (mon.0) 4144 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi059-86598-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T21:52:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:38 smithi139 bash[43301]: cluster 2022-04-28T21:52:37.501760+0000 mon.a (mon.0) 4145 : cluster [DBG] osdmap e1435: 8 total, 8 up, 8 in 2022-04-28T21:52:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:39 smithi059 bash[61152]: cluster 2022-04-28T21:52:38.524388+0000 mon.a (mon.0) 4146 : cluster [DBG] osdmap e1436: 8 total, 8 up, 8 in 2022-04-28T21:52:39.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:39 smithi059 bash[61152]: cluster 2022-04-28T21:52:38.616430+0000 mgr.y (mgr.24961) 4003 : cluster [DBG] pgmap v4789: 233 pgs: 32 unknown, 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:39.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:39 smithi059 bash[59745]: cluster 2022-04-28T21:52:38.524388+0000 mon.a (mon.0) 4146 : cluster [DBG] osdmap e1436: 8 total, 8 up, 8 in 2022-04-28T21:52:39.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:39 smithi059 bash[59745]: cluster 2022-04-28T21:52:38.616430+0000 mgr.y (mgr.24961) 4003 : cluster [DBG] pgmap v4789: 233 pgs: 32 unknown, 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:39.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:39 smithi139 bash[43301]: cluster 2022-04-28T21:52:38.524388+0000 mon.a (mon.0) 4146 : cluster [DBG] osdmap e1436: 8 total, 8 up, 8 in 2022-04-28T21:52:39.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:39 smithi139 bash[43301]: cluster 2022-04-28T21:52:38.616430+0000 mgr.y (mgr.24961) 4003 : cluster [DBG] pgmap v4789: 233 pgs: 32 unknown, 201 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:41 smithi059 bash[61152]: cluster 2022-04-28T21:52:40.618398+0000 mgr.y (mgr.24961) 4004 : cluster [DBG] pgmap v4790: 233 pgs: 9 unknown, 224 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:41 smithi059 bash[59745]: cluster 2022-04-28T21:52:40.618398+0000 mgr.y (mgr.24961) 4004 : cluster [DBG] pgmap v4790: 233 pgs: 9 unknown, 224 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:41 smithi139 bash[43301]: cluster 2022-04-28T21:52:40.618398+0000 mgr.y (mgr.24961) 4004 : cluster [DBG] pgmap v4790: 233 pgs: 9 unknown, 224 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:42.105Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:43.992 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:43 smithi059 bash[59745]: cluster 2022-04-28T21:52:42.619635+0000 mgr.y (mgr.24961) 4005 : cluster [DBG] pgmap v4791: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 835 B/s rd, 167 B/s wr, 1 op/s 2022-04-28T21:52:43.992 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:43 smithi059 bash[61152]: cluster 2022-04-28T21:52:42.619635+0000 mgr.y (mgr.24961) 4005 : cluster [DBG] pgmap v4791: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 835 B/s rd, 167 B/s wr, 1 op/s 2022-04-28T21:52:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:43 smithi139 bash[43301]: cluster 2022-04-28T21:52:42.619635+0000 mgr.y (mgr.24961) 4005 : cluster [DBG] pgmap v4791: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 835 B/s rd, 167 B/s wr, 1 op/s 2022-04-28T21:52:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:43.991Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:44 smithi059 bash[61152]: cluster 2022-04-28T21:52:44.621081+0000 mgr.y (mgr.24961) 4006 : cluster [DBG] pgmap v4792: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 127 B/s wr, 0 op/s 2022-04-28T21:52:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:44 smithi059 bash[59745]: cluster 2022-04-28T21:52:44.621081+0000 mgr.y (mgr.24961) 4006 : cluster [DBG] pgmap v4792: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 127 B/s wr, 0 op/s 2022-04-28T21:52:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:44 smithi139 bash[43301]: cluster 2022-04-28T21:52:44.621081+0000 mgr.y (mgr.24961) 4006 : cluster [DBG] pgmap v4792: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 127 B/s wr, 0 op/s 2022-04-28T21:52:48.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:47 smithi059 bash[61152]: cluster 2022-04-28T21:52:46.622795+0000 mgr.y (mgr.24961) 4007 : cluster [DBG] pgmap v4793: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 561 B/s rd, 112 B/s wr, 0 op/s 2022-04-28T21:52:48.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:47 smithi059 bash[59745]: cluster 2022-04-28T21:52:46.622795+0000 mgr.y (mgr.24961) 4007 : cluster [DBG] pgmap v4793: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 561 B/s rd, 112 B/s wr, 0 op/s 2022-04-28T21:52:48.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:47 smithi139 bash[43301]: cluster 2022-04-28T21:52:46.622795+0000 mgr.y (mgr.24961) 4007 : cluster [DBG] pgmap v4793: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 561 B/s rd, 112 B/s wr, 0 op/s 2022-04-28T21:52:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:49 smithi059 bash[61152]: cluster 2022-04-28T21:52:48.623620+0000 mgr.y (mgr.24961) 4008 : cluster [DBG] pgmap v4794: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 506 B/s rd, 101 B/s wr, 0 op/s 2022-04-28T21:52:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:49 smithi059 bash[59745]: cluster 2022-04-28T21:52:48.623620+0000 mgr.y (mgr.24961) 4008 : cluster [DBG] pgmap v4794: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 506 B/s rd, 101 B/s wr, 0 op/s 2022-04-28T21:52:50.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:49 smithi139 bash[43301]: cluster 2022-04-28T21:52:48.623620+0000 mgr.y (mgr.24961) 4008 : cluster [DBG] pgmap v4794: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 506 B/s rd, 101 B/s wr, 0 op/s 2022-04-28T21:52:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:51 smithi059 bash[61152]: cluster 2022-04-28T21:52:50.625530+0000 mgr.y (mgr.24961) 4009 : cluster [DBG] pgmap v4795: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:52:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:51 smithi059 bash[59745]: cluster 2022-04-28T21:52:50.625530+0000 mgr.y (mgr.24961) 4009 : cluster [DBG] pgmap v4795: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:52:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:51 smithi139 bash[43301]: cluster 2022-04-28T21:52:50.625530+0000 mgr.y (mgr.24961) 4009 : cluster [DBG] pgmap v4795: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:52:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:52.107Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:53.996 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:53 smithi059 bash[61152]: cluster 2022-04-28T21:52:52.626840+0000 mgr.y (mgr.24961) 4010 : cluster [DBG] pgmap v4796: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:52:53.997 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:52:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:52:53.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:52:53.997 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:53 smithi059 bash[59745]: cluster 2022-04-28T21:52:52.626840+0000 mgr.y (mgr.24961) 4010 : cluster [DBG] pgmap v4796: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:52:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:53 smithi139 bash[43301]: cluster 2022-04-28T21:52:52.626840+0000 mgr.y (mgr.24961) 4010 : cluster [DBG] pgmap v4796: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T21:52:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:54 smithi059 bash[61152]: cluster 2022-04-28T21:52:54.628305+0000 mgr.y (mgr.24961) 4011 : cluster [DBG] pgmap v4797: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:54 smithi059 bash[59745]: cluster 2022-04-28T21:52:54.628305+0000 mgr.y (mgr.24961) 4011 : cluster [DBG] pgmap v4797: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:54 smithi139 bash[43301]: cluster 2022-04-28T21:52:54.628305+0000 mgr.y (mgr.24961) 4011 : cluster [DBG] pgmap v4797: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:57 smithi059 bash[61152]: cluster 2022-04-28T21:52:56.630064+0000 mgr.y (mgr.24961) 4012 : cluster [DBG] pgmap v4798: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:57 smithi059 bash[59745]: cluster 2022-04-28T21:52:56.630064+0000 mgr.y (mgr.24961) 4012 : cluster [DBG] pgmap v4798: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:52:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:57 smithi139 bash[43301]: cluster 2022-04-28T21:52:56.630064+0000 mgr.y (mgr.24961) 4012 : cluster [DBG] pgmap v4798: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:52:59 smithi059 bash[61152]: cluster 2022-04-28T21:52:58.630850+0000 mgr.y (mgr.24961) 4013 : cluster [DBG] pgmap v4799: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:52:59 smithi059 bash[59745]: cluster 2022-04-28T21:52:58.630850+0000 mgr.y (mgr.24961) 4013 : cluster [DBG] pgmap v4799: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:52:59 smithi139 bash[43301]: cluster 2022-04-28T21:52:58.630850+0000 mgr.y (mgr.24961) 4013 : cluster [DBG] pgmap v4799: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:01 smithi059 bash[61152]: cluster 2022-04-28T21:53:00.632816+0000 mgr.y (mgr.24961) 4014 : cluster [DBG] pgmap v4800: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:02.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:01 smithi059 bash[59745]: cluster 2022-04-28T21:53:00.632816+0000 mgr.y (mgr.24961) 4014 : cluster [DBG] pgmap v4800: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:01 smithi139 bash[43301]: cluster 2022-04-28T21:53:00.632816+0000 mgr.y (mgr.24961) 4014 : cluster [DBG] pgmap v4800: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:02.106Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:03.995 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:03 smithi059 bash[61152]: cluster 2022-04-28T21:53:02.634057+0000 mgr.y (mgr.24961) 4015 : cluster [DBG] pgmap v4801: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:03.995 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:03.993Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:03.996 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:03 smithi059 bash[59745]: cluster 2022-04-28T21:53:02.634057+0000 mgr.y (mgr.24961) 4015 : cluster [DBG] pgmap v4801: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:03 smithi139 bash[43301]: cluster 2022-04-28T21:53:02.634057+0000 mgr.y (mgr.24961) 4015 : cluster [DBG] pgmap v4801: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:05.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:04 smithi059 bash[61152]: cluster 2022-04-28T21:53:04.635559+0000 mgr.y (mgr.24961) 4016 : cluster [DBG] pgmap v4802: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:05.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:04 smithi059 bash[59745]: cluster 2022-04-28T21:53:04.635559+0000 mgr.y (mgr.24961) 4016 : cluster [DBG] pgmap v4802: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:04 smithi139 bash[43301]: cluster 2022-04-28T21:53:04.635559+0000 mgr.y (mgr.24961) 4016 : cluster [DBG] pgmap v4802: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:07 smithi059 bash[61152]: cluster 2022-04-28T21:53:06.637487+0000 mgr.y (mgr.24961) 4017 : cluster [DBG] pgmap v4803: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:08.134 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:07 smithi059 bash[59745]: cluster 2022-04-28T21:53:06.637487+0000 mgr.y (mgr.24961) 4017 : cluster [DBG] pgmap v4803: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:08.134 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:07 smithi139 bash[43301]: cluster 2022-04-28T21:53:06.637487+0000 mgr.y (mgr.24961) 4017 : cluster [DBG] pgmap v4803: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:09 smithi059 bash[61152]: cluster 2022-04-28T21:53:08.638423+0000 mgr.y (mgr.24961) 4018 : cluster [DBG] pgmap v4804: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:09 smithi059 bash[59745]: cluster 2022-04-28T21:53:08.638423+0000 mgr.y (mgr.24961) 4018 : cluster [DBG] pgmap v4804: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:09 smithi139 bash[43301]: cluster 2022-04-28T21:53:08.638423+0000 mgr.y (mgr.24961) 4018 : cluster [DBG] pgmap v4804: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:11 smithi059 bash[61152]: cluster 2022-04-28T21:53:10.640430+0000 mgr.y (mgr.24961) 4019 : cluster [DBG] pgmap v4805: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:11 smithi059 bash[59745]: cluster 2022-04-28T21:53:10.640430+0000 mgr.y (mgr.24961) 4019 : cluster [DBG] pgmap v4805: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:11 smithi139 bash[43301]: cluster 2022-04-28T21:53:10.640430+0000 mgr.y (mgr.24961) 4019 : cluster [DBG] pgmap v4805: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:12.108Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:13.996 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:13 smithi059 bash[61152]: cluster 2022-04-28T21:53:12.641740+0000 mgr.y (mgr.24961) 4020 : cluster [DBG] pgmap v4806: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:13.996 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:13 smithi059 bash[59745]: cluster 2022-04-28T21:53:12.641740+0000 mgr.y (mgr.24961) 4020 : cluster [DBG] pgmap v4806: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:13 smithi139 bash[43301]: cluster 2022-04-28T21:53:12.641740+0000 mgr.y (mgr.24961) 4020 : cluster [DBG] pgmap v4806: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:13.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:14 smithi059 bash[61152]: cluster 2022-04-28T21:53:14.643189+0000 mgr.y (mgr.24961) 4021 : cluster [DBG] pgmap v4807: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:14 smithi059 bash[59745]: cluster 2022-04-28T21:53:14.643189+0000 mgr.y (mgr.24961) 4021 : cluster [DBG] pgmap v4807: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:14 smithi139 bash[43301]: cluster 2022-04-28T21:53:14.643189+0000 mgr.y (mgr.24961) 4021 : cluster [DBG] pgmap v4807: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:18.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:17 smithi059 bash[61152]: cluster 2022-04-28T21:53:16.645051+0000 mgr.y (mgr.24961) 4022 : cluster [DBG] pgmap v4808: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:17 smithi059 bash[59745]: cluster 2022-04-28T21:53:16.645051+0000 mgr.y (mgr.24961) 4022 : cluster [DBG] pgmap v4808: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:17 smithi139 bash[43301]: cluster 2022-04-28T21:53:16.645051+0000 mgr.y (mgr.24961) 4022 : cluster [DBG] pgmap v4808: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: cluster 2022-04-28T21:53:18.645894+0000 mgr.y (mgr.24961) 4023 : cluster [DBG] pgmap v4809: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.236919+0000 mon.a (mon.0) 4147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:53:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.239822+0000 mon.a (mon.0) 4148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:53:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.241729+0000 mon.a (mon.0) 4149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:53:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.405037+0000 mon.a (mon.0) 4150 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:53:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.419568+0000 mon.a (mon.0) 4151 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:53:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.470617+0000 mon.a (mon.0) 4152 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:53:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.471523+0000 mon.a (mon.0) 4153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:53:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.472238+0000 mon.a (mon.0) 4154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:53:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.473022+0000 mon.a (mon.0) 4155 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:53:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[59745]: audit 2022-04-28T21:53:19.549270+0000 mon.a (mon.0) 4156 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:53:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: cluster 2022-04-28T21:53:18.645894+0000 mgr.y (mgr.24961) 4023 : cluster [DBG] pgmap v4809: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.236919+0000 mon.a (mon.0) 4147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:53:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.239822+0000 mon.a (mon.0) 4148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:53:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.241729+0000 mon.a (mon.0) 4149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:53:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.405037+0000 mon.a (mon.0) 4150 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:53:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.419568+0000 mon.a (mon.0) 4151 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:53:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.470617+0000 mon.a (mon.0) 4152 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:53:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.471523+0000 mon.a (mon.0) 4153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:53:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.472238+0000 mon.a (mon.0) 4154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:53:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.473022+0000 mon.a (mon.0) 4155 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:53:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:19 smithi059 bash[61152]: audit 2022-04-28T21:53:19.549270+0000 mon.a (mon.0) 4156 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:53:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: cluster 2022-04-28T21:53:18.645894+0000 mgr.y (mgr.24961) 4023 : cluster [DBG] pgmap v4809: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.236919+0000 mon.a (mon.0) 4147 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:53:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.239822+0000 mon.a (mon.0) 4148 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:53:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.241729+0000 mon.a (mon.0) 4149 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:53:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.405037+0000 mon.a (mon.0) 4150 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:53:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.419568+0000 mon.a (mon.0) 4151 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:53:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.470617+0000 mon.a (mon.0) 4152 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:53:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.471523+0000 mon.a (mon.0) 4153 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:53:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.472238+0000 mon.a (mon.0) 4154 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:53:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.473022+0000 mon.a (mon.0) 4155 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:53:20.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:19 smithi139 bash[43301]: audit 2022-04-28T21:53:19.549270+0000 mon.a (mon.0) 4156 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:53:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:20 smithi059 bash[61152]: cephadm 2022-04-28T21:53:19.243469+0000 mgr.y (mgr.24961) 4024 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:53:21.101 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:20 smithi059 bash[59745]: cephadm 2022-04-28T21:53:19.243469+0000 mgr.y (mgr.24961) 4024 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:53:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:20 smithi139 bash[43301]: cephadm 2022-04-28T21:53:19.243469+0000 mgr.y (mgr.24961) 4024 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:53:22.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:21 smithi059 bash[59745]: cluster 2022-04-28T21:53:20.647878+0000 mgr.y (mgr.24961) 4025 : cluster [DBG] pgmap v4810: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:21 smithi059 bash[61152]: cluster 2022-04-28T21:53:20.647878+0000 mgr.y (mgr.24961) 4025 : cluster [DBG] pgmap v4810: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:21 smithi139 bash[43301]: cluster 2022-04-28T21:53:20.647878+0000 mgr.y (mgr.24961) 4025 : cluster [DBG] pgmap v4810: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:22.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:22.109Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:23.996 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:23 smithi059 bash[61152]: cluster 2022-04-28T21:53:22.649216+0000 mgr.y (mgr.24961) 4026 : cluster [DBG] pgmap v4811: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:23.996 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:23 smithi059 bash[59745]: cluster 2022-04-28T21:53:22.649216+0000 mgr.y (mgr.24961) 4026 : cluster [DBG] pgmap v4811: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:23 smithi139 bash[43301]: cluster 2022-04-28T21:53:22.649216+0000 mgr.y (mgr.24961) 4026 : cluster [DBG] pgmap v4811: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:23.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:25 smithi059 bash[61152]: cluster 2022-04-28T21:53:24.650669+0000 mgr.y (mgr.24961) 4027 : cluster [DBG] pgmap v4812: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:24 smithi059 bash[59745]: cluster 2022-04-28T21:53:24.650669+0000 mgr.y (mgr.24961) 4027 : cluster [DBG] pgmap v4812: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:24 smithi139 bash[43301]: cluster 2022-04-28T21:53:24.650669+0000 mgr.y (mgr.24961) 4027 : cluster [DBG] pgmap v4812: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:28.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:27 smithi059 bash[61152]: cluster 2022-04-28T21:53:26.652495+0000 mgr.y (mgr.24961) 4028 : cluster [DBG] pgmap v4813: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:28.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:27 smithi059 bash[59745]: cluster 2022-04-28T21:53:26.652495+0000 mgr.y (mgr.24961) 4028 : cluster [DBG] pgmap v4813: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:27 smithi139 bash[43301]: cluster 2022-04-28T21:53:26.652495+0000 mgr.y (mgr.24961) 4028 : cluster [DBG] pgmap v4813: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:29 smithi059 bash[61152]: cluster 2022-04-28T21:53:28.653326+0000 mgr.y (mgr.24961) 4029 : cluster [DBG] pgmap v4814: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:29 smithi059 bash[59745]: cluster 2022-04-28T21:53:28.653326+0000 mgr.y (mgr.24961) 4029 : cluster [DBG] pgmap v4814: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:29 smithi139 bash[43301]: cluster 2022-04-28T21:53:28.653326+0000 mgr.y (mgr.24961) 4029 : cluster [DBG] pgmap v4814: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:31 smithi059 bash[61152]: cluster 2022-04-28T21:53:30.655270+0000 mgr.y (mgr.24961) 4030 : cluster [DBG] pgmap v4815: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:31 smithi059 bash[59745]: cluster 2022-04-28T21:53:30.655270+0000 mgr.y (mgr.24961) 4030 : cluster [DBG] pgmap v4815: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:31 smithi139 bash[43301]: cluster 2022-04-28T21:53:30.655270+0000 mgr.y (mgr.24961) 4030 : cluster [DBG] pgmap v4815: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:32.110Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:33.997 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:33 smithi059 bash[61152]: cluster 2022-04-28T21:53:32.656542+0000 mgr.y (mgr.24961) 4031 : cluster [DBG] pgmap v4816: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:33.998 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:33 smithi059 bash[59745]: cluster 2022-04-28T21:53:32.656542+0000 mgr.y (mgr.24961) 4031 : cluster [DBG] pgmap v4816: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:34.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:33 smithi139 bash[43301]: cluster 2022-04-28T21:53:32.656542+0000 mgr.y (mgr.24961) 4031 : cluster [DBG] pgmap v4816: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:33.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:35 smithi059 bash[61152]: cluster 2022-04-28T21:53:34.658071+0000 mgr.y (mgr.24961) 4032 : cluster [DBG] pgmap v4817: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:35.356 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:35 smithi059 bash[59745]: cluster 2022-04-28T21:53:34.658071+0000 mgr.y (mgr.24961) 4032 : cluster [DBG] pgmap v4817: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:35 smithi139 bash[43301]: cluster 2022-04-28T21:53:34.658071+0000 mgr.y (mgr.24961) 4032 : cluster [DBG] pgmap v4817: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:38.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:37 smithi059 bash[61152]: cluster 2022-04-28T21:53:36.659906+0000 mgr.y (mgr.24961) 4033 : cluster [DBG] pgmap v4818: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:37 smithi059 bash[59745]: cluster 2022-04-28T21:53:36.659906+0000 mgr.y (mgr.24961) 4033 : cluster [DBG] pgmap v4818: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:37 smithi139 bash[43301]: cluster 2022-04-28T21:53:36.659906+0000 mgr.y (mgr.24961) 4033 : cluster [DBG] pgmap v4818: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:39 smithi059 bash[61152]: cluster 2022-04-28T21:53:38.660742+0000 mgr.y (mgr.24961) 4034 : cluster [DBG] pgmap v4819: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:39 smithi059 bash[59745]: cluster 2022-04-28T21:53:38.660742+0000 mgr.y (mgr.24961) 4034 : cluster [DBG] pgmap v4819: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:40.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:39 smithi139 bash[43301]: cluster 2022-04-28T21:53:38.660742+0000 mgr.y (mgr.24961) 4034 : cluster [DBG] pgmap v4819: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:41 smithi059 bash[61152]: cluster 2022-04-28T21:53:40.662738+0000 mgr.y (mgr.24961) 4035 : cluster [DBG] pgmap v4820: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:42.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:41 smithi059 bash[59745]: cluster 2022-04-28T21:53:40.662738+0000 mgr.y (mgr.24961) 4035 : cluster [DBG] pgmap v4820: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:41 smithi139 bash[43301]: cluster 2022-04-28T21:53:40.662738+0000 mgr.y (mgr.24961) 4035 : cluster [DBG] pgmap v4820: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:42.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:43.997 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:43 smithi059 bash[61152]: cluster 2022-04-28T21:53:42.664048+0000 mgr.y (mgr.24961) 4036 : cluster [DBG] pgmap v4821: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:43.998 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:43 smithi059 bash[59745]: cluster 2022-04-28T21:53:42.664048+0000 mgr.y (mgr.24961) 4036 : cluster [DBG] pgmap v4821: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:43 smithi139 bash[43301]: cluster 2022-04-28T21:53:42.664048+0000 mgr.y (mgr.24961) 4036 : cluster [DBG] pgmap v4821: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:44.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:43 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:43.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:45 smithi059 bash[61152]: cluster 2022-04-28T21:53:44.665505+0000 mgr.y (mgr.24961) 4037 : cluster [DBG] pgmap v4822: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:45 smithi059 bash[59745]: cluster 2022-04-28T21:53:44.665505+0000 mgr.y (mgr.24961) 4037 : cluster [DBG] pgmap v4822: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:45 smithi139 bash[43301]: cluster 2022-04-28T21:53:44.665505+0000 mgr.y (mgr.24961) 4037 : cluster [DBG] pgmap v4822: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:48.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:47 smithi059 bash[61152]: cluster 2022-04-28T21:53:46.667385+0000 mgr.y (mgr.24961) 4038 : cluster [DBG] pgmap v4823: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:47 smithi059 bash[59745]: cluster 2022-04-28T21:53:46.667385+0000 mgr.y (mgr.24961) 4038 : cluster [DBG] pgmap v4823: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:48.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:47 smithi139 bash[43301]: cluster 2022-04-28T21:53:46.667385+0000 mgr.y (mgr.24961) 4038 : cluster [DBG] pgmap v4823: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:49 smithi059 bash[61152]: cluster 2022-04-28T21:53:48.668234+0000 mgr.y (mgr.24961) 4039 : cluster [DBG] pgmap v4824: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:49 smithi059 bash[59745]: cluster 2022-04-28T21:53:48.668234+0000 mgr.y (mgr.24961) 4039 : cluster [DBG] pgmap v4824: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:50.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:49 smithi139 bash[43301]: cluster 2022-04-28T21:53:48.668234+0000 mgr.y (mgr.24961) 4039 : cluster [DBG] pgmap v4824: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:51 smithi059 bash[61152]: cluster 2022-04-28T21:53:50.670149+0000 mgr.y (mgr.24961) 4040 : cluster [DBG] pgmap v4825: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:52.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:51 smithi059 bash[59745]: cluster 2022-04-28T21:53:50.670149+0000 mgr.y (mgr.24961) 4040 : cluster [DBG] pgmap v4825: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:51 smithi139 bash[43301]: cluster 2022-04-28T21:53:50.670149+0000 mgr.y (mgr.24961) 4040 : cluster [DBG] pgmap v4825: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:52.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:53.996 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:53 smithi059 bash[61152]: cluster 2022-04-28T21:53:52.671433+0000 mgr.y (mgr.24961) 4041 : cluster [DBG] pgmap v4826: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:53.997 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:53 smithi059 bash[59745]: cluster 2022-04-28T21:53:52.671433+0000 mgr.y (mgr.24961) 4041 : cluster [DBG] pgmap v4826: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:53.998 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:53:53 smithi059 bash[72035]: level=error ts=2022-04-28T21:53:53.995Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:53:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:53 smithi139 bash[43301]: cluster 2022-04-28T21:53:52.671433+0000 mgr.y (mgr.24961) 4041 : cluster [DBG] pgmap v4826: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:55 smithi059 bash[61152]: cluster 2022-04-28T21:53:54.672941+0000 mgr.y (mgr.24961) 4042 : cluster [DBG] pgmap v4827: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:55 smithi059 bash[59745]: cluster 2022-04-28T21:53:54.672941+0000 mgr.y (mgr.24961) 4042 : cluster [DBG] pgmap v4827: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:55 smithi139 bash[43301]: cluster 2022-04-28T21:53:54.672941+0000 mgr.y (mgr.24961) 4042 : cluster [DBG] pgmap v4827: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:58.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:57 smithi059 bash[61152]: cluster 2022-04-28T21:53:56.674659+0000 mgr.y (mgr.24961) 4043 : cluster [DBG] pgmap v4828: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:57 smithi059 bash[59745]: cluster 2022-04-28T21:53:56.674659+0000 mgr.y (mgr.24961) 4043 : cluster [DBG] pgmap v4828: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:53:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:57 smithi139 bash[43301]: cluster 2022-04-28T21:53:56.674659+0000 mgr.y (mgr.24961) 4043 : cluster [DBG] pgmap v4828: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:53:59 smithi059 bash[61152]: cluster 2022-04-28T21:53:58.675513+0000 mgr.y (mgr.24961) 4044 : cluster [DBG] pgmap v4829: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:53:59 smithi059 bash[59745]: cluster 2022-04-28T21:53:58.675513+0000 mgr.y (mgr.24961) 4044 : cluster [DBG] pgmap v4829: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:53:59 smithi139 bash[43301]: cluster 2022-04-28T21:53:58.675513+0000 mgr.y (mgr.24961) 4044 : cluster [DBG] pgmap v4829: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:01 smithi059 bash[61152]: cluster 2022-04-28T21:54:00.677485+0000 mgr.y (mgr.24961) 4045 : cluster [DBG] pgmap v4830: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:01 smithi059 bash[59745]: cluster 2022-04-28T21:54:00.677485+0000 mgr.y (mgr.24961) 4045 : cluster [DBG] pgmap v4830: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:01 smithi139 bash[43301]: cluster 2022-04-28T21:54:00.677485+0000 mgr.y (mgr.24961) 4045 : cluster [DBG] pgmap v4830: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:02.111Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:03 smithi059 bash[61152]: cluster 2022-04-28T21:54:02.678787+0000 mgr.y (mgr.24961) 4046 : cluster [DBG] pgmap v4831: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:03 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:03.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:04.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:03 smithi059 bash[59745]: cluster 2022-04-28T21:54:02.678787+0000 mgr.y (mgr.24961) 4046 : cluster [DBG] pgmap v4831: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:04.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:03 smithi139 bash[43301]: cluster 2022-04-28T21:54:02.678787+0000 mgr.y (mgr.24961) 4046 : cluster [DBG] pgmap v4831: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:05 smithi059 bash[61152]: cluster 2022-04-28T21:54:04.680278+0000 mgr.y (mgr.24961) 4047 : cluster [DBG] pgmap v4832: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:05 smithi059 bash[59745]: cluster 2022-04-28T21:54:04.680278+0000 mgr.y (mgr.24961) 4047 : cluster [DBG] pgmap v4832: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:05 smithi139 bash[43301]: cluster 2022-04-28T21:54:04.680278+0000 mgr.y (mgr.24961) 4047 : cluster [DBG] pgmap v4832: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:07 smithi059 bash[61152]: cluster 2022-04-28T21:54:06.682075+0000 mgr.y (mgr.24961) 4048 : cluster [DBG] pgmap v4833: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:07 smithi059 bash[59745]: cluster 2022-04-28T21:54:06.682075+0000 mgr.y (mgr.24961) 4048 : cluster [DBG] pgmap v4833: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:07 smithi139 bash[43301]: cluster 2022-04-28T21:54:06.682075+0000 mgr.y (mgr.24961) 4048 : cluster [DBG] pgmap v4833: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:09 smithi059 bash[61152]: cluster 2022-04-28T21:54:08.682914+0000 mgr.y (mgr.24961) 4049 : cluster [DBG] pgmap v4834: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:10.186 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:09 smithi059 bash[59745]: cluster 2022-04-28T21:54:08.682914+0000 mgr.y (mgr.24961) 4049 : cluster [DBG] pgmap v4834: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:10.187 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:09 smithi139 bash[43301]: cluster 2022-04-28T21:54:08.682914+0000 mgr.y (mgr.24961) 4049 : cluster [DBG] pgmap v4834: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:12.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:11 smithi059 bash[61152]: cluster 2022-04-28T21:54:10.684942+0000 mgr.y (mgr.24961) 4050 : cluster [DBG] pgmap v4835: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:11 smithi059 bash[59745]: cluster 2022-04-28T21:54:10.684942+0000 mgr.y (mgr.24961) 4050 : cluster [DBG] pgmap v4835: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:11 smithi139 bash[43301]: cluster 2022-04-28T21:54:10.684942+0000 mgr.y (mgr.24961) 4050 : cluster [DBG] pgmap v4835: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:12.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:13 smithi059 bash[61152]: cluster 2022-04-28T21:54:12.686240+0000 mgr.y (mgr.24961) 4051 : cluster [DBG] pgmap v4836: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:14.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:13 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:13.996Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:14.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:13 smithi059 bash[59745]: cluster 2022-04-28T21:54:12.686240+0000 mgr.y (mgr.24961) 4051 : cluster [DBG] pgmap v4836: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:13 smithi139 bash[43301]: cluster 2022-04-28T21:54:12.686240+0000 mgr.y (mgr.24961) 4051 : cluster [DBG] pgmap v4836: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:15 smithi059 bash[61152]: cluster 2022-04-28T21:54:14.687747+0000 mgr.y (mgr.24961) 4052 : cluster [DBG] pgmap v4837: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:15 smithi059 bash[59745]: cluster 2022-04-28T21:54:14.687747+0000 mgr.y (mgr.24961) 4052 : cluster [DBG] pgmap v4837: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:15 smithi139 bash[43301]: cluster 2022-04-28T21:54:14.687747+0000 mgr.y (mgr.24961) 4052 : cluster [DBG] pgmap v4837: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:17 smithi059 bash[61152]: cluster 2022-04-28T21:54:16.689520+0000 mgr.y (mgr.24961) 4053 : cluster [DBG] pgmap v4838: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:17 smithi059 bash[59745]: cluster 2022-04-28T21:54:16.689520+0000 mgr.y (mgr.24961) 4053 : cluster [DBG] pgmap v4838: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:17 smithi139 bash[43301]: cluster 2022-04-28T21:54:16.689520+0000 mgr.y (mgr.24961) 4053 : cluster [DBG] pgmap v4838: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: cluster 2022-04-28T21:54:18.690352+0000 mgr.y (mgr.24961) 4054 : cluster [DBG] pgmap v4839: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.409868+0000 mon.a (mon.0) 4157 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:54:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.425899+0000 mon.a (mon.0) 4158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:54:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.480613+0000 mon.a (mon.0) 4159 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:54:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.481269+0000 mon.a (mon.0) 4160 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:54:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.481863+0000 mon.a (mon.0) 4161 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:54:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.482434+0000 mon.a (mon.0) 4162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:54:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.558705+0000 mon.a (mon.0) 4163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:54:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.561708+0000 mon.a (mon.0) 4164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:54:20.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[59745]: audit 2022-04-28T21:54:19.563732+0000 mon.a (mon.0) 4165 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:54:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: cluster 2022-04-28T21:54:18.690352+0000 mgr.y (mgr.24961) 4054 : cluster [DBG] pgmap v4839: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:20.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.409868+0000 mon.a (mon.0) 4157 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:54:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.425899+0000 mon.a (mon.0) 4158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:54:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.480613+0000 mon.a (mon.0) 4159 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:54:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.481269+0000 mon.a (mon.0) 4160 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:54:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.481863+0000 mon.a (mon.0) 4161 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:54:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.482434+0000 mon.a (mon.0) 4162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:54:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.558705+0000 mon.a (mon.0) 4163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:54:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.561708+0000 mon.a (mon.0) 4164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:54:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:19 smithi059 bash[61152]: audit 2022-04-28T21:54:19.563732+0000 mon.a (mon.0) 4165 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:54:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: cluster 2022-04-28T21:54:18.690352+0000 mgr.y (mgr.24961) 4054 : cluster [DBG] pgmap v4839: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:20.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.409868+0000 mon.a (mon.0) 4157 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:54:20.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.425899+0000 mon.a (mon.0) 4158 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:54:20.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.480613+0000 mon.a (mon.0) 4159 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:54:20.152 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.481269+0000 mon.a (mon.0) 4160 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:54:20.153 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.481863+0000 mon.a (mon.0) 4161 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T21:54:20.153 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.482434+0000 mon.a (mon.0) 4162 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T21:54:20.153 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.558705+0000 mon.a (mon.0) 4163 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:54:20.153 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.561708+0000 mon.a (mon.0) 4164 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:54:20.153 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:19 smithi139 bash[43301]: audit 2022-04-28T21:54:19.563732+0000 mon.a (mon.0) 4165 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:54:21.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:20 smithi059 bash[61152]: cephadm 2022-04-28T21:54:19.565509+0000 mgr.y (mgr.24961) 4055 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:54:21.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:20 smithi059 bash[61152]: audit 2022-04-28T21:54:19.900109+0000 mon.a (mon.0) 4166 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:54:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:20 smithi059 bash[59745]: cephadm 2022-04-28T21:54:19.565509+0000 mgr.y (mgr.24961) 4055 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:54:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:20 smithi059 bash[59745]: audit 2022-04-28T21:54:19.900109+0000 mon.a (mon.0) 4166 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:54:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:20 smithi139 bash[43301]: cephadm 2022-04-28T21:54:19.565509+0000 mgr.y (mgr.24961) 4055 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:54:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:20 smithi139 bash[43301]: audit 2022-04-28T21:54:19.900109+0000 mon.a (mon.0) 4166 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:54:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:21 smithi059 bash[61152]: cluster 2022-04-28T21:54:20.692337+0000 mgr.y (mgr.24961) 4056 : cluster [DBG] pgmap v4840: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:22.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:21 smithi059 bash[59745]: cluster 2022-04-28T21:54:20.692337+0000 mgr.y (mgr.24961) 4056 : cluster [DBG] pgmap v4840: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:21 smithi139 bash[43301]: cluster 2022-04-28T21:54:20.692337+0000 mgr.y (mgr.24961) 4056 : cluster [DBG] pgmap v4840: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:22.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:22.112Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:23 smithi059 bash[61152]: cluster 2022-04-28T21:54:22.693600+0000 mgr.y (mgr.24961) 4057 : cluster [DBG] pgmap v4841: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:24.093 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:23 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:23.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:23 smithi059 bash[59745]: cluster 2022-04-28T21:54:22.693600+0000 mgr.y (mgr.24961) 4057 : cluster [DBG] pgmap v4841: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:23 smithi139 bash[43301]: cluster 2022-04-28T21:54:22.693600+0000 mgr.y (mgr.24961) 4057 : cluster [DBG] pgmap v4841: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:25 smithi059 bash[61152]: cluster 2022-04-28T21:54:24.695019+0000 mgr.y (mgr.24961) 4058 : cluster [DBG] pgmap v4842: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:25 smithi059 bash[59745]: cluster 2022-04-28T21:54:24.695019+0000 mgr.y (mgr.24961) 4058 : cluster [DBG] pgmap v4842: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:25 smithi139 bash[43301]: cluster 2022-04-28T21:54:24.695019+0000 mgr.y (mgr.24961) 4058 : cluster [DBG] pgmap v4842: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:27 smithi059 bash[61152]: cluster 2022-04-28T21:54:26.696846+0000 mgr.y (mgr.24961) 4059 : cluster [DBG] pgmap v4843: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:27 smithi059 bash[59745]: cluster 2022-04-28T21:54:26.696846+0000 mgr.y (mgr.24961) 4059 : cluster [DBG] pgmap v4843: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:27 smithi139 bash[43301]: cluster 2022-04-28T21:54:26.696846+0000 mgr.y (mgr.24961) 4059 : cluster [DBG] pgmap v4843: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:29 smithi059 bash[61152]: cluster 2022-04-28T21:54:28.697690+0000 mgr.y (mgr.24961) 4060 : cluster [DBG] pgmap v4844: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:29 smithi059 bash[59745]: cluster 2022-04-28T21:54:28.697690+0000 mgr.y (mgr.24961) 4060 : cluster [DBG] pgmap v4844: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:29 smithi139 bash[43301]: cluster 2022-04-28T21:54:28.697690+0000 mgr.y (mgr.24961) 4060 : cluster [DBG] pgmap v4844: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:31 smithi059 bash[59745]: cluster 2022-04-28T21:54:30.699707+0000 mgr.y (mgr.24961) 4061 : cluster [DBG] pgmap v4845: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:31 smithi059 bash[61152]: cluster 2022-04-28T21:54:30.699707+0000 mgr.y (mgr.24961) 4061 : cluster [DBG] pgmap v4845: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:32.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:31 smithi139 bash[43301]: cluster 2022-04-28T21:54:30.699707+0000 mgr.y (mgr.24961) 4061 : cluster [DBG] pgmap v4845: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:32.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:33 smithi059 bash[61152]: cluster 2022-04-28T21:54:32.700967+0000 mgr.y (mgr.24961) 4062 : cluster [DBG] pgmap v4846: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:33 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:33.998Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:33 smithi059 bash[59745]: cluster 2022-04-28T21:54:32.700967+0000 mgr.y (mgr.24961) 4062 : cluster [DBG] pgmap v4846: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:34.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:33 smithi139 bash[43301]: cluster 2022-04-28T21:54:32.700967+0000 mgr.y (mgr.24961) 4062 : cluster [DBG] pgmap v4846: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:35 smithi059 bash[61152]: cluster 2022-04-28T21:54:34.702424+0000 mgr.y (mgr.24961) 4063 : cluster [DBG] pgmap v4847: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:35 smithi059 bash[59745]: cluster 2022-04-28T21:54:34.702424+0000 mgr.y (mgr.24961) 4063 : cluster [DBG] pgmap v4847: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:35 smithi139 bash[43301]: cluster 2022-04-28T21:54:34.702424+0000 mgr.y (mgr.24961) 4063 : cluster [DBG] pgmap v4847: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:37 smithi059 bash[61152]: cluster 2022-04-28T21:54:36.704226+0000 mgr.y (mgr.24961) 4064 : cluster [DBG] pgmap v4848: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:37 smithi059 bash[59745]: cluster 2022-04-28T21:54:36.704226+0000 mgr.y (mgr.24961) 4064 : cluster [DBG] pgmap v4848: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:37 smithi139 bash[43301]: cluster 2022-04-28T21:54:36.704226+0000 mgr.y (mgr.24961) 4064 : cluster [DBG] pgmap v4848: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:39 smithi059 bash[61152]: cluster 2022-04-28T21:54:38.705068+0000 mgr.y (mgr.24961) 4065 : cluster [DBG] pgmap v4849: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:39 smithi059 bash[59745]: cluster 2022-04-28T21:54:38.705068+0000 mgr.y (mgr.24961) 4065 : cluster [DBG] pgmap v4849: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:40.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:39 smithi139 bash[43301]: cluster 2022-04-28T21:54:38.705068+0000 mgr.y (mgr.24961) 4065 : cluster [DBG] pgmap v4849: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:42.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:41 smithi059 bash[61152]: cluster 2022-04-28T21:54:40.707029+0000 mgr.y (mgr.24961) 4066 : cluster [DBG] pgmap v4850: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:42.121 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:41 smithi059 bash[59745]: cluster 2022-04-28T21:54:40.707029+0000 mgr.y (mgr.24961) 4066 : cluster [DBG] pgmap v4850: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:42.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:41 smithi139 bash[43301]: cluster 2022-04-28T21:54:40.707029+0000 mgr.y (mgr.24961) 4066 : cluster [DBG] pgmap v4850: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:42.113Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:43 smithi059 bash[61152]: cluster 2022-04-28T21:54:42.708357+0000 mgr.y (mgr.24961) 4067 : cluster [DBG] pgmap v4851: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:44 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:44.000Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:43 smithi059 bash[59745]: cluster 2022-04-28T21:54:42.708357+0000 mgr.y (mgr.24961) 4067 : cluster [DBG] pgmap v4851: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:43 smithi139 bash[43301]: cluster 2022-04-28T21:54:42.708357+0000 mgr.y (mgr.24961) 4067 : cluster [DBG] pgmap v4851: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:45 smithi059 bash[61152]: cluster 2022-04-28T21:54:44.709757+0000 mgr.y (mgr.24961) 4068 : cluster [DBG] pgmap v4852: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:45 smithi059 bash[59745]: cluster 2022-04-28T21:54:44.709757+0000 mgr.y (mgr.24961) 4068 : cluster [DBG] pgmap v4852: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:45 smithi139 bash[43301]: cluster 2022-04-28T21:54:44.709757+0000 mgr.y (mgr.24961) 4068 : cluster [DBG] pgmap v4852: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:48.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:47 smithi059 bash[59745]: cluster 2022-04-28T21:54:46.711565+0000 mgr.y (mgr.24961) 4069 : cluster [DBG] pgmap v4853: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:48.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:47 smithi059 bash[61152]: cluster 2022-04-28T21:54:46.711565+0000 mgr.y (mgr.24961) 4069 : cluster [DBG] pgmap v4853: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:48.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:47 smithi139 bash[43301]: cluster 2022-04-28T21:54:46.711565+0000 mgr.y (mgr.24961) 4069 : cluster [DBG] pgmap v4853: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:50.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:49 smithi059 bash[61152]: cluster 2022-04-28T21:54:48.712393+0000 mgr.y (mgr.24961) 4070 : cluster [DBG] pgmap v4854: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:50.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:49 smithi059 bash[59745]: cluster 2022-04-28T21:54:48.712393+0000 mgr.y (mgr.24961) 4070 : cluster [DBG] pgmap v4854: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:50.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:49 smithi139 bash[43301]: cluster 2022-04-28T21:54:48.712393+0000 mgr.y (mgr.24961) 4070 : cluster [DBG] pgmap v4854: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:52.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:51 smithi059 bash[61152]: cluster 2022-04-28T21:54:50.714318+0000 mgr.y (mgr.24961) 4071 : cluster [DBG] pgmap v4855: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:51 smithi059 bash[59745]: cluster 2022-04-28T21:54:50.714318+0000 mgr.y (mgr.24961) 4071 : cluster [DBG] pgmap v4855: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:51 smithi139 bash[43301]: cluster 2022-04-28T21:54:50.714318+0000 mgr.y (mgr.24961) 4071 : cluster [DBG] pgmap v4855: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:52.115Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:54.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:53 smithi059 bash[61152]: cluster 2022-04-28T21:54:52.715612+0000 mgr.y (mgr.24961) 4072 : cluster [DBG] pgmap v4856: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:54.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:54:54 smithi059 bash[72035]: level=error ts=2022-04-28T21:54:54.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:54:54.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:53 smithi059 bash[59745]: cluster 2022-04-28T21:54:52.715612+0000 mgr.y (mgr.24961) 4072 : cluster [DBG] pgmap v4856: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:54.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:53 smithi139 bash[43301]: cluster 2022-04-28T21:54:52.715612+0000 mgr.y (mgr.24961) 4072 : cluster [DBG] pgmap v4856: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:55 smithi059 bash[59745]: cluster 2022-04-28T21:54:54.717027+0000 mgr.y (mgr.24961) 4073 : cluster [DBG] pgmap v4857: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:55.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:55 smithi059 bash[61152]: cluster 2022-04-28T21:54:54.717027+0000 mgr.y (mgr.24961) 4073 : cluster [DBG] pgmap v4857: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:55 smithi139 bash[43301]: cluster 2022-04-28T21:54:54.717027+0000 mgr.y (mgr.24961) 4073 : cluster [DBG] pgmap v4857: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:57 smithi059 bash[61152]: cluster 2022-04-28T21:54:56.718762+0000 mgr.y (mgr.24961) 4074 : cluster [DBG] pgmap v4858: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:57 smithi059 bash[59745]: cluster 2022-04-28T21:54:56.718762+0000 mgr.y (mgr.24961) 4074 : cluster [DBG] pgmap v4858: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:54:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:57 smithi139 bash[43301]: cluster 2022-04-28T21:54:56.718762+0000 mgr.y (mgr.24961) 4074 : cluster [DBG] pgmap v4858: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:00.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:54:59 smithi059 bash[61152]: cluster 2022-04-28T21:54:58.719605+0000 mgr.y (mgr.24961) 4075 : cluster [DBG] pgmap v4859: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:00.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:54:59 smithi059 bash[59745]: cluster 2022-04-28T21:54:58.719605+0000 mgr.y (mgr.24961) 4075 : cluster [DBG] pgmap v4859: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:54:59 smithi139 bash[43301]: cluster 2022-04-28T21:54:58.719605+0000 mgr.y (mgr.24961) 4075 : cluster [DBG] pgmap v4859: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:02.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:01 smithi059 bash[61152]: cluster 2022-04-28T21:55:00.721556+0000 mgr.y (mgr.24961) 4076 : cluster [DBG] pgmap v4860: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:02.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:01 smithi059 bash[59745]: cluster 2022-04-28T21:55:00.721556+0000 mgr.y (mgr.24961) 4076 : cluster [DBG] pgmap v4860: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:01 smithi139 bash[43301]: cluster 2022-04-28T21:55:00.721556+0000 mgr.y (mgr.24961) 4076 : cluster [DBG] pgmap v4860: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:02.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:04.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:03 smithi059 bash[61152]: cluster 2022-04-28T21:55:02.722829+0000 mgr.y (mgr.24961) 4077 : cluster [DBG] pgmap v4861: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:04.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:04 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:04.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:04.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:03 smithi059 bash[59745]: cluster 2022-04-28T21:55:02.722829+0000 mgr.y (mgr.24961) 4077 : cluster [DBG] pgmap v4861: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:04.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:03 smithi139 bash[43301]: cluster 2022-04-28T21:55:02.722829+0000 mgr.y (mgr.24961) 4077 : cluster [DBG] pgmap v4861: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:05 smithi059 bash[59745]: cluster 2022-04-28T21:55:04.724254+0000 mgr.y (mgr.24961) 4078 : cluster [DBG] pgmap v4862: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:05.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:05 smithi059 bash[61152]: cluster 2022-04-28T21:55:04.724254+0000 mgr.y (mgr.24961) 4078 : cluster [DBG] pgmap v4862: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:05 smithi139 bash[43301]: cluster 2022-04-28T21:55:04.724254+0000 mgr.y (mgr.24961) 4078 : cluster [DBG] pgmap v4862: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:08.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:07 smithi059 bash[61152]: cluster 2022-04-28T21:55:06.726044+0000 mgr.y (mgr.24961) 4079 : cluster [DBG] pgmap v4863: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:08.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:07 smithi059 bash[59745]: cluster 2022-04-28T21:55:06.726044+0000 mgr.y (mgr.24961) 4079 : cluster [DBG] pgmap v4863: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:07 smithi139 bash[43301]: cluster 2022-04-28T21:55:06.726044+0000 mgr.y (mgr.24961) 4079 : cluster [DBG] pgmap v4863: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:09 smithi059 bash[61152]: cluster 2022-04-28T21:55:08.726874+0000 mgr.y (mgr.24961) 4080 : cluster [DBG] pgmap v4864: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:09 smithi059 bash[59745]: cluster 2022-04-28T21:55:08.726874+0000 mgr.y (mgr.24961) 4080 : cluster [DBG] pgmap v4864: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:09 smithi139 bash[43301]: cluster 2022-04-28T21:55:08.726874+0000 mgr.y (mgr.24961) 4080 : cluster [DBG] pgmap v4864: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:12.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:11 smithi059 bash[61152]: cluster 2022-04-28T21:55:10.728948+0000 mgr.y (mgr.24961) 4081 : cluster [DBG] pgmap v4865: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:12.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:11 smithi059 bash[59745]: cluster 2022-04-28T21:55:10.728948+0000 mgr.y (mgr.24961) 4081 : cluster [DBG] pgmap v4865: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:12.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:11 smithi139 bash[43301]: cluster 2022-04-28T21:55:10.728948+0000 mgr.y (mgr.24961) 4081 : cluster [DBG] pgmap v4865: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:12.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:14.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:13 smithi059 bash[61152]: cluster 2022-04-28T21:55:12.730263+0000 mgr.y (mgr.24961) 4082 : cluster [DBG] pgmap v4866: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:14 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:14.002Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:13 smithi059 bash[59745]: cluster 2022-04-28T21:55:12.730263+0000 mgr.y (mgr.24961) 4082 : cluster [DBG] pgmap v4866: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:13 smithi139 bash[43301]: cluster 2022-04-28T21:55:12.730263+0000 mgr.y (mgr.24961) 4082 : cluster [DBG] pgmap v4866: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:15.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:15 smithi059 bash[59745]: cluster 2022-04-28T21:55:14.731731+0000 mgr.y (mgr.24961) 4083 : cluster [DBG] pgmap v4867: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:15.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:15 smithi059 bash[61152]: cluster 2022-04-28T21:55:14.731731+0000 mgr.y (mgr.24961) 4083 : cluster [DBG] pgmap v4867: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:15 smithi139 bash[43301]: cluster 2022-04-28T21:55:14.731731+0000 mgr.y (mgr.24961) 4083 : cluster [DBG] pgmap v4867: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:17 smithi059 bash[61152]: cluster 2022-04-28T21:55:16.733497+0000 mgr.y (mgr.24961) 4084 : cluster [DBG] pgmap v4868: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:17 smithi059 bash[59745]: cluster 2022-04-28T21:55:16.733497+0000 mgr.y (mgr.24961) 4084 : cluster [DBG] pgmap v4868: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:17 smithi139 bash[43301]: cluster 2022-04-28T21:55:16.733497+0000 mgr.y (mgr.24961) 4084 : cluster [DBG] pgmap v4868: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:20.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: cluster 2022-04-28T21:55:18.734340+0000 mgr.y (mgr.24961) 4085 : cluster [DBG] pgmap v4869: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: audit 2022-04-28T21:55:19.415089+0000 mon.a (mon.0) 4167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:55:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: audit 2022-04-28T21:55:19.432289+0000 mon.a (mon.0) 4168 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:55:20.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: audit 2022-04-28T21:55:19.490608+0000 mon.a (mon.0) 4169 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:55:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: audit 2022-04-28T21:55:19.491146+0000 mon.a (mon.0) 4170 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:55:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: audit 2022-04-28T21:55:19.491715+0000 mon.a (mon.0) 4171 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:55:20.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[59745]: audit 2022-04-28T21:55:19.492237+0000 mon.a (mon.0) 4172 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:55:20.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: cluster 2022-04-28T21:55:18.734340+0000 mgr.y (mgr.24961) 4085 : cluster [DBG] pgmap v4869: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: audit 2022-04-28T21:55:19.415089+0000 mon.a (mon.0) 4167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:55:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: audit 2022-04-28T21:55:19.432289+0000 mon.a (mon.0) 4168 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:55:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: audit 2022-04-28T21:55:19.490608+0000 mon.a (mon.0) 4169 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:55:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: audit 2022-04-28T21:55:19.491146+0000 mon.a (mon.0) 4170 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:55:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: audit 2022-04-28T21:55:19.491715+0000 mon.a (mon.0) 4171 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:55:20.096 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:19 smithi059 bash[61152]: audit 2022-04-28T21:55:19.492237+0000 mon.a (mon.0) 4172 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:55:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: cluster 2022-04-28T21:55:18.734340+0000 mgr.y (mgr.24961) 4085 : cluster [DBG] pgmap v4869: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: audit 2022-04-28T21:55:19.415089+0000 mon.a (mon.0) 4167 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:55:20.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: audit 2022-04-28T21:55:19.432289+0000 mon.a (mon.0) 4168 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:55:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: audit 2022-04-28T21:55:19.490608+0000 mon.a (mon.0) 4169 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T21:55:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: audit 2022-04-28T21:55:19.491146+0000 mon.a (mon.0) 4170 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T21:55:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: audit 2022-04-28T21:55:19.491715+0000 mon.a (mon.0) 4171 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2022-04-28T21:55:20.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:19 smithi139 bash[43301]: audit 2022-04-28T21:55:19.492237+0000 mon.a (mon.0) 4172 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.2", "id": [4, 0]}]: dispatch 2022-04-28T21:55:21.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[59745]: audit 2022-04-28T21:55:19.907661+0000 mon.a (mon.0) 4173 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:55:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[59745]: audit 2022-04-28T21:55:19.910561+0000 mon.a (mon.0) 4174 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:55:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[59745]: audit 2022-04-28T21:55:19.912436+0000 mon.a (mon.0) 4175 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:55:21.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[59745]: audit 2022-04-28T21:55:20.224857+0000 mon.a (mon.0) 4176 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:55:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[61152]: audit 2022-04-28T21:55:19.907661+0000 mon.a (mon.0) 4173 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:55:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[61152]: audit 2022-04-28T21:55:19.910561+0000 mon.a (mon.0) 4174 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:55:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[61152]: audit 2022-04-28T21:55:19.912436+0000 mon.a (mon.0) 4175 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:55:21.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:20 smithi059 bash[61152]: audit 2022-04-28T21:55:20.224857+0000 mon.a (mon.0) 4176 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:55:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:20 smithi139 bash[43301]: audit 2022-04-28T21:55:19.907661+0000 mon.a (mon.0) 4173 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:55:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:20 smithi139 bash[43301]: audit 2022-04-28T21:55:19.910561+0000 mon.a (mon.0) 4174 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:55:21.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:20 smithi139 bash[43301]: audit 2022-04-28T21:55:19.912436+0000 mon.a (mon.0) 4175 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:55:21.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:20 smithi139 bash[43301]: audit 2022-04-28T21:55:20.224857+0000 mon.a (mon.0) 4176 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:55:22.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:21 smithi059 bash[61152]: cephadm 2022-04-28T21:55:19.914146+0000 mgr.y (mgr.24961) 4086 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:55:22.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:21 smithi059 bash[61152]: cluster 2022-04-28T21:55:20.736327+0000 mgr.y (mgr.24961) 4087 : cluster [DBG] pgmap v4870: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:22.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:21 smithi059 bash[59745]: cephadm 2022-04-28T21:55:19.914146+0000 mgr.y (mgr.24961) 4086 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:55:22.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:21 smithi059 bash[59745]: cluster 2022-04-28T21:55:20.736327+0000 mgr.y (mgr.24961) 4087 : cluster [DBG] pgmap v4870: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:21 smithi139 bash[43301]: cephadm 2022-04-28T21:55:19.914146+0000 mgr.y (mgr.24961) 4086 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:55:22.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:21 smithi139 bash[43301]: cluster 2022-04-28T21:55:20.736327+0000 mgr.y (mgr.24961) 4087 : cluster [DBG] pgmap v4870: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:22.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:22.116Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:23 smithi059 bash[61152]: cluster 2022-04-28T21:55:22.737591+0000 mgr.y (mgr.24961) 4088 : cluster [DBG] pgmap v4871: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:24 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:24.003Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:23 smithi059 bash[59745]: cluster 2022-04-28T21:55:22.737591+0000 mgr.y (mgr.24961) 4088 : cluster [DBG] pgmap v4871: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:23 smithi139 bash[43301]: cluster 2022-04-28T21:55:22.737591+0000 mgr.y (mgr.24961) 4088 : cluster [DBG] pgmap v4871: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:25 smithi059 bash[61152]: cluster 2022-04-28T21:55:24.739222+0000 mgr.y (mgr.24961) 4089 : cluster [DBG] pgmap v4872: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:25 smithi059 bash[59745]: cluster 2022-04-28T21:55:24.739222+0000 mgr.y (mgr.24961) 4089 : cluster [DBG] pgmap v4872: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:25.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:25 smithi139 bash[43301]: cluster 2022-04-28T21:55:24.739222+0000 mgr.y (mgr.24961) 4089 : cluster [DBG] pgmap v4872: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:28.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:27 smithi059 bash[61152]: cluster 2022-04-28T21:55:26.741022+0000 mgr.y (mgr.24961) 4090 : cluster [DBG] pgmap v4873: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:28.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:27 smithi059 bash[59745]: cluster 2022-04-28T21:55:26.741022+0000 mgr.y (mgr.24961) 4090 : cluster [DBG] pgmap v4873: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:28.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:27 smithi139 bash[43301]: cluster 2022-04-28T21:55:26.741022+0000 mgr.y (mgr.24961) 4090 : cluster [DBG] pgmap v4873: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:30.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:29 smithi059 bash[61152]: cluster 2022-04-28T21:55:28.741843+0000 mgr.y (mgr.24961) 4091 : cluster [DBG] pgmap v4874: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:30.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:29 smithi059 bash[59745]: cluster 2022-04-28T21:55:28.741843+0000 mgr.y (mgr.24961) 4091 : cluster [DBG] pgmap v4874: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:30.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:29 smithi139 bash[43301]: cluster 2022-04-28T21:55:28.741843+0000 mgr.y (mgr.24961) 4091 : cluster [DBG] pgmap v4874: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:32.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:31 smithi059 bash[61152]: cluster 2022-04-28T21:55:30.743910+0000 mgr.y (mgr.24961) 4092 : cluster [DBG] pgmap v4875: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:32.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:31 smithi059 bash[59745]: cluster 2022-04-28T21:55:30.743910+0000 mgr.y (mgr.24961) 4092 : cluster [DBG] pgmap v4875: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:32.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:31 smithi139 bash[43301]: cluster 2022-04-28T21:55:30.743910+0000 mgr.y (mgr.24961) 4092 : cluster [DBG] pgmap v4875: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:32.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:32.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:34.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:33 smithi059 bash[61152]: cluster 2022-04-28T21:55:32.745196+0000 mgr.y (mgr.24961) 4093 : cluster [DBG] pgmap v4876: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:34.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:34 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:34.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:34.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:33 smithi059 bash[59745]: cluster 2022-04-28T21:55:32.745196+0000 mgr.y (mgr.24961) 4093 : cluster [DBG] pgmap v4876: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:34.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:33 smithi139 bash[43301]: cluster 2022-04-28T21:55:32.745196+0000 mgr.y (mgr.24961) 4093 : cluster [DBG] pgmap v4876: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:35 smithi059 bash[61152]: cluster 2022-04-28T21:55:34.746677+0000 mgr.y (mgr.24961) 4094 : cluster [DBG] pgmap v4877: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:35 smithi059 bash[59745]: cluster 2022-04-28T21:55:34.746677+0000 mgr.y (mgr.24961) 4094 : cluster [DBG] pgmap v4877: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:35 smithi139 bash[43301]: cluster 2022-04-28T21:55:34.746677+0000 mgr.y (mgr.24961) 4094 : cluster [DBG] pgmap v4877: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:38.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:37 smithi059 bash[61152]: cluster 2022-04-28T21:55:36.748678+0000 mgr.y (mgr.24961) 4095 : cluster [DBG] pgmap v4878: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:38.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:37 smithi059 bash[59745]: cluster 2022-04-28T21:55:36.748678+0000 mgr.y (mgr.24961) 4095 : cluster [DBG] pgmap v4878: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:38.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:37 smithi139 bash[43301]: cluster 2022-04-28T21:55:36.748678+0000 mgr.y (mgr.24961) 4095 : cluster [DBG] pgmap v4878: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:40.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:39 smithi059 bash[61152]: cluster 2022-04-28T21:55:38.749509+0000 mgr.y (mgr.24961) 4096 : cluster [DBG] pgmap v4879: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:40.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:39 smithi059 bash[59745]: cluster 2022-04-28T21:55:38.749509+0000 mgr.y (mgr.24961) 4096 : cluster [DBG] pgmap v4879: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:40.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:39 smithi139 bash[43301]: cluster 2022-04-28T21:55:38.749509+0000 mgr.y (mgr.24961) 4096 : cluster [DBG] pgmap v4879: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:42.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:41 smithi059 bash[59745]: cluster 2022-04-28T21:55:40.751433+0000 mgr.y (mgr.24961) 4097 : cluster [DBG] pgmap v4880: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:42.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:41 smithi059 bash[61152]: cluster 2022-04-28T21:55:40.751433+0000 mgr.y (mgr.24961) 4097 : cluster [DBG] pgmap v4880: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:42.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:41 smithi139 bash[43301]: cluster 2022-04-28T21:55:40.751433+0000 mgr.y (mgr.24961) 4097 : cluster [DBG] pgmap v4880: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:42.117Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:44.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:43 smithi059 bash[61152]: cluster 2022-04-28T21:55:42.752824+0000 mgr.y (mgr.24961) 4098 : cluster [DBG] pgmap v4881: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:44.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:44 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:44.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:44.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:43 smithi059 bash[59745]: cluster 2022-04-28T21:55:42.752824+0000 mgr.y (mgr.24961) 4098 : cluster [DBG] pgmap v4881: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:44.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:43 smithi139 bash[43301]: cluster 2022-04-28T21:55:42.752824+0000 mgr.y (mgr.24961) 4098 : cluster [DBG] pgmap v4881: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:45 smithi059 bash[61152]: cluster 2022-04-28T21:55:44.754247+0000 mgr.y (mgr.24961) 4099 : cluster [DBG] pgmap v4882: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:45 smithi059 bash[59745]: cluster 2022-04-28T21:55:44.754247+0000 mgr.y (mgr.24961) 4099 : cluster [DBG] pgmap v4882: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:45 smithi139 bash[43301]: cluster 2022-04-28T21:55:44.754247+0000 mgr.y (mgr.24961) 4099 : cluster [DBG] pgmap v4882: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:48.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:47 smithi059 bash[61152]: cluster 2022-04-28T21:55:46.756020+0000 mgr.y (mgr.24961) 4100 : cluster [DBG] pgmap v4883: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:48.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:47 smithi059 bash[59745]: cluster 2022-04-28T21:55:46.756020+0000 mgr.y (mgr.24961) 4100 : cluster [DBG] pgmap v4883: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:48.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:47 smithi139 bash[43301]: cluster 2022-04-28T21:55:46.756020+0000 mgr.y (mgr.24961) 4100 : cluster [DBG] pgmap v4883: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:50.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:49 smithi059 bash[61152]: cluster 2022-04-28T21:55:48.756864+0000 mgr.y (mgr.24961) 4101 : cluster [DBG] pgmap v4884: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:50.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:49 smithi059 bash[59745]: cluster 2022-04-28T21:55:48.756864+0000 mgr.y (mgr.24961) 4101 : cluster [DBG] pgmap v4884: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:50.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:49 smithi139 bash[43301]: cluster 2022-04-28T21:55:48.756864+0000 mgr.y (mgr.24961) 4101 : cluster [DBG] pgmap v4884: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:52.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:51 smithi059 bash[61152]: cluster 2022-04-28T21:55:50.758769+0000 mgr.y (mgr.24961) 4102 : cluster [DBG] pgmap v4885: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:52.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:51 smithi059 bash[59745]: cluster 2022-04-28T21:55:50.758769+0000 mgr.y (mgr.24961) 4102 : cluster [DBG] pgmap v4885: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:52.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:51 smithi139 bash[43301]: cluster 2022-04-28T21:55:50.758769+0000 mgr.y (mgr.24961) 4102 : cluster [DBG] pgmap v4885: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:52.118Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:54.094 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:53 smithi059 bash[61152]: cluster 2022-04-28T21:55:52.760095+0000 mgr.y (mgr.24961) 4103 : cluster [DBG] pgmap v4886: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:54.095 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:55:54 smithi059 bash[72035]: level=error ts=2022-04-28T21:55:54.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:55:54.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:53 smithi059 bash[59745]: cluster 2022-04-28T21:55:52.760095+0000 mgr.y (mgr.24961) 4103 : cluster [DBG] pgmap v4886: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:54.122 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:53 smithi139 bash[43301]: cluster 2022-04-28T21:55:52.760095+0000 mgr.y (mgr.24961) 4103 : cluster [DBG] pgmap v4886: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:55 smithi059 bash[61152]: cluster 2022-04-28T21:55:54.761520+0000 mgr.y (mgr.24961) 4104 : cluster [DBG] pgmap v4887: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:55 smithi059 bash[59745]: cluster 2022-04-28T21:55:54.761520+0000 mgr.y (mgr.24961) 4104 : cluster [DBG] pgmap v4887: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:55 smithi139 bash[43301]: cluster 2022-04-28T21:55:54.761520+0000 mgr.y (mgr.24961) 4104 : cluster [DBG] pgmap v4887: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:58.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:57 smithi059 bash[61152]: cluster 2022-04-28T21:55:56.763311+0000 mgr.y (mgr.24961) 4105 : cluster [DBG] pgmap v4888: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:58.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:57 smithi059 bash[59745]: cluster 2022-04-28T21:55:56.763311+0000 mgr.y (mgr.24961) 4105 : cluster [DBG] pgmap v4888: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:55:58.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:57 smithi139 bash[43301]: cluster 2022-04-28T21:55:56.763311+0000 mgr.y (mgr.24961) 4105 : cluster [DBG] pgmap v4888: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:00.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:55:59 smithi059 bash[61152]: cluster 2022-04-28T21:55:58.764198+0000 mgr.y (mgr.24961) 4106 : cluster [DBG] pgmap v4889: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:00.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:55:59 smithi059 bash[59745]: cluster 2022-04-28T21:55:58.764198+0000 mgr.y (mgr.24961) 4106 : cluster [DBG] pgmap v4889: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:00.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:55:59 smithi139 bash[43301]: cluster 2022-04-28T21:55:58.764198+0000 mgr.y (mgr.24961) 4106 : cluster [DBG] pgmap v4889: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:02.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:01 smithi059 bash[61152]: cluster 2022-04-28T21:56:00.766265+0000 mgr.y (mgr.24961) 4107 : cluster [DBG] pgmap v4890: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:02.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:01 smithi059 bash[59745]: cluster 2022-04-28T21:56:00.766265+0000 mgr.y (mgr.24961) 4107 : cluster [DBG] pgmap v4890: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:02.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:01 smithi139 bash[43301]: cluster 2022-04-28T21:56:00.766265+0000 mgr.y (mgr.24961) 4107 : cluster [DBG] pgmap v4890: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:02.595 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:02.119Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:03.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:02 smithi059 bash[61152]: cluster 2022-04-28T21:56:02.767572+0000 mgr.y (mgr.24961) 4108 : cluster [DBG] pgmap v4891: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:03.379 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:02 smithi059 bash[59745]: cluster 2022-04-28T21:56:02.767572+0000 mgr.y (mgr.24961) 4108 : cluster [DBG] pgmap v4891: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:03.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:02 smithi139 bash[43301]: cluster 2022-04-28T21:56:02.767572+0000 mgr.y (mgr.24961) 4108 : cluster [DBG] pgmap v4891: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:04 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:04.004Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:05.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:05 smithi059 bash[61152]: cluster 2022-04-28T21:56:04.768991+0000 mgr.y (mgr.24961) 4109 : cluster [DBG] pgmap v4892: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:05.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:05 smithi059 bash[59745]: cluster 2022-04-28T21:56:04.768991+0000 mgr.y (mgr.24961) 4109 : cluster [DBG] pgmap v4892: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:05 smithi139 bash[43301]: cluster 2022-04-28T21:56:04.768991+0000 mgr.y (mgr.24961) 4109 : cluster [DBG] pgmap v4892: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:08.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:07 smithi059 bash[61152]: cluster 2022-04-28T21:56:06.770799+0000 mgr.y (mgr.24961) 4110 : cluster [DBG] pgmap v4893: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:08.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:07 smithi059 bash[59745]: cluster 2022-04-28T21:56:06.770799+0000 mgr.y (mgr.24961) 4110 : cluster [DBG] pgmap v4893: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:08.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:07 smithi139 bash[43301]: cluster 2022-04-28T21:56:06.770799+0000 mgr.y (mgr.24961) 4110 : cluster [DBG] pgmap v4893: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:10.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:09 smithi059 bash[61152]: cluster 2022-04-28T21:56:08.771610+0000 mgr.y (mgr.24961) 4111 : cluster [DBG] pgmap v4894: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:10.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:09 smithi059 bash[59745]: cluster 2022-04-28T21:56:08.771610+0000 mgr.y (mgr.24961) 4111 : cluster [DBG] pgmap v4894: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:10.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:09 smithi139 bash[43301]: cluster 2022-04-28T21:56:08.771610+0000 mgr.y (mgr.24961) 4111 : cluster [DBG] pgmap v4894: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:10 smithi059 bash[61152]: cluster 2022-04-28T21:56:10.773543+0000 mgr.y (mgr.24961) 4112 : cluster [DBG] pgmap v4895: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:10 smithi059 bash[59745]: cluster 2022-04-28T21:56:10.773543+0000 mgr.y (mgr.24961) 4112 : cluster [DBG] pgmap v4895: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:11.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:10 smithi139 bash[43301]: cluster 2022-04-28T21:56:10.773543+0000 mgr.y (mgr.24961) 4112 : cluster [DBG] pgmap v4895: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:12.595 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:12.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:14.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:13 smithi059 bash[59745]: cluster 2022-04-28T21:56:12.774790+0000 mgr.y (mgr.24961) 4113 : cluster [DBG] pgmap v4896: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:14.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:14 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:14.006Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:14.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:13 smithi059 bash[61152]: cluster 2022-04-28T21:56:12.774790+0000 mgr.y (mgr.24961) 4113 : cluster [DBG] pgmap v4896: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:14.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:13 smithi139 bash[43301]: cluster 2022-04-28T21:56:12.774790+0000 mgr.y (mgr.24961) 4113 : cluster [DBG] pgmap v4896: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:15 smithi059 bash[61152]: cluster 2022-04-28T21:56:14.776291+0000 mgr.y (mgr.24961) 4114 : cluster [DBG] pgmap v4897: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:15 smithi059 bash[59745]: cluster 2022-04-28T21:56:14.776291+0000 mgr.y (mgr.24961) 4114 : cluster [DBG] pgmap v4897: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:15 smithi139 bash[43301]: cluster 2022-04-28T21:56:14.776291+0000 mgr.y (mgr.24961) 4114 : cluster [DBG] pgmap v4897: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:18.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:17 smithi059 bash[61152]: cluster 2022-04-28T21:56:16.778035+0000 mgr.y (mgr.24961) 4115 : cluster [DBG] pgmap v4898: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:18.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:17 smithi059 bash[59745]: cluster 2022-04-28T21:56:16.778035+0000 mgr.y (mgr.24961) 4115 : cluster [DBG] pgmap v4898: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:18.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:17 smithi139 bash[43301]: cluster 2022-04-28T21:56:16.778035+0000 mgr.y (mgr.24961) 4115 : cluster [DBG] pgmap v4898: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:18 smithi139 bash[43301]: cluster 2022-04-28T21:56:18.778853+0000 mgr.y (mgr.24961) 4116 : cluster [DBG] pgmap v4899: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:18 smithi059 bash[61152]: cluster 2022-04-28T21:56:18.778853+0000 mgr.y (mgr.24961) 4116 : cluster [DBG] pgmap v4899: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:18 smithi059 bash[59745]: cluster 2022-04-28T21:56:18.778853+0000 mgr.y (mgr.24961) 4116 : cluster [DBG] pgmap v4899: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:20.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[59745]: audit 2022-04-28T21:56:19.420308+0000 mon.a (mon.0) 4177 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:56:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[59745]: audit 2022-04-28T21:56:19.438778+0000 mon.a (mon.0) 4178 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:56:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[59745]: audit 2022-04-28T21:56:19.500450+0000 mon.a (mon.0) 4179 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:56:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[59745]: audit 2022-04-28T21:56:19.501235+0000 mon.a (mon.0) 4180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:56:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[59745]: audit 2022-04-28T21:56:19.501744+0000 mon.a (mon.0) 4181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:56:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[59745]: audit 2022-04-28T21:56:19.502209+0000 mon.a (mon.0) 4182 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]: dispatch 2022-04-28T21:56:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[61152]: audit 2022-04-28T21:56:19.420308+0000 mon.a (mon.0) 4177 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:56:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[61152]: audit 2022-04-28T21:56:19.438778+0000 mon.a (mon.0) 4178 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:56:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[61152]: audit 2022-04-28T21:56:19.500450+0000 mon.a (mon.0) 4179 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:56:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[61152]: audit 2022-04-28T21:56:19.501235+0000 mon.a (mon.0) 4180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:56:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[61152]: audit 2022-04-28T21:56:19.501744+0000 mon.a (mon.0) 4181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:56:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:20 smithi059 bash[61152]: audit 2022-04-28T21:56:19.502209+0000 mon.a (mon.0) 4182 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]: dispatch 2022-04-28T21:56:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:20 smithi139 bash[43301]: audit 2022-04-28T21:56:19.420308+0000 mon.a (mon.0) 4177 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:56:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:20 smithi139 bash[43301]: audit 2022-04-28T21:56:19.438778+0000 mon.a (mon.0) 4178 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:56:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:20 smithi139 bash[43301]: audit 2022-04-28T21:56:19.500450+0000 mon.a (mon.0) 4179 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:56:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:20 smithi139 bash[43301]: audit 2022-04-28T21:56:19.501235+0000 mon.a (mon.0) 4180 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T21:56:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:20 smithi139 bash[43301]: audit 2022-04-28T21:56:19.501744+0000 mon.a (mon.0) 4181 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2022-04-28T21:56:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:20 smithi139 bash[43301]: audit 2022-04-28T21:56:19.502209+0000 mon.a (mon.0) 4182 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]: dispatch 2022-04-28T21:56:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[61152]: audit 2022-04-28T21:56:20.232060+0000 mon.a (mon.0) 4183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:56:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[61152]: audit 2022-04-28T21:56:20.235002+0000 mon.a (mon.0) 4184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:56:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[61152]: audit 2022-04-28T21:56:20.236982+0000 mon.a (mon.0) 4185 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:56:21.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[61152]: cephadm 2022-04-28T21:56:20.238715+0000 mgr.y (mgr.24961) 4117 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:56:21.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[61152]: audit 2022-04-28T21:56:20.556610+0000 mon.a (mon.0) 4186 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:56:21.348 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[61152]: cluster 2022-04-28T21:56:20.780813+0000 mgr.y (mgr.24961) 4118 : cluster [DBG] pgmap v4900: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:21.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[59745]: audit 2022-04-28T21:56:20.232060+0000 mon.a (mon.0) 4183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:56:21.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[59745]: audit 2022-04-28T21:56:20.235002+0000 mon.a (mon.0) 4184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:56:21.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[59745]: audit 2022-04-28T21:56:20.236982+0000 mon.a (mon.0) 4185 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:56:21.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[59745]: cephadm 2022-04-28T21:56:20.238715+0000 mgr.y (mgr.24961) 4117 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:56:21.349 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[59745]: audit 2022-04-28T21:56:20.556610+0000 mon.a (mon.0) 4186 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:56:21.350 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:21 smithi059 bash[59745]: cluster 2022-04-28T21:56:20.780813+0000 mgr.y (mgr.24961) 4118 : cluster [DBG] pgmap v4900: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:21.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:21 smithi139 bash[43301]: audit 2022-04-28T21:56:20.232060+0000 mon.a (mon.0) 4183 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:56:21.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:21 smithi139 bash[43301]: audit 2022-04-28T21:56:20.235002+0000 mon.a (mon.0) 4184 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:56:21.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:21 smithi139 bash[43301]: audit 2022-04-28T21:56:20.236982+0000 mon.a (mon.0) 4185 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:56:21.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:21 smithi139 bash[43301]: cephadm 2022-04-28T21:56:20.238715+0000 mgr.y (mgr.24961) 4117 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:56:21.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:21 smithi139 bash[43301]: audit 2022-04-28T21:56:20.556610+0000 mon.a (mon.0) 4186 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:56:21.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:21 smithi139 bash[43301]: cluster 2022-04-28T21:56:20.780813+0000 mgr.y (mgr.24961) 4118 : cluster [DBG] pgmap v4900: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:22.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:22.120Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:24.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:23 smithi059 bash[61152]: cluster 2022-04-28T21:56:22.782196+0000 mgr.y (mgr.24961) 4119 : cluster [DBG] pgmap v4901: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:24.092 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:24 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:24.005Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:24.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:23 smithi059 bash[59745]: cluster 2022-04-28T21:56:22.782196+0000 mgr.y (mgr.24961) 4119 : cluster [DBG] pgmap v4901: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:24.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:23 smithi139 bash[43301]: cluster 2022-04-28T21:56:22.782196+0000 mgr.y (mgr.24961) 4119 : cluster [DBG] pgmap v4901: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:25.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:25 smithi059 bash[61152]: cluster 2022-04-28T21:56:24.783683+0000 mgr.y (mgr.24961) 4120 : cluster [DBG] pgmap v4902: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:25.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:25 smithi059 bash[59745]: cluster 2022-04-28T21:56:24.783683+0000 mgr.y (mgr.24961) 4120 : cluster [DBG] pgmap v4902: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:25.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:25 smithi139 bash[43301]: cluster 2022-04-28T21:56:24.783683+0000 mgr.y (mgr.24961) 4120 : cluster [DBG] pgmap v4902: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:26 smithi059 bash[61152]: cluster 2022-04-28T21:56:26.785424+0000 mgr.y (mgr.24961) 4121 : cluster [DBG] pgmap v4903: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:27.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:26 smithi059 bash[59745]: cluster 2022-04-28T21:56:26.785424+0000 mgr.y (mgr.24961) 4121 : cluster [DBG] pgmap v4903: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:26 smithi139 bash[43301]: cluster 2022-04-28T21:56:26.785424+0000 mgr.y (mgr.24961) 4121 : cluster [DBG] pgmap v4903: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:28 smithi059 bash[61152]: cluster 2022-04-28T21:56:28.786239+0000 mgr.y (mgr.24961) 4122 : cluster [DBG] pgmap v4904: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:28 smithi059 bash[59745]: cluster 2022-04-28T21:56:28.786239+0000 mgr.y (mgr.24961) 4122 : cluster [DBG] pgmap v4904: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:28 smithi139 bash[43301]: cluster 2022-04-28T21:56:28.786239+0000 mgr.y (mgr.24961) 4122 : cluster [DBG] pgmap v4904: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:30 smithi059 bash[61152]: cluster 2022-04-28T21:56:30.788198+0000 mgr.y (mgr.24961) 4123 : cluster [DBG] pgmap v4905: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:31.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:30 smithi059 bash[59745]: cluster 2022-04-28T21:56:30.788198+0000 mgr.y (mgr.24961) 4123 : cluster [DBG] pgmap v4905: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:30 smithi139 bash[43301]: cluster 2022-04-28T21:56:30.788198+0000 mgr.y (mgr.24961) 4123 : cluster [DBG] pgmap v4905: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:32.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:32 smithi059 bash[59745]: cluster 2022-04-28T21:56:32.789490+0000 mgr.y (mgr.24961) 4124 : cluster [DBG] pgmap v4906: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:33.372 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:32 smithi059 bash[61152]: cluster 2022-04-28T21:56:32.789490+0000 mgr.y (mgr.24961) 4124 : cluster [DBG] pgmap v4906: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:33.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:32 smithi139 bash[43301]: cluster 2022-04-28T21:56:32.789490+0000 mgr.y (mgr.24961) 4124 : cluster [DBG] pgmap v4906: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:34 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:34.007Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:35.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:35 smithi059 bash[61152]: cluster 2022-04-28T21:56:34.790518+0000 mgr.y (mgr.24961) 4125 : cluster [DBG] pgmap v4907: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:35 smithi059 bash[59745]: cluster 2022-04-28T21:56:34.790518+0000 mgr.y (mgr.24961) 4125 : cluster [DBG] pgmap v4907: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:35.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:35 smithi139 bash[43301]: cluster 2022-04-28T21:56:34.790518+0000 mgr.y (mgr.24961) 4125 : cluster [DBG] pgmap v4907: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:36 smithi059 bash[59745]: cluster 2022-04-28T21:56:36.792338+0000 mgr.y (mgr.24961) 4126 : cluster [DBG] pgmap v4908: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:37.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:36 smithi059 bash[61152]: cluster 2022-04-28T21:56:36.792338+0000 mgr.y (mgr.24961) 4126 : cluster [DBG] pgmap v4908: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:36 smithi139 bash[43301]: cluster 2022-04-28T21:56:36.792338+0000 mgr.y (mgr.24961) 4126 : cluster [DBG] pgmap v4908: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:38 smithi059 bash[61152]: cluster 2022-04-28T21:56:38.793137+0000 mgr.y (mgr.24961) 4127 : cluster [DBG] pgmap v4909: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:38 smithi059 bash[59745]: cluster 2022-04-28T21:56:38.793137+0000 mgr.y (mgr.24961) 4127 : cluster [DBG] pgmap v4909: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:38 smithi139 bash[43301]: cluster 2022-04-28T21:56:38.793137+0000 mgr.y (mgr.24961) 4127 : cluster [DBG] pgmap v4909: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:40 smithi059 bash[61152]: cluster 2022-04-28T21:56:40.795014+0000 mgr.y (mgr.24961) 4128 : cluster [DBG] pgmap v4910: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:40 smithi059 bash[59745]: cluster 2022-04-28T21:56:40.795014+0000 mgr.y (mgr.24961) 4128 : cluster [DBG] pgmap v4910: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:40 smithi139 bash[43301]: cluster 2022-04-28T21:56:40.795014+0000 mgr.y (mgr.24961) 4128 : cluster [DBG] pgmap v4910: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:42.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:42.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:42 smithi059 bash[61152]: cluster 2022-04-28T21:56:42.796276+0000 mgr.y (mgr.24961) 4129 : cluster [DBG] pgmap v4911: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:42 smithi059 bash[59745]: cluster 2022-04-28T21:56:42.796276+0000 mgr.y (mgr.24961) 4129 : cluster [DBG] pgmap v4911: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:42 smithi139 bash[43301]: cluster 2022-04-28T21:56:42.796276+0000 mgr.y (mgr.24961) 4129 : cluster [DBG] pgmap v4911: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:44 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:44.008Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:45 smithi059 bash[61152]: cluster 2022-04-28T21:56:44.797779+0000 mgr.y (mgr.24961) 4130 : cluster [DBG] pgmap v4912: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:45 smithi059 bash[59745]: cluster 2022-04-28T21:56:44.797779+0000 mgr.y (mgr.24961) 4130 : cluster [DBG] pgmap v4912: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:45 smithi139 bash[43301]: cluster 2022-04-28T21:56:44.797779+0000 mgr.y (mgr.24961) 4130 : cluster [DBG] pgmap v4912: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:46 smithi059 bash[61152]: cluster 2022-04-28T21:56:46.799576+0000 mgr.y (mgr.24961) 4131 : cluster [DBG] pgmap v4913: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:46 smithi059 bash[59745]: cluster 2022-04-28T21:56:46.799576+0000 mgr.y (mgr.24961) 4131 : cluster [DBG] pgmap v4913: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:46 smithi139 bash[43301]: cluster 2022-04-28T21:56:46.799576+0000 mgr.y (mgr.24961) 4131 : cluster [DBG] pgmap v4913: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:49.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:48 smithi059 bash[61152]: cluster 2022-04-28T21:56:48.800303+0000 mgr.y (mgr.24961) 4132 : cluster [DBG] pgmap v4914: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:49.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:48 smithi059 bash[59745]: cluster 2022-04-28T21:56:48.800303+0000 mgr.y (mgr.24961) 4132 : cluster [DBG] pgmap v4914: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:49.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:48 smithi139 bash[43301]: cluster 2022-04-28T21:56:48.800303+0000 mgr.y (mgr.24961) 4132 : cluster [DBG] pgmap v4914: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:51.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:50 smithi059 bash[61152]: cluster 2022-04-28T21:56:50.802232+0000 mgr.y (mgr.24961) 4133 : cluster [DBG] pgmap v4915: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:51.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:50 smithi059 bash[59745]: cluster 2022-04-28T21:56:50.802232+0000 mgr.y (mgr.24961) 4133 : cluster [DBG] pgmap v4915: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:50 smithi139 bash[43301]: cluster 2022-04-28T21:56:50.802232+0000 mgr.y (mgr.24961) 4133 : cluster [DBG] pgmap v4915: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:52.122Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:52 smithi059 bash[59745]: cluster 2022-04-28T21:56:52.803535+0000 mgr.y (mgr.24961) 4134 : cluster [DBG] pgmap v4916: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:52 smithi059 bash[61152]: cluster 2022-04-28T21:56:52.803535+0000 mgr.y (mgr.24961) 4134 : cluster [DBG] pgmap v4916: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:52 smithi139 bash[43301]: cluster 2022-04-28T21:56:52.803535+0000 mgr.y (mgr.24961) 4134 : cluster [DBG] pgmap v4916: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:54.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:56:54 smithi059 bash[72035]: level=error ts=2022-04-28T21:56:54.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:56:55.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:55 smithi059 bash[61152]: cluster 2022-04-28T21:56:54.804929+0000 mgr.y (mgr.24961) 4135 : cluster [DBG] pgmap v4917: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:55.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:55 smithi059 bash[59745]: cluster 2022-04-28T21:56:54.804929+0000 mgr.y (mgr.24961) 4135 : cluster [DBG] pgmap v4917: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:55.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:55 smithi139 bash[43301]: cluster 2022-04-28T21:56:54.804929+0000 mgr.y (mgr.24961) 4135 : cluster [DBG] pgmap v4917: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:57.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:56 smithi059 bash[61152]: cluster 2022-04-28T21:56:56.806782+0000 mgr.y (mgr.24961) 4136 : cluster [DBG] pgmap v4918: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:57.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:56 smithi059 bash[59745]: cluster 2022-04-28T21:56:56.806782+0000 mgr.y (mgr.24961) 4136 : cluster [DBG] pgmap v4918: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:57.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:56 smithi139 bash[43301]: cluster 2022-04-28T21:56:56.806782+0000 mgr.y (mgr.24961) 4136 : cluster [DBG] pgmap v4918: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:56:58 smithi059 bash[61152]: cluster 2022-04-28T21:56:58.807672+0000 mgr.y (mgr.24961) 4137 : cluster [DBG] pgmap v4919: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:59.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:56:58 smithi059 bash[59745]: cluster 2022-04-28T21:56:58.807672+0000 mgr.y (mgr.24961) 4137 : cluster [DBG] pgmap v4919: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:56:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:56:58 smithi139 bash[43301]: cluster 2022-04-28T21:56:58.807672+0000 mgr.y (mgr.24961) 4137 : cluster [DBG] pgmap v4919: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:01.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:00 smithi059 bash[61152]: cluster 2022-04-28T21:57:00.809605+0000 mgr.y (mgr.24961) 4138 : cluster [DBG] pgmap v4920: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:00 smithi059 bash[59745]: cluster 2022-04-28T21:57:00.809605+0000 mgr.y (mgr.24961) 4138 : cluster [DBG] pgmap v4920: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:01.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:00 smithi139 bash[43301]: cluster 2022-04-28T21:57:00.809605+0000 mgr.y (mgr.24961) 4138 : cluster [DBG] pgmap v4920: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:02.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:02.121Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:03.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:02 smithi059 bash[59745]: cluster 2022-04-28T21:57:02.810885+0000 mgr.y (mgr.24961) 4139 : cluster [DBG] pgmap v4921: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:02 smithi059 bash[61152]: cluster 2022-04-28T21:57:02.810885+0000 mgr.y (mgr.24961) 4139 : cluster [DBG] pgmap v4921: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:03.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:02 smithi139 bash[43301]: cluster 2022-04-28T21:57:02.810885+0000 mgr.y (mgr.24961) 4139 : cluster [DBG] pgmap v4921: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:04 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:04.009Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:05.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:05 smithi059 bash[59745]: cluster 2022-04-28T21:57:04.812324+0000 mgr.y (mgr.24961) 4140 : cluster [DBG] pgmap v4922: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:05.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:05 smithi059 bash[61152]: cluster 2022-04-28T21:57:04.812324+0000 mgr.y (mgr.24961) 4140 : cluster [DBG] pgmap v4922: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:05 smithi139 bash[43301]: cluster 2022-04-28T21:57:04.812324+0000 mgr.y (mgr.24961) 4140 : cluster [DBG] pgmap v4922: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:07.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:06 smithi059 bash[61152]: cluster 2022-04-28T21:57:06.814119+0000 mgr.y (mgr.24961) 4141 : cluster [DBG] pgmap v4923: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:07.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:06 smithi059 bash[59745]: cluster 2022-04-28T21:57:06.814119+0000 mgr.y (mgr.24961) 4141 : cluster [DBG] pgmap v4923: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:07.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:06 smithi139 bash[43301]: cluster 2022-04-28T21:57:06.814119+0000 mgr.y (mgr.24961) 4141 : cluster [DBG] pgmap v4923: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:09.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:08 smithi059 bash[61152]: cluster 2022-04-28T21:57:08.814912+0000 mgr.y (mgr.24961) 4142 : cluster [DBG] pgmap v4924: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:09.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:08 smithi059 bash[59745]: cluster 2022-04-28T21:57:08.814912+0000 mgr.y (mgr.24961) 4142 : cluster [DBG] pgmap v4924: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:09.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:08 smithi139 bash[43301]: cluster 2022-04-28T21:57:08.814912+0000 mgr.y (mgr.24961) 4142 : cluster [DBG] pgmap v4924: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:11.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:10 smithi059 bash[61152]: cluster 2022-04-28T21:57:10.816791+0000 mgr.y (mgr.24961) 4143 : cluster [DBG] pgmap v4925: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:11.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:10 smithi059 bash[59745]: cluster 2022-04-28T21:57:10.816791+0000 mgr.y (mgr.24961) 4143 : cluster [DBG] pgmap v4925: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:11.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:10 smithi139 bash[43301]: cluster 2022-04-28T21:57:10.816791+0000 mgr.y (mgr.24961) 4143 : cluster [DBG] pgmap v4925: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:12.123Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:13.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:12 smithi059 bash[61152]: cluster 2022-04-28T21:57:12.818082+0000 mgr.y (mgr.24961) 4144 : cluster [DBG] pgmap v4926: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:12 smithi059 bash[59745]: cluster 2022-04-28T21:57:12.818082+0000 mgr.y (mgr.24961) 4144 : cluster [DBG] pgmap v4926: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:13.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:12 smithi139 bash[43301]: cluster 2022-04-28T21:57:12.818082+0000 mgr.y (mgr.24961) 4144 : cluster [DBG] pgmap v4926: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:14 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:14.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:15.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:15 smithi139 bash[43301]: cluster 2022-04-28T21:57:14.819652+0000 mgr.y (mgr.24961) 4145 : cluster [DBG] pgmap v4927: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:15.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:15 smithi059 bash[61152]: cluster 2022-04-28T21:57:14.819652+0000 mgr.y (mgr.24961) 4145 : cluster [DBG] pgmap v4927: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:15.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:15 smithi059 bash[59745]: cluster 2022-04-28T21:57:14.819652+0000 mgr.y (mgr.24961) 4145 : cluster [DBG] pgmap v4927: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:17.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:16 smithi059 bash[61152]: cluster 2022-04-28T21:57:16.821475+0000 mgr.y (mgr.24961) 4146 : cluster [DBG] pgmap v4928: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:16 smithi059 bash[59745]: cluster 2022-04-28T21:57:16.821475+0000 mgr.y (mgr.24961) 4146 : cluster [DBG] pgmap v4928: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:17.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:16 smithi139 bash[43301]: cluster 2022-04-28T21:57:16.821475+0000 mgr.y (mgr.24961) 4146 : cluster [DBG] pgmap v4928: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:19.346 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:18 smithi059 bash[61152]: cluster 2022-04-28T21:57:18.822359+0000 mgr.y (mgr.24961) 4147 : cluster [DBG] pgmap v4929: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:19.347 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:18 smithi059 bash[59745]: cluster 2022-04-28T21:57:18.822359+0000 mgr.y (mgr.24961) 4147 : cluster [DBG] pgmap v4929: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:19.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:18 smithi139 bash[43301]: cluster 2022-04-28T21:57:18.822359+0000 mgr.y (mgr.24961) 4147 : cluster [DBG] pgmap v4929: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[59745]: audit 2022-04-28T21:57:19.425287+0000 mon.a (mon.0) 4187 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:57:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[59745]: audit 2022-04-28T21:57:19.445395+0000 mon.a (mon.0) 4188 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:57:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[59745]: audit 2022-04-28T21:57:19.510553+0000 mon.a (mon.0) 4189 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:57:20.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[59745]: audit 2022-04-28T21:57:19.511222+0000 mon.a (mon.0) 4190 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:57:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[59745]: audit 2022-04-28T21:57:19.512017+0000 mon.a (mon.0) 4191 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:57:20.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[59745]: audit 2022-04-28T21:57:19.512724+0000 mon.a (mon.0) 4192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:57:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[61152]: audit 2022-04-28T21:57:19.425287+0000 mon.a (mon.0) 4187 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:57:20.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[61152]: audit 2022-04-28T21:57:19.445395+0000 mon.a (mon.0) 4188 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:57:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[61152]: audit 2022-04-28T21:57:19.510553+0000 mon.a (mon.0) 4189 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:57:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[61152]: audit 2022-04-28T21:57:19.511222+0000 mon.a (mon.0) 4190 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:57:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[61152]: audit 2022-04-28T21:57:19.512017+0000 mon.a (mon.0) 4191 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:57:20.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:20 smithi059 bash[61152]: audit 2022-04-28T21:57:19.512724+0000 mon.a (mon.0) 4192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:57:20.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:20 smithi139 bash[43301]: audit 2022-04-28T21:57:19.425287+0000 mon.a (mon.0) 4187 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:57:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:20 smithi139 bash[43301]: audit 2022-04-28T21:57:19.445395+0000 mon.a (mon.0) 4188 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:57:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:20 smithi139 bash[43301]: audit 2022-04-28T21:57:19.510553+0000 mon.a (mon.0) 4189 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T21:57:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:20 smithi139 bash[43301]: audit 2022-04-28T21:57:19.511222+0000 mon.a (mon.0) 4190 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2022-04-28T21:57:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:20 smithi139 bash[43301]: audit 2022-04-28T21:57:19.512017+0000 mon.a (mon.0) 4191 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T21:57:20.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:20 smithi139 bash[43301]: audit 2022-04-28T21:57:19.512724+0000 mon.a (mon.0) 4192 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1e", "id": [4, 2]}]: dispatch 2022-04-28T21:57:21.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:21 smithi139 bash[43301]: audit 2022-04-28T21:57:20.564998+0000 mon.a (mon.0) 4193 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:57:21.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:21 smithi139 bash[43301]: audit 2022-04-28T21:57:20.567923+0000 mon.a (mon.0) 4194 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:57:21.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:21 smithi139 bash[43301]: audit 2022-04-28T21:57:20.569882+0000 mon.a (mon.0) 4195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:57:21.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:21 smithi139 bash[43301]: cephadm 2022-04-28T21:57:20.571645+0000 mgr.y (mgr.24961) 4148 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:57:21.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:21 smithi139 bash[43301]: cluster 2022-04-28T21:57:20.823450+0000 mgr.y (mgr.24961) 4149 : cluster [DBG] pgmap v4930: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:21.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:21 smithi139 bash[43301]: audit 2022-04-28T21:57:20.895829+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:57:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[59745]: audit 2022-04-28T21:57:20.564998+0000 mon.a (mon.0) 4193 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:57:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[59745]: audit 2022-04-28T21:57:20.567923+0000 mon.a (mon.0) 4194 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:57:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[59745]: audit 2022-04-28T21:57:20.569882+0000 mon.a (mon.0) 4195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:57:21.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[59745]: cephadm 2022-04-28T21:57:20.571645+0000 mgr.y (mgr.24961) 4148 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:57:21.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[59745]: cluster 2022-04-28T21:57:20.823450+0000 mgr.y (mgr.24961) 4149 : cluster [DBG] pgmap v4930: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:21.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[59745]: audit 2022-04-28T21:57:20.895829+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:57:21.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[61152]: audit 2022-04-28T21:57:20.564998+0000 mon.a (mon.0) 4193 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:57:21.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[61152]: audit 2022-04-28T21:57:20.567923+0000 mon.a (mon.0) 4194 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:57:21.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[61152]: audit 2022-04-28T21:57:20.569882+0000 mon.a (mon.0) 4195 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:57:21.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[61152]: cephadm 2022-04-28T21:57:20.571645+0000 mgr.y (mgr.24961) 4148 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:57:21.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[61152]: cluster 2022-04-28T21:57:20.823450+0000 mgr.y (mgr.24961) 4149 : cluster [DBG] pgmap v4930: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:21.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:21 smithi059 bash[61152]: audit 2022-04-28T21:57:20.895829+0000 mon.a (mon.0) 4196 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:57:22.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:22.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:23.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:22 smithi059 bash[61152]: cluster 2022-04-28T21:57:22.824859+0000 mgr.y (mgr.24961) 4150 : cluster [DBG] pgmap v4931: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:23.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:22 smithi059 bash[59745]: cluster 2022-04-28T21:57:22.824859+0000 mgr.y (mgr.24961) 4150 : cluster [DBG] pgmap v4931: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:23.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:22 smithi139 bash[43301]: cluster 2022-04-28T21:57:22.824859+0000 mgr.y (mgr.24961) 4150 : cluster [DBG] pgmap v4931: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:24.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:24 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:24.010Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:25.372 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:25 smithi139 bash[43301]: cluster 2022-04-28T21:57:24.826329+0000 mgr.y (mgr.24961) 4151 : cluster [DBG] pgmap v4932: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:25.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:25 smithi059 bash[59745]: cluster 2022-04-28T21:57:24.826329+0000 mgr.y (mgr.24961) 4151 : cluster [DBG] pgmap v4932: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:25.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:25 smithi059 bash[61152]: cluster 2022-04-28T21:57:24.826329+0000 mgr.y (mgr.24961) 4151 : cluster [DBG] pgmap v4932: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:27.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:26 smithi059 bash[59745]: cluster 2022-04-28T21:57:26.828115+0000 mgr.y (mgr.24961) 4152 : cluster [DBG] pgmap v4933: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:27.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:26 smithi059 bash[61152]: cluster 2022-04-28T21:57:26.828115+0000 mgr.y (mgr.24961) 4152 : cluster [DBG] pgmap v4933: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:27.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:26 smithi139 bash[43301]: cluster 2022-04-28T21:57:26.828115+0000 mgr.y (mgr.24961) 4152 : cluster [DBG] pgmap v4933: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:29.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:28 smithi059 bash[59745]: cluster 2022-04-28T21:57:28.828978+0000 mgr.y (mgr.24961) 4153 : cluster [DBG] pgmap v4934: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:29.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:28 smithi059 bash[61152]: cluster 2022-04-28T21:57:28.828978+0000 mgr.y (mgr.24961) 4153 : cluster [DBG] pgmap v4934: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:29.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:28 smithi139 bash[43301]: cluster 2022-04-28T21:57:28.828978+0000 mgr.y (mgr.24961) 4153 : cluster [DBG] pgmap v4934: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:31.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:30 smithi059 bash[61152]: cluster 2022-04-28T21:57:30.830979+0000 mgr.y (mgr.24961) 4154 : cluster [DBG] pgmap v4935: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:31.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:30 smithi059 bash[59745]: cluster 2022-04-28T21:57:30.830979+0000 mgr.y (mgr.24961) 4154 : cluster [DBG] pgmap v4935: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:31.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:30 smithi139 bash[43301]: cluster 2022-04-28T21:57:30.830979+0000 mgr.y (mgr.24961) 4154 : cluster [DBG] pgmap v4935: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:32.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:32.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:33.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:32 smithi059 bash[59745]: cluster 2022-04-28T21:57:32.832329+0000 mgr.y (mgr.24961) 4155 : cluster [DBG] pgmap v4936: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:33.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:32 smithi059 bash[61152]: cluster 2022-04-28T21:57:32.832329+0000 mgr.y (mgr.24961) 4155 : cluster [DBG] pgmap v4936: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:33.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:32 smithi139 bash[43301]: cluster 2022-04-28T21:57:32.832329+0000 mgr.y (mgr.24961) 4155 : cluster [DBG] pgmap v4936: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:34 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:34.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:35.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:35 smithi139 bash[43301]: cluster 2022-04-28T21:57:34.833768+0000 mgr.y (mgr.24961) 4156 : cluster [DBG] pgmap v4937: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:35.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:35 smithi059 bash[59745]: cluster 2022-04-28T21:57:34.833768+0000 mgr.y (mgr.24961) 4156 : cluster [DBG] pgmap v4937: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:35.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:35 smithi059 bash[61152]: cluster 2022-04-28T21:57:34.833768+0000 mgr.y (mgr.24961) 4156 : cluster [DBG] pgmap v4937: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:37.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:36 smithi059 bash[59745]: cluster 2022-04-28T21:57:36.835683+0000 mgr.y (mgr.24961) 4157 : cluster [DBG] pgmap v4938: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:37.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:36 smithi059 bash[61152]: cluster 2022-04-28T21:57:36.835683+0000 mgr.y (mgr.24961) 4157 : cluster [DBG] pgmap v4938: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:37.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:36 smithi139 bash[43301]: cluster 2022-04-28T21:57:36.835683+0000 mgr.y (mgr.24961) 4157 : cluster [DBG] pgmap v4938: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:39.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:38 smithi059 bash[59745]: cluster 2022-04-28T21:57:38.836622+0000 mgr.y (mgr.24961) 4158 : cluster [DBG] pgmap v4939: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:39.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:38 smithi059 bash[61152]: cluster 2022-04-28T21:57:38.836622+0000 mgr.y (mgr.24961) 4158 : cluster [DBG] pgmap v4939: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:39.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:38 smithi139 bash[43301]: cluster 2022-04-28T21:57:38.836622+0000 mgr.y (mgr.24961) 4158 : cluster [DBG] pgmap v4939: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:40 smithi059 bash[59745]: cluster 2022-04-28T21:57:40.838614+0000 mgr.y (mgr.24961) 4159 : cluster [DBG] pgmap v4940: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:40 smithi059 bash[61152]: cluster 2022-04-28T21:57:40.838614+0000 mgr.y (mgr.24961) 4159 : cluster [DBG] pgmap v4940: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:40 smithi139 bash[43301]: cluster 2022-04-28T21:57:40.838614+0000 mgr.y (mgr.24961) 4159 : cluster [DBG] pgmap v4940: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:42.125Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:42 smithi059 bash[61152]: cluster 2022-04-28T21:57:42.839940+0000 mgr.y (mgr.24961) 4160 : cluster [DBG] pgmap v4941: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:42 smithi059 bash[59745]: cluster 2022-04-28T21:57:42.839940+0000 mgr.y (mgr.24961) 4160 : cluster [DBG] pgmap v4941: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:42 smithi139 bash[43301]: cluster 2022-04-28T21:57:42.839940+0000 mgr.y (mgr.24961) 4160 : cluster [DBG] pgmap v4941: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:44.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:44 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:44.012Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:45 smithi139 bash[43301]: cluster 2022-04-28T21:57:44.841335+0000 mgr.y (mgr.24961) 4161 : cluster [DBG] pgmap v4942: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:45.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:45 smithi059 bash[59745]: cluster 2022-04-28T21:57:44.841335+0000 mgr.y (mgr.24961) 4161 : cluster [DBG] pgmap v4942: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:45.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:45 smithi059 bash[61152]: cluster 2022-04-28T21:57:44.841335+0000 mgr.y (mgr.24961) 4161 : cluster [DBG] pgmap v4942: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:47.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:46 smithi059 bash[59745]: cluster 2022-04-28T21:57:46.843166+0000 mgr.y (mgr.24961) 4162 : cluster [DBG] pgmap v4943: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:47.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:46 smithi059 bash[61152]: cluster 2022-04-28T21:57:46.843166+0000 mgr.y (mgr.24961) 4162 : cluster [DBG] pgmap v4943: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:46 smithi139 bash[43301]: cluster 2022-04-28T21:57:46.843166+0000 mgr.y (mgr.24961) 4162 : cluster [DBG] pgmap v4943: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:49.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:48 smithi059 bash[59745]: cluster 2022-04-28T21:57:48.844024+0000 mgr.y (mgr.24961) 4163 : cluster [DBG] pgmap v4944: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:48 smithi059 bash[61152]: cluster 2022-04-28T21:57:48.844024+0000 mgr.y (mgr.24961) 4163 : cluster [DBG] pgmap v4944: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:49.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:48 smithi139 bash[43301]: cluster 2022-04-28T21:57:48.844024+0000 mgr.y (mgr.24961) 4163 : cluster [DBG] pgmap v4944: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:51.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:50 smithi059 bash[59745]: cluster 2022-04-28T21:57:50.846041+0000 mgr.y (mgr.24961) 4164 : cluster [DBG] pgmap v4945: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:51.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:50 smithi059 bash[61152]: cluster 2022-04-28T21:57:50.846041+0000 mgr.y (mgr.24961) 4164 : cluster [DBG] pgmap v4945: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:50 smithi139 bash[43301]: cluster 2022-04-28T21:57:50.846041+0000 mgr.y (mgr.24961) 4164 : cluster [DBG] pgmap v4945: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:52.126Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:53.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:52 smithi059 bash[61152]: cluster 2022-04-28T21:57:52.847368+0000 mgr.y (mgr.24961) 4165 : cluster [DBG] pgmap v4946: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:53.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:52 smithi059 bash[59745]: cluster 2022-04-28T21:57:52.847368+0000 mgr.y (mgr.24961) 4165 : cluster [DBG] pgmap v4946: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:52 smithi139 bash[43301]: cluster 2022-04-28T21:57:52.847368+0000 mgr.y (mgr.24961) 4165 : cluster [DBG] pgmap v4946: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:54.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:57:54 smithi059 bash[72035]: level=error ts=2022-04-28T21:57:54.013Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:57:55.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:55 smithi139 bash[43301]: cluster 2022-04-28T21:57:54.848805+0000 mgr.y (mgr.24961) 4166 : cluster [DBG] pgmap v4947: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:55.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:55 smithi059 bash[59745]: cluster 2022-04-28T21:57:54.848805+0000 mgr.y (mgr.24961) 4166 : cluster [DBG] pgmap v4947: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:55.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:55 smithi059 bash[61152]: cluster 2022-04-28T21:57:54.848805+0000 mgr.y (mgr.24961) 4166 : cluster [DBG] pgmap v4947: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:57.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:56 smithi059 bash[59745]: cluster 2022-04-28T21:57:56.850541+0000 mgr.y (mgr.24961) 4167 : cluster [DBG] pgmap v4948: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:57.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:56 smithi059 bash[61152]: cluster 2022-04-28T21:57:56.850541+0000 mgr.y (mgr.24961) 4167 : cluster [DBG] pgmap v4948: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:57.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:56 smithi139 bash[43301]: cluster 2022-04-28T21:57:56.850541+0000 mgr.y (mgr.24961) 4167 : cluster [DBG] pgmap v4948: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:59.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:57:58 smithi059 bash[59745]: cluster 2022-04-28T21:57:58.851316+0000 mgr.y (mgr.24961) 4168 : cluster [DBG] pgmap v4949: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:59.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:57:58 smithi059 bash[61152]: cluster 2022-04-28T21:57:58.851316+0000 mgr.y (mgr.24961) 4168 : cluster [DBG] pgmap v4949: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:57:59.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:57:58 smithi139 bash[43301]: cluster 2022-04-28T21:57:58.851316+0000 mgr.y (mgr.24961) 4168 : cluster [DBG] pgmap v4949: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:01.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:00 smithi059 bash[61152]: cluster 2022-04-28T21:58:00.853255+0000 mgr.y (mgr.24961) 4169 : cluster [DBG] pgmap v4950: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:01.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:00 smithi059 bash[59745]: cluster 2022-04-28T21:58:00.853255+0000 mgr.y (mgr.24961) 4169 : cluster [DBG] pgmap v4950: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:01.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:00 smithi139 bash[43301]: cluster 2022-04-28T21:58:00.853255+0000 mgr.y (mgr.24961) 4169 : cluster [DBG] pgmap v4950: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:02.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:02.127Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:03.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:02 smithi059 bash[59745]: cluster 2022-04-28T21:58:02.854529+0000 mgr.y (mgr.24961) 4170 : cluster [DBG] pgmap v4951: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:03.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:02 smithi059 bash[61152]: cluster 2022-04-28T21:58:02.854529+0000 mgr.y (mgr.24961) 4170 : cluster [DBG] pgmap v4951: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:03.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:02 smithi139 bash[43301]: cluster 2022-04-28T21:58:02.854529+0000 mgr.y (mgr.24961) 4170 : cluster [DBG] pgmap v4951: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:04 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:04.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:05.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:05 smithi139 bash[43301]: cluster 2022-04-28T21:58:04.855964+0000 mgr.y (mgr.24961) 4171 : cluster [DBG] pgmap v4952: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:05.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:05 smithi059 bash[61152]: cluster 2022-04-28T21:58:04.855964+0000 mgr.y (mgr.24961) 4171 : cluster [DBG] pgmap v4952: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:05.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:05 smithi059 bash[59745]: cluster 2022-04-28T21:58:04.855964+0000 mgr.y (mgr.24961) 4171 : cluster [DBG] pgmap v4952: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:08.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:07 smithi059 bash[59745]: cluster 2022-04-28T21:58:06.857710+0000 mgr.y (mgr.24961) 4172 : cluster [DBG] pgmap v4953: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:08.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:07 smithi059 bash[61152]: cluster 2022-04-28T21:58:06.857710+0000 mgr.y (mgr.24961) 4172 : cluster [DBG] pgmap v4953: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:08.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:07 smithi139 bash[43301]: cluster 2022-04-28T21:58:06.857710+0000 mgr.y (mgr.24961) 4172 : cluster [DBG] pgmap v4953: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:10.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:10 smithi139 bash[43301]: cluster 2022-04-28T21:58:08.858549+0000 mgr.y (mgr.24961) 4173 : cluster [DBG] pgmap v4954: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:10.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:10 smithi059 bash[61152]: cluster 2022-04-28T21:58:08.858549+0000 mgr.y (mgr.24961) 4173 : cluster [DBG] pgmap v4954: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:10 smithi059 bash[59745]: cluster 2022-04-28T21:58:08.858549+0000 mgr.y (mgr.24961) 4173 : cluster [DBG] pgmap v4954: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:12.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:12 smithi139 bash[43301]: cluster 2022-04-28T21:58:10.860563+0000 mgr.y (mgr.24961) 4174 : cluster [DBG] pgmap v4955: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:12 smithi059 bash[61152]: cluster 2022-04-28T21:58:10.860563+0000 mgr.y (mgr.24961) 4174 : cluster [DBG] pgmap v4955: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:12 smithi059 bash[59745]: cluster 2022-04-28T21:58:10.860563+0000 mgr.y (mgr.24961) 4174 : cluster [DBG] pgmap v4955: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:12.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:12.124Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:14 smithi059 bash[61152]: cluster 2022-04-28T21:58:12.861849+0000 mgr.y (mgr.24961) 4175 : cluster [DBG] pgmap v4956: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:14 smithi059 bash[59745]: cluster 2022-04-28T21:58:12.861849+0000 mgr.y (mgr.24961) 4175 : cluster [DBG] pgmap v4956: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:14 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:14.015Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:14.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:14 smithi139 bash[43301]: cluster 2022-04-28T21:58:12.861849+0000 mgr.y (mgr.24961) 4175 : cluster [DBG] pgmap v4956: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:16.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:16 smithi059 bash[59745]: cluster 2022-04-28T21:58:14.863379+0000 mgr.y (mgr.24961) 4176 : cluster [DBG] pgmap v4957: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:16 smithi059 bash[61152]: cluster 2022-04-28T21:58:14.863379+0000 mgr.y (mgr.24961) 4176 : cluster [DBG] pgmap v4957: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:16 smithi139 bash[43301]: cluster 2022-04-28T21:58:14.863379+0000 mgr.y (mgr.24961) 4176 : cluster [DBG] pgmap v4957: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:18 smithi059 bash[61152]: cluster 2022-04-28T21:58:16.865227+0000 mgr.y (mgr.24961) 4177 : cluster [DBG] pgmap v4958: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:18 smithi059 bash[59745]: cluster 2022-04-28T21:58:16.865227+0000 mgr.y (mgr.24961) 4177 : cluster [DBG] pgmap v4958: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:18 smithi139 bash[43301]: cluster 2022-04-28T21:58:16.865227+0000 mgr.y (mgr.24961) 4177 : cluster [DBG] pgmap v4958: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: cluster 2022-04-28T21:58:18.866004+0000 mgr.y (mgr.24961) 4178 : cluster [DBG] pgmap v4959: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: audit 2022-04-28T21:58:19.430331+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:58:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: audit 2022-04-28T21:58:19.451885+0000 mon.a (mon.0) 4198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:58:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: audit 2022-04-28T21:58:19.520791+0000 mon.a (mon.0) 4199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:58:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: audit 2022-04-28T21:58:19.521680+0000 mon.a (mon.0) 4200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:58:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: audit 2022-04-28T21:58:19.522485+0000 mon.a (mon.0) 4201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:58:20.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[61152]: audit 2022-04-28T21:58:19.523282+0000 mon.a (mon.0) 4202 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]: dispatch 2022-04-28T21:58:20.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: cluster 2022-04-28T21:58:18.866004+0000 mgr.y (mgr.24961) 4178 : cluster [DBG] pgmap v4959: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:20.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: audit 2022-04-28T21:58:19.430331+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:58:20.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: audit 2022-04-28T21:58:19.451885+0000 mon.a (mon.0) 4198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:58:20.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: audit 2022-04-28T21:58:19.520791+0000 mon.a (mon.0) 4199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:58:20.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: audit 2022-04-28T21:58:19.521680+0000 mon.a (mon.0) 4200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:58:20.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: audit 2022-04-28T21:58:19.522485+0000 mon.a (mon.0) 4201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:58:20.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:20 smithi059 bash[59745]: audit 2022-04-28T21:58:19.523282+0000 mon.a (mon.0) 4202 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]: dispatch 2022-04-28T21:58:20.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: cluster 2022-04-28T21:58:18.866004+0000 mgr.y (mgr.24961) 4178 : cluster [DBG] pgmap v4959: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:20.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: audit 2022-04-28T21:58:19.430331+0000 mon.a (mon.0) 4197 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:58:20.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: audit 2022-04-28T21:58:19.451885+0000 mon.a (mon.0) 4198 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:58:20.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: audit 2022-04-28T21:58:19.520791+0000 mon.a (mon.0) 4199 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T21:58:20.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: audit 2022-04-28T21:58:19.521680+0000 mon.a (mon.0) 4200 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T21:58:20.627 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: audit 2022-04-28T21:58:19.522485+0000 mon.a (mon.0) 4201 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2022-04-28T21:58:20.627 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:20 smithi139 bash[43301]: audit 2022-04-28T21:58:19.523282+0000 mon.a (mon.0) 4202 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]: dispatch 2022-04-28T21:58:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[59745]: audit 2022-04-28T21:58:20.903832+0000 mon.a (mon.0) 4203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:58:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[59745]: audit 2022-04-28T21:58:20.906707+0000 mon.a (mon.0) 4204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:58:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[59745]: audit 2022-04-28T21:58:20.908691+0000 mon.a (mon.0) 4205 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:58:21.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[59745]: audit 2022-04-28T21:58:20.911857+0000 mon.a (mon.0) 4206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:58:21.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[59745]: audit 2022-04-28T21:58:20.912470+0000 mon.a (mon.0) 4207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:58:21.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[61152]: audit 2022-04-28T21:58:20.903832+0000 mon.a (mon.0) 4203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:58:21.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[61152]: audit 2022-04-28T21:58:20.906707+0000 mon.a (mon.0) 4204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:58:21.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[61152]: audit 2022-04-28T21:58:20.908691+0000 mon.a (mon.0) 4205 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:58:21.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[61152]: audit 2022-04-28T21:58:20.911857+0000 mon.a (mon.0) 4206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:58:21.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:21 smithi059 bash[61152]: audit 2022-04-28T21:58:20.912470+0000 mon.a (mon.0) 4207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:58:21.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:21 smithi139 bash[43301]: audit 2022-04-28T21:58:20.903832+0000 mon.a (mon.0) 4203 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:58:21.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:21 smithi139 bash[43301]: audit 2022-04-28T21:58:20.906707+0000 mon.a (mon.0) 4204 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:58:21.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:21 smithi139 bash[43301]: audit 2022-04-28T21:58:20.908691+0000 mon.a (mon.0) 4205 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:58:21.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:21 smithi139 bash[43301]: audit 2022-04-28T21:58:20.911857+0000 mon.a (mon.0) 4206 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:58:21.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:21 smithi139 bash[43301]: audit 2022-04-28T21:58:20.912470+0000 mon.a (mon.0) 4207 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T21:58:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:22 smithi059 bash[59745]: cluster 2022-04-28T21:58:20.867987+0000 mgr.y (mgr.24961) 4179 : cluster [DBG] pgmap v4960: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:22 smithi059 bash[59745]: cephadm 2022-04-28T21:58:20.910389+0000 mgr.y (mgr.24961) 4180 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:58:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:22 smithi059 bash[61152]: cluster 2022-04-28T21:58:20.867987+0000 mgr.y (mgr.24961) 4179 : cluster [DBG] pgmap v4960: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:22.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:22 smithi059 bash[61152]: cephadm 2022-04-28T21:58:20.910389+0000 mgr.y (mgr.24961) 4180 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:58:22.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:22.128Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:22 smithi139 bash[43301]: cluster 2022-04-28T21:58:20.867987+0000 mgr.y (mgr.24961) 4179 : cluster [DBG] pgmap v4960: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:22.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:22 smithi139 bash[43301]: cephadm 2022-04-28T21:58:20.910389+0000 mgr.y (mgr.24961) 4180 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:58:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:24 smithi059 bash[59745]: cluster 2022-04-28T21:58:22.869250+0000 mgr.y (mgr.24961) 4181 : cluster [DBG] pgmap v4961: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:24.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:24 smithi059 bash[61152]: cluster 2022-04-28T21:58:22.869250+0000 mgr.y (mgr.24961) 4181 : cluster [DBG] pgmap v4961: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:24.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:24 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:24.014Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:24 smithi139 bash[43301]: cluster 2022-04-28T21:58:22.869250+0000 mgr.y (mgr.24961) 4181 : cluster [DBG] pgmap v4961: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:26.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[59745]: cluster 2022-04-28T21:58:24.871180+0000 mgr.y (mgr.24961) 4182 : cluster [DBG] pgmap v4962: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[59745]: audit 2022-04-28T21:58:25.032566+0000 mon.a (mon.0) 4208 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[59745]: audit 2022-04-28T21:58:25.235517+0000 mon.a (mon.0) 4209 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[59745]: audit 2022-04-28T21:58:25.525587+0000 mon.a (mon.0) 4210 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[61152]: cluster 2022-04-28T21:58:24.871180+0000 mgr.y (mgr.24961) 4182 : cluster [DBG] pgmap v4962: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:26.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[61152]: audit 2022-04-28T21:58:25.032566+0000 mon.a (mon.0) 4208 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[61152]: audit 2022-04-28T21:58:25.235517+0000 mon.a (mon.0) 4209 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.344 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:26 smithi059 bash[61152]: audit 2022-04-28T21:58:25.525587+0000 mon.a (mon.0) 4210 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:26 smithi139 bash[43301]: cluster 2022-04-28T21:58:24.871180+0000 mgr.y (mgr.24961) 4182 : cluster [DBG] pgmap v4962: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:26.375 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:26 smithi139 bash[43301]: audit 2022-04-28T21:58:25.032566+0000 mon.a (mon.0) 4208 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:26 smithi139 bash[43301]: audit 2022-04-28T21:58:25.235517+0000 mon.a (mon.0) 4209 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:26.376 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:26 smithi139 bash[43301]: audit 2022-04-28T21:58:25.525587+0000 mon.a (mon.0) 4210 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:58:28.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:28 smithi059 bash[61152]: cluster 2022-04-28T21:58:26.873053+0000 mgr.y (mgr.24961) 4183 : cluster [DBG] pgmap v4963: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:28.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:28 smithi059 bash[59745]: cluster 2022-04-28T21:58:26.873053+0000 mgr.y (mgr.24961) 4183 : cluster [DBG] pgmap v4963: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:28.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:28 smithi139 bash[43301]: cluster 2022-04-28T21:58:26.873053+0000 mgr.y (mgr.24961) 4183 : cluster [DBG] pgmap v4963: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:30.597 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:30 smithi059 bash[61152]: cluster 2022-04-28T21:58:28.873831+0000 mgr.y (mgr.24961) 4184 : cluster [DBG] pgmap v4964: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:30.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:30 smithi059 bash[59745]: cluster 2022-04-28T21:58:28.873831+0000 mgr.y (mgr.24961) 4184 : cluster [DBG] pgmap v4964: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:30 smithi139 bash[43301]: cluster 2022-04-28T21:58:28.873831+0000 mgr.y (mgr.24961) 4184 : cluster [DBG] pgmap v4964: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:32.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:32 smithi059 bash[61152]: cluster 2022-04-28T21:58:30.875793+0000 mgr.y (mgr.24961) 4185 : cluster [DBG] pgmap v4965: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:32.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:32.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:32 smithi059 bash[59745]: cluster 2022-04-28T21:58:30.875793+0000 mgr.y (mgr.24961) 4185 : cluster [DBG] pgmap v4965: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:32.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:32 smithi139 bash[43301]: cluster 2022-04-28T21:58:30.875793+0000 mgr.y (mgr.24961) 4185 : cluster [DBG] pgmap v4965: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:34.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:34 smithi059 bash[61152]: cluster 2022-04-28T21:58:32.877060+0000 mgr.y (mgr.24961) 4186 : cluster [DBG] pgmap v4966: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:34.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:34 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:34.017Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:34.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:34 smithi059 bash[59745]: cluster 2022-04-28T21:58:32.877060+0000 mgr.y (mgr.24961) 4186 : cluster [DBG] pgmap v4966: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:34.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:34 smithi139 bash[43301]: cluster 2022-04-28T21:58:32.877060+0000 mgr.y (mgr.24961) 4186 : cluster [DBG] pgmap v4966: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:36 smithi059 bash[61152]: cluster 2022-04-28T21:58:34.878540+0000 mgr.y (mgr.24961) 4187 : cluster [DBG] pgmap v4967: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:36.626 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:36 smithi059 bash[59745]: cluster 2022-04-28T21:58:34.878540+0000 mgr.y (mgr.24961) 4187 : cluster [DBG] pgmap v4967: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:36.627 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:36 smithi139 bash[43301]: cluster 2022-04-28T21:58:34.878540+0000 mgr.y (mgr.24961) 4187 : cluster [DBG] pgmap v4967: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:38 smithi059 bash[61152]: cluster 2022-04-28T21:58:36.880397+0000 mgr.y (mgr.24961) 4188 : cluster [DBG] pgmap v4968: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:38 smithi059 bash[59745]: cluster 2022-04-28T21:58:36.880397+0000 mgr.y (mgr.24961) 4188 : cluster [DBG] pgmap v4968: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:38.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:38 smithi139 bash[43301]: cluster 2022-04-28T21:58:36.880397+0000 mgr.y (mgr.24961) 4188 : cluster [DBG] pgmap v4968: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:40 smithi059 bash[61152]: cluster 2022-04-28T21:58:38.881129+0000 mgr.y (mgr.24961) 4189 : cluster [DBG] pgmap v4969: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:40 smithi059 bash[59745]: cluster 2022-04-28T21:58:38.881129+0000 mgr.y (mgr.24961) 4189 : cluster [DBG] pgmap v4969: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:40 smithi139 bash[43301]: cluster 2022-04-28T21:58:38.881129+0000 mgr.y (mgr.24961) 4189 : cluster [DBG] pgmap v4969: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:42.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:42 smithi059 bash[61152]: cluster 2022-04-28T21:58:40.883083+0000 mgr.y (mgr.24961) 4190 : cluster [DBG] pgmap v4970: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:42.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:42.130Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:42.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:42 smithi059 bash[59745]: cluster 2022-04-28T21:58:40.883083+0000 mgr.y (mgr.24961) 4190 : cluster [DBG] pgmap v4970: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:42.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:42 smithi139 bash[43301]: cluster 2022-04-28T21:58:40.883083+0000 mgr.y (mgr.24961) 4190 : cluster [DBG] pgmap v4970: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:44 smithi059 bash[61152]: cluster 2022-04-28T21:58:42.884369+0000 mgr.y (mgr.24961) 4191 : cluster [DBG] pgmap v4971: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:44.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:44 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:44.018Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:44 smithi059 bash[59745]: cluster 2022-04-28T21:58:42.884369+0000 mgr.y (mgr.24961) 4191 : cluster [DBG] pgmap v4971: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:44.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:44 smithi139 bash[43301]: cluster 2022-04-28T21:58:42.884369+0000 mgr.y (mgr.24961) 4191 : cluster [DBG] pgmap v4971: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:46 smithi059 bash[61152]: cluster 2022-04-28T21:58:44.885805+0000 mgr.y (mgr.24961) 4192 : cluster [DBG] pgmap v4972: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:46 smithi059 bash[59745]: cluster 2022-04-28T21:58:44.885805+0000 mgr.y (mgr.24961) 4192 : cluster [DBG] pgmap v4972: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:46 smithi139 bash[43301]: cluster 2022-04-28T21:58:44.885805+0000 mgr.y (mgr.24961) 4192 : cluster [DBG] pgmap v4972: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:48 smithi059 bash[61152]: cluster 2022-04-28T21:58:46.887614+0000 mgr.y (mgr.24961) 4193 : cluster [DBG] pgmap v4973: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:48 smithi059 bash[59745]: cluster 2022-04-28T21:58:46.887614+0000 mgr.y (mgr.24961) 4193 : cluster [DBG] pgmap v4973: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:48.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:48 smithi139 bash[43301]: cluster 2022-04-28T21:58:46.887614+0000 mgr.y (mgr.24961) 4193 : cluster [DBG] pgmap v4973: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:50 smithi059 bash[61152]: cluster 2022-04-28T21:58:48.888395+0000 mgr.y (mgr.24961) 4194 : cluster [DBG] pgmap v4974: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:50.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:50 smithi059 bash[59745]: cluster 2022-04-28T21:58:48.888395+0000 mgr.y (mgr.24961) 4194 : cluster [DBG] pgmap v4974: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:50.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:50 smithi139 bash[43301]: cluster 2022-04-28T21:58:48.888395+0000 mgr.y (mgr.24961) 4194 : cluster [DBG] pgmap v4974: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:52 smithi059 bash[61152]: cluster 2022-04-28T21:58:50.890394+0000 mgr.y (mgr.24961) 4195 : cluster [DBG] pgmap v4975: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:52.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:52 smithi059 bash[59745]: cluster 2022-04-28T21:58:50.890394+0000 mgr.y (mgr.24961) 4195 : cluster [DBG] pgmap v4975: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:52 smithi139 bash[43301]: cluster 2022-04-28T21:58:50.890394+0000 mgr.y (mgr.24961) 4195 : cluster [DBG] pgmap v4975: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:54.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:54 smithi059 bash[61152]: cluster 2022-04-28T21:58:52.891732+0000 mgr.y (mgr.24961) 4196 : cluster [DBG] pgmap v4976: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:54.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:58:54 smithi059 bash[72035]: level=error ts=2022-04-28T21:58:54.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:58:54.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:54 smithi059 bash[59745]: cluster 2022-04-28T21:58:52.891732+0000 mgr.y (mgr.24961) 4196 : cluster [DBG] pgmap v4976: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:54.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:54 smithi139 bash[43301]: cluster 2022-04-28T21:58:52.891732+0000 mgr.y (mgr.24961) 4196 : cluster [DBG] pgmap v4976: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:56.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:56 smithi059 bash[59745]: cluster 2022-04-28T21:58:54.893134+0000 mgr.y (mgr.24961) 4197 : cluster [DBG] pgmap v4977: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:56 smithi059 bash[61152]: cluster 2022-04-28T21:58:54.893134+0000 mgr.y (mgr.24961) 4197 : cluster [DBG] pgmap v4977: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:56 smithi139 bash[43301]: cluster 2022-04-28T21:58:54.893134+0000 mgr.y (mgr.24961) 4197 : cluster [DBG] pgmap v4977: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:58:58 smithi059 bash[61152]: cluster 2022-04-28T21:58:56.894942+0000 mgr.y (mgr.24961) 4198 : cluster [DBG] pgmap v4978: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:58:58 smithi059 bash[59745]: cluster 2022-04-28T21:58:56.894942+0000 mgr.y (mgr.24961) 4198 : cluster [DBG] pgmap v4978: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:58:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:58:58 smithi139 bash[43301]: cluster 2022-04-28T21:58:56.894942+0000 mgr.y (mgr.24961) 4198 : cluster [DBG] pgmap v4978: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:00 smithi059 bash[61152]: cluster 2022-04-28T21:58:58.895748+0000 mgr.y (mgr.24961) 4199 : cluster [DBG] pgmap v4979: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:00 smithi059 bash[59745]: cluster 2022-04-28T21:58:58.895748+0000 mgr.y (mgr.24961) 4199 : cluster [DBG] pgmap v4979: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:00 smithi139 bash[43301]: cluster 2022-04-28T21:58:58.895748+0000 mgr.y (mgr.24961) 4199 : cluster [DBG] pgmap v4979: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:02.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:02 smithi059 bash[61152]: cluster 2022-04-28T21:59:00.897673+0000 mgr.y (mgr.24961) 4200 : cluster [DBG] pgmap v4980: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:02.594 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:02 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:02.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:02.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:02 smithi059 bash[59745]: cluster 2022-04-28T21:59:00.897673+0000 mgr.y (mgr.24961) 4200 : cluster [DBG] pgmap v4980: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:02.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:02 smithi139 bash[43301]: cluster 2022-04-28T21:59:00.897673+0000 mgr.y (mgr.24961) 4200 : cluster [DBG] pgmap v4980: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:04 smithi059 bash[61152]: cluster 2022-04-28T21:59:02.898924+0000 mgr.y (mgr.24961) 4201 : cluster [DBG] pgmap v4981: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:04 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:04.019Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:04 smithi059 bash[59745]: cluster 2022-04-28T21:59:02.898924+0000 mgr.y (mgr.24961) 4201 : cluster [DBG] pgmap v4981: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:04.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:04 smithi139 bash[43301]: cluster 2022-04-28T21:59:02.898924+0000 mgr.y (mgr.24961) 4201 : cluster [DBG] pgmap v4981: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:06 smithi059 bash[61152]: cluster 2022-04-28T21:59:04.900369+0000 mgr.y (mgr.24961) 4202 : cluster [DBG] pgmap v4982: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:06 smithi059 bash[59745]: cluster 2022-04-28T21:59:04.900369+0000 mgr.y (mgr.24961) 4202 : cluster [DBG] pgmap v4982: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:06.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:06 smithi139 bash[43301]: cluster 2022-04-28T21:59:04.900369+0000 mgr.y (mgr.24961) 4202 : cluster [DBG] pgmap v4982: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:08 smithi059 bash[61152]: cluster 2022-04-28T21:59:06.902223+0000 mgr.y (mgr.24961) 4203 : cluster [DBG] pgmap v4983: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:08 smithi059 bash[59745]: cluster 2022-04-28T21:59:06.902223+0000 mgr.y (mgr.24961) 4203 : cluster [DBG] pgmap v4983: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:08.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:08 smithi139 bash[43301]: cluster 2022-04-28T21:59:06.902223+0000 mgr.y (mgr.24961) 4203 : cluster [DBG] pgmap v4983: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:10 smithi059 bash[61152]: cluster 2022-04-28T21:59:08.903029+0000 mgr.y (mgr.24961) 4204 : cluster [DBG] pgmap v4984: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:10 smithi059 bash[59745]: cluster 2022-04-28T21:59:08.903029+0000 mgr.y (mgr.24961) 4204 : cluster [DBG] pgmap v4984: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:10 smithi139 bash[43301]: cluster 2022-04-28T21:59:08.903029+0000 mgr.y (mgr.24961) 4204 : cluster [DBG] pgmap v4984: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:12 smithi059 bash[61152]: cluster 2022-04-28T21:59:10.904970+0000 mgr.y (mgr.24961) 4205 : cluster [DBG] pgmap v4985: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:12 smithi059 bash[59745]: cluster 2022-04-28T21:59:10.904970+0000 mgr.y (mgr.24961) 4205 : cluster [DBG] pgmap v4985: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:12 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:12.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:12 smithi139 bash[43301]: cluster 2022-04-28T21:59:10.904970+0000 mgr.y (mgr.24961) 4205 : cluster [DBG] pgmap v4985: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:14.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:14 smithi059 bash[59745]: cluster 2022-04-28T21:59:12.906226+0000 mgr.y (mgr.24961) 4206 : cluster [DBG] pgmap v4986: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:14.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:14 smithi059 bash[61152]: cluster 2022-04-28T21:59:12.906226+0000 mgr.y (mgr.24961) 4206 : cluster [DBG] pgmap v4986: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:14 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:14.020Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:14 smithi139 bash[43301]: cluster 2022-04-28T21:59:12.906226+0000 mgr.y (mgr.24961) 4206 : cluster [DBG] pgmap v4986: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:16.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:16 smithi059 bash[61152]: cluster 2022-04-28T21:59:14.907744+0000 mgr.y (mgr.24961) 4207 : cluster [DBG] pgmap v4987: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:16 smithi059 bash[59745]: cluster 2022-04-28T21:59:14.907744+0000 mgr.y (mgr.24961) 4207 : cluster [DBG] pgmap v4987: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:16 smithi139 bash[43301]: cluster 2022-04-28T21:59:14.907744+0000 mgr.y (mgr.24961) 4207 : cluster [DBG] pgmap v4987: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:18 smithi059 bash[61152]: cluster 2022-04-28T21:59:16.909538+0000 mgr.y (mgr.24961) 4208 : cluster [DBG] pgmap v4988: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:18 smithi059 bash[59745]: cluster 2022-04-28T21:59:16.909538+0000 mgr.y (mgr.24961) 4208 : cluster [DBG] pgmap v4988: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:18 smithi139 bash[43301]: cluster 2022-04-28T21:59:16.909538+0000 mgr.y (mgr.24961) 4208 : cluster [DBG] pgmap v4988: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: cluster 2022-04-28T21:59:18.910404+0000 mgr.y (mgr.24961) 4209 : cluster [DBG] pgmap v4989: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: audit 2022-04-28T21:59:19.435643+0000 mon.a (mon.0) 4211 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:59:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: audit 2022-04-28T21:59:19.458765+0000 mon.a (mon.0) 4212 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:59:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: audit 2022-04-28T21:59:19.531715+0000 mon.a (mon.0) 4213 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:59:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: audit 2022-04-28T21:59:19.532296+0000 mon.a (mon.0) 4214 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:59:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: audit 2022-04-28T21:59:19.532880+0000 mon.a (mon.0) 4215 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:59:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[61152]: audit 2022-04-28T21:59:19.533386+0000 mon.a (mon.0) 4216 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.d", "id": [4, 2]}]: dispatch 2022-04-28T21:59:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: cluster 2022-04-28T21:59:18.910404+0000 mgr.y (mgr.24961) 4209 : cluster [DBG] pgmap v4989: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: audit 2022-04-28T21:59:19.435643+0000 mon.a (mon.0) 4211 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:59:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: audit 2022-04-28T21:59:19.458765+0000 mon.a (mon.0) 4212 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:59:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: audit 2022-04-28T21:59:19.531715+0000 mon.a (mon.0) 4213 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:59:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: audit 2022-04-28T21:59:19.532296+0000 mon.a (mon.0) 4214 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:59:20.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: audit 2022-04-28T21:59:19.532880+0000 mon.a (mon.0) 4215 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:59:20.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:20 smithi059 bash[59745]: audit 2022-04-28T21:59:19.533386+0000 mon.a (mon.0) 4216 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.d", "id": [4, 2]}]: dispatch 2022-04-28T21:59:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: cluster 2022-04-28T21:59:18.910404+0000 mgr.y (mgr.24961) 4209 : cluster [DBG] pgmap v4989: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: audit 2022-04-28T21:59:19.435643+0000 mon.a (mon.0) 4211 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T21:59:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: audit 2022-04-28T21:59:19.458765+0000 mon.a (mon.0) 4212 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T21:59:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: audit 2022-04-28T21:59:19.531715+0000 mon.a (mon.0) 4213 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T21:59:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: audit 2022-04-28T21:59:19.532296+0000 mon.a (mon.0) 4214 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T21:59:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: audit 2022-04-28T21:59:19.532880+0000 mon.a (mon.0) 4215 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T21:59:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:20 smithi139 bash[43301]: audit 2022-04-28T21:59:19.533386+0000 mon.a (mon.0) 4216 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.d", "id": [4, 2]}]: dispatch 2022-04-28T21:59:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:22 smithi059 bash[61152]: cluster 2022-04-28T21:59:20.912403+0000 mgr.y (mgr.24961) 4210 : cluster [DBG] pgmap v4990: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:22.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:22 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:22.131Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:22.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:22 smithi059 bash[59745]: cluster 2022-04-28T21:59:20.912403+0000 mgr.y (mgr.24961) 4210 : cluster [DBG] pgmap v4990: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:22 smithi139 bash[43301]: cluster 2022-04-28T21:59:20.912403+0000 mgr.y (mgr.24961) 4210 : cluster [DBG] pgmap v4990: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:24 smithi059 bash[61152]: cluster 2022-04-28T21:59:22.913661+0000 mgr.y (mgr.24961) 4211 : cluster [DBG] pgmap v4991: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:24 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:24.021Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:24 smithi059 bash[59745]: cluster 2022-04-28T21:59:22.913661+0000 mgr.y (mgr.24961) 4211 : cluster [DBG] pgmap v4991: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:24 smithi139 bash[43301]: cluster 2022-04-28T21:59:22.913661+0000 mgr.y (mgr.24961) 4211 : cluster [DBG] pgmap v4991: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[59745]: cluster 2022-04-28T21:59:24.915072+0000 mgr.y (mgr.24961) 4212 : cluster [DBG] pgmap v4992: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[59745]: audit 2022-04-28T21:59:25.534458+0000 mon.a (mon.0) 4217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:59:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[59745]: audit 2022-04-28T21:59:25.538317+0000 mon.a (mon.0) 4218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:59:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[59745]: audit 2022-04-28T21:59:25.541347+0000 mon.a (mon.0) 4219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:59:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[59745]: cephadm 2022-04-28T21:59:25.542894+0000 mgr.y (mgr.24961) 4213 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:59:26.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[59745]: audit 2022-04-28T21:59:25.860361+0000 mon.a (mon.0) 4220 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:59:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[61152]: cluster 2022-04-28T21:59:24.915072+0000 mgr.y (mgr.24961) 4212 : cluster [DBG] pgmap v4992: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:26.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[61152]: audit 2022-04-28T21:59:25.534458+0000 mon.a (mon.0) 4217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:59:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[61152]: audit 2022-04-28T21:59:25.538317+0000 mon.a (mon.0) 4218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:59:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[61152]: audit 2022-04-28T21:59:25.541347+0000 mon.a (mon.0) 4219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:59:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[61152]: cephadm 2022-04-28T21:59:25.542894+0000 mgr.y (mgr.24961) 4213 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:59:26.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:26 smithi059 bash[61152]: audit 2022-04-28T21:59:25.860361+0000 mon.a (mon.0) 4220 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:59:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:26 smithi139 bash[43301]: cluster 2022-04-28T21:59:24.915072+0000 mgr.y (mgr.24961) 4212 : cluster [DBG] pgmap v4992: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:26 smithi139 bash[43301]: audit 2022-04-28T21:59:25.534458+0000 mon.a (mon.0) 4217 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T21:59:26.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:26 smithi139 bash[43301]: audit 2022-04-28T21:59:25.538317+0000 mon.a (mon.0) 4218 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T21:59:26.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:26 smithi139 bash[43301]: audit 2022-04-28T21:59:25.541347+0000 mon.a (mon.0) 4219 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T21:59:26.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:26 smithi139 bash[43301]: cephadm 2022-04-28T21:59:25.542894+0000 mgr.y (mgr.24961) 4213 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T21:59:26.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:26 smithi139 bash[43301]: audit 2022-04-28T21:59:25.860361+0000 mon.a (mon.0) 4220 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T21:59:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:28 smithi059 bash[59745]: cluster 2022-04-28T21:59:26.916892+0000 mgr.y (mgr.24961) 4214 : cluster [DBG] pgmap v4993: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:28 smithi059 bash[61152]: cluster 2022-04-28T21:59:26.916892+0000 mgr.y (mgr.24961) 4214 : cluster [DBG] pgmap v4993: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:28 smithi139 bash[43301]: cluster 2022-04-28T21:59:26.916892+0000 mgr.y (mgr.24961) 4214 : cluster [DBG] pgmap v4993: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:30 smithi059 bash[61152]: cluster 2022-04-28T21:59:28.917668+0000 mgr.y (mgr.24961) 4215 : cluster [DBG] pgmap v4994: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:30 smithi059 bash[59745]: cluster 2022-04-28T21:59:28.917668+0000 mgr.y (mgr.24961) 4215 : cluster [DBG] pgmap v4994: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:30 smithi139 bash[43301]: cluster 2022-04-28T21:59:28.917668+0000 mgr.y (mgr.24961) 4215 : cluster [DBG] pgmap v4994: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:32 smithi059 bash[61152]: cluster 2022-04-28T21:59:30.919625+0000 mgr.y (mgr.24961) 4216 : cluster [DBG] pgmap v4995: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:32.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:32 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:32.133Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:32.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:32 smithi059 bash[59745]: cluster 2022-04-28T21:59:30.919625+0000 mgr.y (mgr.24961) 4216 : cluster [DBG] pgmap v4995: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:32.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:32 smithi139 bash[43301]: cluster 2022-04-28T21:59:30.919625+0000 mgr.y (mgr.24961) 4216 : cluster [DBG] pgmap v4995: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:34.296 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:34 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:34.022Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:34.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:34 smithi059 bash[61152]: cluster 2022-04-28T21:59:32.920858+0000 mgr.y (mgr.24961) 4217 : cluster [DBG] pgmap v4996: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:34.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:34 smithi059 bash[59745]: cluster 2022-04-28T21:59:32.920858+0000 mgr.y (mgr.24961) 4217 : cluster [DBG] pgmap v4996: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:34.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:34 smithi139 bash[43301]: cluster 2022-04-28T21:59:32.920858+0000 mgr.y (mgr.24961) 4217 : cluster [DBG] pgmap v4996: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:36.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:36 smithi059 bash[61152]: cluster 2022-04-28T21:59:34.922254+0000 mgr.y (mgr.24961) 4218 : cluster [DBG] pgmap v4997: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:36.674 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:36 smithi059 bash[59745]: cluster 2022-04-28T21:59:34.922254+0000 mgr.y (mgr.24961) 4218 : cluster [DBG] pgmap v4997: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:36.674 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:36 smithi139 bash[43301]: cluster 2022-04-28T21:59:34.922254+0000 mgr.y (mgr.24961) 4218 : cluster [DBG] pgmap v4997: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:38 smithi059 bash[61152]: cluster 2022-04-28T21:59:36.924037+0000 mgr.y (mgr.24961) 4219 : cluster [DBG] pgmap v4998: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:38.631 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:38 smithi059 bash[59745]: cluster 2022-04-28T21:59:36.924037+0000 mgr.y (mgr.24961) 4219 : cluster [DBG] pgmap v4998: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:38.632 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:38 smithi139 bash[43301]: cluster 2022-04-28T21:59:36.924037+0000 mgr.y (mgr.24961) 4219 : cluster [DBG] pgmap v4998: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:40 smithi059 bash[61152]: cluster 2022-04-28T21:59:38.924901+0000 mgr.y (mgr.24961) 4220 : cluster [DBG] pgmap v4999: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:40.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:40 smithi059 bash[59745]: cluster 2022-04-28T21:59:38.924901+0000 mgr.y (mgr.24961) 4220 : cluster [DBG] pgmap v4999: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:40.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:40 smithi139 bash[43301]: cluster 2022-04-28T21:59:38.924901+0000 mgr.y (mgr.24961) 4220 : cluster [DBG] pgmap v4999: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:42 smithi059 bash[61152]: cluster 2022-04-28T21:59:40.926827+0000 mgr.y (mgr.24961) 4221 : cluster [DBG] pgmap v5000: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:42.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:42 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:42.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:42 smithi059 bash[59745]: cluster 2022-04-28T21:59:40.926827+0000 mgr.y (mgr.24961) 4221 : cluster [DBG] pgmap v5000: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:42 smithi139 bash[43301]: cluster 2022-04-28T21:59:40.926827+0000 mgr.y (mgr.24961) 4221 : cluster [DBG] pgmap v5000: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:44.322 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:44 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:44.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:44 smithi059 bash[61152]: cluster 2022-04-28T21:59:42.928103+0000 mgr.y (mgr.24961) 4222 : cluster [DBG] pgmap v5001: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:44 smithi059 bash[59745]: cluster 2022-04-28T21:59:42.928103+0000 mgr.y (mgr.24961) 4222 : cluster [DBG] pgmap v5001: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:44 smithi139 bash[43301]: cluster 2022-04-28T21:59:42.928103+0000 mgr.y (mgr.24961) 4222 : cluster [DBG] pgmap v5001: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:46 smithi059 bash[61152]: cluster 2022-04-28T21:59:44.929547+0000 mgr.y (mgr.24961) 4223 : cluster [DBG] pgmap v5002: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:46 smithi059 bash[59745]: cluster 2022-04-28T21:59:44.929547+0000 mgr.y (mgr.24961) 4223 : cluster [DBG] pgmap v5002: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:46 smithi139 bash[43301]: cluster 2022-04-28T21:59:44.929547+0000 mgr.y (mgr.24961) 4223 : cluster [DBG] pgmap v5002: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:48 smithi059 bash[61152]: cluster 2022-04-28T21:59:46.931408+0000 mgr.y (mgr.24961) 4224 : cluster [DBG] pgmap v5003: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:48 smithi059 bash[59745]: cluster 2022-04-28T21:59:46.931408+0000 mgr.y (mgr.24961) 4224 : cluster [DBG] pgmap v5003: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:48.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:48 smithi139 bash[43301]: cluster 2022-04-28T21:59:46.931408+0000 mgr.y (mgr.24961) 4224 : cluster [DBG] pgmap v5003: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:50 smithi059 bash[59745]: cluster 2022-04-28T21:59:48.932236+0000 mgr.y (mgr.24961) 4225 : cluster [DBG] pgmap v5004: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:50 smithi059 bash[61152]: cluster 2022-04-28T21:59:48.932236+0000 mgr.y (mgr.24961) 4225 : cluster [DBG] pgmap v5004: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:50.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:50 smithi139 bash[43301]: cluster 2022-04-28T21:59:48.932236+0000 mgr.y (mgr.24961) 4225 : cluster [DBG] pgmap v5004: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:52.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:52 smithi059 bash[61152]: cluster 2022-04-28T21:59:50.934142+0000 mgr.y (mgr.24961) 4226 : cluster [DBG] pgmap v5005: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:52.595 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:52 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:52.134Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:52.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:52 smithi059 bash[59745]: cluster 2022-04-28T21:59:50.934142+0000 mgr.y (mgr.24961) 4226 : cluster [DBG] pgmap v5005: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:52 smithi139 bash[43301]: cluster 2022-04-28T21:59:50.934142+0000 mgr.y (mgr.24961) 4226 : cluster [DBG] pgmap v5005: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 21:59:54 smithi059 bash[72035]: level=error ts=2022-04-28T21:59:54.024Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T21:59:54.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:54 smithi139 bash[43301]: cluster 2022-04-28T21:59:52.935414+0000 mgr.y (mgr.24961) 4227 : cluster [DBG] pgmap v5006: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:54.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:54 smithi059 bash[61152]: cluster 2022-04-28T21:59:52.935414+0000 mgr.y (mgr.24961) 4227 : cluster [DBG] pgmap v5006: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:54.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:54 smithi059 bash[59745]: cluster 2022-04-28T21:59:52.935414+0000 mgr.y (mgr.24961) 4227 : cluster [DBG] pgmap v5006: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:56.626 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:56 smithi139 bash[43301]: cluster 2022-04-28T21:59:54.936790+0000 mgr.y (mgr.24961) 4228 : cluster [DBG] pgmap v5007: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:56 smithi059 bash[61152]: cluster 2022-04-28T21:59:54.936790+0000 mgr.y (mgr.24961) 4228 : cluster [DBG] pgmap v5007: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:56 smithi059 bash[59745]: cluster 2022-04-28T21:59:54.936790+0000 mgr.y (mgr.24961) 4228 : cluster [DBG] pgmap v5007: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 21:59:58 smithi139 bash[43301]: cluster 2022-04-28T21:59:56.938554+0000 mgr.y (mgr.24961) 4229 : cluster [DBG] pgmap v5008: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 21:59:58 smithi059 bash[61152]: cluster 2022-04-28T21:59:56.938554+0000 mgr.y (mgr.24961) 4229 : cluster [DBG] pgmap v5008: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T21:59:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 21:59:58 smithi059 bash[59745]: cluster 2022-04-28T21:59:56.938554+0000 mgr.y (mgr.24961) 4229 : cluster [DBG] pgmap v5008: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:00.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:00 smithi139 bash[43301]: cluster 2022-04-28T21:59:58.939336+0000 mgr.y (mgr.24961) 4230 : cluster [DBG] pgmap v5009: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:00.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:00 smithi139 bash[43301]: cluster 2022-04-28T22:00:00.000120+0000 mon.a (mon.0) 4221 : cluster [INF] overall HEALTH_OK 2022-04-28T22:00:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:00 smithi059 bash[61152]: cluster 2022-04-28T21:59:58.939336+0000 mgr.y (mgr.24961) 4230 : cluster [DBG] pgmap v5009: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:00 smithi059 bash[61152]: cluster 2022-04-28T22:00:00.000120+0000 mon.a (mon.0) 4221 : cluster [INF] overall HEALTH_OK 2022-04-28T22:00:00.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:00 smithi059 bash[59745]: cluster 2022-04-28T21:59:58.939336+0000 mgr.y (mgr.24961) 4230 : cluster [DBG] pgmap v5009: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:00.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:00 smithi059 bash[59745]: cluster 2022-04-28T22:00:00.000120+0000 mon.a (mon.0) 4221 : cluster [INF] overall HEALTH_OK 2022-04-28T22:00:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:02 smithi059 bash[61152]: cluster 2022-04-28T22:00:00.941258+0000 mgr.y (mgr.24961) 4231 : cluster [DBG] pgmap v5010: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:02.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:02.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:02 smithi059 bash[59745]: cluster 2022-04-28T22:00:00.941258+0000 mgr.y (mgr.24961) 4231 : cluster [DBG] pgmap v5010: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:02 smithi139 bash[43301]: cluster 2022-04-28T22:00:00.941258+0000 mgr.y (mgr.24961) 4231 : cluster [DBG] pgmap v5010: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:04.025Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:04 smithi059 bash[61152]: cluster 2022-04-28T22:00:02.942525+0000 mgr.y (mgr.24961) 4232 : cluster [DBG] pgmap v5011: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:04.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:04 smithi059 bash[59745]: cluster 2022-04-28T22:00:02.942525+0000 mgr.y (mgr.24961) 4232 : cluster [DBG] pgmap v5011: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:04 smithi139 bash[43301]: cluster 2022-04-28T22:00:02.942525+0000 mgr.y (mgr.24961) 4232 : cluster [DBG] pgmap v5011: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:06 smithi059 bash[61152]: cluster 2022-04-28T22:00:04.943980+0000 mgr.y (mgr.24961) 4233 : cluster [DBG] pgmap v5012: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:06.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:06 smithi059 bash[59745]: cluster 2022-04-28T22:00:04.943980+0000 mgr.y (mgr.24961) 4233 : cluster [DBG] pgmap v5012: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:06 smithi139 bash[43301]: cluster 2022-04-28T22:00:04.943980+0000 mgr.y (mgr.24961) 4233 : cluster [DBG] pgmap v5012: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:08 smithi059 bash[61152]: cluster 2022-04-28T22:00:06.945842+0000 mgr.y (mgr.24961) 4234 : cluster [DBG] pgmap v5013: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:08 smithi059 bash[59745]: cluster 2022-04-28T22:00:06.945842+0000 mgr.y (mgr.24961) 4234 : cluster [DBG] pgmap v5013: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:08 smithi139 bash[43301]: cluster 2022-04-28T22:00:06.945842+0000 mgr.y (mgr.24961) 4234 : cluster [DBG] pgmap v5013: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:10.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:10 smithi059 bash[61152]: cluster 2022-04-28T22:00:08.946655+0000 mgr.y (mgr.24961) 4235 : cluster [DBG] pgmap v5014: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:10.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:10 smithi059 bash[59745]: cluster 2022-04-28T22:00:08.946655+0000 mgr.y (mgr.24961) 4235 : cluster [DBG] pgmap v5014: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:10.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:10 smithi139 bash[43301]: cluster 2022-04-28T22:00:08.946655+0000 mgr.y (mgr.24961) 4235 : cluster [DBG] pgmap v5014: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:12.400 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:12.137Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:12.400 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:12 smithi059 bash[59745]: cluster 2022-04-28T22:00:10.948607+0000 mgr.y (mgr.24961) 4236 : cluster [DBG] pgmap v5015: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:12 smithi059 bash[61152]: cluster 2022-04-28T22:00:10.948607+0000 mgr.y (mgr.24961) 4236 : cluster [DBG] pgmap v5015: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:12 smithi139 bash[43301]: cluster 2022-04-28T22:00:10.948607+0000 mgr.y (mgr.24961) 4236 : cluster [DBG] pgmap v5015: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:14.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:14.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:14 smithi059 bash[61152]: cluster 2022-04-28T22:00:12.949874+0000 mgr.y (mgr.24961) 4237 : cluster [DBG] pgmap v5016: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:14.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:14 smithi059 bash[59745]: cluster 2022-04-28T22:00:12.949874+0000 mgr.y (mgr.24961) 4237 : cluster [DBG] pgmap v5016: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:14 smithi139 bash[43301]: cluster 2022-04-28T22:00:12.949874+0000 mgr.y (mgr.24961) 4237 : cluster [DBG] pgmap v5016: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:16 smithi059 bash[61152]: cluster 2022-04-28T22:00:14.951225+0000 mgr.y (mgr.24961) 4238 : cluster [DBG] pgmap v5017: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:16 smithi059 bash[59745]: cluster 2022-04-28T22:00:14.951225+0000 mgr.y (mgr.24961) 4238 : cluster [DBG] pgmap v5017: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:16 smithi139 bash[43301]: cluster 2022-04-28T22:00:14.951225+0000 mgr.y (mgr.24961) 4238 : cluster [DBG] pgmap v5017: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:18.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:18 smithi059 bash[61152]: cluster 2022-04-28T22:00:16.953074+0000 mgr.y (mgr.24961) 4239 : cluster [DBG] pgmap v5018: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:18.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:18 smithi059 bash[59745]: cluster 2022-04-28T22:00:16.953074+0000 mgr.y (mgr.24961) 4239 : cluster [DBG] pgmap v5018: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:18 smithi139 bash[43301]: cluster 2022-04-28T22:00:16.953074+0000 mgr.y (mgr.24961) 4239 : cluster [DBG] pgmap v5018: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:20.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: cluster 2022-04-28T22:00:18.953795+0000 mgr.y (mgr.24961) 4240 : cluster [DBG] pgmap v5019: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: audit 2022-04-28T22:00:19.440805+0000 mon.a (mon.0) 4222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:00:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: audit 2022-04-28T22:00:19.465203+0000 mon.a (mon.0) 4223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:00:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: audit 2022-04-28T22:00:19.541344+0000 mon.a (mon.0) 4224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: audit 2022-04-28T22:00:19.541934+0000 mon.a (mon.0) 4225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: audit 2022-04-28T22:00:19.542421+0000 mon.a (mon.0) 4226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[59745]: audit 2022-04-28T22:00:19.542912+0000 mon.a (mon.0) 4227 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T22:00:20.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: cluster 2022-04-28T22:00:18.953795+0000 mgr.y (mgr.24961) 4240 : cluster [DBG] pgmap v5019: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:20.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: audit 2022-04-28T22:00:19.440805+0000 mon.a (mon.0) 4222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:00:20.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: audit 2022-04-28T22:00:19.465203+0000 mon.a (mon.0) 4223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:00:20.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: audit 2022-04-28T22:00:19.541344+0000 mon.a (mon.0) 4224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: audit 2022-04-28T22:00:19.541934+0000 mon.a (mon.0) 4225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: audit 2022-04-28T22:00:19.542421+0000 mon.a (mon.0) 4226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:20 smithi059 bash[61152]: audit 2022-04-28T22:00:19.542912+0000 mon.a (mon.0) 4227 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T22:00:20.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: cluster 2022-04-28T22:00:18.953795+0000 mgr.y (mgr.24961) 4240 : cluster [DBG] pgmap v5019: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:20.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: audit 2022-04-28T22:00:19.440805+0000 mon.a (mon.0) 4222 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:00:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: audit 2022-04-28T22:00:19.465203+0000 mon.a (mon.0) 4223 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:00:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: audit 2022-04-28T22:00:19.541344+0000 mon.a (mon.0) 4224 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: audit 2022-04-28T22:00:19.541934+0000 mon.a (mon.0) 4225 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: audit 2022-04-28T22:00:19.542421+0000 mon.a (mon.0) 4226 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T22:00:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:20 smithi139 bash[43301]: audit 2022-04-28T22:00:19.542912+0000 mon.a (mon.0) 4227 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T22:00:22.427 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:22 smithi059 bash[59745]: cluster 2022-04-28T22:00:20.955797+0000 mgr.y (mgr.24961) 4241 : cluster [DBG] pgmap v5020: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:22.428 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:22.138Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:22 smithi059 bash[61152]: cluster 2022-04-28T22:00:20.955797+0000 mgr.y (mgr.24961) 4241 : cluster [DBG] pgmap v5020: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:22 smithi139 bash[43301]: cluster 2022-04-28T22:00:20.955797+0000 mgr.y (mgr.24961) 4241 : cluster [DBG] pgmap v5020: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:24.026Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:24.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:24 smithi059 bash[61152]: cluster 2022-04-28T22:00:22.957080+0000 mgr.y (mgr.24961) 4242 : cluster [DBG] pgmap v5021: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:24 smithi059 bash[59745]: cluster 2022-04-28T22:00:22.957080+0000 mgr.y (mgr.24961) 4242 : cluster [DBG] pgmap v5021: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:24.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:24 smithi139 bash[43301]: cluster 2022-04-28T22:00:22.957080+0000 mgr.y (mgr.24961) 4242 : cluster [DBG] pgmap v5021: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: cluster 2022-04-28T22:00:24.958517+0000 mgr.y (mgr.24961) 4243 : cluster [DBG] pgmap v5022: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:26.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: audit 2022-04-28T22:00:25.868688+0000 mon.a (mon.0) 4228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:00:26.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: audit 2022-04-28T22:00:25.871553+0000 mon.a (mon.0) 4229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:00:26.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: audit 2022-04-28T22:00:25.873335+0000 mon.a (mon.0) 4230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:00:26.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: cephadm 2022-04-28T22:00:25.875086+0000 mgr.y (mgr.24961) 4244 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:00:26.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: audit 2022-04-28T22:00:26.171535+0000 mon.a (mon.0) 4231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:26.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[59745]: audit 2022-04-28T22:00:26.185723+0000 mon.a (mon.0) 4232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:26.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: cluster 2022-04-28T22:00:24.958517+0000 mgr.y (mgr.24961) 4243 : cluster [DBG] pgmap v5022: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:26.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: audit 2022-04-28T22:00:25.868688+0000 mon.a (mon.0) 4228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:00:26.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: audit 2022-04-28T22:00:25.871553+0000 mon.a (mon.0) 4229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:00:26.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: audit 2022-04-28T22:00:25.873335+0000 mon.a (mon.0) 4230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:00:26.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: cephadm 2022-04-28T22:00:25.875086+0000 mgr.y (mgr.24961) 4244 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:00:26.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: audit 2022-04-28T22:00:26.171535+0000 mon.a (mon.0) 4231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:26.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:26 smithi059 bash[61152]: audit 2022-04-28T22:00:26.185723+0000 mon.a (mon.0) 4232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:26.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: cluster 2022-04-28T22:00:24.958517+0000 mgr.y (mgr.24961) 4243 : cluster [DBG] pgmap v5022: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:26.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: audit 2022-04-28T22:00:25.868688+0000 mon.a (mon.0) 4228 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:00:26.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: audit 2022-04-28T22:00:25.871553+0000 mon.a (mon.0) 4229 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:00:26.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: audit 2022-04-28T22:00:25.873335+0000 mon.a (mon.0) 4230 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:00:26.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: cephadm 2022-04-28T22:00:25.875086+0000 mgr.y (mgr.24961) 4244 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:00:26.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: audit 2022-04-28T22:00:26.171535+0000 mon.a (mon.0) 4231 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:26.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:26 smithi139 bash[43301]: audit 2022-04-28T22:00:26.185723+0000 mon.a (mon.0) 4232 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:27.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:27 smithi059 bash[61152]: audit 2022-04-28T22:00:26.488451+0000 mon.a (mon.0) 4233 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:27 smithi059 bash[59745]: audit 2022-04-28T22:00:26.488451+0000 mon.a (mon.0) 4233 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:27 smithi139 bash[43301]: audit 2022-04-28T22:00:26.488451+0000 mon.a (mon.0) 4233 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:00:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:28 smithi059 bash[61152]: cluster 2022-04-28T22:00:26.960367+0000 mgr.y (mgr.24961) 4245 : cluster [DBG] pgmap v5023: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:28 smithi059 bash[59745]: cluster 2022-04-28T22:00:26.960367+0000 mgr.y (mgr.24961) 4245 : cluster [DBG] pgmap v5023: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:28 smithi139 bash[43301]: cluster 2022-04-28T22:00:26.960367+0000 mgr.y (mgr.24961) 4245 : cluster [DBG] pgmap v5023: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:30 smithi059 bash[61152]: cluster 2022-04-28T22:00:28.961160+0000 mgr.y (mgr.24961) 4246 : cluster [DBG] pgmap v5024: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:30 smithi059 bash[59745]: cluster 2022-04-28T22:00:28.961160+0000 mgr.y (mgr.24961) 4246 : cluster [DBG] pgmap v5024: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:30 smithi139 bash[43301]: cluster 2022-04-28T22:00:28.961160+0000 mgr.y (mgr.24961) 4246 : cluster [DBG] pgmap v5024: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:32.509 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:32.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:32.510 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:32 smithi059 bash[59745]: cluster 2022-04-28T22:00:30.963196+0000 mgr.y (mgr.24961) 4247 : cluster [DBG] pgmap v5025: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:32 smithi059 bash[61152]: cluster 2022-04-28T22:00:30.963196+0000 mgr.y (mgr.24961) 4247 : cluster [DBG] pgmap v5025: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:32 smithi139 bash[43301]: cluster 2022-04-28T22:00:30.963196+0000 mgr.y (mgr.24961) 4247 : cluster [DBG] pgmap v5025: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:34.027Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:34.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:34 smithi059 bash[59745]: cluster 2022-04-28T22:00:32.964580+0000 mgr.y (mgr.24961) 4248 : cluster [DBG] pgmap v5026: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:34 smithi059 bash[61152]: cluster 2022-04-28T22:00:32.964580+0000 mgr.y (mgr.24961) 4248 : cluster [DBG] pgmap v5026: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:34 smithi139 bash[43301]: cluster 2022-04-28T22:00:32.964580+0000 mgr.y (mgr.24961) 4248 : cluster [DBG] pgmap v5026: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:36.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:36 smithi059 bash[59745]: cluster 2022-04-28T22:00:34.966166+0000 mgr.y (mgr.24961) 4249 : cluster [DBG] pgmap v5027: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:36.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:36 smithi059 bash[61152]: cluster 2022-04-28T22:00:34.966166+0000 mgr.y (mgr.24961) 4249 : cluster [DBG] pgmap v5027: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:36.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:36 smithi139 bash[43301]: cluster 2022-04-28T22:00:34.966166+0000 mgr.y (mgr.24961) 4249 : cluster [DBG] pgmap v5027: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:38.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:38 smithi059 bash[61152]: cluster 2022-04-28T22:00:36.968134+0000 mgr.y (mgr.24961) 4250 : cluster [DBG] pgmap v5028: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:38 smithi059 bash[59745]: cluster 2022-04-28T22:00:36.968134+0000 mgr.y (mgr.24961) 4250 : cluster [DBG] pgmap v5028: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:38 smithi139 bash[43301]: cluster 2022-04-28T22:00:36.968134+0000 mgr.y (mgr.24961) 4250 : cluster [DBG] pgmap v5028: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:40 smithi059 bash[61152]: cluster 2022-04-28T22:00:38.968979+0000 mgr.y (mgr.24961) 4251 : cluster [DBG] pgmap v5029: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:40 smithi059 bash[59745]: cluster 2022-04-28T22:00:38.968979+0000 mgr.y (mgr.24961) 4251 : cluster [DBG] pgmap v5029: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:40 smithi139 bash[43301]: cluster 2022-04-28T22:00:38.968979+0000 mgr.y (mgr.24961) 4251 : cluster [DBG] pgmap v5029: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:42.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:41 smithi059 bash[61152]: cluster 2022-04-28T22:00:40.970958+0000 mgr.y (mgr.24961) 4252 : cluster [DBG] pgmap v5030: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:42.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:42.139Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:42.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:41 smithi059 bash[59745]: cluster 2022-04-28T22:00:40.970958+0000 mgr.y (mgr.24961) 4252 : cluster [DBG] pgmap v5030: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:42.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:41 smithi139 bash[43301]: cluster 2022-04-28T22:00:40.970958+0000 mgr.y (mgr.24961) 4252 : cluster [DBG] pgmap v5030: 233 pgs: 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[61152]: cluster 2022-04-28T22:00:41.973293+0000 mon.a (mon.0) 4234 : cluster [DBG] osdmap e1437: 8 total, 8 up, 8 in 2022-04-28T22:00:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[61152]: audit 2022-04-28T22:00:41.976533+0000 mon.b (mon.2) 30 : audit [INF] from='client.? 172.21.9.34:0/3580026237' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T22:00:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[61152]: audit 2022-04-28T22:00:41.984702+0000 mon.a (mon.0) 4235 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T22:00:43.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[61152]: audit 2022-04-28T22:00:42.976392+0000 mon.a (mon.0) 4236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T22:00:43.343 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[61152]: cluster 2022-04-28T22:00:42.976743+0000 mon.a (mon.0) 4237 : cluster [DBG] osdmap e1438: 8 total, 8 up, 8 in 2022-04-28T22:00:43.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[59745]: cluster 2022-04-28T22:00:41.973293+0000 mon.a (mon.0) 4234 : cluster [DBG] osdmap e1437: 8 total, 8 up, 8 in 2022-04-28T22:00:43.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[59745]: audit 2022-04-28T22:00:41.976533+0000 mon.b (mon.2) 30 : audit [INF] from='client.? 172.21.9.34:0/3580026237' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T22:00:43.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[59745]: audit 2022-04-28T22:00:41.984702+0000 mon.a (mon.0) 4235 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T22:00:43.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[59745]: audit 2022-04-28T22:00:42.976392+0000 mon.a (mon.0) 4236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T22:00:43.344 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:42 smithi059 bash[59745]: cluster 2022-04-28T22:00:42.976743+0000 mon.a (mon.0) 4237 : cluster [DBG] osdmap e1438: 8 total, 8 up, 8 in 2022-04-28T22:00:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:42 smithi139 bash[43301]: cluster 2022-04-28T22:00:41.973293+0000 mon.a (mon.0) 4234 : cluster [DBG] osdmap e1437: 8 total, 8 up, 8 in 2022-04-28T22:00:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:42 smithi139 bash[43301]: audit 2022-04-28T22:00:41.976533+0000 mon.b (mon.2) 30 : audit [INF] from='client.? 172.21.9.34:0/3580026237' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T22:00:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:42 smithi139 bash[43301]: audit 2022-04-28T22:00:41.984702+0000 mon.a (mon.0) 4235 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2022-04-28T22:00:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:42 smithi139 bash[43301]: audit 2022-04-28T22:00:42.976392+0000 mon.a (mon.0) 4236 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-senta04-3629860-1","app": "rados","yes_i_really_mean_it": true}]': finished 2022-04-28T22:00:43.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:42 smithi139 bash[43301]: cluster 2022-04-28T22:00:42.976743+0000 mon.a (mon.0) 4237 : cluster [DBG] osdmap e1438: 8 total, 8 up, 8 in 2022-04-28T22:00:44.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:44 smithi059 bash[61152]: cluster 2022-04-28T22:00:42.971856+0000 mgr.y (mgr.24961) 4253 : cluster [DBG] pgmap v5033: 265 pgs: 32 unknown, 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:44.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:44.023Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:44.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:44 smithi059 bash[59745]: cluster 2022-04-28T22:00:42.971856+0000 mgr.y (mgr.24961) 4253 : cluster [DBG] pgmap v5033: 265 pgs: 32 unknown, 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:44.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:44 smithi139 bash[43301]: cluster 2022-04-28T22:00:42.971856+0000 mgr.y (mgr.24961) 4253 : cluster [DBG] pgmap v5033: 265 pgs: 32 unknown, 233 active+clean; 24 KiB data, 916 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:45.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:45 smithi059 bash[61152]: cluster 2022-04-28T22:00:43.997895+0000 mon.a (mon.0) 4238 : cluster [DBG] osdmap e1439: 8 total, 8 up, 8 in 2022-04-28T22:00:45.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:45 smithi059 bash[59745]: cluster 2022-04-28T22:00:43.997895+0000 mon.a (mon.0) 4238 : cluster [DBG] osdmap e1439: 8 total, 8 up, 8 in 2022-04-28T22:00:45.371 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:45 smithi139 bash[43301]: cluster 2022-04-28T22:00:43.997895+0000 mon.a (mon.0) 4238 : cluster [DBG] osdmap e1439: 8 total, 8 up, 8 in 2022-04-28T22:00:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:46 smithi059 bash[61152]: cluster 2022-04-28T22:00:44.973493+0000 mgr.y (mgr.24961) 4254 : cluster [DBG] pgmap v5035: 265 pgs: 17 unknown, 248 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:46 smithi059 bash[59745]: cluster 2022-04-28T22:00:44.973493+0000 mgr.y (mgr.24961) 4254 : cluster [DBG] pgmap v5035: 265 pgs: 17 unknown, 248 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:46 smithi139 bash[43301]: cluster 2022-04-28T22:00:44.973493+0000 mgr.y (mgr.24961) 4254 : cluster [DBG] pgmap v5035: 265 pgs: 17 unknown, 248 active+clean; 24 KiB data, 913 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:00:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:48 smithi059 bash[59745]: cluster 2022-04-28T22:00:46.975578+0000 mgr.y (mgr.24961) 4255 : cluster [DBG] pgmap v5036: 265 pgs: 265 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T22:00:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:48 smithi059 bash[61152]: cluster 2022-04-28T22:00:46.975578+0000 mgr.y (mgr.24961) 4255 : cluster [DBG] pgmap v5036: 265 pgs: 265 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T22:00:48.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:48 smithi139 bash[43301]: cluster 2022-04-28T22:00:46.975578+0000 mgr.y (mgr.24961) 4255 : cluster [DBG] pgmap v5036: 265 pgs: 265 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 170 B/s wr, 1 op/s 2022-04-28T22:00:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:50 smithi059 bash[61152]: cluster 2022-04-28T22:00:48.976468+0000 mgr.y (mgr.24961) 4256 : cluster [DBG] pgmap v5037: 265 pgs: 265 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail; 730 B/s rd, 146 B/s wr, 0 op/s 2022-04-28T22:00:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:50 smithi059 bash[59745]: cluster 2022-04-28T22:00:48.976468+0000 mgr.y (mgr.24961) 4256 : cluster [DBG] pgmap v5037: 265 pgs: 265 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail; 730 B/s rd, 146 B/s wr, 0 op/s 2022-04-28T22:00:50.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:50 smithi139 bash[43301]: cluster 2022-04-28T22:00:48.976468+0000 mgr.y (mgr.24961) 4256 : cluster [DBG] pgmap v5037: 265 pgs: 265 active+clean; 24 KiB data, 910 MiB used, 714 GiB / 715 GiB avail; 730 B/s rd, 146 B/s wr, 0 op/s 2022-04-28T22:00:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:52 smithi059 bash[61152]: cluster 2022-04-28T22:00:50.978607+0000 mgr.y (mgr.24961) 4257 : cluster [DBG] pgmap v5038: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 127 B/s wr, 0 op/s 2022-04-28T22:00:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:52.141Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:52.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:52 smithi059 bash[59745]: cluster 2022-04-28T22:00:50.978607+0000 mgr.y (mgr.24961) 4257 : cluster [DBG] pgmap v5038: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 127 B/s wr, 0 op/s 2022-04-28T22:00:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:52 smithi139 bash[43301]: cluster 2022-04-28T22:00:50.978607+0000 mgr.y (mgr.24961) 4257 : cluster [DBG] pgmap v5038: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 127 B/s wr, 0 op/s 2022-04-28T22:00:54.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:54 smithi059 bash[61152]: cluster 2022-04-28T22:00:52.979974+0000 mgr.y (mgr.24961) 4258 : cluster [DBG] pgmap v5039: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 0 op/s 2022-04-28T22:00:54.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:00:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:00:54.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:00:54.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:54 smithi059 bash[59745]: cluster 2022-04-28T22:00:52.979974+0000 mgr.y (mgr.24961) 4258 : cluster [DBG] pgmap v5039: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 0 op/s 2022-04-28T22:00:54.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:54 smithi139 bash[43301]: cluster 2022-04-28T22:00:52.979974+0000 mgr.y (mgr.24961) 4258 : cluster [DBG] pgmap v5039: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 0 op/s 2022-04-28T22:00:56.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:56 smithi059 bash[61152]: cluster 2022-04-28T22:00:54.981530+0000 mgr.y (mgr.24961) 4259 : cluster [DBG] pgmap v5040: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 466 B/s rd, 93 B/s wr, 0 op/s 2022-04-28T22:00:56.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:56 smithi059 bash[59745]: cluster 2022-04-28T22:00:54.981530+0000 mgr.y (mgr.24961) 4259 : cluster [DBG] pgmap v5040: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 466 B/s rd, 93 B/s wr, 0 op/s 2022-04-28T22:00:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:56 smithi139 bash[43301]: cluster 2022-04-28T22:00:54.981530+0000 mgr.y (mgr.24961) 4259 : cluster [DBG] pgmap v5040: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 466 B/s rd, 93 B/s wr, 0 op/s 2022-04-28T22:00:58.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:00:58 smithi059 bash[61152]: cluster 2022-04-28T22:00:56.983475+0000 mgr.y (mgr.24961) 4260 : cluster [DBG] pgmap v5041: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T22:00:58.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:00:58 smithi059 bash[59745]: cluster 2022-04-28T22:00:56.983475+0000 mgr.y (mgr.24961) 4260 : cluster [DBG] pgmap v5041: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T22:00:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:00:58 smithi139 bash[43301]: cluster 2022-04-28T22:00:56.983475+0000 mgr.y (mgr.24961) 4260 : cluster [DBG] pgmap v5041: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail; 426 B/s rd, 85 B/s wr, 0 op/s 2022-04-28T22:01:00.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:00 smithi059 bash[61152]: cluster 2022-04-28T22:00:58.984408+0000 mgr.y (mgr.24961) 4261 : cluster [DBG] pgmap v5042: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:00.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:00 smithi059 bash[59745]: cluster 2022-04-28T22:00:58.984408+0000 mgr.y (mgr.24961) 4261 : cluster [DBG] pgmap v5042: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:00 smithi139 bash[43301]: cluster 2022-04-28T22:00:58.984408+0000 mgr.y (mgr.24961) 4261 : cluster [DBG] pgmap v5042: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:02.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:02 smithi059 bash[61152]: cluster 2022-04-28T22:01:00.986566+0000 mgr.y (mgr.24961) 4262 : cluster [DBG] pgmap v5043: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:02.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:02.142Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:02.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:02 smithi059 bash[59745]: cluster 2022-04-28T22:01:00.986566+0000 mgr.y (mgr.24961) 4262 : cluster [DBG] pgmap v5043: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:02.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:02 smithi139 bash[43301]: cluster 2022-04-28T22:01:00.986566+0000 mgr.y (mgr.24961) 4262 : cluster [DBG] pgmap v5043: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:04.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:04 smithi059 bash[61152]: cluster 2022-04-28T22:01:02.988041+0000 mgr.y (mgr.24961) 4263 : cluster [DBG] pgmap v5044: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:04.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:04.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:04.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:04 smithi059 bash[59745]: cluster 2022-04-28T22:01:02.988041+0000 mgr.y (mgr.24961) 4263 : cluster [DBG] pgmap v5044: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:04.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:04 smithi139 bash[43301]: cluster 2022-04-28T22:01:02.988041+0000 mgr.y (mgr.24961) 4263 : cluster [DBG] pgmap v5044: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:06.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:06 smithi059 bash[61152]: cluster 2022-04-28T22:01:04.989784+0000 mgr.y (mgr.24961) 4264 : cluster [DBG] pgmap v5045: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:06.630 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:06 smithi059 bash[59745]: cluster 2022-04-28T22:01:04.989784+0000 mgr.y (mgr.24961) 4264 : cluster [DBG] pgmap v5045: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:06.631 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:06 smithi139 bash[43301]: cluster 2022-04-28T22:01:04.989784+0000 mgr.y (mgr.24961) 4264 : cluster [DBG] pgmap v5045: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:08 smithi059 bash[61152]: cluster 2022-04-28T22:01:06.991592+0000 mgr.y (mgr.24961) 4265 : cluster [DBG] pgmap v5046: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:08 smithi059 bash[59745]: cluster 2022-04-28T22:01:06.991592+0000 mgr.y (mgr.24961) 4265 : cluster [DBG] pgmap v5046: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:08.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:08 smithi139 bash[43301]: cluster 2022-04-28T22:01:06.991592+0000 mgr.y (mgr.24961) 4265 : cluster [DBG] pgmap v5046: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:10 smithi059 bash[59745]: cluster 2022-04-28T22:01:08.992393+0000 mgr.y (mgr.24961) 4266 : cluster [DBG] pgmap v5047: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:10.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:10 smithi059 bash[61152]: cluster 2022-04-28T22:01:08.992393+0000 mgr.y (mgr.24961) 4266 : cluster [DBG] pgmap v5047: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:10 smithi139 bash[43301]: cluster 2022-04-28T22:01:08.992393+0000 mgr.y (mgr.24961) 4266 : cluster [DBG] pgmap v5047: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:12 smithi059 bash[61152]: cluster 2022-04-28T22:01:10.994515+0000 mgr.y (mgr.24961) 4267 : cluster [DBG] pgmap v5048: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:12.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:12.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:12 smithi059 bash[59745]: cluster 2022-04-28T22:01:10.994515+0000 mgr.y (mgr.24961) 4267 : cluster [DBG] pgmap v5048: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:12 smithi139 bash[43301]: cluster 2022-04-28T22:01:10.994515+0000 mgr.y (mgr.24961) 4267 : cluster [DBG] pgmap v5048: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:14 smithi059 bash[61152]: cluster 2022-04-28T22:01:12.995922+0000 mgr.y (mgr.24961) 4268 : cluster [DBG] pgmap v5049: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:14.029Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:14.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:14 smithi059 bash[59745]: cluster 2022-04-28T22:01:12.995922+0000 mgr.y (mgr.24961) 4268 : cluster [DBG] pgmap v5049: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:14 smithi139 bash[43301]: cluster 2022-04-28T22:01:12.995922+0000 mgr.y (mgr.24961) 4268 : cluster [DBG] pgmap v5049: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:16 smithi059 bash[59745]: cluster 2022-04-28T22:01:14.997933+0000 mgr.y (mgr.24961) 4269 : cluster [DBG] pgmap v5050: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:16.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:16 smithi059 bash[61152]: cluster 2022-04-28T22:01:14.997933+0000 mgr.y (mgr.24961) 4269 : cluster [DBG] pgmap v5050: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:16 smithi139 bash[43301]: cluster 2022-04-28T22:01:14.997933+0000 mgr.y (mgr.24961) 4269 : cluster [DBG] pgmap v5050: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:18.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:18 smithi059 bash[61152]: cluster 2022-04-28T22:01:16.999777+0000 mgr.y (mgr.24961) 4270 : cluster [DBG] pgmap v5051: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:18 smithi059 bash[59745]: cluster 2022-04-28T22:01:16.999777+0000 mgr.y (mgr.24961) 4270 : cluster [DBG] pgmap v5051: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:18 smithi139 bash[43301]: cluster 2022-04-28T22:01:16.999777+0000 mgr.y (mgr.24961) 4270 : cluster [DBG] pgmap v5051: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: cluster 2022-04-28T22:01:19.000593+0000 mgr.y (mgr.24961) 4271 : cluster [DBG] pgmap v5052: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: audit 2022-04-28T22:01:19.446051+0000 mon.a (mon.0) 4239 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:01:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: audit 2022-04-28T22:01:19.473540+0000 mon.a (mon.0) 4240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:01:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: audit 2022-04-28T22:01:19.552162+0000 mon.a (mon.0) 4241 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T22:01:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: audit 2022-04-28T22:01:19.553778+0000 mon.a (mon.0) 4242 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T22:01:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: audit 2022-04-28T22:01:19.554430+0000 mon.a (mon.0) 4243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:01:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[61152]: audit 2022-04-28T22:01:19.555061+0000 mon.a (mon.0) 4244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T22:01:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: cluster 2022-04-28T22:01:19.000593+0000 mgr.y (mgr.24961) 4271 : cluster [DBG] pgmap v5052: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: audit 2022-04-28T22:01:19.446051+0000 mon.a (mon.0) 4239 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:01:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: audit 2022-04-28T22:01:19.473540+0000 mon.a (mon.0) 4240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:01:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: audit 2022-04-28T22:01:19.552162+0000 mon.a (mon.0) 4241 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T22:01:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: audit 2022-04-28T22:01:19.553778+0000 mon.a (mon.0) 4242 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T22:01:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: audit 2022-04-28T22:01:19.554430+0000 mon.a (mon.0) 4243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:01:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:20 smithi059 bash[59745]: audit 2022-04-28T22:01:19.555061+0000 mon.a (mon.0) 4244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T22:01:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: cluster 2022-04-28T22:01:19.000593+0000 mgr.y (mgr.24961) 4271 : cluster [DBG] pgmap v5052: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: audit 2022-04-28T22:01:19.446051+0000 mon.a (mon.0) 4239 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:01:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: audit 2022-04-28T22:01:19.473540+0000 mon.a (mon.0) 4240 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:01:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: audit 2022-04-28T22:01:19.552162+0000 mon.a (mon.0) 4241 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2022-04-28T22:01:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: audit 2022-04-28T22:01:19.553778+0000 mon.a (mon.0) 4242 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2022-04-28T22:01:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: audit 2022-04-28T22:01:19.554430+0000 mon.a (mon.0) 4243 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:01:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:20 smithi139 bash[43301]: audit 2022-04-28T22:01:19.555061+0000 mon.a (mon.0) 4244 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1c", "id": [4, 2]}]: dispatch 2022-04-28T22:01:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:22 smithi059 bash[61152]: cluster 2022-04-28T22:01:21.002664+0000 mgr.y (mgr.24961) 4272 : cluster [DBG] pgmap v5053: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:22.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:22.143Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:22 smithi059 bash[59745]: cluster 2022-04-28T22:01:21.002664+0000 mgr.y (mgr.24961) 4272 : cluster [DBG] pgmap v5053: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:22 smithi139 bash[43301]: cluster 2022-04-28T22:01:21.002664+0000 mgr.y (mgr.24961) 4272 : cluster [DBG] pgmap v5053: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:24 smithi059 bash[61152]: cluster 2022-04-28T22:01:23.003857+0000 mgr.y (mgr.24961) 4273 : cluster [DBG] pgmap v5054: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:24.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:24 smithi059 bash[59745]: cluster 2022-04-28T22:01:23.003857+0000 mgr.y (mgr.24961) 4273 : cluster [DBG] pgmap v5054: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:24 smithi139 bash[43301]: cluster 2022-04-28T22:01:23.003857+0000 mgr.y (mgr.24961) 4273 : cluster [DBG] pgmap v5054: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:26 smithi059 bash[59745]: cluster 2022-04-28T22:01:25.005515+0000 mgr.y (mgr.24961) 4274 : cluster [DBG] pgmap v5055: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:26 smithi059 bash[61152]: cluster 2022-04-28T22:01:25.005515+0000 mgr.y (mgr.24961) 4274 : cluster [DBG] pgmap v5055: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:26 smithi139 bash[43301]: cluster 2022-04-28T22:01:25.005515+0000 mgr.y (mgr.24961) 4274 : cluster [DBG] pgmap v5055: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[61152]: audit 2022-04-28T22:01:26.496181+0000 mon.a (mon.0) 4245 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:01:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[61152]: audit 2022-04-28T22:01:26.499112+0000 mon.a (mon.0) 4246 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:01:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[61152]: audit 2022-04-28T22:01:26.501158+0000 mon.a (mon.0) 4247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:01:27.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[61152]: cephadm 2022-04-28T22:01:26.502886+0000 mgr.y (mgr.24961) 4275 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:01:27.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[61152]: audit 2022-04-28T22:01:26.822660+0000 mon.a (mon.0) 4248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:01:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[59745]: audit 2022-04-28T22:01:26.496181+0000 mon.a (mon.0) 4245 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:01:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[59745]: audit 2022-04-28T22:01:26.499112+0000 mon.a (mon.0) 4246 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:01:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[59745]: audit 2022-04-28T22:01:26.501158+0000 mon.a (mon.0) 4247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:01:27.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[59745]: cephadm 2022-04-28T22:01:26.502886+0000 mgr.y (mgr.24961) 4275 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:01:27.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:27 smithi059 bash[59745]: audit 2022-04-28T22:01:26.822660+0000 mon.a (mon.0) 4248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:01:27.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:27 smithi139 bash[43301]: audit 2022-04-28T22:01:26.496181+0000 mon.a (mon.0) 4245 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:01:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:27 smithi139 bash[43301]: audit 2022-04-28T22:01:26.499112+0000 mon.a (mon.0) 4246 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:01:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:27 smithi139 bash[43301]: audit 2022-04-28T22:01:26.501158+0000 mon.a (mon.0) 4247 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:01:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:27 smithi139 bash[43301]: cephadm 2022-04-28T22:01:26.502886+0000 mgr.y (mgr.24961) 4275 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:01:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:27 smithi139 bash[43301]: audit 2022-04-28T22:01:26.822660+0000 mon.a (mon.0) 4248 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:01:29.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:28 smithi059 bash[61152]: cluster 2022-04-28T22:01:27.007290+0000 mgr.y (mgr.24961) 4276 : cluster [DBG] pgmap v5056: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:29.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:28 smithi059 bash[59745]: cluster 2022-04-28T22:01:27.007290+0000 mgr.y (mgr.24961) 4276 : cluster [DBG] pgmap v5056: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:29.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:28 smithi139 bash[43301]: cluster 2022-04-28T22:01:27.007290+0000 mgr.y (mgr.24961) 4276 : cluster [DBG] pgmap v5056: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:31.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:30 smithi059 bash[61152]: cluster 2022-04-28T22:01:29.008346+0000 mgr.y (mgr.24961) 4277 : cluster [DBG] pgmap v5057: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:30 smithi059 bash[59745]: cluster 2022-04-28T22:01:29.008346+0000 mgr.y (mgr.24961) 4277 : cluster [DBG] pgmap v5057: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:31.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:30 smithi139 bash[43301]: cluster 2022-04-28T22:01:29.008346+0000 mgr.y (mgr.24961) 4277 : cluster [DBG] pgmap v5057: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:32.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:33.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:32 smithi059 bash[61152]: cluster 2022-04-28T22:01:31.009595+0000 mgr.y (mgr.24961) 4278 : cluster [DBG] pgmap v5058: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:33.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:32 smithi059 bash[59745]: cluster 2022-04-28T22:01:31.009595+0000 mgr.y (mgr.24961) 4278 : cluster [DBG] pgmap v5058: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:33.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:32 smithi139 bash[43301]: cluster 2022-04-28T22:01:31.009595+0000 mgr.y (mgr.24961) 4278 : cluster [DBG] pgmap v5058: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:34.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:35.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:34 smithi139 bash[43301]: cluster 2022-04-28T22:01:33.010708+0000 mgr.y (mgr.24961) 4279 : cluster [DBG] pgmap v5059: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:35.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:34 smithi059 bash[61152]: cluster 2022-04-28T22:01:33.010708+0000 mgr.y (mgr.24961) 4279 : cluster [DBG] pgmap v5059: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:35.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:34 smithi059 bash[59745]: cluster 2022-04-28T22:01:33.010708+0000 mgr.y (mgr.24961) 4279 : cluster [DBG] pgmap v5059: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:36 smithi139 bash[43301]: cluster 2022-04-28T22:01:35.012395+0000 mgr.y (mgr.24961) 4280 : cluster [DBG] pgmap v5060: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:36 smithi059 bash[61152]: cluster 2022-04-28T22:01:35.012395+0000 mgr.y (mgr.24961) 4280 : cluster [DBG] pgmap v5060: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:36 smithi059 bash[59745]: cluster 2022-04-28T22:01:35.012395+0000 mgr.y (mgr.24961) 4280 : cluster [DBG] pgmap v5060: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:38 smithi139 bash[43301]: cluster 2022-04-28T22:01:37.014153+0000 mgr.y (mgr.24961) 4281 : cluster [DBG] pgmap v5061: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:39.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:38 smithi059 bash[61152]: cluster 2022-04-28T22:01:37.014153+0000 mgr.y (mgr.24961) 4281 : cluster [DBG] pgmap v5061: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:38 smithi059 bash[59745]: cluster 2022-04-28T22:01:37.014153+0000 mgr.y (mgr.24961) 4281 : cluster [DBG] pgmap v5061: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:41.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:40 smithi059 bash[61152]: cluster 2022-04-28T22:01:39.015154+0000 mgr.y (mgr.24961) 4282 : cluster [DBG] pgmap v5062: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:41.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:40 smithi059 bash[59745]: cluster 2022-04-28T22:01:39.015154+0000 mgr.y (mgr.24961) 4282 : cluster [DBG] pgmap v5062: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:41.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:40 smithi139 bash[43301]: cluster 2022-04-28T22:01:39.015154+0000 mgr.y (mgr.24961) 4282 : cluster [DBG] pgmap v5062: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:42.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:42.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:43.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:42 smithi059 bash[61152]: cluster 2022-04-28T22:01:41.017288+0000 mgr.y (mgr.24961) 4283 : cluster [DBG] pgmap v5063: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:43.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:42 smithi059 bash[59745]: cluster 2022-04-28T22:01:41.017288+0000 mgr.y (mgr.24961) 4283 : cluster [DBG] pgmap v5063: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:43.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:42 smithi139 bash[43301]: cluster 2022-04-28T22:01:41.017288+0000 mgr.y (mgr.24961) 4283 : cluster [DBG] pgmap v5063: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:44.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:45.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:44 smithi059 bash[61152]: cluster 2022-04-28T22:01:43.018559+0000 mgr.y (mgr.24961) 4284 : cluster [DBG] pgmap v5064: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:45.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:44 smithi059 bash[59745]: cluster 2022-04-28T22:01:43.018559+0000 mgr.y (mgr.24961) 4284 : cluster [DBG] pgmap v5064: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:45.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:44 smithi139 bash[43301]: cluster 2022-04-28T22:01:43.018559+0000 mgr.y (mgr.24961) 4284 : cluster [DBG] pgmap v5064: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:47.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:46 smithi059 bash[61152]: cluster 2022-04-28T22:01:45.020315+0000 mgr.y (mgr.24961) 4285 : cluster [DBG] pgmap v5065: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:47.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:46 smithi059 bash[59745]: cluster 2022-04-28T22:01:45.020315+0000 mgr.y (mgr.24961) 4285 : cluster [DBG] pgmap v5065: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:47.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:46 smithi139 bash[43301]: cluster 2022-04-28T22:01:45.020315+0000 mgr.y (mgr.24961) 4285 : cluster [DBG] pgmap v5065: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:49.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:48 smithi059 bash[61152]: cluster 2022-04-28T22:01:47.021746+0000 mgr.y (mgr.24961) 4286 : cluster [DBG] pgmap v5066: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:49.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:48 smithi059 bash[59745]: cluster 2022-04-28T22:01:47.021746+0000 mgr.y (mgr.24961) 4286 : cluster [DBG] pgmap v5066: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:49.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:48 smithi139 bash[43301]: cluster 2022-04-28T22:01:47.021746+0000 mgr.y (mgr.24961) 4286 : cluster [DBG] pgmap v5066: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:51.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:50 smithi059 bash[61152]: cluster 2022-04-28T22:01:49.022821+0000 mgr.y (mgr.24961) 4287 : cluster [DBG] pgmap v5067: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:51.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:50 smithi059 bash[59745]: cluster 2022-04-28T22:01:49.022821+0000 mgr.y (mgr.24961) 4287 : cluster [DBG] pgmap v5067: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:51.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:50 smithi139 bash[43301]: cluster 2022-04-28T22:01:49.022821+0000 mgr.y (mgr.24961) 4287 : cluster [DBG] pgmap v5067: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:52.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:53.341 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:52 smithi059 bash[59745]: cluster 2022-04-28T22:01:51.025201+0000 mgr.y (mgr.24961) 4288 : cluster [DBG] pgmap v5068: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:53.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:52 smithi059 bash[61152]: cluster 2022-04-28T22:01:51.025201+0000 mgr.y (mgr.24961) 4288 : cluster [DBG] pgmap v5068: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:53.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:52 smithi139 bash[43301]: cluster 2022-04-28T22:01:51.025201+0000 mgr.y (mgr.24961) 4288 : cluster [DBG] pgmap v5068: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:54.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:01:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:01:54.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:01:55.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:54 smithi059 bash[61152]: cluster 2022-04-28T22:01:53.026063+0000 mgr.y (mgr.24961) 4289 : cluster [DBG] pgmap v5069: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:55.380 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:54 smithi059 bash[59745]: cluster 2022-04-28T22:01:53.026063+0000 mgr.y (mgr.24961) 4289 : cluster [DBG] pgmap v5069: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:55.380 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:54 smithi139 bash[43301]: cluster 2022-04-28T22:01:53.026063+0000 mgr.y (mgr.24961) 4289 : cluster [DBG] pgmap v5069: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:56.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:55 smithi059 bash[61152]: cluster 2022-04-28T22:01:55.027409+0000 mgr.y (mgr.24961) 4290 : cluster [DBG] pgmap v5070: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:56.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:55 smithi059 bash[59745]: cluster 2022-04-28T22:01:55.027409+0000 mgr.y (mgr.24961) 4290 : cluster [DBG] pgmap v5070: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:56.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:55 smithi139 bash[43301]: cluster 2022-04-28T22:01:55.027409+0000 mgr.y (mgr.24961) 4290 : cluster [DBG] pgmap v5070: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:58.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:01:58 smithi059 bash[61152]: cluster 2022-04-28T22:01:57.029160+0000 mgr.y (mgr.24961) 4291 : cluster [DBG] pgmap v5071: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:58.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:01:58 smithi059 bash[59745]: cluster 2022-04-28T22:01:57.029160+0000 mgr.y (mgr.24961) 4291 : cluster [DBG] pgmap v5071: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:01:58.370 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:01:58 smithi139 bash[43301]: cluster 2022-04-28T22:01:57.029160+0000 mgr.y (mgr.24961) 4291 : cluster [DBG] pgmap v5071: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:00.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:00 smithi059 bash[59745]: cluster 2022-04-28T22:01:59.030182+0000 mgr.y (mgr.24961) 4292 : cluster [DBG] pgmap v5072: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:00.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:00 smithi059 bash[61152]: cluster 2022-04-28T22:01:59.030182+0000 mgr.y (mgr.24961) 4292 : cluster [DBG] pgmap v5072: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:00 smithi139 bash[43301]: cluster 2022-04-28T22:01:59.030182+0000 mgr.y (mgr.24961) 4292 : cluster [DBG] pgmap v5072: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:02 smithi059 bash[61152]: cluster 2022-04-28T22:02:01.032521+0000 mgr.y (mgr.24961) 4293 : cluster [DBG] pgmap v5073: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:02.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:02.145Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:02 smithi059 bash[59745]: cluster 2022-04-28T22:02:01.032521+0000 mgr.y (mgr.24961) 4293 : cluster [DBG] pgmap v5073: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:02.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:02 smithi139 bash[43301]: cluster 2022-04-28T22:02:01.032521+0000 mgr.y (mgr.24961) 4293 : cluster [DBG] pgmap v5073: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:04.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:04 smithi059 bash[61152]: cluster 2022-04-28T22:02:03.033402+0000 mgr.y (mgr.24961) 4294 : cluster [DBG] pgmap v5074: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:04.030Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:04.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:04 smithi059 bash[59745]: cluster 2022-04-28T22:02:03.033402+0000 mgr.y (mgr.24961) 4294 : cluster [DBG] pgmap v5074: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:04.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:04 smithi139 bash[43301]: cluster 2022-04-28T22:02:03.033402+0000 mgr.y (mgr.24961) 4294 : cluster [DBG] pgmap v5074: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:06.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:06 smithi059 bash[61152]: cluster 2022-04-28T22:02:05.035134+0000 mgr.y (mgr.24961) 4295 : cluster [DBG] pgmap v5075: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:06.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:06 smithi059 bash[59745]: cluster 2022-04-28T22:02:05.035134+0000 mgr.y (mgr.24961) 4295 : cluster [DBG] pgmap v5075: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:06.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:06 smithi139 bash[43301]: cluster 2022-04-28T22:02:05.035134+0000 mgr.y (mgr.24961) 4295 : cluster [DBG] pgmap v5075: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:08.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:08 smithi059 bash[61152]: cluster 2022-04-28T22:02:07.036929+0000 mgr.y (mgr.24961) 4296 : cluster [DBG] pgmap v5076: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:08.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:08 smithi059 bash[59745]: cluster 2022-04-28T22:02:07.036929+0000 mgr.y (mgr.24961) 4296 : cluster [DBG] pgmap v5076: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:08.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:08 smithi139 bash[43301]: cluster 2022-04-28T22:02:07.036929+0000 mgr.y (mgr.24961) 4296 : cluster [DBG] pgmap v5076: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:10.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:10 smithi059 bash[61152]: cluster 2022-04-28T22:02:09.037990+0000 mgr.y (mgr.24961) 4297 : cluster [DBG] pgmap v5077: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:10.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:10 smithi059 bash[59745]: cluster 2022-04-28T22:02:09.037990+0000 mgr.y (mgr.24961) 4297 : cluster [DBG] pgmap v5077: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:10 smithi139 bash[43301]: cluster 2022-04-28T22:02:09.037990+0000 mgr.y (mgr.24961) 4297 : cluster [DBG] pgmap v5077: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:12 smithi059 bash[61152]: cluster 2022-04-28T22:02:11.039930+0000 mgr.y (mgr.24961) 4298 : cluster [DBG] pgmap v5078: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:12.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:12.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:12 smithi059 bash[59745]: cluster 2022-04-28T22:02:11.039930+0000 mgr.y (mgr.24961) 4298 : cluster [DBG] pgmap v5078: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:12.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:12 smithi139 bash[43301]: cluster 2022-04-28T22:02:11.039930+0000 mgr.y (mgr.24961) 4298 : cluster [DBG] pgmap v5078: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:14.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:14 smithi059 bash[61152]: cluster 2022-04-28T22:02:13.040767+0000 mgr.y (mgr.24961) 4299 : cluster [DBG] pgmap v5079: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:14.032Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:14.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:14 smithi059 bash[59745]: cluster 2022-04-28T22:02:13.040767+0000 mgr.y (mgr.24961) 4299 : cluster [DBG] pgmap v5079: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:14.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:14 smithi139 bash[43301]: cluster 2022-04-28T22:02:13.040767+0000 mgr.y (mgr.24961) 4299 : cluster [DBG] pgmap v5079: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:16.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:16 smithi059 bash[61152]: cluster 2022-04-28T22:02:15.042616+0000 mgr.y (mgr.24961) 4300 : cluster [DBG] pgmap v5080: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:16.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:16 smithi059 bash[59745]: cluster 2022-04-28T22:02:15.042616+0000 mgr.y (mgr.24961) 4300 : cluster [DBG] pgmap v5080: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:16.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:16 smithi139 bash[43301]: cluster 2022-04-28T22:02:15.042616+0000 mgr.y (mgr.24961) 4300 : cluster [DBG] pgmap v5080: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:18.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:18 smithi059 bash[61152]: cluster 2022-04-28T22:02:17.044553+0000 mgr.y (mgr.24961) 4301 : cluster [DBG] pgmap v5081: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:18.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:18 smithi059 bash[59745]: cluster 2022-04-28T22:02:17.044553+0000 mgr.y (mgr.24961) 4301 : cluster [DBG] pgmap v5081: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:18.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:18 smithi139 bash[43301]: cluster 2022-04-28T22:02:17.044553+0000 mgr.y (mgr.24961) 4301 : cluster [DBG] pgmap v5081: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: cluster 2022-04-28T22:02:19.045654+0000 mgr.y (mgr.24961) 4302 : cluster [DBG] pgmap v5082: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: audit 2022-04-28T22:02:19.451451+0000 mon.a (mon.0) 4249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:02:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: audit 2022-04-28T22:02:19.480370+0000 mon.a (mon.0) 4250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:02:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: audit 2022-04-28T22:02:19.563467+0000 mon.a (mon.0) 4251 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T22:02:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: audit 2022-04-28T22:02:19.564012+0000 mon.a (mon.0) 4252 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T22:02:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: audit 2022-04-28T22:02:19.564787+0000 mon.a (mon.0) 4253 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T22:02:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[61152]: audit 2022-04-28T22:02:19.565569+0000 mon.a (mon.0) 4254 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T22:02:20.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: cluster 2022-04-28T22:02:19.045654+0000 mgr.y (mgr.24961) 4302 : cluster [DBG] pgmap v5082: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:20.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: audit 2022-04-28T22:02:19.451451+0000 mon.a (mon.0) 4249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:02:20.599 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: audit 2022-04-28T22:02:19.480370+0000 mon.a (mon.0) 4250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:02:20.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: audit 2022-04-28T22:02:19.563467+0000 mon.a (mon.0) 4251 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T22:02:20.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: audit 2022-04-28T22:02:19.564012+0000 mon.a (mon.0) 4252 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T22:02:20.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: audit 2022-04-28T22:02:19.564787+0000 mon.a (mon.0) 4253 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T22:02:20.600 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:20 smithi059 bash[59745]: audit 2022-04-28T22:02:19.565569+0000 mon.a (mon.0) 4254 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T22:02:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: cluster 2022-04-28T22:02:19.045654+0000 mgr.y (mgr.24961) 4302 : cluster [DBG] pgmap v5082: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: audit 2022-04-28T22:02:19.451451+0000 mon.a (mon.0) 4249 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:02:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: audit 2022-04-28T22:02:19.480370+0000 mon.a (mon.0) 4250 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:02:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: audit 2022-04-28T22:02:19.563467+0000 mon.a (mon.0) 4251 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2022-04-28T22:02:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: audit 2022-04-28T22:02:19.564012+0000 mon.a (mon.0) 4252 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T22:02:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: audit 2022-04-28T22:02:19.564787+0000 mon.a (mon.0) 4253 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2022-04-28T22:02:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:20 smithi139 bash[43301]: audit 2022-04-28T22:02:19.565569+0000 mon.a (mon.0) 4254 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.15", "id": [4, 2]}]: dispatch 2022-04-28T22:02:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:22 smithi059 bash[61152]: cluster 2022-04-28T22:02:21.048022+0000 mgr.y (mgr.24961) 4303 : cluster [DBG] pgmap v5083: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:22.617 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:22.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:22.618 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:22 smithi059 bash[59745]: cluster 2022-04-28T22:02:21.048022+0000 mgr.y (mgr.24961) 4303 : cluster [DBG] pgmap v5083: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:22 smithi139 bash[43301]: cluster 2022-04-28T22:02:21.048022+0000 mgr.y (mgr.24961) 4303 : cluster [DBG] pgmap v5083: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:24.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:24 smithi059 bash[61152]: cluster 2022-04-28T22:02:23.048855+0000 mgr.y (mgr.24961) 4304 : cluster [DBG] pgmap v5084: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:24.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:24.033Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:24.345 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:24 smithi059 bash[59745]: cluster 2022-04-28T22:02:23.048855+0000 mgr.y (mgr.24961) 4304 : cluster [DBG] pgmap v5084: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:24 smithi139 bash[43301]: cluster 2022-04-28T22:02:23.048855+0000 mgr.y (mgr.24961) 4304 : cluster [DBG] pgmap v5084: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:26 smithi059 bash[61152]: cluster 2022-04-28T22:02:25.050559+0000 mgr.y (mgr.24961) 4305 : cluster [DBG] pgmap v5085: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:26 smithi059 bash[59745]: cluster 2022-04-28T22:02:25.050559+0000 mgr.y (mgr.24961) 4305 : cluster [DBG] pgmap v5085: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:26 smithi139 bash[43301]: cluster 2022-04-28T22:02:25.050559+0000 mgr.y (mgr.24961) 4305 : cluster [DBG] pgmap v5085: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:27.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:26.829191+0000 mon.a (mon.0) 4255 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:02:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:26.832010+0000 mon.a (mon.0) 4256 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:02:27.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:26.833787+0000 mon.a (mon.0) 4257 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:02:27.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: cephadm 2022-04-28T22:02:26.835504+0000 mgr.y (mgr.24961) 4306 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:02:27.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.121238+0000 mon.a (mon.0) 4258 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.122087+0000 mon.a (mon.0) 4259 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.122699+0000 mon.a (mon.0) 4260 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.123275+0000 mon.a (mon.0) 4261 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.123880+0000 mon.a (mon.0) 4262 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.124468+0000 mon.a (mon.0) 4263 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.125188+0000 mon.a (mon.0) 4264 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.596 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.126017+0000 mon.a (mon.0) 4265 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.126749+0000 mon.a (mon.0) 4266 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T22:02:27.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.127479+0000 mon.a (mon.0) 4267 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.597 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.136353+0000 mon.a (mon.0) 4268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.598 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.137122+0000 mon.a (mon.0) 4269 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.598 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:26.829191+0000 mon.a (mon.0) 4255 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:02:27.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:26.832010+0000 mon.a (mon.0) 4256 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:02:27.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:26.833787+0000 mon.a (mon.0) 4257 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:02:27.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: cephadm 2022-04-28T22:02:26.835504+0000 mgr.y (mgr.24961) 4306 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:02:27.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.121238+0000 mon.a (mon.0) 4258 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.599 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.122087+0000 mon.a (mon.0) 4259 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.122699+0000 mon.a (mon.0) 4260 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.123275+0000 mon.a (mon.0) 4261 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.600 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.123880+0000 mon.a (mon.0) 4262 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.124468+0000 mon.a (mon.0) 4263 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.125188+0000 mon.a (mon.0) 4264 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.126017+0000 mon.a (mon.0) 4265 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.601 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.126749+0000 mon.a (mon.0) 4266 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T22:02:27.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.127479+0000 mon.a (mon.0) 4267 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.136353+0000 mon.a (mon.0) 4268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.602 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.137122+0000 mon.a (mon.0) 4269 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.137836+0000 mon.a (mon.0) 4270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.138563+0000 mon.a (mon.0) 4271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.603 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.139284+0000 mon.a (mon.0) 4272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.604 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.140026+0000 mon.a (mon.0) 4273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.604 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.140770+0000 mon.a (mon.0) 4274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.604 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.141496+0000 mon.a (mon.0) 4275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.604 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.142233+0000 mon.a (mon.0) 4276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T22:02:27.604 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.143044+0000 mon.a (mon.0) 4277 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.605 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[59745]: audit 2022-04-28T22:02:27.150302+0000 mon.a (mon.0) 4278 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:02:27.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.137836+0000 mon.a (mon.0) 4270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.138563+0000 mon.a (mon.0) 4271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.139284+0000 mon.a (mon.0) 4272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.605 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.140026+0000 mon.a (mon.0) 4273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.140770+0000 mon.a (mon.0) 4274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.141496+0000 mon.a (mon.0) 4275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.142233+0000 mon.a (mon.0) 4276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T22:02:27.606 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.143044+0000 mon.a (mon.0) 4277 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.607 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:27 smithi059 bash[61152]: audit 2022-04-28T22:02:27.150302+0000 mon.a (mon.0) 4278 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:02:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:26.829191+0000 mon.a (mon.0) 4255 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:02:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:26.832010+0000 mon.a (mon.0) 4256 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:02:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:26.833787+0000 mon.a (mon.0) 4257 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:02:27.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: cephadm 2022-04-28T22:02:26.835504+0000 mgr.y (mgr.24961) 4306 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:02:27.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.121238+0000 mon.a (mon.0) 4258 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.122087+0000 mon.a (mon.0) 4259 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.4","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.122699+0000 mon.a (mon.0) 4260 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.123275+0000 mon.a (mon.0) 4261 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.5","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.123880+0000 mon.a (mon.0) 4262 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.124468+0000 mon.a (mon.0) 4263 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.6","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.125188+0000 mon.a (mon.0) 4264 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.126017+0000 mon.a (mon.0) 4265 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.7","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.126749+0000 mon.a (mon.0) 4266 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi139.oefiza","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.127479+0000 mon.a (mon.0) 4267 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi139", "name": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.136353+0000 mon.a (mon.0) 4268 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.137122+0000 mon.a (mon.0) 4269 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.0","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.623 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.137836+0000 mon.a (mon.0) 4270 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.138563+0000 mon.a (mon.0) 4271 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.1","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.139284+0000 mon.a (mon.0) 4272 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.140026+0000 mon.a (mon.0) 4273 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.2","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.140770+0000 mon.a (mon.0) 4274 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target_autotune"}]: dispatch 2022-04-28T22:02:27.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.141496+0000 mon.a (mon.0) 4275 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "osd.3","key": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.142233+0000 mon.a (mon.0) 4276 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mds.foo.smithi059.tnizcj","key": "mds_cache_memory_limit"}]: dispatch 2022-04-28T22:02:27.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.143044+0000 mon.a (mon.0) 4277 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi059", "name": "osd_memory_target"}]: dispatch 2022-04-28T22:02:27.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:27 smithi139 bash[43301]: audit 2022-04-28T22:02:27.150302+0000 mon.a (mon.0) 4278 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:02:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:28 smithi059 bash[61152]: cluster 2022-04-28T22:02:27.051701+0000 mgr.y (mgr.24961) 4307 : cluster [DBG] pgmap v5086: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:28 smithi059 bash[59745]: cluster 2022-04-28T22:02:27.051701+0000 mgr.y (mgr.24961) 4307 : cluster [DBG] pgmap v5086: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:28.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:28 smithi139 bash[43301]: cluster 2022-04-28T22:02:27.051701+0000 mgr.y (mgr.24961) 4307 : cluster [DBG] pgmap v5086: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:30 smithi059 bash[61152]: cluster 2022-04-28T22:02:29.052976+0000 mgr.y (mgr.24961) 4308 : cluster [DBG] pgmap v5087: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:30 smithi059 bash[59745]: cluster 2022-04-28T22:02:29.052976+0000 mgr.y (mgr.24961) 4308 : cluster [DBG] pgmap v5087: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:30 smithi139 bash[43301]: cluster 2022-04-28T22:02:29.052976+0000 mgr.y (mgr.24961) 4308 : cluster [DBG] pgmap v5087: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:32 smithi059 bash[61152]: cluster 2022-04-28T22:02:31.055031+0000 mgr.y (mgr.24961) 4309 : cluster [DBG] pgmap v5088: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:32.147Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:32.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:32 smithi059 bash[59745]: cluster 2022-04-28T22:02:31.055031+0000 mgr.y (mgr.24961) 4309 : cluster [DBG] pgmap v5088: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:32.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:32 smithi139 bash[43301]: cluster 2022-04-28T22:02:31.055031+0000 mgr.y (mgr.24961) 4309 : cluster [DBG] pgmap v5088: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:34 smithi059 bash[61152]: cluster 2022-04-28T22:02:33.055866+0000 mgr.y (mgr.24961) 4310 : cluster [DBG] pgmap v5089: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:34.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:34.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:34 smithi059 bash[59745]: cluster 2022-04-28T22:02:33.055866+0000 mgr.y (mgr.24961) 4310 : cluster [DBG] pgmap v5089: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:34.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:34 smithi139 bash[43301]: cluster 2022-04-28T22:02:33.055866+0000 mgr.y (mgr.24961) 4310 : cluster [DBG] pgmap v5089: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:36.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:36 smithi059 bash[61152]: cluster 2022-04-28T22:02:35.057606+0000 mgr.y (mgr.24961) 4311 : cluster [DBG] pgmap v5090: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:36.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:36 smithi059 bash[59745]: cluster 2022-04-28T22:02:35.057606+0000 mgr.y (mgr.24961) 4311 : cluster [DBG] pgmap v5090: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:36.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:36 smithi139 bash[43301]: cluster 2022-04-28T22:02:35.057606+0000 mgr.y (mgr.24961) 4311 : cluster [DBG] pgmap v5090: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:38.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:38 smithi059 bash[61152]: cluster 2022-04-28T22:02:37.059369+0000 mgr.y (mgr.24961) 4312 : cluster [DBG] pgmap v5091: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:38 smithi059 bash[59745]: cluster 2022-04-28T22:02:37.059369+0000 mgr.y (mgr.24961) 4312 : cluster [DBG] pgmap v5091: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:38.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:38 smithi139 bash[43301]: cluster 2022-04-28T22:02:37.059369+0000 mgr.y (mgr.24961) 4312 : cluster [DBG] pgmap v5091: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:40.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:40 smithi059 bash[61152]: cluster 2022-04-28T22:02:39.060727+0000 mgr.y (mgr.24961) 4313 : cluster [DBG] pgmap v5092: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:40.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:40 smithi059 bash[59745]: cluster 2022-04-28T22:02:39.060727+0000 mgr.y (mgr.24961) 4313 : cluster [DBG] pgmap v5092: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:40 smithi139 bash[43301]: cluster 2022-04-28T22:02:39.060727+0000 mgr.y (mgr.24961) 4313 : cluster [DBG] pgmap v5092: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:42 smithi059 bash[61152]: cluster 2022-04-28T22:02:41.062784+0000 mgr.y (mgr.24961) 4314 : cluster [DBG] pgmap v5093: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:42.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:42.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:42 smithi059 bash[59745]: cluster 2022-04-28T22:02:41.062784+0000 mgr.y (mgr.24961) 4314 : cluster [DBG] pgmap v5093: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:42 smithi139 bash[43301]: cluster 2022-04-28T22:02:41.062784+0000 mgr.y (mgr.24961) 4314 : cluster [DBG] pgmap v5093: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:44.311 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:44.034Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:44.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:44 smithi059 bash[61152]: cluster 2022-04-28T22:02:43.063645+0000 mgr.y (mgr.24961) 4315 : cluster [DBG] pgmap v5094: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:44.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:44 smithi059 bash[59745]: cluster 2022-04-28T22:02:43.063645+0000 mgr.y (mgr.24961) 4315 : cluster [DBG] pgmap v5094: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:44 smithi139 bash[43301]: cluster 2022-04-28T22:02:43.063645+0000 mgr.y (mgr.24961) 4315 : cluster [DBG] pgmap v5094: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:46.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:46 smithi059 bash[61152]: cluster 2022-04-28T22:02:45.065472+0000 mgr.y (mgr.24961) 4316 : cluster [DBG] pgmap v5095: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:46 smithi059 bash[59745]: cluster 2022-04-28T22:02:45.065472+0000 mgr.y (mgr.24961) 4316 : cluster [DBG] pgmap v5095: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:46 smithi139 bash[43301]: cluster 2022-04-28T22:02:45.065472+0000 mgr.y (mgr.24961) 4316 : cluster [DBG] pgmap v5095: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:48.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:48 smithi059 bash[61152]: cluster 2022-04-28T22:02:47.067151+0000 mgr.y (mgr.24961) 4317 : cluster [DBG] pgmap v5096: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:48.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:48 smithi059 bash[59745]: cluster 2022-04-28T22:02:47.067151+0000 mgr.y (mgr.24961) 4317 : cluster [DBG] pgmap v5096: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:48.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:48 smithi139 bash[43301]: cluster 2022-04-28T22:02:47.067151+0000 mgr.y (mgr.24961) 4317 : cluster [DBG] pgmap v5096: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:50.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:50 smithi059 bash[61152]: cluster 2022-04-28T22:02:49.068513+0000 mgr.y (mgr.24961) 4318 : cluster [DBG] pgmap v5097: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:50 smithi059 bash[59745]: cluster 2022-04-28T22:02:49.068513+0000 mgr.y (mgr.24961) 4318 : cluster [DBG] pgmap v5097: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:50.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:50 smithi139 bash[43301]: cluster 2022-04-28T22:02:49.068513+0000 mgr.y (mgr.24961) 4318 : cluster [DBG] pgmap v5097: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:52.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:52 smithi059 bash[61152]: cluster 2022-04-28T22:02:51.070603+0000 mgr.y (mgr.24961) 4319 : cluster [DBG] pgmap v5098: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:52.149Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:52.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:52 smithi059 bash[59745]: cluster 2022-04-28T22:02:51.070603+0000 mgr.y (mgr.24961) 4319 : cluster [DBG] pgmap v5098: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:52 smithi139 bash[43301]: cluster 2022-04-28T22:02:51.070603+0000 mgr.y (mgr.24961) 4319 : cluster [DBG] pgmap v5098: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:54.340 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:02:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:02:54.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:02:54.340 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:54 smithi059 bash[59745]: cluster 2022-04-28T22:02:53.071423+0000 mgr.y (mgr.24961) 4320 : cluster [DBG] pgmap v5099: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:54.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:54 smithi059 bash[61152]: cluster 2022-04-28T22:02:53.071423+0000 mgr.y (mgr.24961) 4320 : cluster [DBG] pgmap v5099: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:54.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:54 smithi139 bash[43301]: cluster 2022-04-28T22:02:53.071423+0000 mgr.y (mgr.24961) 4320 : cluster [DBG] pgmap v5099: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:56 smithi139 bash[43301]: cluster 2022-04-28T22:02:55.073257+0000 mgr.y (mgr.24961) 4321 : cluster [DBG] pgmap v5100: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:56 smithi059 bash[61152]: cluster 2022-04-28T22:02:55.073257+0000 mgr.y (mgr.24961) 4321 : cluster [DBG] pgmap v5100: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:56 smithi059 bash[59745]: cluster 2022-04-28T22:02:55.073257+0000 mgr.y (mgr.24961) 4321 : cluster [DBG] pgmap v5100: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:02:58 smithi139 bash[43301]: cluster 2022-04-28T22:02:57.074834+0000 mgr.y (mgr.24961) 4322 : cluster [DBG] pgmap v5101: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:02:58 smithi059 bash[61152]: cluster 2022-04-28T22:02:57.074834+0000 mgr.y (mgr.24961) 4322 : cluster [DBG] pgmap v5101: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:02:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:02:58 smithi059 bash[59745]: cluster 2022-04-28T22:02:57.074834+0000 mgr.y (mgr.24961) 4322 : cluster [DBG] pgmap v5101: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:00 smithi139 bash[43301]: cluster 2022-04-28T22:02:59.076150+0000 mgr.y (mgr.24961) 4323 : cluster [DBG] pgmap v5102: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:00 smithi059 bash[61152]: cluster 2022-04-28T22:02:59.076150+0000 mgr.y (mgr.24961) 4323 : cluster [DBG] pgmap v5102: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:00 smithi059 bash[59745]: cluster 2022-04-28T22:02:59.076150+0000 mgr.y (mgr.24961) 4323 : cluster [DBG] pgmap v5102: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:02 smithi059 bash[61152]: cluster 2022-04-28T22:03:01.078330+0000 mgr.y (mgr.24961) 4324 : cluster [DBG] pgmap v5103: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:02.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:02.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:02 smithi059 bash[59745]: cluster 2022-04-28T22:03:01.078330+0000 mgr.y (mgr.24961) 4324 : cluster [DBG] pgmap v5103: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:02.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:02 smithi139 bash[43301]: cluster 2022-04-28T22:03:01.078330+0000 mgr.y (mgr.24961) 4324 : cluster [DBG] pgmap v5103: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:04.035Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:04.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:04 smithi139 bash[43301]: cluster 2022-04-28T22:03:03.079273+0000 mgr.y (mgr.24961) 4325 : cluster [DBG] pgmap v5104: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:04.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:04 smithi059 bash[61152]: cluster 2022-04-28T22:03:03.079273+0000 mgr.y (mgr.24961) 4325 : cluster [DBG] pgmap v5104: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:04.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:04 smithi059 bash[59745]: cluster 2022-04-28T22:03:03.079273+0000 mgr.y (mgr.24961) 4325 : cluster [DBG] pgmap v5104: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:06.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:06 smithi059 bash[61152]: cluster 2022-04-28T22:03:05.081146+0000 mgr.y (mgr.24961) 4326 : cluster [DBG] pgmap v5105: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:06.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:06 smithi059 bash[59745]: cluster 2022-04-28T22:03:05.081146+0000 mgr.y (mgr.24961) 4326 : cluster [DBG] pgmap v5105: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:06 smithi139 bash[43301]: cluster 2022-04-28T22:03:05.081146+0000 mgr.y (mgr.24961) 4326 : cluster [DBG] pgmap v5105: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:08 smithi059 bash[61152]: cluster 2022-04-28T22:03:07.082787+0000 mgr.y (mgr.24961) 4327 : cluster [DBG] pgmap v5106: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:08 smithi059 bash[59745]: cluster 2022-04-28T22:03:07.082787+0000 mgr.y (mgr.24961) 4327 : cluster [DBG] pgmap v5106: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:08 smithi139 bash[43301]: cluster 2022-04-28T22:03:07.082787+0000 mgr.y (mgr.24961) 4327 : cluster [DBG] pgmap v5106: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:10 smithi059 bash[61152]: cluster 2022-04-28T22:03:09.084144+0000 mgr.y (mgr.24961) 4328 : cluster [DBG] pgmap v5107: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:10.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:10 smithi059 bash[59745]: cluster 2022-04-28T22:03:09.084144+0000 mgr.y (mgr.24961) 4328 : cluster [DBG] pgmap v5107: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:10.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:10 smithi139 bash[43301]: cluster 2022-04-28T22:03:09.084144+0000 mgr.y (mgr.24961) 4328 : cluster [DBG] pgmap v5107: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:12.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:12 smithi059 bash[61152]: cluster 2022-04-28T22:03:11.086236+0000 mgr.y (mgr.24961) 4329 : cluster [DBG] pgmap v5108: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:12.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:12.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:12 smithi059 bash[59745]: cluster 2022-04-28T22:03:11.086236+0000 mgr.y (mgr.24961) 4329 : cluster [DBG] pgmap v5108: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:12.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:12 smithi139 bash[43301]: cluster 2022-04-28T22:03:11.086236+0000 mgr.y (mgr.24961) 4329 : cluster [DBG] pgmap v5108: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:14.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:14.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:14 smithi059 bash[59745]: cluster 2022-04-28T22:03:13.086995+0000 mgr.y (mgr.24961) 4330 : cluster [DBG] pgmap v5109: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:14.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:14 smithi059 bash[61152]: cluster 2022-04-28T22:03:13.086995+0000 mgr.y (mgr.24961) 4330 : cluster [DBG] pgmap v5109: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:14 smithi139 bash[43301]: cluster 2022-04-28T22:03:13.086995+0000 mgr.y (mgr.24961) 4330 : cluster [DBG] pgmap v5109: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:16.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:16 smithi059 bash[59745]: cluster 2022-04-28T22:03:15.088836+0000 mgr.y (mgr.24961) 4331 : cluster [DBG] pgmap v5110: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:16 smithi059 bash[61152]: cluster 2022-04-28T22:03:15.088836+0000 mgr.y (mgr.24961) 4331 : cluster [DBG] pgmap v5110: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:16.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:16 smithi139 bash[43301]: cluster 2022-04-28T22:03:15.088836+0000 mgr.y (mgr.24961) 4331 : cluster [DBG] pgmap v5110: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:18 smithi059 bash[61152]: cluster 2022-04-28T22:03:17.090449+0000 mgr.y (mgr.24961) 4332 : cluster [DBG] pgmap v5111: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:18.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:18 smithi059 bash[59745]: cluster 2022-04-28T22:03:17.090449+0000 mgr.y (mgr.24961) 4332 : cluster [DBG] pgmap v5111: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:18.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:18 smithi139 bash[43301]: cluster 2022-04-28T22:03:17.090449+0000 mgr.y (mgr.24961) 4332 : cluster [DBG] pgmap v5111: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: cluster 2022-04-28T22:03:19.091779+0000 mgr.y (mgr.24961) 4333 : cluster [DBG] pgmap v5112: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: audit 2022-04-28T22:03:19.457017+0000 mon.a (mon.0) 4279 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:03:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: audit 2022-04-28T22:03:19.485708+0000 mon.a (mon.0) 4280 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:03:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: audit 2022-04-28T22:03:19.574301+0000 mon.a (mon.0) 4281 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: audit 2022-04-28T22:03:19.575088+0000 mon.a (mon.0) 4282 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: audit 2022-04-28T22:03:19.575791+0000 mon.a (mon.0) 4283 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[61152]: audit 2022-04-28T22:03:19.576438+0000 mon.a (mon.0) 4284 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T22:03:20.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: cluster 2022-04-28T22:03:19.091779+0000 mgr.y (mgr.24961) 4333 : cluster [DBG] pgmap v5112: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:20.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: audit 2022-04-28T22:03:19.457017+0000 mon.a (mon.0) 4279 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:03:20.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: audit 2022-04-28T22:03:19.485708+0000 mon.a (mon.0) 4280 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:03:20.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: audit 2022-04-28T22:03:19.574301+0000 mon.a (mon.0) 4281 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: audit 2022-04-28T22:03:19.575088+0000 mon.a (mon.0) 4282 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.846 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: audit 2022-04-28T22:03:19.575791+0000 mon.a (mon.0) 4283 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.847 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:20 smithi059 bash[59745]: audit 2022-04-28T22:03:19.576438+0000 mon.a (mon.0) 4284 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T22:03:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: cluster 2022-04-28T22:03:19.091779+0000 mgr.y (mgr.24961) 4333 : cluster [DBG] pgmap v5112: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: audit 2022-04-28T22:03:19.457017+0000 mon.a (mon.0) 4279 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:03:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: audit 2022-04-28T22:03:19.485708+0000 mon.a (mon.0) 4280 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:03:20.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: audit 2022-04-28T22:03:19.574301+0000 mon.a (mon.0) 4281 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: audit 2022-04-28T22:03:19.575088+0000 mon.a (mon.0) 4282 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: audit 2022-04-28T22:03:19.575791+0000 mon.a (mon.0) 4283 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2022-04-28T22:03:20.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:20 smithi139 bash[43301]: audit 2022-04-28T22:03:19.576438+0000 mon.a (mon.0) 4284 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]: dispatch 2022-04-28T22:03:22.416 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:22.150Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:22 smithi059 bash[61152]: cluster 2022-04-28T22:03:21.093871+0000 mgr.y (mgr.24961) 4334 : cluster [DBG] pgmap v5113: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:22.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:22 smithi059 bash[59745]: cluster 2022-04-28T22:03:21.093871+0000 mgr.y (mgr.24961) 4334 : cluster [DBG] pgmap v5113: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:22 smithi139 bash[43301]: cluster 2022-04-28T22:03:21.093871+0000 mgr.y (mgr.24961) 4334 : cluster [DBG] pgmap v5113: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:24.037Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:24.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:24 smithi059 bash[59745]: cluster 2022-04-28T22:03:23.094692+0000 mgr.y (mgr.24961) 4335 : cluster [DBG] pgmap v5114: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:24.854 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:24 smithi059 bash[61152]: cluster 2022-04-28T22:03:23.094692+0000 mgr.y (mgr.24961) 4335 : cluster [DBG] pgmap v5114: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:24.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:24 smithi139 bash[43301]: cluster 2022-04-28T22:03:23.094692+0000 mgr.y (mgr.24961) 4335 : cluster [DBG] pgmap v5114: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:26.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:26 smithi059 bash[59745]: cluster 2022-04-28T22:03:25.096500+0000 mgr.y (mgr.24961) 4336 : cluster [DBG] pgmap v5115: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:26.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:26 smithi059 bash[61152]: cluster 2022-04-28T22:03:25.096500+0000 mgr.y (mgr.24961) 4336 : cluster [DBG] pgmap v5115: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:26.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:26 smithi139 bash[43301]: cluster 2022-04-28T22:03:25.096500+0000 mgr.y (mgr.24961) 4336 : cluster [DBG] pgmap v5115: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:27 smithi059 bash[59745]: audit 2022-04-28T22:03:27.157642+0000 mon.a (mon.0) 4285 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:03:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:27 smithi059 bash[59745]: audit 2022-04-28T22:03:27.160525+0000 mon.a (mon.0) 4286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:03:27.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:27 smithi059 bash[59745]: audit 2022-04-28T22:03:27.162334+0000 mon.a (mon.0) 4287 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:03:27.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:27 smithi059 bash[61152]: audit 2022-04-28T22:03:27.157642+0000 mon.a (mon.0) 4285 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:03:27.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:27 smithi059 bash[61152]: audit 2022-04-28T22:03:27.160525+0000 mon.a (mon.0) 4286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:03:27.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:27 smithi059 bash[61152]: audit 2022-04-28T22:03:27.162334+0000 mon.a (mon.0) 4287 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:03:27.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:27 smithi139 bash[43301]: audit 2022-04-28T22:03:27.157642+0000 mon.a (mon.0) 4285 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:03:27.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:27 smithi139 bash[43301]: audit 2022-04-28T22:03:27.160525+0000 mon.a (mon.0) 4286 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:03:27.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:27 smithi139 bash[43301]: audit 2022-04-28T22:03:27.162334+0000 mon.a (mon.0) 4287 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:03:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:28 smithi059 bash[61152]: cluster 2022-04-28T22:03:27.098087+0000 mgr.y (mgr.24961) 4337 : cluster [DBG] pgmap v5116: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:28 smithi059 bash[61152]: cephadm 2022-04-28T22:03:27.164019+0000 mgr.y (mgr.24961) 4338 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:03:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:28 smithi059 bash[61152]: audit 2022-04-28T22:03:27.466417+0000 mon.a (mon.0) 4288 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:03:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:28 smithi059 bash[59745]: cluster 2022-04-28T22:03:27.098087+0000 mgr.y (mgr.24961) 4337 : cluster [DBG] pgmap v5116: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:28 smithi059 bash[59745]: cephadm 2022-04-28T22:03:27.164019+0000 mgr.y (mgr.24961) 4338 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:03:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:28 smithi059 bash[59745]: audit 2022-04-28T22:03:27.466417+0000 mon.a (mon.0) 4288 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:03:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:28 smithi139 bash[43301]: cluster 2022-04-28T22:03:27.098087+0000 mgr.y (mgr.24961) 4337 : cluster [DBG] pgmap v5116: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:28 smithi139 bash[43301]: cephadm 2022-04-28T22:03:27.164019+0000 mgr.y (mgr.24961) 4338 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:03:28.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:28 smithi139 bash[43301]: audit 2022-04-28T22:03:27.466417+0000 mon.a (mon.0) 4288 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:03:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:30 smithi059 bash[61152]: cluster 2022-04-28T22:03:29.099473+0000 mgr.y (mgr.24961) 4339 : cluster [DBG] pgmap v5117: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:30 smithi059 bash[59745]: cluster 2022-04-28T22:03:29.099473+0000 mgr.y (mgr.24961) 4339 : cluster [DBG] pgmap v5117: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:30 smithi139 bash[43301]: cluster 2022-04-28T22:03:29.099473+0000 mgr.y (mgr.24961) 4339 : cluster [DBG] pgmap v5117: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:32.477 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:32.151Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:32.478 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:32 smithi059 bash[59745]: cluster 2022-04-28T22:03:31.101595+0000 mgr.y (mgr.24961) 4340 : cluster [DBG] pgmap v5118: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:32 smithi059 bash[61152]: cluster 2022-04-28T22:03:31.101595+0000 mgr.y (mgr.24961) 4340 : cluster [DBG] pgmap v5118: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:32 smithi139 bash[43301]: cluster 2022-04-28T22:03:31.101595+0000 mgr.y (mgr.24961) 4340 : cluster [DBG] pgmap v5118: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:34.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:34 smithi059 bash[61152]: cluster 2022-04-28T22:03:33.102418+0000 mgr.y (mgr.24961) 4341 : cluster [DBG] pgmap v5119: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:34.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:34 smithi059 bash[59745]: cluster 2022-04-28T22:03:33.102418+0000 mgr.y (mgr.24961) 4341 : cluster [DBG] pgmap v5119: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:34.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:34 smithi139 bash[43301]: cluster 2022-04-28T22:03:33.102418+0000 mgr.y (mgr.24961) 4341 : cluster [DBG] pgmap v5119: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:36.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:36 smithi059 bash[61152]: cluster 2022-04-28T22:03:35.104253+0000 mgr.y (mgr.24961) 4342 : cluster [DBG] pgmap v5120: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:36 smithi059 bash[59745]: cluster 2022-04-28T22:03:35.104253+0000 mgr.y (mgr.24961) 4342 : cluster [DBG] pgmap v5120: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:36.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:36 smithi139 bash[43301]: cluster 2022-04-28T22:03:35.104253+0000 mgr.y (mgr.24961) 4342 : cluster [DBG] pgmap v5120: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:38.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:38 smithi059 bash[61152]: cluster 2022-04-28T22:03:37.106024+0000 mgr.y (mgr.24961) 4343 : cluster [DBG] pgmap v5121: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:38 smithi059 bash[59745]: cluster 2022-04-28T22:03:37.106024+0000 mgr.y (mgr.24961) 4343 : cluster [DBG] pgmap v5121: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:38.874 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:38 smithi139 bash[43301]: cluster 2022-04-28T22:03:37.106024+0000 mgr.y (mgr.24961) 4343 : cluster [DBG] pgmap v5121: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:40 smithi059 bash[61152]: cluster 2022-04-28T22:03:39.107355+0000 mgr.y (mgr.24961) 4344 : cluster [DBG] pgmap v5122: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:40 smithi059 bash[59745]: cluster 2022-04-28T22:03:39.107355+0000 mgr.y (mgr.24961) 4344 : cluster [DBG] pgmap v5122: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:40 smithi139 bash[43301]: cluster 2022-04-28T22:03:39.107355+0000 mgr.y (mgr.24961) 4344 : cluster [DBG] pgmap v5122: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:42.503 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:42.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:42 smithi059 bash[61152]: cluster 2022-04-28T22:03:41.109448+0000 mgr.y (mgr.24961) 4345 : cluster [DBG] pgmap v5123: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:42.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:42 smithi059 bash[59745]: cluster 2022-04-28T22:03:41.109448+0000 mgr.y (mgr.24961) 4345 : cluster [DBG] pgmap v5123: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:42 smithi139 bash[43301]: cluster 2022-04-28T22:03:41.109448+0000 mgr.y (mgr.24961) 4345 : cluster [DBG] pgmap v5123: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:44.038Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:44 smithi059 bash[59745]: cluster 2022-04-28T22:03:43.110264+0000 mgr.y (mgr.24961) 4346 : cluster [DBG] pgmap v5124: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:44 smithi059 bash[61152]: cluster 2022-04-28T22:03:43.110264+0000 mgr.y (mgr.24961) 4346 : cluster [DBG] pgmap v5124: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:44.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:44 smithi139 bash[43301]: cluster 2022-04-28T22:03:43.110264+0000 mgr.y (mgr.24961) 4346 : cluster [DBG] pgmap v5124: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:46.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:46 smithi059 bash[61152]: cluster 2022-04-28T22:03:45.112118+0000 mgr.y (mgr.24961) 4347 : cluster [DBG] pgmap v5125: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:46 smithi059 bash[59745]: cluster 2022-04-28T22:03:45.112118+0000 mgr.y (mgr.24961) 4347 : cluster [DBG] pgmap v5125: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:46 smithi139 bash[43301]: cluster 2022-04-28T22:03:45.112118+0000 mgr.y (mgr.24961) 4347 : cluster [DBG] pgmap v5125: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:48 smithi059 bash[61152]: cluster 2022-04-28T22:03:47.113782+0000 mgr.y (mgr.24961) 4348 : cluster [DBG] pgmap v5126: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:48 smithi059 bash[59745]: cluster 2022-04-28T22:03:47.113782+0000 mgr.y (mgr.24961) 4348 : cluster [DBG] pgmap v5126: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:48 smithi139 bash[43301]: cluster 2022-04-28T22:03:47.113782+0000 mgr.y (mgr.24961) 4348 : cluster [DBG] pgmap v5126: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:50.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:50 smithi059 bash[61152]: cluster 2022-04-28T22:03:49.115134+0000 mgr.y (mgr.24961) 4349 : cluster [DBG] pgmap v5127: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:50 smithi059 bash[59745]: cluster 2022-04-28T22:03:49.115134+0000 mgr.y (mgr.24961) 4349 : cluster [DBG] pgmap v5127: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:50 smithi139 bash[43301]: cluster 2022-04-28T22:03:49.115134+0000 mgr.y (mgr.24961) 4349 : cluster [DBG] pgmap v5127: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:52.530 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:52.153Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:52.531 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:52 smithi059 bash[59745]: cluster 2022-04-28T22:03:51.117213+0000 mgr.y (mgr.24961) 4350 : cluster [DBG] pgmap v5128: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:52 smithi059 bash[61152]: cluster 2022-04-28T22:03:51.117213+0000 mgr.y (mgr.24961) 4350 : cluster [DBG] pgmap v5128: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:52.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:52 smithi139 bash[43301]: cluster 2022-04-28T22:03:51.117213+0000 mgr.y (mgr.24961) 4350 : cluster [DBG] pgmap v5128: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:54.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:03:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:03:54.039Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:03:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:54 smithi059 bash[61152]: cluster 2022-04-28T22:03:53.118041+0000 mgr.y (mgr.24961) 4351 : cluster [DBG] pgmap v5129: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:54.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:54 smithi059 bash[59745]: cluster 2022-04-28T22:03:53.118041+0000 mgr.y (mgr.24961) 4351 : cluster [DBG] pgmap v5129: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:54 smithi139 bash[43301]: cluster 2022-04-28T22:03:53.118041+0000 mgr.y (mgr.24961) 4351 : cluster [DBG] pgmap v5129: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:56 smithi059 bash[61152]: cluster 2022-04-28T22:03:55.120034+0000 mgr.y (mgr.24961) 4352 : cluster [DBG] pgmap v5130: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:56 smithi059 bash[59745]: cluster 2022-04-28T22:03:55.120034+0000 mgr.y (mgr.24961) 4352 : cluster [DBG] pgmap v5130: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:56 smithi139 bash[43301]: cluster 2022-04-28T22:03:55.120034+0000 mgr.y (mgr.24961) 4352 : cluster [DBG] pgmap v5130: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:03:58 smithi059 bash[61152]: cluster 2022-04-28T22:03:57.121678+0000 mgr.y (mgr.24961) 4353 : cluster [DBG] pgmap v5131: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:03:58 smithi059 bash[59745]: cluster 2022-04-28T22:03:57.121678+0000 mgr.y (mgr.24961) 4353 : cluster [DBG] pgmap v5131: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:03:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:03:58 smithi139 bash[43301]: cluster 2022-04-28T22:03:57.121678+0000 mgr.y (mgr.24961) 4353 : cluster [DBG] pgmap v5131: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:00 smithi059 bash[61152]: cluster 2022-04-28T22:03:59.122957+0000 mgr.y (mgr.24961) 4354 : cluster [DBG] pgmap v5132: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:00 smithi059 bash[59745]: cluster 2022-04-28T22:03:59.122957+0000 mgr.y (mgr.24961) 4354 : cluster [DBG] pgmap v5132: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:00 smithi139 bash[43301]: cluster 2022-04-28T22:03:59.122957+0000 mgr.y (mgr.24961) 4354 : cluster [DBG] pgmap v5132: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:02.562 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:02.154Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:02 smithi059 bash[61152]: cluster 2022-04-28T22:04:01.125023+0000 mgr.y (mgr.24961) 4355 : cluster [DBG] pgmap v5133: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:02 smithi059 bash[59745]: cluster 2022-04-28T22:04:01.125023+0000 mgr.y (mgr.24961) 4355 : cluster [DBG] pgmap v5133: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:02 smithi139 bash[43301]: cluster 2022-04-28T22:04:01.125023+0000 mgr.y (mgr.24961) 4355 : cluster [DBG] pgmap v5133: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:04.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:04 smithi059 bash[61152]: cluster 2022-04-28T22:04:03.125854+0000 mgr.y (mgr.24961) 4356 : cluster [DBG] pgmap v5134: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:04.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:04 smithi059 bash[59745]: cluster 2022-04-28T22:04:03.125854+0000 mgr.y (mgr.24961) 4356 : cluster [DBG] pgmap v5134: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:04 smithi139 bash[43301]: cluster 2022-04-28T22:04:03.125854+0000 mgr.y (mgr.24961) 4356 : cluster [DBG] pgmap v5134: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:06 smithi059 bash[61152]: cluster 2022-04-28T22:04:05.127741+0000 mgr.y (mgr.24961) 4357 : cluster [DBG] pgmap v5135: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:06 smithi059 bash[59745]: cluster 2022-04-28T22:04:05.127741+0000 mgr.y (mgr.24961) 4357 : cluster [DBG] pgmap v5135: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:06 smithi139 bash[43301]: cluster 2022-04-28T22:04:05.127741+0000 mgr.y (mgr.24961) 4357 : cluster [DBG] pgmap v5135: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:08 smithi059 bash[61152]: cluster 2022-04-28T22:04:07.129360+0000 mgr.y (mgr.24961) 4358 : cluster [DBG] pgmap v5136: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:08 smithi059 bash[59745]: cluster 2022-04-28T22:04:07.129360+0000 mgr.y (mgr.24961) 4358 : cluster [DBG] pgmap v5136: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:08 smithi139 bash[43301]: cluster 2022-04-28T22:04:07.129360+0000 mgr.y (mgr.24961) 4358 : cluster [DBG] pgmap v5136: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:10 smithi059 bash[61152]: cluster 2022-04-28T22:04:09.130661+0000 mgr.y (mgr.24961) 4359 : cluster [DBG] pgmap v5137: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:10 smithi059 bash[59745]: cluster 2022-04-28T22:04:09.130661+0000 mgr.y (mgr.24961) 4359 : cluster [DBG] pgmap v5137: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:10.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:10 smithi139 bash[43301]: cluster 2022-04-28T22:04:09.130661+0000 mgr.y (mgr.24961) 4359 : cluster [DBG] pgmap v5137: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:12.588 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:12.155Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:12 smithi059 bash[59745]: cluster 2022-04-28T22:04:11.132754+0000 mgr.y (mgr.24961) 4360 : cluster [DBG] pgmap v5138: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:12.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:12 smithi059 bash[61152]: cluster 2022-04-28T22:04:11.132754+0000 mgr.y (mgr.24961) 4360 : cluster [DBG] pgmap v5138: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:12 smithi139 bash[43301]: cluster 2022-04-28T22:04:11.132754+0000 mgr.y (mgr.24961) 4360 : cluster [DBG] pgmap v5138: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:14.040Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:14 smithi139 bash[43301]: cluster 2022-04-28T22:04:13.133584+0000 mgr.y (mgr.24961) 4361 : cluster [DBG] pgmap v5139: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:15.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:14 smithi059 bash[61152]: cluster 2022-04-28T22:04:13.133584+0000 mgr.y (mgr.24961) 4361 : cluster [DBG] pgmap v5139: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:15.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:14 smithi059 bash[59745]: cluster 2022-04-28T22:04:13.133584+0000 mgr.y (mgr.24961) 4361 : cluster [DBG] pgmap v5139: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:16 smithi139 bash[43301]: cluster 2022-04-28T22:04:15.135661+0000 mgr.y (mgr.24961) 4362 : cluster [DBG] pgmap v5140: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:17.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:16 smithi059 bash[61152]: cluster 2022-04-28T22:04:15.135661+0000 mgr.y (mgr.24961) 4362 : cluster [DBG] pgmap v5140: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:17.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:16 smithi059 bash[59745]: cluster 2022-04-28T22:04:15.135661+0000 mgr.y (mgr.24961) 4362 : cluster [DBG] pgmap v5140: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:18 smithi139 bash[43301]: cluster 2022-04-28T22:04:17.137121+0000 mgr.y (mgr.24961) 4363 : cluster [DBG] pgmap v5141: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:19.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:18 smithi059 bash[61152]: cluster 2022-04-28T22:04:17.137121+0000 mgr.y (mgr.24961) 4363 : cluster [DBG] pgmap v5141: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:19.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:18 smithi059 bash[59745]: cluster 2022-04-28T22:04:17.137121+0000 mgr.y (mgr.24961) 4363 : cluster [DBG] pgmap v5141: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:20.248 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: cluster 2022-04-28T22:04:19.138476+0000 mgr.y (mgr.24961) 4364 : cluster [DBG] pgmap v5142: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:20.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: audit 2022-04-28T22:04:19.462306+0000 mon.a (mon.0) 4289 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:04:20.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: audit 2022-04-28T22:04:19.491937+0000 mon.a (mon.0) 4290 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:04:20.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: audit 2022-04-28T22:04:19.585230+0000 mon.a (mon.0) 4291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:04:20.249 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: audit 2022-04-28T22:04:19.585781+0000 mon.a (mon.0) 4292 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2022-04-28T22:04:20.250 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: audit 2022-04-28T22:04:19.586162+0000 mon.a (mon.0) 4293 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:04:20.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[59745]: audit 2022-04-28T22:04:19.586540+0000 mon.a (mon.0) 4294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T22:04:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: cluster 2022-04-28T22:04:19.138476+0000 mgr.y (mgr.24961) 4364 : cluster [DBG] pgmap v5142: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: audit 2022-04-28T22:04:19.462306+0000 mon.a (mon.0) 4289 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:04:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: audit 2022-04-28T22:04:19.491937+0000 mon.a (mon.0) 4290 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:04:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: audit 2022-04-28T22:04:19.585230+0000 mon.a (mon.0) 4291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:04:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: audit 2022-04-28T22:04:19.585781+0000 mon.a (mon.0) 4292 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2022-04-28T22:04:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: audit 2022-04-28T22:04:19.586162+0000 mon.a (mon.0) 4293 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:04:20.594 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:20 smithi059 bash[61152]: audit 2022-04-28T22:04:19.586540+0000 mon.a (mon.0) 4294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T22:04:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: cluster 2022-04-28T22:04:19.138476+0000 mgr.y (mgr.24961) 4364 : cluster [DBG] pgmap v5142: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: audit 2022-04-28T22:04:19.462306+0000 mon.a (mon.0) 4289 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:04:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: audit 2022-04-28T22:04:19.491937+0000 mon.a (mon.0) 4290 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:04:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: audit 2022-04-28T22:04:19.585230+0000 mon.a (mon.0) 4291 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:04:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: audit 2022-04-28T22:04:19.585781+0000 mon.a (mon.0) 4292 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 5]}]: dispatch 2022-04-28T22:04:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: audit 2022-04-28T22:04:19.586162+0000 mon.a (mon.0) 4293 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:04:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:20 smithi139 bash[43301]: audit 2022-04-28T22:04:19.586540+0000 mon.a (mon.0) 4294 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2022-04-28T22:04:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:22 smithi059 bash[61152]: cluster 2022-04-28T22:04:21.140556+0000 mgr.y (mgr.24961) 4365 : cluster [DBG] pgmap v5143: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:22.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:22.156Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:22.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:22 smithi059 bash[59745]: cluster 2022-04-28T22:04:21.140556+0000 mgr.y (mgr.24961) 4365 : cluster [DBG] pgmap v5143: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:22 smithi139 bash[43301]: cluster 2022-04-28T22:04:21.140556+0000 mgr.y (mgr.24961) 4365 : cluster [DBG] pgmap v5143: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:24.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:24 smithi059 bash[61152]: cluster 2022-04-28T22:04:23.141389+0000 mgr.y (mgr.24961) 4366 : cluster [DBG] pgmap v5144: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:24.042Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:24.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:24 smithi059 bash[59745]: cluster 2022-04-28T22:04:23.141389+0000 mgr.y (mgr.24961) 4366 : cluster [DBG] pgmap v5144: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:24 smithi139 bash[43301]: cluster 2022-04-28T22:04:23.141389+0000 mgr.y (mgr.24961) 4366 : cluster [DBG] pgmap v5144: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:26 smithi059 bash[61152]: cluster 2022-04-28T22:04:25.143479+0000 mgr.y (mgr.24961) 4367 : cluster [DBG] pgmap v5145: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:26 smithi059 bash[59745]: cluster 2022-04-28T22:04:25.143479+0000 mgr.y (mgr.24961) 4367 : cluster [DBG] pgmap v5145: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:26 smithi139 bash[43301]: cluster 2022-04-28T22:04:25.143479+0000 mgr.y (mgr.24961) 4367 : cluster [DBG] pgmap v5145: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[61152]: cluster 2022-04-28T22:04:27.144934+0000 mgr.y (mgr.24961) 4368 : cluster [DBG] pgmap v5146: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[61152]: audit 2022-04-28T22:04:27.475609+0000 mon.a (mon.0) 4295 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:04:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[61152]: audit 2022-04-28T22:04:27.478456+0000 mon.a (mon.0) 4296 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:04:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[61152]: audit 2022-04-28T22:04:27.481189+0000 mon.a (mon.0) 4297 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:04:28.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[61152]: cephadm 2022-04-28T22:04:27.484087+0000 mgr.y (mgr.24961) 4369 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:04:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[61152]: audit 2022-04-28T22:04:27.775608+0000 mon.a (mon.0) 4298 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:04:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[59745]: cluster 2022-04-28T22:04:27.144934+0000 mgr.y (mgr.24961) 4368 : cluster [DBG] pgmap v5146: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[59745]: audit 2022-04-28T22:04:27.475609+0000 mon.a (mon.0) 4295 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:04:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[59745]: audit 2022-04-28T22:04:27.478456+0000 mon.a (mon.0) 4296 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:04:28.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[59745]: audit 2022-04-28T22:04:27.481189+0000 mon.a (mon.0) 4297 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:04:28.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[59745]: cephadm 2022-04-28T22:04:27.484087+0000 mgr.y (mgr.24961) 4369 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:04:28.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:28 smithi059 bash[59745]: audit 2022-04-28T22:04:27.775608+0000 mon.a (mon.0) 4298 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:04:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:28 smithi139 bash[43301]: cluster 2022-04-28T22:04:27.144934+0000 mgr.y (mgr.24961) 4368 : cluster [DBG] pgmap v5146: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:28 smithi139 bash[43301]: audit 2022-04-28T22:04:27.475609+0000 mon.a (mon.0) 4295 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:04:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:28 smithi139 bash[43301]: audit 2022-04-28T22:04:27.478456+0000 mon.a (mon.0) 4296 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:04:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:28 smithi139 bash[43301]: audit 2022-04-28T22:04:27.481189+0000 mon.a (mon.0) 4297 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:04:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:28 smithi139 bash[43301]: cephadm 2022-04-28T22:04:27.484087+0000 mgr.y (mgr.24961) 4369 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:04:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:28 smithi139 bash[43301]: audit 2022-04-28T22:04:27.775608+0000 mon.a (mon.0) 4298 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:04:30.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:30 smithi059 bash[61152]: cluster 2022-04-28T22:04:29.146233+0000 mgr.y (mgr.24961) 4370 : cluster [DBG] pgmap v5147: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:30.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:30 smithi059 bash[59745]: cluster 2022-04-28T22:04:29.146233+0000 mgr.y (mgr.24961) 4370 : cluster [DBG] pgmap v5147: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:30 smithi139 bash[43301]: cluster 2022-04-28T22:04:29.146233+0000 mgr.y (mgr.24961) 4370 : cluster [DBG] pgmap v5147: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:32 smithi059 bash[61152]: cluster 2022-04-28T22:04:31.148350+0000 mgr.y (mgr.24961) 4371 : cluster [DBG] pgmap v5148: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:32.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:32.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:32 smithi059 bash[59745]: cluster 2022-04-28T22:04:31.148350+0000 mgr.y (mgr.24961) 4371 : cluster [DBG] pgmap v5148: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:32.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:32 smithi139 bash[43301]: cluster 2022-04-28T22:04:31.148350+0000 mgr.y (mgr.24961) 4371 : cluster [DBG] pgmap v5148: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:34.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:34 smithi059 bash[61152]: cluster 2022-04-28T22:04:33.149191+0000 mgr.y (mgr.24961) 4372 : cluster [DBG] pgmap v5149: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:34.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:34.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:34.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:34 smithi059 bash[59745]: cluster 2022-04-28T22:04:33.149191+0000 mgr.y (mgr.24961) 4372 : cluster [DBG] pgmap v5149: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:34.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:34 smithi139 bash[43301]: cluster 2022-04-28T22:04:33.149191+0000 mgr.y (mgr.24961) 4372 : cluster [DBG] pgmap v5149: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:36.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:36 smithi059 bash[61152]: cluster 2022-04-28T22:04:35.151605+0000 mgr.y (mgr.24961) 4373 : cluster [DBG] pgmap v5150: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:36.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:36 smithi059 bash[59745]: cluster 2022-04-28T22:04:35.151605+0000 mgr.y (mgr.24961) 4373 : cluster [DBG] pgmap v5150: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:36.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:36 smithi139 bash[43301]: cluster 2022-04-28T22:04:35.151605+0000 mgr.y (mgr.24961) 4373 : cluster [DBG] pgmap v5150: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:38.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:38 smithi059 bash[61152]: cluster 2022-04-28T22:04:37.152928+0000 mgr.y (mgr.24961) 4374 : cluster [DBG] pgmap v5151: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:38.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:38 smithi059 bash[59745]: cluster 2022-04-28T22:04:37.152928+0000 mgr.y (mgr.24961) 4374 : cluster [DBG] pgmap v5151: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:38.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:38 smithi139 bash[43301]: cluster 2022-04-28T22:04:37.152928+0000 mgr.y (mgr.24961) 4374 : cluster [DBG] pgmap v5151: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:40.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:40 smithi059 bash[61152]: cluster 2022-04-28T22:04:39.154289+0000 mgr.y (mgr.24961) 4375 : cluster [DBG] pgmap v5152: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:40.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:40 smithi059 bash[59745]: cluster 2022-04-28T22:04:39.154289+0000 mgr.y (mgr.24961) 4375 : cluster [DBG] pgmap v5152: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:40 smithi139 bash[43301]: cluster 2022-04-28T22:04:39.154289+0000 mgr.y (mgr.24961) 4375 : cluster [DBG] pgmap v5152: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:42 smithi059 bash[61152]: cluster 2022-04-28T22:04:41.156432+0000 mgr.y (mgr.24961) 4376 : cluster [DBG] pgmap v5153: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:42 smithi059 bash[59745]: cluster 2022-04-28T22:04:41.156432+0000 mgr.y (mgr.24961) 4376 : cluster [DBG] pgmap v5153: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:42.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:42.157Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:42 smithi139 bash[43301]: cluster 2022-04-28T22:04:41.156432+0000 mgr.y (mgr.24961) 4376 : cluster [DBG] pgmap v5153: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:44.310 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:44.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:44.310 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:44 smithi059 bash[59745]: cluster 2022-04-28T22:04:43.157366+0000 mgr.y (mgr.24961) 4377 : cluster [DBG] pgmap v5154: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:44.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:44 smithi059 bash[61152]: cluster 2022-04-28T22:04:43.157366+0000 mgr.y (mgr.24961) 4377 : cluster [DBG] pgmap v5154: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:44.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:44 smithi139 bash[43301]: cluster 2022-04-28T22:04:43.157366+0000 mgr.y (mgr.24961) 4377 : cluster [DBG] pgmap v5154: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:46.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:46 smithi059 bash[61152]: cluster 2022-04-28T22:04:45.159627+0000 mgr.y (mgr.24961) 4378 : cluster [DBG] pgmap v5155: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:46.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:46 smithi059 bash[59745]: cluster 2022-04-28T22:04:45.159627+0000 mgr.y (mgr.24961) 4378 : cluster [DBG] pgmap v5155: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:46.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:46 smithi139 bash[43301]: cluster 2022-04-28T22:04:45.159627+0000 mgr.y (mgr.24961) 4378 : cluster [DBG] pgmap v5155: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:48.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:48 smithi059 bash[61152]: cluster 2022-04-28T22:04:47.161018+0000 mgr.y (mgr.24961) 4379 : cluster [DBG] pgmap v5156: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:48.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:48 smithi059 bash[59745]: cluster 2022-04-28T22:04:47.161018+0000 mgr.y (mgr.24961) 4379 : cluster [DBG] pgmap v5156: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:48.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:48 smithi139 bash[43301]: cluster 2022-04-28T22:04:47.161018+0000 mgr.y (mgr.24961) 4379 : cluster [DBG] pgmap v5156: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:50.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:50 smithi059 bash[61152]: cluster 2022-04-28T22:04:49.162338+0000 mgr.y (mgr.24961) 4380 : cluster [DBG] pgmap v5157: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:50.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:50 smithi059 bash[59745]: cluster 2022-04-28T22:04:49.162338+0000 mgr.y (mgr.24961) 4380 : cluster [DBG] pgmap v5157: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:50.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:50 smithi139 bash[43301]: cluster 2022-04-28T22:04:49.162338+0000 mgr.y (mgr.24961) 4380 : cluster [DBG] pgmap v5157: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:52 smithi059 bash[61152]: cluster 2022-04-28T22:04:51.164419+0000 mgr.y (mgr.24961) 4381 : cluster [DBG] pgmap v5158: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:52.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:52.159Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:52.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:52 smithi059 bash[59745]: cluster 2022-04-28T22:04:51.164419+0000 mgr.y (mgr.24961) 4381 : cluster [DBG] pgmap v5158: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:52.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:52 smithi139 bash[43301]: cluster 2022-04-28T22:04:51.164419+0000 mgr.y (mgr.24961) 4381 : cluster [DBG] pgmap v5158: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:04:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:04:54.043Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:04:54.595 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:54 smithi059 bash[61152]: cluster 2022-04-28T22:04:53.165261+0000 mgr.y (mgr.24961) 4382 : cluster [DBG] pgmap v5159: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:54.595 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:54 smithi059 bash[59745]: cluster 2022-04-28T22:04:53.165261+0000 mgr.y (mgr.24961) 4382 : cluster [DBG] pgmap v5159: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:54.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:54 smithi139 bash[43301]: cluster 2022-04-28T22:04:53.165261+0000 mgr.y (mgr.24961) 4382 : cluster [DBG] pgmap v5159: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:56.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:56 smithi139 bash[43301]: cluster 2022-04-28T22:04:55.167340+0000 mgr.y (mgr.24961) 4383 : cluster [DBG] pgmap v5160: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:56 smithi059 bash[61152]: cluster 2022-04-28T22:04:55.167340+0000 mgr.y (mgr.24961) 4383 : cluster [DBG] pgmap v5160: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:56 smithi059 bash[59745]: cluster 2022-04-28T22:04:55.167340+0000 mgr.y (mgr.24961) 4383 : cluster [DBG] pgmap v5160: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:58.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:04:58 smithi139 bash[43301]: cluster 2022-04-28T22:04:57.168745+0000 mgr.y (mgr.24961) 4384 : cluster [DBG] pgmap v5161: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:58.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:04:58 smithi059 bash[61152]: cluster 2022-04-28T22:04:57.168745+0000 mgr.y (mgr.24961) 4384 : cluster [DBG] pgmap v5161: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:04:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:04:58 smithi059 bash[59745]: cluster 2022-04-28T22:04:57.168745+0000 mgr.y (mgr.24961) 4384 : cluster [DBG] pgmap v5161: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:00 smithi139 bash[43301]: cluster 2022-04-28T22:04:59.170084+0000 mgr.y (mgr.24961) 4385 : cluster [DBG] pgmap v5162: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:00.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:00 smithi059 bash[59745]: cluster 2022-04-28T22:04:59.170084+0000 mgr.y (mgr.24961) 4385 : cluster [DBG] pgmap v5162: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:00 smithi059 bash[61152]: cluster 2022-04-28T22:04:59.170084+0000 mgr.y (mgr.24961) 4385 : cluster [DBG] pgmap v5162: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:02 smithi059 bash[61152]: cluster 2022-04-28T22:05:01.172220+0000 mgr.y (mgr.24961) 4386 : cluster [DBG] pgmap v5163: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:02.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:02.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:02 smithi059 bash[59745]: cluster 2022-04-28T22:05:01.172220+0000 mgr.y (mgr.24961) 4386 : cluster [DBG] pgmap v5163: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:02.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:02 smithi139 bash[43301]: cluster 2022-04-28T22:05:01.172220+0000 mgr.y (mgr.24961) 4386 : cluster [DBG] pgmap v5163: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:04.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:04.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:04 smithi059 bash[61152]: cluster 2022-04-28T22:05:03.173062+0000 mgr.y (mgr.24961) 4387 : cluster [DBG] pgmap v5164: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:04.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:04 smithi059 bash[59745]: cluster 2022-04-28T22:05:03.173062+0000 mgr.y (mgr.24961) 4387 : cluster [DBG] pgmap v5164: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:04 smithi139 bash[43301]: cluster 2022-04-28T22:05:03.173062+0000 mgr.y (mgr.24961) 4387 : cluster [DBG] pgmap v5164: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:06 smithi059 bash[61152]: cluster 2022-04-28T22:05:05.175159+0000 mgr.y (mgr.24961) 4388 : cluster [DBG] pgmap v5165: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:06 smithi059 bash[59745]: cluster 2022-04-28T22:05:05.175159+0000 mgr.y (mgr.24961) 4388 : cluster [DBG] pgmap v5165: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:06 smithi139 bash[43301]: cluster 2022-04-28T22:05:05.175159+0000 mgr.y (mgr.24961) 4388 : cluster [DBG] pgmap v5165: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:08.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:08 smithi059 bash[61152]: cluster 2022-04-28T22:05:07.176578+0000 mgr.y (mgr.24961) 4389 : cluster [DBG] pgmap v5166: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:08 smithi059 bash[59745]: cluster 2022-04-28T22:05:07.176578+0000 mgr.y (mgr.24961) 4389 : cluster [DBG] pgmap v5166: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:08 smithi139 bash[43301]: cluster 2022-04-28T22:05:07.176578+0000 mgr.y (mgr.24961) 4389 : cluster [DBG] pgmap v5166: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:10 smithi059 bash[61152]: cluster 2022-04-28T22:05:09.178087+0000 mgr.y (mgr.24961) 4390 : cluster [DBG] pgmap v5167: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:10 smithi059 bash[59745]: cluster 2022-04-28T22:05:09.178087+0000 mgr.y (mgr.24961) 4390 : cluster [DBG] pgmap v5167: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:10.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:10 smithi139 bash[43301]: cluster 2022-04-28T22:05:09.178087+0000 mgr.y (mgr.24961) 4390 : cluster [DBG] pgmap v5167: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:12.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:12 smithi059 bash[61152]: cluster 2022-04-28T22:05:11.180010+0000 mgr.y (mgr.24961) 4391 : cluster [DBG] pgmap v5168: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:12.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:12.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:12.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:12 smithi059 bash[59745]: cluster 2022-04-28T22:05:11.180010+0000 mgr.y (mgr.24961) 4391 : cluster [DBG] pgmap v5168: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:12 smithi139 bash[43301]: cluster 2022-04-28T22:05:11.180010+0000 mgr.y (mgr.24961) 4391 : cluster [DBG] pgmap v5168: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:14.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:14.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:14.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:14 smithi059 bash[61152]: cluster 2022-04-28T22:05:13.180795+0000 mgr.y (mgr.24961) 4392 : cluster [DBG] pgmap v5169: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:14.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:14 smithi059 bash[59745]: cluster 2022-04-28T22:05:13.180795+0000 mgr.y (mgr.24961) 4392 : cluster [DBG] pgmap v5169: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:14 smithi139 bash[43301]: cluster 2022-04-28T22:05:13.180795+0000 mgr.y (mgr.24961) 4392 : cluster [DBG] pgmap v5169: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:16 smithi059 bash[61152]: cluster 2022-04-28T22:05:15.182855+0000 mgr.y (mgr.24961) 4393 : cluster [DBG] pgmap v5170: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:16 smithi059 bash[59745]: cluster 2022-04-28T22:05:15.182855+0000 mgr.y (mgr.24961) 4393 : cluster [DBG] pgmap v5170: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:16 smithi139 bash[43301]: cluster 2022-04-28T22:05:15.182855+0000 mgr.y (mgr.24961) 4393 : cluster [DBG] pgmap v5170: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:18.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:18 smithi059 bash[61152]: cluster 2022-04-28T22:05:17.184295+0000 mgr.y (mgr.24961) 4394 : cluster [DBG] pgmap v5171: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:18.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:18 smithi059 bash[59745]: cluster 2022-04-28T22:05:17.184295+0000 mgr.y (mgr.24961) 4394 : cluster [DBG] pgmap v5171: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:18 smithi139 bash[43301]: cluster 2022-04-28T22:05:17.184295+0000 mgr.y (mgr.24961) 4394 : cluster [DBG] pgmap v5171: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: cluster 2022-04-28T22:05:19.185832+0000 mgr.y (mgr.24961) 4395 : cluster [DBG] pgmap v5172: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: audit 2022-04-28T22:05:19.467721+0000 mon.a (mon.0) 4299 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:05:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: audit 2022-04-28T22:05:19.496701+0000 mon.a (mon.0) 4300 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:05:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: audit 2022-04-28T22:05:19.595558+0000 mon.a (mon.0) 4301 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T22:05:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: audit 2022-04-28T22:05:19.596297+0000 mon.a (mon.0) 4302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:05:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: audit 2022-04-28T22:05:19.597072+0000 mon.a (mon.0) 4303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T22:05:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[61152]: audit 2022-04-28T22:05:19.597864+0000 mon.a (mon.0) 4304 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T22:05:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: cluster 2022-04-28T22:05:19.185832+0000 mgr.y (mgr.24961) 4395 : cluster [DBG] pgmap v5172: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: audit 2022-04-28T22:05:19.467721+0000 mon.a (mon.0) 4299 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:05:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: audit 2022-04-28T22:05:19.496701+0000 mon.a (mon.0) 4300 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:05:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: audit 2022-04-28T22:05:19.595558+0000 mon.a (mon.0) 4301 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T22:05:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: audit 2022-04-28T22:05:19.596297+0000 mon.a (mon.0) 4302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:05:20.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: audit 2022-04-28T22:05:19.597072+0000 mon.a (mon.0) 4303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T22:05:20.845 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:20 smithi059 bash[59745]: audit 2022-04-28T22:05:19.597864+0000 mon.a (mon.0) 4304 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T22:05:20.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: cluster 2022-04-28T22:05:19.185832+0000 mgr.y (mgr.24961) 4395 : cluster [DBG] pgmap v5172: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: audit 2022-04-28T22:05:19.467721+0000 mon.a (mon.0) 4299 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:05:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: audit 2022-04-28T22:05:19.496701+0000 mon.a (mon.0) 4300 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:05:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: audit 2022-04-28T22:05:19.595558+0000 mon.a (mon.0) 4301 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T22:05:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: audit 2022-04-28T22:05:19.596297+0000 mon.a (mon.0) 4302 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:05:20.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: audit 2022-04-28T22:05:19.597072+0000 mon.a (mon.0) 4303 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2022-04-28T22:05:20.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:20 smithi139 bash[43301]: audit 2022-04-28T22:05:19.597864+0000 mon.a (mon.0) 4304 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.17", "id": [4, 2]}]: dispatch 2022-04-28T22:05:22.422 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:22.160Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:22.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:22 smithi059 bash[59745]: cluster 2022-04-28T22:05:21.187739+0000 mgr.y (mgr.24961) 4396 : cluster [DBG] pgmap v5173: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:22.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:22 smithi059 bash[61152]: cluster 2022-04-28T22:05:21.187739+0000 mgr.y (mgr.24961) 4396 : cluster [DBG] pgmap v5173: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:22 smithi139 bash[43301]: cluster 2022-04-28T22:05:21.187739+0000 mgr.y (mgr.24961) 4396 : cluster [DBG] pgmap v5173: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:24.045Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:24.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:24 smithi059 bash[61152]: cluster 2022-04-28T22:05:23.188584+0000 mgr.y (mgr.24961) 4397 : cluster [DBG] pgmap v5174: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:24 smithi059 bash[59745]: cluster 2022-04-28T22:05:23.188584+0000 mgr.y (mgr.24961) 4397 : cluster [DBG] pgmap v5174: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:24.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:24 smithi139 bash[43301]: cluster 2022-04-28T22:05:23.188584+0000 mgr.y (mgr.24961) 4397 : cluster [DBG] pgmap v5174: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:26 smithi059 bash[61152]: cluster 2022-04-28T22:05:25.190643+0000 mgr.y (mgr.24961) 4398 : cluster [DBG] pgmap v5175: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:26 smithi059 bash[59745]: cluster 2022-04-28T22:05:25.190643+0000 mgr.y (mgr.24961) 4398 : cluster [DBG] pgmap v5175: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:26.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:26 smithi139 bash[43301]: cluster 2022-04-28T22:05:25.190643+0000 mgr.y (mgr.24961) 4398 : cluster [DBG] pgmap v5175: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[59745]: cluster 2022-04-28T22:05:27.191991+0000 mgr.y (mgr.24961) 4399 : cluster [DBG] pgmap v5176: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[59745]: audit 2022-04-28T22:05:27.783409+0000 mon.a (mon.0) 4305 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:05:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[59745]: audit 2022-04-28T22:05:27.786167+0000 mon.a (mon.0) 4306 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:05:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[59745]: audit 2022-04-28T22:05:27.788023+0000 mon.a (mon.0) 4307 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:05:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[59745]: cephadm 2022-04-28T22:05:27.789640+0000 mgr.y (mgr.24961) 4400 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:05:28.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[59745]: audit 2022-04-28T22:05:28.093914+0000 mon.a (mon.0) 4308 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:05:28.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[61152]: cluster 2022-04-28T22:05:27.191991+0000 mgr.y (mgr.24961) 4399 : cluster [DBG] pgmap v5176: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:28.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[61152]: audit 2022-04-28T22:05:27.783409+0000 mon.a (mon.0) 4305 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:05:28.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[61152]: audit 2022-04-28T22:05:27.786167+0000 mon.a (mon.0) 4306 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:05:28.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[61152]: audit 2022-04-28T22:05:27.788023+0000 mon.a (mon.0) 4307 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:05:28.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[61152]: cephadm 2022-04-28T22:05:27.789640+0000 mgr.y (mgr.24961) 4400 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:05:28.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:28 smithi059 bash[61152]: audit 2022-04-28T22:05:28.093914+0000 mon.a (mon.0) 4308 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:05:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:28 smithi139 bash[43301]: cluster 2022-04-28T22:05:27.191991+0000 mgr.y (mgr.24961) 4399 : cluster [DBG] pgmap v5176: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:28.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:28 smithi139 bash[43301]: audit 2022-04-28T22:05:27.783409+0000 mon.a (mon.0) 4305 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:05:28.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:28 smithi139 bash[43301]: audit 2022-04-28T22:05:27.786167+0000 mon.a (mon.0) 4306 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:05:28.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:28 smithi139 bash[43301]: audit 2022-04-28T22:05:27.788023+0000 mon.a (mon.0) 4307 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:05:28.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:28 smithi139 bash[43301]: cephadm 2022-04-28T22:05:27.789640+0000 mgr.y (mgr.24961) 4400 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:05:28.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:28 smithi139 bash[43301]: audit 2022-04-28T22:05:28.093914+0000 mon.a (mon.0) 4308 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:05:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:30 smithi059 bash[61152]: cluster 2022-04-28T22:05:29.193501+0000 mgr.y (mgr.24961) 4401 : cluster [DBG] pgmap v5177: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:30 smithi059 bash[59745]: cluster 2022-04-28T22:05:29.193501+0000 mgr.y (mgr.24961) 4401 : cluster [DBG] pgmap v5177: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:30 smithi139 bash[43301]: cluster 2022-04-28T22:05:29.193501+0000 mgr.y (mgr.24961) 4401 : cluster [DBG] pgmap v5177: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:32.455 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:32.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:32.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:32 smithi059 bash[61152]: cluster 2022-04-28T22:05:31.195415+0000 mgr.y (mgr.24961) 4402 : cluster [DBG] pgmap v5178: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:32 smithi059 bash[59745]: cluster 2022-04-28T22:05:31.195415+0000 mgr.y (mgr.24961) 4402 : cluster [DBG] pgmap v5178: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:32 smithi139 bash[43301]: cluster 2022-04-28T22:05:31.195415+0000 mgr.y (mgr.24961) 4402 : cluster [DBG] pgmap v5178: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:34.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:34.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:34.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:34 smithi059 bash[61152]: cluster 2022-04-28T22:05:33.196264+0000 mgr.y (mgr.24961) 4403 : cluster [DBG] pgmap v5179: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:34.883 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:34 smithi059 bash[59745]: cluster 2022-04-28T22:05:33.196264+0000 mgr.y (mgr.24961) 4403 : cluster [DBG] pgmap v5179: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:34.883 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:34 smithi139 bash[43301]: cluster 2022-04-28T22:05:33.196264+0000 mgr.y (mgr.24961) 4403 : cluster [DBG] pgmap v5179: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:36.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:36 smithi059 bash[61152]: cluster 2022-04-28T22:05:35.198327+0000 mgr.y (mgr.24961) 4404 : cluster [DBG] pgmap v5180: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:36 smithi059 bash[59745]: cluster 2022-04-28T22:05:35.198327+0000 mgr.y (mgr.24961) 4404 : cluster [DBG] pgmap v5180: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:36.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:36 smithi139 bash[43301]: cluster 2022-04-28T22:05:35.198327+0000 mgr.y (mgr.24961) 4404 : cluster [DBG] pgmap v5180: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:38.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:38 smithi059 bash[61152]: cluster 2022-04-28T22:05:37.199747+0000 mgr.y (mgr.24961) 4405 : cluster [DBG] pgmap v5181: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:38 smithi059 bash[59745]: cluster 2022-04-28T22:05:37.199747+0000 mgr.y (mgr.24961) 4405 : cluster [DBG] pgmap v5181: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:38 smithi139 bash[43301]: cluster 2022-04-28T22:05:37.199747+0000 mgr.y (mgr.24961) 4405 : cluster [DBG] pgmap v5181: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:40.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:40 smithi059 bash[61152]: cluster 2022-04-28T22:05:39.201190+0000 mgr.y (mgr.24961) 4406 : cluster [DBG] pgmap v5182: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:40 smithi059 bash[59745]: cluster 2022-04-28T22:05:39.201190+0000 mgr.y (mgr.24961) 4406 : cluster [DBG] pgmap v5182: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:40 smithi139 bash[43301]: cluster 2022-04-28T22:05:39.201190+0000 mgr.y (mgr.24961) 4406 : cluster [DBG] pgmap v5182: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:42.479 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:42.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:42 smithi059 bash[61152]: cluster 2022-04-28T22:05:41.203063+0000 mgr.y (mgr.24961) 4407 : cluster [DBG] pgmap v5183: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:42.862 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:42 smithi059 bash[59745]: cluster 2022-04-28T22:05:41.203063+0000 mgr.y (mgr.24961) 4407 : cluster [DBG] pgmap v5183: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:42 smithi139 bash[43301]: cluster 2022-04-28T22:05:41.203063+0000 mgr.y (mgr.24961) 4407 : cluster [DBG] pgmap v5183: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:44.343 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:44.047Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:44 smithi059 bash[61152]: cluster 2022-04-28T22:05:43.204048+0000 mgr.y (mgr.24961) 4408 : cluster [DBG] pgmap v5184: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:44 smithi059 bash[59745]: cluster 2022-04-28T22:05:43.204048+0000 mgr.y (mgr.24961) 4408 : cluster [DBG] pgmap v5184: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:44.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:44 smithi139 bash[43301]: cluster 2022-04-28T22:05:43.204048+0000 mgr.y (mgr.24961) 4408 : cluster [DBG] pgmap v5184: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:46.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:46 smithi059 bash[61152]: cluster 2022-04-28T22:05:45.206121+0000 mgr.y (mgr.24961) 4409 : cluster [DBG] pgmap v5185: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:46 smithi059 bash[59745]: cluster 2022-04-28T22:05:45.206121+0000 mgr.y (mgr.24961) 4409 : cluster [DBG] pgmap v5185: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:46 smithi139 bash[43301]: cluster 2022-04-28T22:05:45.206121+0000 mgr.y (mgr.24961) 4409 : cluster [DBG] pgmap v5185: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:48 smithi059 bash[61152]: cluster 2022-04-28T22:05:47.207536+0000 mgr.y (mgr.24961) 4410 : cluster [DBG] pgmap v5186: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:48 smithi059 bash[59745]: cluster 2022-04-28T22:05:47.207536+0000 mgr.y (mgr.24961) 4410 : cluster [DBG] pgmap v5186: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:48 smithi139 bash[43301]: cluster 2022-04-28T22:05:47.207536+0000 mgr.y (mgr.24961) 4410 : cluster [DBG] pgmap v5186: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:50.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:50 smithi059 bash[61152]: cluster 2022-04-28T22:05:49.209076+0000 mgr.y (mgr.24961) 4411 : cluster [DBG] pgmap v5187: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:50 smithi059 bash[59745]: cluster 2022-04-28T22:05:49.209076+0000 mgr.y (mgr.24961) 4411 : cluster [DBG] pgmap v5187: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:50.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:50 smithi139 bash[43301]: cluster 2022-04-28T22:05:49.209076+0000 mgr.y (mgr.24961) 4411 : cluster [DBG] pgmap v5187: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:52.505 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:52.161Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:52 smithi059 bash[61152]: cluster 2022-04-28T22:05:51.210959+0000 mgr.y (mgr.24961) 4412 : cluster [DBG] pgmap v5188: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:52 smithi059 bash[59745]: cluster 2022-04-28T22:05:51.210959+0000 mgr.y (mgr.24961) 4412 : cluster [DBG] pgmap v5188: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:52.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:52 smithi139 bash[43301]: cluster 2022-04-28T22:05:51.210959+0000 mgr.y (mgr.24961) 4412 : cluster [DBG] pgmap v5188: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:54.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:05:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:05:54.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:05:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:54 smithi059 bash[61152]: cluster 2022-04-28T22:05:53.211861+0000 mgr.y (mgr.24961) 4413 : cluster [DBG] pgmap v5189: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:54.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:54 smithi059 bash[59745]: cluster 2022-04-28T22:05:53.211861+0000 mgr.y (mgr.24961) 4413 : cluster [DBG] pgmap v5189: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:54 smithi139 bash[43301]: cluster 2022-04-28T22:05:53.211861+0000 mgr.y (mgr.24961) 4413 : cluster [DBG] pgmap v5189: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:56 smithi059 bash[61152]: cluster 2022-04-28T22:05:55.213934+0000 mgr.y (mgr.24961) 4414 : cluster [DBG] pgmap v5190: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:56 smithi059 bash[59745]: cluster 2022-04-28T22:05:55.213934+0000 mgr.y (mgr.24961) 4414 : cluster [DBG] pgmap v5190: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:56 smithi139 bash[43301]: cluster 2022-04-28T22:05:55.213934+0000 mgr.y (mgr.24961) 4414 : cluster [DBG] pgmap v5190: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:05:58 smithi059 bash[61152]: cluster 2022-04-28T22:05:57.215287+0000 mgr.y (mgr.24961) 4415 : cluster [DBG] pgmap v5191: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:05:58 smithi059 bash[59745]: cluster 2022-04-28T22:05:57.215287+0000 mgr.y (mgr.24961) 4415 : cluster [DBG] pgmap v5191: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:05:58.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:05:58 smithi139 bash[43301]: cluster 2022-04-28T22:05:57.215287+0000 mgr.y (mgr.24961) 4415 : cluster [DBG] pgmap v5191: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:00.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:00 smithi059 bash[61152]: cluster 2022-04-28T22:05:59.216753+0000 mgr.y (mgr.24961) 4416 : cluster [DBG] pgmap v5192: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:00.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:00 smithi059 bash[59745]: cluster 2022-04-28T22:05:59.216753+0000 mgr.y (mgr.24961) 4416 : cluster [DBG] pgmap v5192: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:00.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:00 smithi139 bash[43301]: cluster 2022-04-28T22:05:59.216753+0000 mgr.y (mgr.24961) 4416 : cluster [DBG] pgmap v5192: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:02.531 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:02.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:02.531 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:02 smithi059 bash[59745]: cluster 2022-04-28T22:06:01.218737+0000 mgr.y (mgr.24961) 4417 : cluster [DBG] pgmap v5193: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:02.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:02 smithi059 bash[61152]: cluster 2022-04-28T22:06:01.218737+0000 mgr.y (mgr.24961) 4417 : cluster [DBG] pgmap v5193: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:02 smithi139 bash[43301]: cluster 2022-04-28T22:06:01.218737+0000 mgr.y (mgr.24961) 4417 : cluster [DBG] pgmap v5193: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:04.048Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:04.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:04 smithi059 bash[61152]: cluster 2022-04-28T22:06:03.219646+0000 mgr.y (mgr.24961) 4418 : cluster [DBG] pgmap v5194: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:04.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:04 smithi059 bash[59745]: cluster 2022-04-28T22:06:03.219646+0000 mgr.y (mgr.24961) 4418 : cluster [DBG] pgmap v5194: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:04.872 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:04 smithi139 bash[43301]: cluster 2022-04-28T22:06:03.219646+0000 mgr.y (mgr.24961) 4418 : cluster [DBG] pgmap v5194: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:06.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:06 smithi059 bash[61152]: cluster 2022-04-28T22:06:05.221693+0000 mgr.y (mgr.24961) 4419 : cluster [DBG] pgmap v5195: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:06 smithi059 bash[59745]: cluster 2022-04-28T22:06:05.221693+0000 mgr.y (mgr.24961) 4419 : cluster [DBG] pgmap v5195: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:06 smithi139 bash[43301]: cluster 2022-04-28T22:06:05.221693+0000 mgr.y (mgr.24961) 4419 : cluster [DBG] pgmap v5195: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:08 smithi059 bash[61152]: cluster 2022-04-28T22:06:07.223055+0000 mgr.y (mgr.24961) 4420 : cluster [DBG] pgmap v5196: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:08 smithi059 bash[59745]: cluster 2022-04-28T22:06:07.223055+0000 mgr.y (mgr.24961) 4420 : cluster [DBG] pgmap v5196: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:08 smithi139 bash[43301]: cluster 2022-04-28T22:06:07.223055+0000 mgr.y (mgr.24961) 4420 : cluster [DBG] pgmap v5196: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:10.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:10 smithi059 bash[61152]: cluster 2022-04-28T22:06:09.224615+0000 mgr.y (mgr.24961) 4421 : cluster [DBG] pgmap v5197: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:10 smithi059 bash[59745]: cluster 2022-04-28T22:06:09.224615+0000 mgr.y (mgr.24961) 4421 : cluster [DBG] pgmap v5197: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:10.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:10 smithi139 bash[43301]: cluster 2022-04-28T22:06:09.224615+0000 mgr.y (mgr.24961) 4421 : cluster [DBG] pgmap v5197: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:12.555 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:12.162Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:12.556 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:12 smithi059 bash[59745]: cluster 2022-04-28T22:06:11.226492+0000 mgr.y (mgr.24961) 4422 : cluster [DBG] pgmap v5198: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:12 smithi059 bash[61152]: cluster 2022-04-28T22:06:11.226492+0000 mgr.y (mgr.24961) 4422 : cluster [DBG] pgmap v5198: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:12 smithi139 bash[43301]: cluster 2022-04-28T22:06:11.226492+0000 mgr.y (mgr.24961) 4422 : cluster [DBG] pgmap v5198: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:14.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:14.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:14 smithi059 bash[61152]: cluster 2022-04-28T22:06:13.227336+0000 mgr.y (mgr.24961) 4423 : cluster [DBG] pgmap v5199: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:14.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:14 smithi059 bash[59745]: cluster 2022-04-28T22:06:13.227336+0000 mgr.y (mgr.24961) 4423 : cluster [DBG] pgmap v5199: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:14 smithi139 bash[43301]: cluster 2022-04-28T22:06:13.227336+0000 mgr.y (mgr.24961) 4423 : cluster [DBG] pgmap v5199: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:16 smithi059 bash[61152]: cluster 2022-04-28T22:06:15.229402+0000 mgr.y (mgr.24961) 4424 : cluster [DBG] pgmap v5200: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:16.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:16 smithi059 bash[59745]: cluster 2022-04-28T22:06:15.229402+0000 mgr.y (mgr.24961) 4424 : cluster [DBG] pgmap v5200: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:16 smithi139 bash[43301]: cluster 2022-04-28T22:06:15.229402+0000 mgr.y (mgr.24961) 4424 : cluster [DBG] pgmap v5200: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:18 smithi059 bash[61152]: cluster 2022-04-28T22:06:17.230755+0000 mgr.y (mgr.24961) 4425 : cluster [DBG] pgmap v5201: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:18.914 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:18 smithi059 bash[59745]: cluster 2022-04-28T22:06:17.230755+0000 mgr.y (mgr.24961) 4425 : cluster [DBG] pgmap v5201: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:18.915 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:18 smithi139 bash[43301]: cluster 2022-04-28T22:06:17.230755+0000 mgr.y (mgr.24961) 4425 : cluster [DBG] pgmap v5201: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: cluster 2022-04-28T22:06:19.232273+0000 mgr.y (mgr.24961) 4426 : cluster [DBG] pgmap v5202: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: audit 2022-04-28T22:06:19.472859+0000 mon.a (mon.0) 4309 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:06:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: audit 2022-04-28T22:06:19.503247+0000 mon.a (mon.0) 4310 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:06:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: audit 2022-04-28T22:06:19.606281+0000 mon.a (mon.0) 4311 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T22:06:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: audit 2022-04-28T22:06:19.606830+0000 mon.a (mon.0) 4312 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T22:06:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: audit 2022-04-28T22:06:19.607333+0000 mon.a (mon.0) 4313 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T22:06:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[61152]: audit 2022-04-28T22:06:19.607824+0000 mon.a (mon.0) 4314 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T22:06:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: cluster 2022-04-28T22:06:19.232273+0000 mgr.y (mgr.24961) 4426 : cluster [DBG] pgmap v5202: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: audit 2022-04-28T22:06:19.472859+0000 mon.a (mon.0) 4309 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:06:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: audit 2022-04-28T22:06:19.503247+0000 mon.a (mon.0) 4310 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:06:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: audit 2022-04-28T22:06:19.606281+0000 mon.a (mon.0) 4311 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T22:06:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: audit 2022-04-28T22:06:19.606830+0000 mon.a (mon.0) 4312 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T22:06:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: audit 2022-04-28T22:06:19.607333+0000 mon.a (mon.0) 4313 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T22:06:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:20 smithi059 bash[59745]: audit 2022-04-28T22:06:19.607824+0000 mon.a (mon.0) 4314 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T22:06:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: cluster 2022-04-28T22:06:19.232273+0000 mgr.y (mgr.24961) 4426 : cluster [DBG] pgmap v5202: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: audit 2022-04-28T22:06:19.472859+0000 mon.a (mon.0) 4309 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:06:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: audit 2022-04-28T22:06:19.503247+0000 mon.a (mon.0) 4310 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:06:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: audit 2022-04-28T22:06:19.606281+0000 mon.a (mon.0) 4311 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2022-04-28T22:06:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: audit 2022-04-28T22:06:19.606830+0000 mon.a (mon.0) 4312 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2022-04-28T22:06:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: audit 2022-04-28T22:06:19.607333+0000 mon.a (mon.0) 4313 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2022-04-28T22:06:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:20 smithi139 bash[43301]: audit 2022-04-28T22:06:19.607824+0000 mon.a (mon.0) 4314 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1d", "id": [4, 2]}]: dispatch 2022-04-28T22:06:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:22 smithi059 bash[61152]: cluster 2022-04-28T22:06:21.234178+0000 mgr.y (mgr.24961) 4427 : cluster [DBG] pgmap v5203: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:22.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:22.163Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:22 smithi059 bash[59745]: cluster 2022-04-28T22:06:21.234178+0000 mgr.y (mgr.24961) 4427 : cluster [DBG] pgmap v5203: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:22 smithi139 bash[43301]: cluster 2022-04-28T22:06:21.234178+0000 mgr.y (mgr.24961) 4427 : cluster [DBG] pgmap v5203: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:24.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:24 smithi059 bash[61152]: cluster 2022-04-28T22:06:23.234999+0000 mgr.y (mgr.24961) 4428 : cluster [DBG] pgmap v5204: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:24.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:24.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:24.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:24 smithi059 bash[59745]: cluster 2022-04-28T22:06:23.234999+0000 mgr.y (mgr.24961) 4428 : cluster [DBG] pgmap v5204: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:24 smithi139 bash[43301]: cluster 2022-04-28T22:06:23.234999+0000 mgr.y (mgr.24961) 4428 : cluster [DBG] pgmap v5204: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:26.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:26 smithi059 bash[61152]: cluster 2022-04-28T22:06:25.237028+0000 mgr.y (mgr.24961) 4429 : cluster [DBG] pgmap v5205: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:26 smithi059 bash[59745]: cluster 2022-04-28T22:06:25.237028+0000 mgr.y (mgr.24961) 4429 : cluster [DBG] pgmap v5205: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:26.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:26 smithi139 bash[43301]: cluster 2022-04-28T22:06:25.237028+0000 mgr.y (mgr.24961) 4429 : cluster [DBG] pgmap v5205: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[59745]: cluster 2022-04-28T22:06:27.238435+0000 mgr.y (mgr.24961) 4430 : cluster [DBG] pgmap v5206: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[59745]: audit 2022-04-28T22:06:28.100510+0000 mon.a (mon.0) 4315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:06:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[59745]: audit 2022-04-28T22:06:28.103271+0000 mon.a (mon.0) 4316 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:06:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[59745]: audit 2022-04-28T22:06:28.105095+0000 mon.a (mon.0) 4317 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:06:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[61152]: cluster 2022-04-28T22:06:27.238435+0000 mgr.y (mgr.24961) 4430 : cluster [DBG] pgmap v5206: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[61152]: audit 2022-04-28T22:06:28.100510+0000 mon.a (mon.0) 4315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:06:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[61152]: audit 2022-04-28T22:06:28.103271+0000 mon.a (mon.0) 4316 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:06:28.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:28 smithi059 bash[61152]: audit 2022-04-28T22:06:28.105095+0000 mon.a (mon.0) 4317 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:06:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:28 smithi139 bash[43301]: cluster 2022-04-28T22:06:27.238435+0000 mgr.y (mgr.24961) 4430 : cluster [DBG] pgmap v5206: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:28 smithi139 bash[43301]: audit 2022-04-28T22:06:28.100510+0000 mon.a (mon.0) 4315 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:06:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:28 smithi139 bash[43301]: audit 2022-04-28T22:06:28.103271+0000 mon.a (mon.0) 4316 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:06:28.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:28 smithi139 bash[43301]: audit 2022-04-28T22:06:28.105095+0000 mon.a (mon.0) 4317 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:06:29.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:29 smithi059 bash[61152]: cephadm 2022-04-28T22:06:28.106697+0000 mgr.y (mgr.24961) 4431 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:06:29.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:29 smithi059 bash[61152]: audit 2022-04-28T22:06:28.404728+0000 mon.a (mon.0) 4318 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:06:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:29 smithi059 bash[59745]: cephadm 2022-04-28T22:06:28.106697+0000 mgr.y (mgr.24961) 4431 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:06:29.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:29 smithi059 bash[59745]: audit 2022-04-28T22:06:28.404728+0000 mon.a (mon.0) 4318 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:06:29.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:29 smithi139 bash[43301]: cephadm 2022-04-28T22:06:28.106697+0000 mgr.y (mgr.24961) 4431 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:06:29.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:29 smithi139 bash[43301]: audit 2022-04-28T22:06:28.404728+0000 mon.a (mon.0) 4318 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:06:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:30 smithi059 bash[61152]: cluster 2022-04-28T22:06:29.239932+0000 mgr.y (mgr.24961) 4432 : cluster [DBG] pgmap v5207: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:30 smithi059 bash[59745]: cluster 2022-04-28T22:06:29.239932+0000 mgr.y (mgr.24961) 4432 : cluster [DBG] pgmap v5207: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:30.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:30 smithi139 bash[43301]: cluster 2022-04-28T22:06:29.239932+0000 mgr.y (mgr.24961) 4432 : cluster [DBG] pgmap v5207: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:32.419 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:32.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:32.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:32 smithi059 bash[61152]: cluster 2022-04-28T22:06:31.241847+0000 mgr.y (mgr.24961) 4433 : cluster [DBG] pgmap v5208: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:32 smithi059 bash[59745]: cluster 2022-04-28T22:06:31.241847+0000 mgr.y (mgr.24961) 4433 : cluster [DBG] pgmap v5208: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:32 smithi139 bash[43301]: cluster 2022-04-28T22:06:31.241847+0000 mgr.y (mgr.24961) 4433 : cluster [DBG] pgmap v5208: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:34.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:34.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:34 smithi059 bash[61152]: cluster 2022-04-28T22:06:33.242672+0000 mgr.y (mgr.24961) 4434 : cluster [DBG] pgmap v5209: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:34.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:34 smithi059 bash[59745]: cluster 2022-04-28T22:06:33.242672+0000 mgr.y (mgr.24961) 4434 : cluster [DBG] pgmap v5209: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:34.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:34 smithi139 bash[43301]: cluster 2022-04-28T22:06:33.242672+0000 mgr.y (mgr.24961) 4434 : cluster [DBG] pgmap v5209: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:36.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:36 smithi059 bash[59745]: cluster 2022-04-28T22:06:35.244782+0000 mgr.y (mgr.24961) 4435 : cluster [DBG] pgmap v5210: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:36.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:36 smithi059 bash[61152]: cluster 2022-04-28T22:06:35.244782+0000 mgr.y (mgr.24961) 4435 : cluster [DBG] pgmap v5210: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:36.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:36 smithi139 bash[43301]: cluster 2022-04-28T22:06:35.244782+0000 mgr.y (mgr.24961) 4435 : cluster [DBG] pgmap v5210: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:38.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:38 smithi059 bash[61152]: cluster 2022-04-28T22:06:37.246207+0000 mgr.y (mgr.24961) 4436 : cluster [DBG] pgmap v5211: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:38.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:38 smithi059 bash[59745]: cluster 2022-04-28T22:06:37.246207+0000 mgr.y (mgr.24961) 4436 : cluster [DBG] pgmap v5211: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:38.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:38 smithi139 bash[43301]: cluster 2022-04-28T22:06:37.246207+0000 mgr.y (mgr.24961) 4436 : cluster [DBG] pgmap v5211: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:40.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:40 smithi059 bash[61152]: cluster 2022-04-28T22:06:39.247723+0000 mgr.y (mgr.24961) 4437 : cluster [DBG] pgmap v5212: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:40.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:40 smithi059 bash[59745]: cluster 2022-04-28T22:06:39.247723+0000 mgr.y (mgr.24961) 4437 : cluster [DBG] pgmap v5212: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:40.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:40 smithi139 bash[43301]: cluster 2022-04-28T22:06:39.247723+0000 mgr.y (mgr.24961) 4437 : cluster [DBG] pgmap v5212: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:42.448 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:42.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:42.448 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:42 smithi059 bash[59745]: cluster 2022-04-28T22:06:41.249743+0000 mgr.y (mgr.24961) 4438 : cluster [DBG] pgmap v5213: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:42.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:42 smithi059 bash[61152]: cluster 2022-04-28T22:06:41.249743+0000 mgr.y (mgr.24961) 4438 : cluster [DBG] pgmap v5213: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:42.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:42 smithi139 bash[43301]: cluster 2022-04-28T22:06:41.249743+0000 mgr.y (mgr.24961) 4438 : cluster [DBG] pgmap v5213: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:44.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:44.051Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:44.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:44 smithi059 bash[61152]: cluster 2022-04-28T22:06:43.250703+0000 mgr.y (mgr.24961) 4439 : cluster [DBG] pgmap v5214: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:44 smithi059 bash[59745]: cluster 2022-04-28T22:06:43.250703+0000 mgr.y (mgr.24961) 4439 : cluster [DBG] pgmap v5214: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:44.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:44 smithi139 bash[43301]: cluster 2022-04-28T22:06:43.250703+0000 mgr.y (mgr.24961) 4439 : cluster [DBG] pgmap v5214: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:46.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:46 smithi059 bash[61152]: cluster 2022-04-28T22:06:45.252845+0000 mgr.y (mgr.24961) 4440 : cluster [DBG] pgmap v5215: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:46.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:46 smithi059 bash[59745]: cluster 2022-04-28T22:06:45.252845+0000 mgr.y (mgr.24961) 4440 : cluster [DBG] pgmap v5215: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:46.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:46 smithi139 bash[43301]: cluster 2022-04-28T22:06:45.252845+0000 mgr.y (mgr.24961) 4440 : cluster [DBG] pgmap v5215: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:48 smithi059 bash[61152]: cluster 2022-04-28T22:06:47.254270+0000 mgr.y (mgr.24961) 4441 : cluster [DBG] pgmap v5216: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:48 smithi059 bash[59745]: cluster 2022-04-28T22:06:47.254270+0000 mgr.y (mgr.24961) 4441 : cluster [DBG] pgmap v5216: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:48.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:48 smithi139 bash[43301]: cluster 2022-04-28T22:06:47.254270+0000 mgr.y (mgr.24961) 4441 : cluster [DBG] pgmap v5216: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:50.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:50 smithi059 bash[61152]: cluster 2022-04-28T22:06:49.255784+0000 mgr.y (mgr.24961) 4442 : cluster [DBG] pgmap v5217: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:50 smithi059 bash[59745]: cluster 2022-04-28T22:06:49.255784+0000 mgr.y (mgr.24961) 4442 : cluster [DBG] pgmap v5217: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:50.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:50 smithi139 bash[43301]: cluster 2022-04-28T22:06:49.255784+0000 mgr.y (mgr.24961) 4442 : cluster [DBG] pgmap v5217: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:52.473 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:52.164Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:52.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:52 smithi059 bash[61152]: cluster 2022-04-28T22:06:51.257693+0000 mgr.y (mgr.24961) 4443 : cluster [DBG] pgmap v5218: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:52.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:52 smithi059 bash[59745]: cluster 2022-04-28T22:06:51.257693+0000 mgr.y (mgr.24961) 4443 : cluster [DBG] pgmap v5218: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:52.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:52 smithi139 bash[43301]: cluster 2022-04-28T22:06:51.257693+0000 mgr.y (mgr.24961) 4443 : cluster [DBG] pgmap v5218: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:06:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:06:54.053Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:06:54.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:54 smithi059 bash[61152]: cluster 2022-04-28T22:06:53.258564+0000 mgr.y (mgr.24961) 4444 : cluster [DBG] pgmap v5219: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:54.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:54 smithi059 bash[59745]: cluster 2022-04-28T22:06:53.258564+0000 mgr.y (mgr.24961) 4444 : cluster [DBG] pgmap v5219: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:54.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:54 smithi139 bash[43301]: cluster 2022-04-28T22:06:53.258564+0000 mgr.y (mgr.24961) 4444 : cluster [DBG] pgmap v5219: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:56.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:56 smithi059 bash[61152]: cluster 2022-04-28T22:06:55.260679+0000 mgr.y (mgr.24961) 4445 : cluster [DBG] pgmap v5220: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:56 smithi059 bash[59745]: cluster 2022-04-28T22:06:55.260679+0000 mgr.y (mgr.24961) 4445 : cluster [DBG] pgmap v5220: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:56 smithi139 bash[43301]: cluster 2022-04-28T22:06:55.260679+0000 mgr.y (mgr.24961) 4445 : cluster [DBG] pgmap v5220: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:58.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:06:58 smithi059 bash[59745]: cluster 2022-04-28T22:06:57.262034+0000 mgr.y (mgr.24961) 4446 : cluster [DBG] pgmap v5221: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:58.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:06:58 smithi059 bash[61152]: cluster 2022-04-28T22:06:57.262034+0000 mgr.y (mgr.24961) 4446 : cluster [DBG] pgmap v5221: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:06:58.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:06:58 smithi139 bash[43301]: cluster 2022-04-28T22:06:57.262034+0000 mgr.y (mgr.24961) 4446 : cluster [DBG] pgmap v5221: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:00 smithi059 bash[61152]: cluster 2022-04-28T22:06:59.263571+0000 mgr.y (mgr.24961) 4447 : cluster [DBG] pgmap v5222: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:00 smithi059 bash[59745]: cluster 2022-04-28T22:06:59.263571+0000 mgr.y (mgr.24961) 4447 : cluster [DBG] pgmap v5222: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:00.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:00 smithi139 bash[43301]: cluster 2022-04-28T22:06:59.263571+0000 mgr.y (mgr.24961) 4447 : cluster [DBG] pgmap v5222: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:02.500 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:02.165Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:02.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:02 smithi059 bash[61152]: cluster 2022-04-28T22:07:01.265486+0000 mgr.y (mgr.24961) 4448 : cluster [DBG] pgmap v5223: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:02.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:02 smithi059 bash[59745]: cluster 2022-04-28T22:07:01.265486+0000 mgr.y (mgr.24961) 4448 : cluster [DBG] pgmap v5223: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:02 smithi139 bash[43301]: cluster 2022-04-28T22:07:01.265486+0000 mgr.y (mgr.24961) 4448 : cluster [DBG] pgmap v5223: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:04.050Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:04.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:04 smithi059 bash[61152]: cluster 2022-04-28T22:07:03.266309+0000 mgr.y (mgr.24961) 4449 : cluster [DBG] pgmap v5224: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:04.857 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:04 smithi059 bash[59745]: cluster 2022-04-28T22:07:03.266309+0000 mgr.y (mgr.24961) 4449 : cluster [DBG] pgmap v5224: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:04.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:04 smithi139 bash[43301]: cluster 2022-04-28T22:07:03.266309+0000 mgr.y (mgr.24961) 4449 : cluster [DBG] pgmap v5224: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:06 smithi059 bash[61152]: cluster 2022-04-28T22:07:05.268367+0000 mgr.y (mgr.24961) 4450 : cluster [DBG] pgmap v5225: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:06 smithi059 bash[59745]: cluster 2022-04-28T22:07:05.268367+0000 mgr.y (mgr.24961) 4450 : cluster [DBG] pgmap v5225: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:06 smithi139 bash[43301]: cluster 2022-04-28T22:07:05.268367+0000 mgr.y (mgr.24961) 4450 : cluster [DBG] pgmap v5225: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:08 smithi059 bash[61152]: cluster 2022-04-28T22:07:07.269768+0000 mgr.y (mgr.24961) 4451 : cluster [DBG] pgmap v5226: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:08 smithi059 bash[59745]: cluster 2022-04-28T22:07:07.269768+0000 mgr.y (mgr.24961) 4451 : cluster [DBG] pgmap v5226: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:08 smithi139 bash[43301]: cluster 2022-04-28T22:07:07.269768+0000 mgr.y (mgr.24961) 4451 : cluster [DBG] pgmap v5226: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:10 smithi059 bash[61152]: cluster 2022-04-28T22:07:09.271256+0000 mgr.y (mgr.24961) 4452 : cluster [DBG] pgmap v5227: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:10 smithi059 bash[59745]: cluster 2022-04-28T22:07:09.271256+0000 mgr.y (mgr.24961) 4452 : cluster [DBG] pgmap v5227: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:10.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:10 smithi139 bash[43301]: cluster 2022-04-28T22:07:09.271256+0000 mgr.y (mgr.24961) 4452 : cluster [DBG] pgmap v5227: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:12.527 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:12.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:12.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:12 smithi059 bash[61152]: cluster 2022-04-28T22:07:11.273285+0000 mgr.y (mgr.24961) 4453 : cluster [DBG] pgmap v5228: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:12 smithi059 bash[59745]: cluster 2022-04-28T22:07:11.273285+0000 mgr.y (mgr.24961) 4453 : cluster [DBG] pgmap v5228: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:12 smithi139 bash[43301]: cluster 2022-04-28T22:07:11.273285+0000 mgr.y (mgr.24961) 4453 : cluster [DBG] pgmap v5228: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:14.052Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:14.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:14 smithi059 bash[59745]: cluster 2022-04-28T22:07:13.274254+0000 mgr.y (mgr.24961) 4454 : cluster [DBG] pgmap v5229: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:14.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:14 smithi059 bash[61152]: cluster 2022-04-28T22:07:13.274254+0000 mgr.y (mgr.24961) 4454 : cluster [DBG] pgmap v5229: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:14.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:14 smithi139 bash[43301]: cluster 2022-04-28T22:07:13.274254+0000 mgr.y (mgr.24961) 4454 : cluster [DBG] pgmap v5229: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:16 smithi059 bash[59745]: cluster 2022-04-28T22:07:15.276386+0000 mgr.y (mgr.24961) 4455 : cluster [DBG] pgmap v5230: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:16.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:16 smithi059 bash[61152]: cluster 2022-04-28T22:07:15.276386+0000 mgr.y (mgr.24961) 4455 : cluster [DBG] pgmap v5230: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:16 smithi139 bash[43301]: cluster 2022-04-28T22:07:15.276386+0000 mgr.y (mgr.24961) 4455 : cluster [DBG] pgmap v5230: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:18 smithi059 bash[59745]: cluster 2022-04-28T22:07:17.277761+0000 mgr.y (mgr.24961) 4456 : cluster [DBG] pgmap v5231: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:18.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:18 smithi059 bash[61152]: cluster 2022-04-28T22:07:17.277761+0000 mgr.y (mgr.24961) 4456 : cluster [DBG] pgmap v5231: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:18 smithi139 bash[43301]: cluster 2022-04-28T22:07:17.277761+0000 mgr.y (mgr.24961) 4456 : cluster [DBG] pgmap v5231: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: cluster 2022-04-28T22:07:19.279253+0000 mgr.y (mgr.24961) 4457 : cluster [DBG] pgmap v5232: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: audit 2022-04-28T22:07:19.478195+0000 mon.a (mon.0) 4319 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:07:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: audit 2022-04-28T22:07:19.509109+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:07:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: audit 2022-04-28T22:07:19.616378+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: audit 2022-04-28T22:07:19.616952+0000 mon.a (mon.0) 4322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: audit 2022-04-28T22:07:19.617229+0000 mon.a (mon.0) 4323 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[61152]: audit 2022-04-28T22:07:19.617530+0000 mon.a (mon.0) 4324 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.14", "id": [4, 0]}]: dispatch 2022-04-28T22:07:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: cluster 2022-04-28T22:07:19.279253+0000 mgr.y (mgr.24961) 4457 : cluster [DBG] pgmap v5232: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: audit 2022-04-28T22:07:19.478195+0000 mon.a (mon.0) 4319 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:07:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: audit 2022-04-28T22:07:19.509109+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:07:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: audit 2022-04-28T22:07:19.616378+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: audit 2022-04-28T22:07:19.616952+0000 mon.a (mon.0) 4322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: audit 2022-04-28T22:07:19.617229+0000 mon.a (mon.0) 4323 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:20 smithi059 bash[59745]: audit 2022-04-28T22:07:19.617530+0000 mon.a (mon.0) 4324 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.14", "id": [4, 0]}]: dispatch 2022-04-28T22:07:20.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: cluster 2022-04-28T22:07:19.279253+0000 mgr.y (mgr.24961) 4457 : cluster [DBG] pgmap v5232: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: audit 2022-04-28T22:07:19.478195+0000 mon.a (mon.0) 4319 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:07:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: audit 2022-04-28T22:07:19.509109+0000 mon.a (mon.0) 4320 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:07:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: audit 2022-04-28T22:07:19.616378+0000 mon.a (mon.0) 4321 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: audit 2022-04-28T22:07:19.616952+0000 mon.a (mon.0) 4322 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: audit 2022-04-28T22:07:19.617229+0000 mon.a (mon.0) 4323 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2022-04-28T22:07:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:20 smithi139 bash[43301]: audit 2022-04-28T22:07:19.617530+0000 mon.a (mon.0) 4324 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.14", "id": [4, 0]}]: dispatch 2022-04-28T22:07:22.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:22 smithi059 bash[61152]: cluster 2022-04-28T22:07:21.281129+0000 mgr.y (mgr.24961) 4458 : cluster [DBG] pgmap v5233: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:22.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:22.166Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:22.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:22 smithi059 bash[59745]: cluster 2022-04-28T22:07:21.281129+0000 mgr.y (mgr.24961) 4458 : cluster [DBG] pgmap v5233: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:22.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:22 smithi139 bash[43301]: cluster 2022-04-28T22:07:21.281129+0000 mgr.y (mgr.24961) 4458 : cluster [DBG] pgmap v5233: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:24.321 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:24.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:24.322 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:24 smithi059 bash[59745]: cluster 2022-04-28T22:07:23.281949+0000 mgr.y (mgr.24961) 4459 : cluster [DBG] pgmap v5234: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:24.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:24 smithi059 bash[61152]: cluster 2022-04-28T22:07:23.281949+0000 mgr.y (mgr.24961) 4459 : cluster [DBG] pgmap v5234: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:24.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:24 smithi139 bash[43301]: cluster 2022-04-28T22:07:23.281949+0000 mgr.y (mgr.24961) 4459 : cluster [DBG] pgmap v5234: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:26.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:26 smithi059 bash[61152]: cluster 2022-04-28T22:07:25.284006+0000 mgr.y (mgr.24961) 4460 : cluster [DBG] pgmap v5235: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:26.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:26 smithi059 bash[59745]: cluster 2022-04-28T22:07:25.284006+0000 mgr.y (mgr.24961) 4460 : cluster [DBG] pgmap v5235: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:26 smithi139 bash[43301]: cluster 2022-04-28T22:07:25.284006+0000 mgr.y (mgr.24961) 4460 : cluster [DBG] pgmap v5235: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:28.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:28 smithi059 bash[61152]: cluster 2022-04-28T22:07:27.285374+0000 mgr.y (mgr.24961) 4461 : cluster [DBG] pgmap v5236: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:28.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:28 smithi059 bash[59745]: cluster 2022-04-28T22:07:27.285374+0000 mgr.y (mgr.24961) 4461 : cluster [DBG] pgmap v5236: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:28 smithi139 bash[43301]: cluster 2022-04-28T22:07:27.285374+0000 mgr.y (mgr.24961) 4461 : cluster [DBG] pgmap v5236: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[61152]: audit 2022-04-28T22:07:28.413846+0000 mon.a (mon.0) 4325 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:07:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[61152]: audit 2022-04-28T22:07:28.416701+0000 mon.a (mon.0) 4326 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:07:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[61152]: audit 2022-04-28T22:07:28.418606+0000 mon.a (mon.0) 4327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:07:29.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[61152]: cephadm 2022-04-28T22:07:28.420365+0000 mgr.y (mgr.24961) 4462 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:07:29.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[61152]: audit 2022-04-28T22:07:28.737349+0000 mon.a (mon.0) 4328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:07:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[59745]: audit 2022-04-28T22:07:28.413846+0000 mon.a (mon.0) 4325 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:07:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[59745]: audit 2022-04-28T22:07:28.416701+0000 mon.a (mon.0) 4326 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:07:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[59745]: audit 2022-04-28T22:07:28.418606+0000 mon.a (mon.0) 4327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:07:29.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[59745]: cephadm 2022-04-28T22:07:28.420365+0000 mgr.y (mgr.24961) 4462 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:07:29.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:29 smithi059 bash[59745]: audit 2022-04-28T22:07:28.737349+0000 mon.a (mon.0) 4328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:07:29.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:29 smithi139 bash[43301]: audit 2022-04-28T22:07:28.413846+0000 mon.a (mon.0) 4325 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:07:29.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:29 smithi139 bash[43301]: audit 2022-04-28T22:07:28.416701+0000 mon.a (mon.0) 4326 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:07:29.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:29 smithi139 bash[43301]: audit 2022-04-28T22:07:28.418606+0000 mon.a (mon.0) 4327 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:07:29.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:29 smithi139 bash[43301]: cephadm 2022-04-28T22:07:28.420365+0000 mgr.y (mgr.24961) 4462 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:07:29.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:29 smithi139 bash[43301]: audit 2022-04-28T22:07:28.737349+0000 mon.a (mon.0) 4328 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:07:31.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:30 smithi059 bash[61152]: cluster 2022-04-28T22:07:29.286855+0000 mgr.y (mgr.24961) 4463 : cluster [DBG] pgmap v5237: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:31.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:30 smithi059 bash[59745]: cluster 2022-04-28T22:07:29.286855+0000 mgr.y (mgr.24961) 4463 : cluster [DBG] pgmap v5237: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:31.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:30 smithi139 bash[43301]: cluster 2022-04-28T22:07:29.286855+0000 mgr.y (mgr.24961) 4463 : cluster [DBG] pgmap v5237: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:32.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:33.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:32 smithi059 bash[61152]: cluster 2022-04-28T22:07:31.288832+0000 mgr.y (mgr.24961) 4464 : cluster [DBG] pgmap v5238: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:33.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:32 smithi059 bash[59745]: cluster 2022-04-28T22:07:31.288832+0000 mgr.y (mgr.24961) 4464 : cluster [DBG] pgmap v5238: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:33.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:32 smithi139 bash[43301]: cluster 2022-04-28T22:07:31.288832+0000 mgr.y (mgr.24961) 4464 : cluster [DBG] pgmap v5238: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:34.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:35.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:34 smithi059 bash[61152]: cluster 2022-04-28T22:07:33.289616+0000 mgr.y (mgr.24961) 4465 : cluster [DBG] pgmap v5239: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:35.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:34 smithi059 bash[59745]: cluster 2022-04-28T22:07:33.289616+0000 mgr.y (mgr.24961) 4465 : cluster [DBG] pgmap v5239: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:35.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:34 smithi139 bash[43301]: cluster 2022-04-28T22:07:33.289616+0000 mgr.y (mgr.24961) 4465 : cluster [DBG] pgmap v5239: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:37.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:36 smithi059 bash[61152]: cluster 2022-04-28T22:07:35.291758+0000 mgr.y (mgr.24961) 4466 : cluster [DBG] pgmap v5240: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:37.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:36 smithi059 bash[59745]: cluster 2022-04-28T22:07:35.291758+0000 mgr.y (mgr.24961) 4466 : cluster [DBG] pgmap v5240: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:36 smithi139 bash[43301]: cluster 2022-04-28T22:07:35.291758+0000 mgr.y (mgr.24961) 4466 : cluster [DBG] pgmap v5240: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:39.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:38 smithi059 bash[61152]: cluster 2022-04-28T22:07:37.293166+0000 mgr.y (mgr.24961) 4467 : cluster [DBG] pgmap v5241: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:39.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:38 smithi059 bash[59745]: cluster 2022-04-28T22:07:37.293166+0000 mgr.y (mgr.24961) 4467 : cluster [DBG] pgmap v5241: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:38 smithi139 bash[43301]: cluster 2022-04-28T22:07:37.293166+0000 mgr.y (mgr.24961) 4467 : cluster [DBG] pgmap v5241: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:41.095 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:40 smithi059 bash[61152]: cluster 2022-04-28T22:07:39.294587+0000 mgr.y (mgr.24961) 4468 : cluster [DBG] pgmap v5242: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:41.096 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:40 smithi059 bash[59745]: cluster 2022-04-28T22:07:39.294587+0000 mgr.y (mgr.24961) 4468 : cluster [DBG] pgmap v5242: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:41.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:40 smithi139 bash[43301]: cluster 2022-04-28T22:07:39.294587+0000 mgr.y (mgr.24961) 4468 : cluster [DBG] pgmap v5242: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:42.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:42.168Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:43.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:42 smithi059 bash[61152]: cluster 2022-04-28T22:07:41.296572+0000 mgr.y (mgr.24961) 4469 : cluster [DBG] pgmap v5243: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:43.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:42 smithi059 bash[59745]: cluster 2022-04-28T22:07:41.296572+0000 mgr.y (mgr.24961) 4469 : cluster [DBG] pgmap v5243: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:43.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:42 smithi139 bash[43301]: cluster 2022-04-28T22:07:41.296572+0000 mgr.y (mgr.24961) 4469 : cluster [DBG] pgmap v5243: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:44.054Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:45.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:44 smithi059 bash[61152]: cluster 2022-04-28T22:07:43.297420+0000 mgr.y (mgr.24961) 4470 : cluster [DBG] pgmap v5244: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:45.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:44 smithi059 bash[59745]: cluster 2022-04-28T22:07:43.297420+0000 mgr.y (mgr.24961) 4470 : cluster [DBG] pgmap v5244: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:45.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:44 smithi139 bash[43301]: cluster 2022-04-28T22:07:43.297420+0000 mgr.y (mgr.24961) 4470 : cluster [DBG] pgmap v5244: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:47.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:46 smithi059 bash[61152]: cluster 2022-04-28T22:07:45.299535+0000 mgr.y (mgr.24961) 4471 : cluster [DBG] pgmap v5245: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:47.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:46 smithi059 bash[59745]: cluster 2022-04-28T22:07:45.299535+0000 mgr.y (mgr.24961) 4471 : cluster [DBG] pgmap v5245: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:47.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:46 smithi139 bash[43301]: cluster 2022-04-28T22:07:45.299535+0000 mgr.y (mgr.24961) 4471 : cluster [DBG] pgmap v5245: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:49.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:48 smithi059 bash[61152]: cluster 2022-04-28T22:07:47.300873+0000 mgr.y (mgr.24961) 4472 : cluster [DBG] pgmap v5246: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:49.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:48 smithi059 bash[59745]: cluster 2022-04-28T22:07:47.300873+0000 mgr.y (mgr.24961) 4472 : cluster [DBG] pgmap v5246: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:49.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:48 smithi139 bash[43301]: cluster 2022-04-28T22:07:47.300873+0000 mgr.y (mgr.24961) 4472 : cluster [DBG] pgmap v5246: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:51.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:50 smithi059 bash[61152]: cluster 2022-04-28T22:07:49.302452+0000 mgr.y (mgr.24961) 4473 : cluster [DBG] pgmap v5247: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:51.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:50 smithi059 bash[59745]: cluster 2022-04-28T22:07:49.302452+0000 mgr.y (mgr.24961) 4473 : cluster [DBG] pgmap v5247: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:51.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:50 smithi139 bash[43301]: cluster 2022-04-28T22:07:49.302452+0000 mgr.y (mgr.24961) 4473 : cluster [DBG] pgmap v5247: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:52.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:52.169Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:53.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:52 smithi059 bash[61152]: cluster 2022-04-28T22:07:51.304403+0000 mgr.y (mgr.24961) 4474 : cluster [DBG] pgmap v5248: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:53.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:52 smithi059 bash[59745]: cluster 2022-04-28T22:07:51.304403+0000 mgr.y (mgr.24961) 4474 : cluster [DBG] pgmap v5248: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:53.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:52 smithi139 bash[43301]: cluster 2022-04-28T22:07:51.304403+0000 mgr.y (mgr.24961) 4474 : cluster [DBG] pgmap v5248: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:54.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:07:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:07:54.055Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:07:55.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:54 smithi059 bash[61152]: cluster 2022-04-28T22:07:53.305239+0000 mgr.y (mgr.24961) 4475 : cluster [DBG] pgmap v5249: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:55.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:54 smithi059 bash[59745]: cluster 2022-04-28T22:07:53.305239+0000 mgr.y (mgr.24961) 4475 : cluster [DBG] pgmap v5249: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:55.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:54 smithi139 bash[43301]: cluster 2022-04-28T22:07:53.305239+0000 mgr.y (mgr.24961) 4475 : cluster [DBG] pgmap v5249: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:57.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:56 smithi059 bash[61152]: cluster 2022-04-28T22:07:55.307310+0000 mgr.y (mgr.24961) 4476 : cluster [DBG] pgmap v5250: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:57.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:56 smithi059 bash[59745]: cluster 2022-04-28T22:07:55.307310+0000 mgr.y (mgr.24961) 4476 : cluster [DBG] pgmap v5250: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:57.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:56 smithi139 bash[43301]: cluster 2022-04-28T22:07:55.307310+0000 mgr.y (mgr.24961) 4476 : cluster [DBG] pgmap v5250: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:59.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:07:58 smithi059 bash[61152]: cluster 2022-04-28T22:07:57.308663+0000 mgr.y (mgr.24961) 4477 : cluster [DBG] pgmap v5251: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:59.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:07:58 smithi059 bash[59745]: cluster 2022-04-28T22:07:57.308663+0000 mgr.y (mgr.24961) 4477 : cluster [DBG] pgmap v5251: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:07:59.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:07:58 smithi139 bash[43301]: cluster 2022-04-28T22:07:57.308663+0000 mgr.y (mgr.24961) 4477 : cluster [DBG] pgmap v5251: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:01.091 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:00 smithi059 bash[59745]: cluster 2022-04-28T22:07:59.310126+0000 mgr.y (mgr.24961) 4478 : cluster [DBG] pgmap v5252: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:01.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:00 smithi059 bash[61152]: cluster 2022-04-28T22:07:59.310126+0000 mgr.y (mgr.24961) 4478 : cluster [DBG] pgmap v5252: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:01.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:00 smithi139 bash[43301]: cluster 2022-04-28T22:07:59.310126+0000 mgr.y (mgr.24961) 4478 : cluster [DBG] pgmap v5252: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:02.595 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:02.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:03.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:02 smithi059 bash[61152]: cluster 2022-04-28T22:08:01.312086+0000 mgr.y (mgr.24961) 4479 : cluster [DBG] pgmap v5253: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:03.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:02 smithi059 bash[59745]: cluster 2022-04-28T22:08:01.312086+0000 mgr.y (mgr.24961) 4479 : cluster [DBG] pgmap v5253: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:03.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:02 smithi139 bash[43301]: cluster 2022-04-28T22:08:01.312086+0000 mgr.y (mgr.24961) 4479 : cluster [DBG] pgmap v5253: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:04.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:04.057Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:05.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:04 smithi059 bash[61152]: cluster 2022-04-28T22:08:03.312943+0000 mgr.y (mgr.24961) 4480 : cluster [DBG] pgmap v5254: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:05.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:04 smithi059 bash[59745]: cluster 2022-04-28T22:08:03.312943+0000 mgr.y (mgr.24961) 4480 : cluster [DBG] pgmap v5254: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:05.123 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:04 smithi139 bash[43301]: cluster 2022-04-28T22:08:03.312943+0000 mgr.y (mgr.24961) 4480 : cluster [DBG] pgmap v5254: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:07.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:06 smithi059 bash[61152]: cluster 2022-04-28T22:08:05.315057+0000 mgr.y (mgr.24961) 4481 : cluster [DBG] pgmap v5255: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:07.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:06 smithi059 bash[59745]: cluster 2022-04-28T22:08:05.315057+0000 mgr.y (mgr.24961) 4481 : cluster [DBG] pgmap v5255: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:07.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:06 smithi139 bash[43301]: cluster 2022-04-28T22:08:05.315057+0000 mgr.y (mgr.24961) 4481 : cluster [DBG] pgmap v5255: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:09.091 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:08 smithi059 bash[61152]: cluster 2022-04-28T22:08:07.316436+0000 mgr.y (mgr.24961) 4482 : cluster [DBG] pgmap v5256: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:09.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:08 smithi059 bash[59745]: cluster 2022-04-28T22:08:07.316436+0000 mgr.y (mgr.24961) 4482 : cluster [DBG] pgmap v5256: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:09.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:08 smithi139 bash[43301]: cluster 2022-04-28T22:08:07.316436+0000 mgr.y (mgr.24961) 4482 : cluster [DBG] pgmap v5256: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:11.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:10 smithi139 bash[43301]: cluster 2022-04-28T22:08:09.317906+0000 mgr.y (mgr.24961) 4483 : cluster [DBG] pgmap v5257: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:11.342 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:10 smithi059 bash[61152]: cluster 2022-04-28T22:08:09.317906+0000 mgr.y (mgr.24961) 4483 : cluster [DBG] pgmap v5257: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:11.343 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:10 smithi059 bash[59745]: cluster 2022-04-28T22:08:09.317906+0000 mgr.y (mgr.24961) 4483 : cluster [DBG] pgmap v5257: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:12.591 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:12.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:13.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:12 smithi139 bash[43301]: cluster 2022-04-28T22:08:11.319924+0000 mgr.y (mgr.24961) 4484 : cluster [DBG] pgmap v5258: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:13.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:12 smithi059 bash[61152]: cluster 2022-04-28T22:08:11.319924+0000 mgr.y (mgr.24961) 4484 : cluster [DBG] pgmap v5258: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:13.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:12 smithi059 bash[59745]: cluster 2022-04-28T22:08:11.319924+0000 mgr.y (mgr.24961) 4484 : cluster [DBG] pgmap v5258: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:14.056Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:15.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:14 smithi139 bash[43301]: cluster 2022-04-28T22:08:13.320738+0000 mgr.y (mgr.24961) 4485 : cluster [DBG] pgmap v5259: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:15.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:14 smithi059 bash[61152]: cluster 2022-04-28T22:08:13.320738+0000 mgr.y (mgr.24961) 4485 : cluster [DBG] pgmap v5259: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:15.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:14 smithi059 bash[59745]: cluster 2022-04-28T22:08:13.320738+0000 mgr.y (mgr.24961) 4485 : cluster [DBG] pgmap v5259: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:17.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:16 smithi139 bash[43301]: cluster 2022-04-28T22:08:15.322874+0000 mgr.y (mgr.24961) 4486 : cluster [DBG] pgmap v5260: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:17.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:16 smithi059 bash[61152]: cluster 2022-04-28T22:08:15.322874+0000 mgr.y (mgr.24961) 4486 : cluster [DBG] pgmap v5260: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:17.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:16 smithi059 bash[59745]: cluster 2022-04-28T22:08:15.322874+0000 mgr.y (mgr.24961) 4486 : cluster [DBG] pgmap v5260: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:19.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:18 smithi139 bash[43301]: cluster 2022-04-28T22:08:17.324277+0000 mgr.y (mgr.24961) 4487 : cluster [DBG] pgmap v5261: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:19.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:18 smithi059 bash[61152]: cluster 2022-04-28T22:08:17.324277+0000 mgr.y (mgr.24961) 4487 : cluster [DBG] pgmap v5261: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:19.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:18 smithi059 bash[59745]: cluster 2022-04-28T22:08:17.324277+0000 mgr.y (mgr.24961) 4487 : cluster [DBG] pgmap v5261: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: cluster 2022-04-28T22:08:19.326222+0000 mgr.y (mgr.24961) 4488 : cluster [DBG] pgmap v5262: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: audit 2022-04-28T22:08:19.483611+0000 mon.a (mon.0) 4329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:08:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: audit 2022-04-28T22:08:19.515563+0000 mon.a (mon.0) 4330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:08:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: audit 2022-04-28T22:08:19.625651+0000 mon.a (mon.0) 4331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:08:20.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: audit 2022-04-28T22:08:19.626346+0000 mon.a (mon.0) 4332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T22:08:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: audit 2022-04-28T22:08:19.626997+0000 mon.a (mon.0) 4333 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T22:08:20.593 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[61152]: audit 2022-04-28T22:08:19.627726+0000 mon.a (mon.0) 4334 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T22:08:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: cluster 2022-04-28T22:08:19.326222+0000 mgr.y (mgr.24961) 4488 : cluster [DBG] pgmap v5262: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:20.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: audit 2022-04-28T22:08:19.483611+0000 mon.a (mon.0) 4329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:08:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: audit 2022-04-28T22:08:19.515563+0000 mon.a (mon.0) 4330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:08:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: audit 2022-04-28T22:08:19.625651+0000 mon.a (mon.0) 4331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:08:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: audit 2022-04-28T22:08:19.626346+0000 mon.a (mon.0) 4332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T22:08:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: audit 2022-04-28T22:08:19.626997+0000 mon.a (mon.0) 4333 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T22:08:20.594 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:20 smithi059 bash[59745]: audit 2022-04-28T22:08:19.627726+0000 mon.a (mon.0) 4334 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T22:08:20.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: cluster 2022-04-28T22:08:19.326222+0000 mgr.y (mgr.24961) 4488 : cluster [DBG] pgmap v5262: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:20.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: audit 2022-04-28T22:08:19.483611+0000 mon.a (mon.0) 4329 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:08:20.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: audit 2022-04-28T22:08:19.515563+0000 mon.a (mon.0) 4330 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:08:20.624 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: audit 2022-04-28T22:08:19.625651+0000 mon.a (mon.0) 4331 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2022-04-28T22:08:20.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: audit 2022-04-28T22:08:19.626346+0000 mon.a (mon.0) 4332 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2022-04-28T22:08:20.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: audit 2022-04-28T22:08:19.626997+0000 mon.a (mon.0) 4333 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2022-04-28T22:08:20.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:20 smithi139 bash[43301]: audit 2022-04-28T22:08:19.627726+0000 mon.a (mon.0) 4334 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.8", "id": [4, 2]}]: dispatch 2022-04-28T22:08:22.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:22 smithi059 bash[61152]: cluster 2022-04-28T22:08:21.328231+0000 mgr.y (mgr.24961) 4489 : cluster [DBG] pgmap v5263: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:22.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:22.171Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:22.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:22 smithi059 bash[59745]: cluster 2022-04-28T22:08:21.328231+0000 mgr.y (mgr.24961) 4489 : cluster [DBG] pgmap v5263: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:22.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:22 smithi139 bash[43301]: cluster 2022-04-28T22:08:21.328231+0000 mgr.y (mgr.24961) 4489 : cluster [DBG] pgmap v5263: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:24.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:24.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:24 smithi139 bash[43301]: cluster 2022-04-28T22:08:23.329086+0000 mgr.y (mgr.24961) 4490 : cluster [DBG] pgmap v5264: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:24.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:24 smithi059 bash[61152]: cluster 2022-04-28T22:08:23.329086+0000 mgr.y (mgr.24961) 4490 : cluster [DBG] pgmap v5264: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:24 smithi059 bash[59745]: cluster 2022-04-28T22:08:23.329086+0000 mgr.y (mgr.24961) 4490 : cluster [DBG] pgmap v5264: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:26.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:26 smithi139 bash[43301]: cluster 2022-04-28T22:08:25.331090+0000 mgr.y (mgr.24961) 4491 : cluster [DBG] pgmap v5265: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:26 smithi059 bash[61152]: cluster 2022-04-28T22:08:25.331090+0000 mgr.y (mgr.24961) 4491 : cluster [DBG] pgmap v5265: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:26 smithi059 bash[59745]: cluster 2022-04-28T22:08:25.331090+0000 mgr.y (mgr.24961) 4491 : cluster [DBG] pgmap v5265: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:28.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:28 smithi139 bash[43301]: cluster 2022-04-28T22:08:27.332464+0000 mgr.y (mgr.24961) 4492 : cluster [DBG] pgmap v5266: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:28.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:28 smithi059 bash[61152]: cluster 2022-04-28T22:08:27.332464+0000 mgr.y (mgr.24961) 4492 : cluster [DBG] pgmap v5266: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:28.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:28 smithi059 bash[59745]: cluster 2022-04-28T22:08:27.332464+0000 mgr.y (mgr.24961) 4492 : cluster [DBG] pgmap v5266: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:29.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:29 smithi139 bash[43301]: audit 2022-04-28T22:08:28.743093+0000 mon.a (mon.0) 4335 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:08:29.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:29 smithi139 bash[43301]: audit 2022-04-28T22:08:28.745946+0000 mon.a (mon.0) 4336 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:08:29.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:29 smithi139 bash[43301]: audit 2022-04-28T22:08:28.747842+0000 mon.a (mon.0) 4337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:08:29.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:29 smithi139 bash[43301]: cephadm 2022-04-28T22:08:28.749472+0000 mgr.y (mgr.24961) 4493 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:08:29.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:29 smithi139 bash[43301]: audit 2022-04-28T22:08:28.750928+0000 mon.a (mon.0) 4338 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T22:08:29.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:29 smithi139 bash[43301]: audit 2022-04-28T22:08:28.751606+0000 mon.a (mon.0) 4339 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T22:08:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[59745]: audit 2022-04-28T22:08:28.743093+0000 mon.a (mon.0) 4335 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:08:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[59745]: audit 2022-04-28T22:08:28.745946+0000 mon.a (mon.0) 4336 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:08:29.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[59745]: audit 2022-04-28T22:08:28.747842+0000 mon.a (mon.0) 4337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:08:29.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[59745]: cephadm 2022-04-28T22:08:28.749472+0000 mgr.y (mgr.24961) 4493 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:08:29.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[59745]: audit 2022-04-28T22:08:28.750928+0000 mon.a (mon.0) 4338 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T22:08:29.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[59745]: audit 2022-04-28T22:08:28.751606+0000 mon.a (mon.0) 4339 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T22:08:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[61152]: audit 2022-04-28T22:08:28.743093+0000 mon.a (mon.0) 4335 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:08:29.844 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[61152]: audit 2022-04-28T22:08:28.745946+0000 mon.a (mon.0) 4336 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:08:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[61152]: audit 2022-04-28T22:08:28.747842+0000 mon.a (mon.0) 4337 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:08:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[61152]: cephadm 2022-04-28T22:08:28.749472+0000 mgr.y (mgr.24961) 4493 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:08:29.845 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[61152]: audit 2022-04-28T22:08:28.750928+0000 mon.a (mon.0) 4338 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T22:08:29.846 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:29 smithi059 bash[61152]: audit 2022-04-28T22:08:28.751606+0000 mon.a (mon.0) 4339 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config get","who": "mon","key": "container_image"}]: dispatch 2022-04-28T22:08:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:30 smithi139 bash[43301]: cluster 2022-04-28T22:08:29.334059+0000 mgr.y (mgr.24961) 4494 : cluster [DBG] pgmap v5267: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:30.841 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:30 smithi059 bash[59745]: cluster 2022-04-28T22:08:29.334059+0000 mgr.y (mgr.24961) 4494 : cluster [DBG] pgmap v5267: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:30.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:30 smithi059 bash[61152]: cluster 2022-04-28T22:08:29.334059+0000 mgr.y (mgr.24961) 4494 : cluster [DBG] pgmap v5267: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:32.591 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:32 smithi059 bash[59745]: cluster 2022-04-28T22:08:31.336025+0000 mgr.y (mgr.24961) 4495 : cluster [DBG] pgmap v5268: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:32.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:32 smithi059 bash[61152]: cluster 2022-04-28T22:08:31.336025+0000 mgr.y (mgr.24961) 4495 : cluster [DBG] pgmap v5268: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:32.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:32.170Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:32.625 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:32 smithi139 bash[43301]: cluster 2022-04-28T22:08:31.336025+0000 mgr.y (mgr.24961) 4495 : cluster [DBG] pgmap v5268: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:34.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:33 smithi059 bash[61152]: audit 2022-04-28T22:08:32.831511+0000 mon.a (mon.0) 4340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:33 smithi059 bash[61152]: audit 2022-04-28T22:08:33.100745+0000 mon.a (mon.0) 4341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.093 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:33 smithi059 bash[61152]: audit 2022-04-28T22:08:33.391891+0000 mon.a (mon.0) 4342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.094 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:33 smithi059 bash[59745]: audit 2022-04-28T22:08:32.831511+0000 mon.a (mon.0) 4340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:33 smithi059 bash[59745]: audit 2022-04-28T22:08:33.100745+0000 mon.a (mon.0) 4341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.095 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:33 smithi059 bash[59745]: audit 2022-04-28T22:08:33.391891+0000 mon.a (mon.0) 4342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.096 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:34.058Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:34.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:33 smithi139 bash[43301]: audit 2022-04-28T22:08:32.831511+0000 mon.a (mon.0) 4340 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:33 smithi139 bash[43301]: audit 2022-04-28T22:08:33.100745+0000 mon.a (mon.0) 4341 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:34.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:33 smithi139 bash[43301]: audit 2022-04-28T22:08:33.391891+0000 mon.a (mon.0) 4342 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:08:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:34 smithi059 bash[61152]: cluster 2022-04-28T22:08:33.336918+0000 mgr.y (mgr.24961) 4496 : cluster [DBG] pgmap v5269: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:35.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:34 smithi059 bash[59745]: cluster 2022-04-28T22:08:33.336918+0000 mgr.y (mgr.24961) 4496 : cluster [DBG] pgmap v5269: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:35.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:34 smithi139 bash[43301]: cluster 2022-04-28T22:08:33.336918+0000 mgr.y (mgr.24961) 4496 : cluster [DBG] pgmap v5269: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:37.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:36 smithi139 bash[43301]: cluster 2022-04-28T22:08:35.339023+0000 mgr.y (mgr.24961) 4497 : cluster [DBG] pgmap v5270: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:37.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:36 smithi059 bash[61152]: cluster 2022-04-28T22:08:35.339023+0000 mgr.y (mgr.24961) 4497 : cluster [DBG] pgmap v5270: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:37.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:36 smithi059 bash[59745]: cluster 2022-04-28T22:08:35.339023+0000 mgr.y (mgr.24961) 4497 : cluster [DBG] pgmap v5270: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:39.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:38 smithi139 bash[43301]: cluster 2022-04-28T22:08:37.340440+0000 mgr.y (mgr.24961) 4498 : cluster [DBG] pgmap v5271: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:39.341 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:38 smithi059 bash[61152]: cluster 2022-04-28T22:08:37.340440+0000 mgr.y (mgr.24961) 4498 : cluster [DBG] pgmap v5271: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:39.342 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:38 smithi059 bash[59745]: cluster 2022-04-28T22:08:37.340440+0000 mgr.y (mgr.24961) 4498 : cluster [DBG] pgmap v5271: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:40.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:40 smithi139 bash[43301]: cluster 2022-04-28T22:08:39.342081+0000 mgr.y (mgr.24961) 4499 : cluster [DBG] pgmap v5272: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:40.847 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:40 smithi059 bash[61152]: cluster 2022-04-28T22:08:39.342081+0000 mgr.y (mgr.24961) 4499 : cluster [DBG] pgmap v5272: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:40.848 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:40 smithi059 bash[59745]: cluster 2022-04-28T22:08:39.342081+0000 mgr.y (mgr.24961) 4499 : cluster [DBG] pgmap v5272: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:42.591 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:42 smithi059 bash[61152]: cluster 2022-04-28T22:08:41.344061+0000 mgr.y (mgr.24961) 4500 : cluster [DBG] pgmap v5273: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:42.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:42 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:42.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:42.592 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:42 smithi059 bash[59745]: cluster 2022-04-28T22:08:41.344061+0000 mgr.y (mgr.24961) 4500 : cluster [DBG] pgmap v5273: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:42.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:42 smithi139 bash[43301]: cluster 2022-04-28T22:08:41.344061+0000 mgr.y (mgr.24961) 4500 : cluster [DBG] pgmap v5273: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:44.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:44 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:44.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:44.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:44 smithi139 bash[43301]: cluster 2022-04-28T22:08:43.344897+0000 mgr.y (mgr.24961) 4501 : cluster [DBG] pgmap v5274: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:44.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:44 smithi059 bash[61152]: cluster 2022-04-28T22:08:43.344897+0000 mgr.y (mgr.24961) 4501 : cluster [DBG] pgmap v5274: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:44.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:44 smithi059 bash[59745]: cluster 2022-04-28T22:08:43.344897+0000 mgr.y (mgr.24961) 4501 : cluster [DBG] pgmap v5274: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:46.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:46 smithi059 bash[61152]: cluster 2022-04-28T22:08:45.347000+0000 mgr.y (mgr.24961) 4502 : cluster [DBG] pgmap v5275: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:46.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:46 smithi059 bash[59745]: cluster 2022-04-28T22:08:45.347000+0000 mgr.y (mgr.24961) 4502 : cluster [DBG] pgmap v5275: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:46.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:46 smithi139 bash[43301]: cluster 2022-04-28T22:08:45.347000+0000 mgr.y (mgr.24961) 4502 : cluster [DBG] pgmap v5275: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:48.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:48 smithi059 bash[61152]: cluster 2022-04-28T22:08:47.348567+0000 mgr.y (mgr.24961) 4503 : cluster [DBG] pgmap v5276: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:48.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:48 smithi059 bash[59745]: cluster 2022-04-28T22:08:47.348567+0000 mgr.y (mgr.24961) 4503 : cluster [DBG] pgmap v5276: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:48.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:48 smithi139 bash[43301]: cluster 2022-04-28T22:08:47.348567+0000 mgr.y (mgr.24961) 4503 : cluster [DBG] pgmap v5276: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:50.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:50 smithi139 bash[43301]: cluster 2022-04-28T22:08:49.350219+0000 mgr.y (mgr.24961) 4504 : cluster [DBG] pgmap v5277: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:50.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:50 smithi059 bash[61152]: cluster 2022-04-28T22:08:49.350219+0000 mgr.y (mgr.24961) 4504 : cluster [DBG] pgmap v5277: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:50.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:50 smithi059 bash[59745]: cluster 2022-04-28T22:08:49.350219+0000 mgr.y (mgr.24961) 4504 : cluster [DBG] pgmap v5277: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:52.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:52 smithi059 bash[61152]: cluster 2022-04-28T22:08:51.352185+0000 mgr.y (mgr.24961) 4505 : cluster [DBG] pgmap v5278: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:52.593 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:52 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:52.172Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:52.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:52 smithi059 bash[59745]: cluster 2022-04-28T22:08:51.352185+0000 mgr.y (mgr.24961) 4505 : cluster [DBG] pgmap v5278: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:52.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:52 smithi139 bash[43301]: cluster 2022-04-28T22:08:51.352185+0000 mgr.y (mgr.24961) 4505 : cluster [DBG] pgmap v5278: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:54.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:08:54 smithi059 bash[72035]: level=error ts=2022-04-28T22:08:54.059Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:08:54.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:54 smithi059 bash[61152]: cluster 2022-04-28T22:08:53.353137+0000 mgr.y (mgr.24961) 4506 : cluster [DBG] pgmap v5279: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:54.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:54 smithi059 bash[59745]: cluster 2022-04-28T22:08:53.353137+0000 mgr.y (mgr.24961) 4506 : cluster [DBG] pgmap v5279: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:54.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:54 smithi139 bash[43301]: cluster 2022-04-28T22:08:53.353137+0000 mgr.y (mgr.24961) 4506 : cluster [DBG] pgmap v5279: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:56.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:56 smithi059 bash[59745]: cluster 2022-04-28T22:08:55.355222+0000 mgr.y (mgr.24961) 4507 : cluster [DBG] pgmap v5280: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:56.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:56 smithi059 bash[61152]: cluster 2022-04-28T22:08:55.355222+0000 mgr.y (mgr.24961) 4507 : cluster [DBG] pgmap v5280: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:56.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:56 smithi139 bash[43301]: cluster 2022-04-28T22:08:55.355222+0000 mgr.y (mgr.24961) 4507 : cluster [DBG] pgmap v5280: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:58.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:08:58 smithi059 bash[61152]: cluster 2022-04-28T22:08:57.356608+0000 mgr.y (mgr.24961) 4508 : cluster [DBG] pgmap v5281: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:58.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:08:58 smithi059 bash[59745]: cluster 2022-04-28T22:08:57.356608+0000 mgr.y (mgr.24961) 4508 : cluster [DBG] pgmap v5281: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:08:58.873 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:08:58 smithi139 bash[43301]: cluster 2022-04-28T22:08:57.356608+0000 mgr.y (mgr.24961) 4508 : cluster [DBG] pgmap v5281: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:00.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:00 smithi139 bash[43301]: cluster 2022-04-28T22:08:59.358118+0000 mgr.y (mgr.24961) 4509 : cluster [DBG] pgmap v5282: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:00.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:00 smithi059 bash[61152]: cluster 2022-04-28T22:08:59.358118+0000 mgr.y (mgr.24961) 4509 : cluster [DBG] pgmap v5282: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:00.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:00 smithi059 bash[59745]: cluster 2022-04-28T22:08:59.358118+0000 mgr.y (mgr.24961) 4509 : cluster [DBG] pgmap v5282: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:02.592 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:02 smithi059 bash[61152]: cluster 2022-04-28T22:09:01.360092+0000 mgr.y (mgr.24961) 4510 : cluster [DBG] pgmap v5283: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:02.592 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:02 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:02.173Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:02.593 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:02 smithi059 bash[59745]: cluster 2022-04-28T22:09:01.360092+0000 mgr.y (mgr.24961) 4510 : cluster [DBG] pgmap v5283: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:02.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:02 smithi139 bash[43301]: cluster 2022-04-28T22:09:01.360092+0000 mgr.y (mgr.24961) 4510 : cluster [DBG] pgmap v5283: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:04.342 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:04 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:04.060Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:04.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:04 smithi059 bash[59745]: cluster 2022-04-28T22:09:03.360948+0000 mgr.y (mgr.24961) 4511 : cluster [DBG] pgmap v5284: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:04.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:04 smithi059 bash[61152]: cluster 2022-04-28T22:09:03.360948+0000 mgr.y (mgr.24961) 4511 : cluster [DBG] pgmap v5284: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:04.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:04 smithi139 bash[43301]: cluster 2022-04-28T22:09:03.360948+0000 mgr.y (mgr.24961) 4511 : cluster [DBG] pgmap v5284: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:06.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:06 smithi059 bash[61152]: cluster 2022-04-28T22:09:05.362986+0000 mgr.y (mgr.24961) 4512 : cluster [DBG] pgmap v5285: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:06.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:06 smithi059 bash[59745]: cluster 2022-04-28T22:09:05.362986+0000 mgr.y (mgr.24961) 4512 : cluster [DBG] pgmap v5285: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:06.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:06 smithi139 bash[43301]: cluster 2022-04-28T22:09:05.362986+0000 mgr.y (mgr.24961) 4512 : cluster [DBG] pgmap v5285: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:08.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:08 smithi059 bash[61152]: cluster 2022-04-28T22:09:07.364422+0000 mgr.y (mgr.24961) 4513 : cluster [DBG] pgmap v5286: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:08.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:08 smithi059 bash[59745]: cluster 2022-04-28T22:09:07.364422+0000 mgr.y (mgr.24961) 4513 : cluster [DBG] pgmap v5286: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:08.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:08 smithi139 bash[43301]: cluster 2022-04-28T22:09:07.364422+0000 mgr.y (mgr.24961) 4513 : cluster [DBG] pgmap v5286: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:10.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:10 smithi139 bash[43301]: cluster 2022-04-28T22:09:09.365995+0000 mgr.y (mgr.24961) 4514 : cluster [DBG] pgmap v5287: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:10.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:10 smithi059 bash[61152]: cluster 2022-04-28T22:09:09.365995+0000 mgr.y (mgr.24961) 4514 : cluster [DBG] pgmap v5287: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:10.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:10 smithi059 bash[59745]: cluster 2022-04-28T22:09:09.365995+0000 mgr.y (mgr.24961) 4514 : cluster [DBG] pgmap v5287: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:12.426 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:12 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:12.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:12.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:12 smithi059 bash[61152]: cluster 2022-04-28T22:09:11.367950+0000 mgr.y (mgr.24961) 4515 : cluster [DBG] pgmap v5288: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:12.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:12 smithi059 bash[59745]: cluster 2022-04-28T22:09:11.367950+0000 mgr.y (mgr.24961) 4515 : cluster [DBG] pgmap v5288: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:12.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:12 smithi139 bash[43301]: cluster 2022-04-28T22:09:11.367950+0000 mgr.y (mgr.24961) 4515 : cluster [DBG] pgmap v5288: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:14.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:14 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:14.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:14.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:14 smithi059 bash[61152]: cluster 2022-04-28T22:09:13.368770+0000 mgr.y (mgr.24961) 4516 : cluster [DBG] pgmap v5289: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:14.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:14 smithi059 bash[59745]: cluster 2022-04-28T22:09:13.368770+0000 mgr.y (mgr.24961) 4516 : cluster [DBG] pgmap v5289: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:14.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:14 smithi139 bash[43301]: cluster 2022-04-28T22:09:13.368770+0000 mgr.y (mgr.24961) 4516 : cluster [DBG] pgmap v5289: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:16.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:16 smithi059 bash[61152]: cluster 2022-04-28T22:09:15.370822+0000 mgr.y (mgr.24961) 4517 : cluster [DBG] pgmap v5290: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:16.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:16 smithi059 bash[59745]: cluster 2022-04-28T22:09:15.370822+0000 mgr.y (mgr.24961) 4517 : cluster [DBG] pgmap v5290: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:16.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:16 smithi139 bash[43301]: cluster 2022-04-28T22:09:15.370822+0000 mgr.y (mgr.24961) 4517 : cluster [DBG] pgmap v5290: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:18.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:18 smithi059 bash[61152]: cluster 2022-04-28T22:09:17.372161+0000 mgr.y (mgr.24961) 4518 : cluster [DBG] pgmap v5291: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:18.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:18 smithi059 bash[59745]: cluster 2022-04-28T22:09:17.372161+0000 mgr.y (mgr.24961) 4518 : cluster [DBG] pgmap v5291: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:18.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:18 smithi139 bash[43301]: cluster 2022-04-28T22:09:17.372161+0000 mgr.y (mgr.24961) 4518 : cluster [DBG] pgmap v5291: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: cluster 2022-04-28T22:09:19.373772+0000 mgr.y (mgr.24961) 4519 : cluster [DBG] pgmap v5292: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: audit 2022-04-28T22:09:19.488896+0000 mon.a (mon.0) 4343 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:09:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: audit 2022-04-28T22:09:19.521901+0000 mon.a (mon.0) 4344 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:09:20.621 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: audit 2022-04-28T22:09:19.636153+0000 mon.a (mon.0) 4345 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:09:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: audit 2022-04-28T22:09:19.636895+0000 mon.a (mon.0) 4346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T22:09:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: audit 2022-04-28T22:09:19.637506+0000 mon.a (mon.0) 4347 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T22:09:20.622 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:20 smithi139 bash[43301]: audit 2022-04-28T22:09:19.638069+0000 mon.a (mon.0) 4348 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T22:09:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: cluster 2022-04-28T22:09:19.373772+0000 mgr.y (mgr.24961) 4519 : cluster [DBG] pgmap v5292: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: audit 2022-04-28T22:09:19.488896+0000 mon.a (mon.0) 4343 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:09:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: audit 2022-04-28T22:09:19.521901+0000 mon.a (mon.0) 4344 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:09:20.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: audit 2022-04-28T22:09:19.636153+0000 mon.a (mon.0) 4345 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:09:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: audit 2022-04-28T22:09:19.636895+0000 mon.a (mon.0) 4346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T22:09:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: audit 2022-04-28T22:09:19.637506+0000 mon.a (mon.0) 4347 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T22:09:20.843 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[61152]: audit 2022-04-28T22:09:19.638069+0000 mon.a (mon.0) 4348 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T22:09:20.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: cluster 2022-04-28T22:09:19.373772+0000 mgr.y (mgr.24961) 4519 : cluster [DBG] pgmap v5292: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: audit 2022-04-28T22:09:19.488896+0000 mon.a (mon.0) 4343 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2022-04-28T22:09:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: audit 2022-04-28T22:09:19.521901+0000 mon.a (mon.0) 4344 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2022-04-28T22:09:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: audit 2022-04-28T22:09:19.636153+0000 mon.a (mon.0) 4345 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2022-04-28T22:09:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: audit 2022-04-28T22:09:19.636895+0000 mon.a (mon.0) 4346 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2022-04-28T22:09:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: audit 2022-04-28T22:09:19.637506+0000 mon.a (mon.0) 4347 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2022-04-28T22:09:20.844 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:20 smithi059 bash[59745]: audit 2022-04-28T22:09:19.638069+0000 mon.a (mon.0) 4348 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2022-04-28T22:09:22.436 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:22 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:22.174Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:22.437 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:22 smithi059 bash[59745]: cluster 2022-04-28T22:09:21.375678+0000 mgr.y (mgr.24961) 4520 : cluster [DBG] pgmap v5293: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:22.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:22 smithi059 bash[61152]: cluster 2022-04-28T22:09:21.375678+0000 mgr.y (mgr.24961) 4520 : cluster [DBG] pgmap v5293: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:22.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:22 smithi139 bash[43301]: cluster 2022-04-28T22:09:21.375678+0000 mgr.y (mgr.24961) 4520 : cluster [DBG] pgmap v5293: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:24.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:24 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:24.061Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:24.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:24 smithi059 bash[61152]: cluster 2022-04-28T22:09:23.376641+0000 mgr.y (mgr.24961) 4521 : cluster [DBG] pgmap v5294: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:24.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:24 smithi059 bash[59745]: cluster 2022-04-28T22:09:23.376641+0000 mgr.y (mgr.24961) 4521 : cluster [DBG] pgmap v5294: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:24.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:24 smithi139 bash[43301]: cluster 2022-04-28T22:09:23.376641+0000 mgr.y (mgr.24961) 4521 : cluster [DBG] pgmap v5294: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:26.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:26 smithi059 bash[61152]: cluster 2022-04-28T22:09:25.378771+0000 mgr.y (mgr.24961) 4522 : cluster [DBG] pgmap v5295: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:26.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:26 smithi059 bash[59745]: cluster 2022-04-28T22:09:25.378771+0000 mgr.y (mgr.24961) 4522 : cluster [DBG] pgmap v5295: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:26.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:26 smithi139 bash[43301]: cluster 2022-04-28T22:09:25.378771+0000 mgr.y (mgr.24961) 4522 : cluster [DBG] pgmap v5295: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:28.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:28 smithi059 bash[61152]: cluster 2022-04-28T22:09:27.380200+0000 mgr.y (mgr.24961) 4523 : cluster [DBG] pgmap v5296: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:28.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:28 smithi059 bash[59745]: cluster 2022-04-28T22:09:27.380200+0000 mgr.y (mgr.24961) 4523 : cluster [DBG] pgmap v5296: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:28.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:28 smithi139 bash[43301]: cluster 2022-04-28T22:09:27.380200+0000 mgr.y (mgr.24961) 4523 : cluster [DBG] pgmap v5296: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:30.620 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:30 smithi139 bash[43301]: cluster 2022-04-28T22:09:29.381865+0000 mgr.y (mgr.24961) 4524 : cluster [DBG] pgmap v5297: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:30.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:30 smithi059 bash[61152]: cluster 2022-04-28T22:09:29.381865+0000 mgr.y (mgr.24961) 4524 : cluster [DBG] pgmap v5297: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:30.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:30 smithi059 bash[59745]: cluster 2022-04-28T22:09:29.381865+0000 mgr.y (mgr.24961) 4524 : cluster [DBG] pgmap v5297: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:32.443 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:32 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:32.175Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:32.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:32 smithi059 bash[61152]: cluster 2022-04-28T22:09:31.383885+0000 mgr.y (mgr.24961) 4525 : cluster [DBG] pgmap v5298: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:32.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:32 smithi059 bash[59745]: cluster 2022-04-28T22:09:31.383885+0000 mgr.y (mgr.24961) 4525 : cluster [DBG] pgmap v5298: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:32.870 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:32 smithi139 bash[43301]: cluster 2022-04-28T22:09:31.383885+0000 mgr.y (mgr.24961) 4525 : cluster [DBG] pgmap v5298: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:33.841 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:33 smithi059 bash[61152]: audit 2022-04-28T22:09:33.401544+0000 mon.a (mon.0) 4349 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:09:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:33 smithi059 bash[61152]: audit 2022-04-28T22:09:33.404441+0000 mon.a (mon.0) 4350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:09:33.842 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:33 smithi059 bash[61152]: audit 2022-04-28T22:09:33.406398+0000 mon.a (mon.0) 4351 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:09:33.842 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:33 smithi059 bash[59745]: audit 2022-04-28T22:09:33.401544+0000 mon.a (mon.0) 4349 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:09:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:33 smithi059 bash[59745]: audit 2022-04-28T22:09:33.404441+0000 mon.a (mon.0) 4350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:09:33.843 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:33 smithi059 bash[59745]: audit 2022-04-28T22:09:33.406398+0000 mon.a (mon.0) 4351 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:09:33.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:33 smithi139 bash[43301]: audit 2022-04-28T22:09:33.401544+0000 mon.a (mon.0) 4349 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2022-04-28T22:09:33.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:33 smithi139 bash[43301]: audit 2022-04-28T22:09:33.404441+0000 mon.a (mon.0) 4350 : audit [DBG] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2022-04-28T22:09:33.871 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:33 smithi139 bash[43301]: audit 2022-04-28T22:09:33.406398+0000 mon.a (mon.0) 4351 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2022-04-28T22:09:34.341 INFO:journalctl@ceph.alertmanager.a.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[72035]: level=error ts=2022-04-28T22:09:34.062Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=14 err="ceph-dashboard/webhook[0]: notify retry canceled due to unrecoverable error after 1 attempts: unexpected status code 404: https://172.21.15.59:8443//api/prometheus_receiver" 2022-04-28T22:09:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[61152]: cluster 2022-04-28T22:09:33.384724+0000 mgr.y (mgr.24961) 4526 : cluster [DBG] pgmap v5299: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[61152]: cephadm 2022-04-28T22:09:33.408142+0000 mgr.y (mgr.24961) 4527 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:09:35.092 INFO:journalctl@ceph.mon.c.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[61152]: audit 2022-04-28T22:09:33.744578+0000 mon.a (mon.0) 4352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:09:35.092 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[59745]: cluster 2022-04-28T22:09:33.384724+0000 mgr.y (mgr.24961) 4526 : cluster [DBG] pgmap v5299: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:35.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[59745]: cephadm 2022-04-28T22:09:33.408142+0000 mgr.y (mgr.24961) 4527 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:09:35.093 INFO:journalctl@ceph.mon.a.smithi059.stdout:Apr 28 22:09:34 smithi059 bash[59745]: audit 2022-04-28T22:09:33.744578+0000 mon.a (mon.0) 4352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-28T22:09:35.120 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:34 smithi139 bash[43301]: cluster 2022-04-28T22:09:33.384724+0000 mgr.y (mgr.24961) 4526 : cluster [DBG] pgmap v5299: 265 pgs: 265 active+clean; 24 KiB data, 908 MiB used, 714 GiB / 715 GiB avail 2022-04-28T22:09:35.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:34 smithi139 bash[43301]: cephadm 2022-04-28T22:09:33.408142+0000 mgr.y (mgr.24961) 4527 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2022-04-28T22:09:35.121 INFO:journalctl@ceph.mon.b.smithi139.stdout:Apr 28 22:09:34 smithi139 bash[43301]: audit 2022-04-28T22:09:33.744578+0000 mon.a (mon.0) 4352 : audit [INF] from='mgr.24961 172.21.15.59:0/3767179148' entity='mgr.y' 2022-04-29T01:58:26.519 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2022-04-29T01:58:26.543 DEBUG:teuthology.task.console_log:Killing console logger for smithi059 2022-04-29T01:58:26.544 DEBUG:teuthology.task.console_log:Killing console logger for smithi139 2022-04-29T01:58:26.544 DEBUG:teuthology.exit:Finished running handlers